Dask/make-demo-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: pandas\n",
      "Version: 1.0.1\n",
      "Summary: Powerful data structures for data analysis, time series, and statistics\n",
      "Home-page: https://pandas.pydata.org\n",
      "Author: None\n",
      "Author-email: None\n",
      "License: BSD\n",
      "Location: /Users/NothingToLose/clepy/myvenv/lib/python3.8/site-packages\n",
      "Requires: numpy, python-dateutil, pytz\n",
      "Required-by: \n",
      "---\n",
      "Name: dask\n",
      "Version: 2.12.0\n",
      "Summary: Parallel PyData with Task Scheduling\n",
      "Home-page: https://github.com/dask/dask/\n",
      "Author: None\n",
      "Author-email: None\n",
      "License: BSD\n",
      "Location: /Users/NothingToLose/clepy/myvenv/lib/python3.8/site-packages\n",
      "Requires: \n",
      "Required-by: \n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip show pandas dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from dask import dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    np.random.randint(0,100,size=(9000000,26)),\n",
    "    columns=list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')\n",
    ")\n",
    "ddf = dd.from_pandas(df, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/NothingToLose/clepy/myvenv/lib/python3.8/site-packages/dask/dataframe/core.py:3073: UserWarning: \n",
      "You did not provide metadata, so Dask is running your function on a small dataset to guess output types. It is possible that Dask will guess incorrectly.\n",
      "To provide an explicit output types or to silence this message, please provide the `meta=` keyword, as described in the map or apply function that you are using.\n",
      "  Before: .apply(func)\n",
      "  After:  .apply(func, meta=('A', 'object'))\n",
      "\n",
      "  warnings.warn(meta_warning(meta))\n"
     ]
    }
   ],
   "source": [
    "ddf['useless_letter'] = ddf['A'].apply(lambda x: np.random.choice(list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')))\n",
    "ddf['alphabet'] = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
    "out_columns = ddf.columns.tolist()\n",
    "out_columns.reverse()\n",
    "ddf = ddf[out_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alphabet</th>\n",
       "      <th>useless_letter</th>\n",
       "      <th>Z</th>\n",
       "      <th>Y</th>\n",
       "      <th>X</th>\n",
       "      <th>W</th>\n",
       "      <th>V</th>\n",
       "      <th>U</th>\n",
       "      <th>T</th>\n",
       "      <th>S</th>\n",
       "      <th>...</th>\n",
       "      <th>J</th>\n",
       "      <th>I</th>\n",
       "      <th>H</th>\n",
       "      <th>G</th>\n",
       "      <th>F</th>\n",
       "      <th>E</th>\n",
       "      <th>D</th>\n",
       "      <th>C</th>\n",
       "      <th>B</th>\n",
       "      <th>A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCDEFGHIJKLMNOPQRSTUVWXYZ</td>\n",
       "      <td>W</td>\n",
       "      <td>1</td>\n",
       "      <td>66</td>\n",
       "      <td>69</td>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>64</td>\n",
       "      <td>83</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>33</td>\n",
       "      <td>57</td>\n",
       "      <td>23</td>\n",
       "      <td>86</td>\n",
       "      <td>48</td>\n",
       "      <td>30</td>\n",
       "      <td>91</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABCDEFGHIJKLMNOPQRSTUVWXYZ</td>\n",
       "      <td>J</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>36</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>75</td>\n",
       "      <td>60</td>\n",
       "      <td>73</td>\n",
       "      <td>...</td>\n",
       "      <td>42</td>\n",
       "      <td>91</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>64</td>\n",
       "      <td>13</td>\n",
       "      <td>43</td>\n",
       "      <td>47</td>\n",
       "      <td>94</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABCDEFGHIJKLMNOPQRSTUVWXYZ</td>\n",
       "      <td>D</td>\n",
       "      <td>40</td>\n",
       "      <td>67</td>\n",
       "      <td>36</td>\n",
       "      <td>54</td>\n",
       "      <td>46</td>\n",
       "      <td>5</td>\n",
       "      <td>57</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>82</td>\n",
       "      <td>76</td>\n",
       "      <td>24</td>\n",
       "      <td>60</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>64</td>\n",
       "      <td>28</td>\n",
       "      <td>26</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABCDEFGHIJKLMNOPQRSTUVWXYZ</td>\n",
       "      <td>D</td>\n",
       "      <td>91</td>\n",
       "      <td>98</td>\n",
       "      <td>8</td>\n",
       "      <td>36</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>29</td>\n",
       "      <td>90</td>\n",
       "      <td>...</td>\n",
       "      <td>88</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>51</td>\n",
       "      <td>52</td>\n",
       "      <td>87</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>19</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABCDEFGHIJKLMNOPQRSTUVWXYZ</td>\n",
       "      <td>H</td>\n",
       "      <td>87</td>\n",
       "      <td>68</td>\n",
       "      <td>73</td>\n",
       "      <td>78</td>\n",
       "      <td>39</td>\n",
       "      <td>67</td>\n",
       "      <td>57</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>51</td>\n",
       "      <td>18</td>\n",
       "      <td>95</td>\n",
       "      <td>71</td>\n",
       "      <td>28</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     alphabet useless_letter   Z   Y   X   W   V   U   T   S  \\\n",
       "0  ABCDEFGHIJKLMNOPQRSTUVWXYZ              W   1  66  69   4  70  64  83  55   \n",
       "1  ABCDEFGHIJKLMNOPQRSTUVWXYZ              J  25  26  36  34   0  75  60  73   \n",
       "2  ABCDEFGHIJKLMNOPQRSTUVWXYZ              D  40  67  36  54  46   5  57  50   \n",
       "3  ABCDEFGHIJKLMNOPQRSTUVWXYZ              D  91  98   8  36  17   3  29  90   \n",
       "4  ABCDEFGHIJKLMNOPQRSTUVWXYZ              H  87  68  73  78  39  67  57  24   \n",
       "\n",
       "   ...   J   I   H   G   F   E   D   C   B   A  \n",
       "0  ...  36  28  33  57  23  86  48  30  91  84  \n",
       "1  ...  42  91   8  24  64  13  43  47  94  11  \n",
       "2  ...  82  76  24  60   3  55  64  28  26  89  \n",
       "3  ...  88  24   7  51  52  87   1   6  19  48  \n",
       "4  ...   9  14   0   2  51  18  95  71  28  13  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/NothingToLose/clepy/Dask/sample_data.csv']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "ddf.to_csv('sample_data.csv', single_file=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "ddf.to_parquet('sample_data_parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}