autox/autox_recommend/demo/HM/HM_AutoX_recall_and_rank.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## import包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:01.011822Z",
     "start_time": "2022-05-14T04:24:01.008279Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../AutoX')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:05.983625Z",
     "start_time": "2022-05-14T04:24:01.014556Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/pandas/compat/_optional.py:138: UserWarning: Pandas requires version '2.7.0' or newer of 'numexpr' (version '2.6.9' currently installed).\n",
      "  warnings.warn(msg, UserWarning)\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:30: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  method='lar', copy_X=True, eps=np.finfo(np.float).eps,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:167: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  method='lar', copy_X=True, eps=np.finfo(np.float).eps,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:284: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  eps=np.finfo(np.float).eps, copy_Gram=True, verbose=0,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:862: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  eps=np.finfo(np.float).eps, copy_X=True, fit_path=True,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:1101: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  eps=np.finfo(np.float).eps, copy_X=True, fit_path=True,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:1127: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  eps=np.finfo(np.float).eps, positive=False):\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:1362: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  max_n_alphas=1000, n_jobs=None, eps=np.finfo(np.float).eps,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:1602: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  max_n_alphas=1000, n_jobs=None, eps=np.finfo(np.float).eps,\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/least_angle.py:1738: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  eps=np.finfo(np.float).eps, copy_X=True, positive=False):\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/decomposition/online_lda.py:29: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  EPS = np.finfo(np.float).eps\n",
      "/home/caihengxing/anaconda3/lib/python3.7/site-packages/sklearn/feature_extraction/image.py:167: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  dtype=np.int):\n"
     ]
    }
   ],
   "source": [
    "from autox import AutoXRecommend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:05.989922Z",
     "start_time": "2022-05-14T04:24:05.986810Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:06.084949Z",
     "start_time": "2022-05-14T04:24:05.993172Z"
    }
   },
   "outputs": [],
   "source": [
    "path = '/home/caihengxing/kaggle/h_and_m/input/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:54.597184Z",
     "start_time": "2022-05-14T04:24:06.087490Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "inter_df = pd.read_csv(path + 'transactions_train.csv', dtype={'article_id': str})\n",
    "user_df = pd.read_csv(path + 'customers.csv')\n",
    "item_df = pd.read_csv(path + 'articles.csv', dtype={'article_id': str})\n",
    "sample_submission = pd.read_csv(path + 'sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:54.615964Z",
     "start_time": "2022-05-14T04:24:54.600329Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_dat</th>\n",
       "      <th>customer_id</th>\n",
       "      <th>article_id</th>\n",
       "      <th>price</th>\n",
       "      <th>sales_channel_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018-09-20</td>\n",
       "      <td>000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...</td>\n",
       "      <td>0663713001</td>\n",
       "      <td>0.050831</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2018-09-20</td>\n",
       "      <td>000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...</td>\n",
       "      <td>0541518023</td>\n",
       "      <td>0.030492</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018-09-20</td>\n",
       "      <td>00007d2de826758b65a93dd24ce629ed66842531df6699...</td>\n",
       "      <td>0505221004</td>\n",
       "      <td>0.015237</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018-09-20</td>\n",
       "      <td>00007d2de826758b65a93dd24ce629ed66842531df6699...</td>\n",
       "      <td>0685687003</td>\n",
       "      <td>0.016932</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-09-20</td>\n",
       "      <td>00007d2de826758b65a93dd24ce629ed66842531df6699...</td>\n",
       "      <td>0685687004</td>\n",
       "      <td>0.016932</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        t_dat                                        customer_id  article_id  \\\n",
       "0  2018-09-20  000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...  0663713001   \n",
       "1  2018-09-20  000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...  0541518023   \n",
       "2  2018-09-20  00007d2de826758b65a93dd24ce629ed66842531df6699...  0505221004   \n",
       "3  2018-09-20  00007d2de826758b65a93dd24ce629ed66842531df6699...  0685687003   \n",
       "4  2018-09-20  00007d2de826758b65a93dd24ce629ed66842531df6699...  0685687004   \n",
       "\n",
       "      price  sales_channel_id  \n",
       "0  0.050831                 2  \n",
       "1  0.030492                 2  \n",
       "2  0.015237                 2  \n",
       "3  0.016932                 2  \n",
       "4  0.016932                 2  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inter_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 配置参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:24:54.694112Z",
     "start_time": "2022-05-14T04:24:54.618079Z"
    }
   },
   "outputs": [],
   "source": [
    "uid = 'customer_id'\n",
    "iid = 'article_id'\n",
    "time_col = 't_dat'\n",
    "recall_num = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 注意时间列需要转成时间格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:25:00.325869Z",
     "start_time": "2022-05-14T04:24:54.696364Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "inter_df[time_col] = pd.to_datetime(inter_df[time_col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:25:00.567717Z",
     "start_time": "2022-05-14T04:25:00.331214Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('2018-09-20 00:00:00'), Timestamp('2020-09-22 00:00:00'))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inter_df[time_col].min(), inter_df[time_col].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:25:00.574221Z",
     "start_time": "2022-05-14T04:25:00.570281Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31788324, 5)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inter_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:25:01.618527Z",
     "start_time": "2022-05-14T04:25:00.576512Z"
    }
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "data_used_time = datetime.datetime.strptime('2020-06-22', '%Y-%m-%d')\n",
    "inter_df = inter_df.loc[inter_df[time_col] > data_used_time]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:25:01.652706Z",
     "start_time": "2022-05-14T04:25:01.620873Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('2020-06-23 00:00:00'), Timestamp('2020-09-22 00:00:00'))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inter_df[time_col].min(), inter_df[time_col].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:25:01.673692Z",
     "start_time": "2022-05-14T04:25:01.654763Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3981458, 5)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inter_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 执行AutoXRecommend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T04:41:28.769411Z",
     "start_time": "2022-05-14T04:25:01.675506Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "popular_recall\n",
      "train\n",
      "2020-09-10 00:00:00 2020-09-16 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 68613/68613 [00:03<00:00, 20461.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HIT:  0.054450377315542246\n",
      "valid\n",
      "2020-09-17 00:00:00 2020-09-22 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 61950/61950 [00:03<00:00, 18052.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HIT:  0.06762614152025388\n",
      "\n",
      "history_recall\n",
      "train\n",
      "2020-09-10 00:00:00 2020-09-16 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 48796/48796 [00:00<00:00, 138152.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid\n",
      "2020-09-17 00:00:00 2020-09-22 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 44687/44687 [00:01<00:00, 42101.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "itemcf_recall\n",
      "train\n",
      "valid\n",
      "\n",
      "binary_recall\n",
      "train\n",
      "valid\n",
      "\n",
      "merge recalls\n",
      "train\n",
      "valid\n",
      "\n",
      "feature engineer\n",
      "train\n",
      "valid\n",
      "train_fe shape: (8140675, 61)\n",
      "valid_fe shape: (7415371, 61)\n",
      "\n",
      "ranker\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\tvalid_0's map@12: 0.769538\n",
      "[200]\tvalid_0's map@12: 0.770037\n",
      "[300]\tvalid_0's map@12: 0.770703\n",
      "[400]\tvalid_0's map@12: 0.771118\n",
      "[500]\tvalid_0's map@12: 0.771073\n",
      "Early stopping, best iteration is:\n",
      "[439]\tvalid_0's map@12: 0.771374\n",
      "defaultdict(<class 'dict'>, {'valid_0': {'map@12': 0.7713743327197474}})\n",
      "                        feature  importance\n",
      "21                 product_code        1417\n",
      "40               article_id_idx        1366\n",
      "1                  binary_score        1323\n",
      "39     latest_purchase_time_sub        1042\n",
      "31  purchase_corr_item_max_time         962\n",
      "27                department_no         851\n",
      "22              product_type_no         807\n",
      "30             garment_group_no         646\n",
      "29                   section_no         543\n",
      "0                  itemcf_score         497\n",
      "20                          age         432\n",
      "3            n_purchase_nunique         428\n",
      "23      graphical_appearance_no         354\n",
      "9                     price_sum         326\n",
      "24            colour_group_code         304\n",
      "32       purchase_corr_item_cnt         240\n",
      "25    perceived_colour_value_id         191\n",
      "26   perceived_colour_master_id         184\n",
      "16         sales_channel_id_sum         176\n",
      "13        sales_channel_id_mean         157\n",
      "\n",
      "local result calculation\n",
      "2020-09-17 00:00:00 2020-09-22 00:00:00\n",
      "mAP Score on Validation set: 0.02752116511050258\n",
      "##############################\n",
      "retrain\n",
      "\n",
      "popular_recall\n",
      "2020-09-17 00:00:00 2020-09-22 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 61950/61950 [00:03<00:00, 17810.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HIT:  0.06762614152025388\n",
      "\n",
      "history_recall\n",
      "2020-09-17 00:00:00 2020-09-22 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 44687/44687 [00:00<00:00, 123855.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "itemcf_recall\n",
      "\n",
      "binary_recall\n",
      "\n",
      "merge recalls\n",
      "\n",
      "feature engineer\n",
      "train_fe shape: (7415371, 61)\n",
      "\n",
      "ranker\n",
      "[100]\tvalid_0's map@12: 0.771972\n",
      "[200]\tvalid_0's map@12: 0.77371\n",
      "[300]\tvalid_0's map@12: 0.774858\n",
      "[400]\tvalid_0's map@12: 0.776024\n",
      "defaultdict(<class 'dict'>, {'valid_0': {'map@12': 0.7762112027212378}})\n",
      "                        feature  importance\n",
      "40               article_id_idx        1598\n",
      "21                 product_code        1408\n",
      "1                  binary_score        1274\n",
      "31  purchase_corr_item_max_time         900\n",
      "39     latest_purchase_time_sub         880\n",
      "22              product_type_no         737\n",
      "0                  itemcf_score         630\n",
      "30             garment_group_no         603\n",
      "27                department_no         601\n",
      "29                   section_no         501\n",
      "20                          age         468\n",
      "23      graphical_appearance_no         424\n",
      "9                     price_sum         361\n",
      "3            n_purchase_nunique         326\n",
      "24            colour_group_code         302\n",
      "26   perceived_colour_master_id         259\n",
      "32       purchase_corr_item_cnt         254\n",
      "13        sales_channel_id_mean         204\n",
      "25    perceived_colour_value_id         202\n",
      "6                    price_mean         160\n"
     ]
    }
   ],
   "source": [
    "autoXRecommend = AutoXRecommend()\n",
    "\n",
    "autoXRecommend.fit(inter_df = inter_df, user_df = user_df, item_df = item_df,\n",
    "                  uid = uid, iid = iid, time_col = time_col,\n",
    "                  recall_num = recall_num,\n",
    "                  debug = True, debug_save_path = './temp_h_and_m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T06:32:34.982264Z",
     "start_time": "2022-05-14T05:10:10.854335Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "popular recall, test\n",
      "2020-09-16 00:00:00 2020-09-22 00:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1371980/1371980 [00:36<00:00, 37501.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "history recall, test\n",
      "\n",
      "itemcf recall, test\n",
      "\n",
      "binary recall, test\n",
      "\n",
      "merge recalls\n",
      "\n",
      "feature engineer\n",
      "customer feature engineer\n",
      "num_cols: ['price', 'sales_channel_id']\n",
      "cat_cols: []\n",
      "interact feature engineer\n",
      "test_fe shape: (118376398, 60)\n",
      "\n",
      "inference\n",
      "[1/23]\n",
      "(5185377, 60)\n",
      "[2/23]\n",
      "(5171404, 60)\n",
      "[3/23]\n",
      "(5179673, 60)\n",
      "[4/23]\n",
      "(5171740, 60)\n",
      "[5/23]\n",
      "(5186864, 60)\n",
      "[6/23]\n",
      "(5172234, 60)\n",
      "[7/23]\n",
      "(5169199, 60)\n",
      "[8/23]\n",
      "(5182743, 60)\n",
      "[9/23]\n",
      "(5183403, 60)\n",
      "[10/23]\n",
      "(5173039, 60)\n",
      "[11/23]\n",
      "(5186632, 60)\n",
      "[12/23]\n",
      "(5171012, 60)\n",
      "[13/23]\n",
      "(5188590, 60)\n",
      "[14/23]\n",
      "(5171528, 60)\n",
      "[15/23]\n",
      "(5175754, 60)\n",
      "[16/23]\n",
      "(5165226, 60)\n",
      "[17/23]\n",
      "(5190626, 60)\n",
      "[18/23]\n",
      "(5168493, 60)\n",
      "[19/23]\n",
      "(5156278, 60)\n",
      "[20/23]\n",
      "(5175952, 60)\n",
      "[21/23]\n",
      "(5180041, 60)\n",
      "[22/23]\n",
      "(5178734, 60)\n",
      "[23/23]\n",
      "(4491856, 60)\n"
     ]
    }
   ],
   "source": [
    "test_users = list(sample_submission[uid])\n",
    "res = autoXRecommend.transform(test_users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T06:32:37.739210Z",
     "start_time": "2022-05-14T06:32:37.727582Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000dbacae5abe5e23885899a1fa44253a17956c6d1c3...</td>\n",
       "      <td>[0568601043, 0924243001, 0918522001, 092424300...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000423b00ade91418cceaf3b26c6af3dd342b51fd051e...</td>\n",
       "      <td>[0918522001, 0924243001, 0918292001, 091552900...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...</td>\n",
       "      <td>[0794321007, 0924243001, 0924243002, 091852200...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00005ca1c9ed5f5146b52ac8639a40ca9d57aeff4d1bd2...</td>\n",
       "      <td>[0924243001, 0918522001, 0918292001, 092424300...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00006413d8573cd20ed7128e53b7b13819fe5cfc2d801f...</td>\n",
       "      <td>[0896152002, 0924243001, 0918522001, 092424300...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         customer_id  \\\n",
       "0  00000dbacae5abe5e23885899a1fa44253a17956c6d1c3...   \n",
       "1  0000423b00ade91418cceaf3b26c6af3dd342b51fd051e...   \n",
       "2  000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...   \n",
       "3  00005ca1c9ed5f5146b52ac8639a40ca9d57aeff4d1bd2...   \n",
       "4  00006413d8573cd20ed7128e53b7b13819fe5cfc2d801f...   \n",
       "\n",
       "                                          prediction  \n",
       "0  [0568601043, 0924243001, 0918522001, 092424300...  \n",
       "1  [0918522001, 0924243001, 0918292001, 091552900...  \n",
       "2  [0794321007, 0924243001, 0924243002, 091852200...  \n",
       "3  [0924243001, 0918522001, 0918292001, 092424300...  \n",
       "4  [0896152002, 0924243001, 0918522001, 092424300...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T07:11:39.370728Z",
     "start_time": "2022-05-14T07:11:33.242873Z"
    }
   },
   "outputs": [],
   "source": [
    "res['prediction'] = res['prediction'].apply(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T07:12:14.585542Z",
     "start_time": "2022-05-14T07:11:42.122820Z"
    }
   },
   "outputs": [],
   "source": [
    "res.to_csv('./AutoX_hm.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T07:13:44.586282Z",
     "start_time": "2022-05-14T07:12:17.229749Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  adding: AutoX_hm.csv (deflated 79%)\n"
     ]
    }
   ],
   "source": [
    "!zip -r AutoX_hm.csv.zip AutoX_hm.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T07:14:16.012266Z",
     "start_time": "2022-05-14T07:14:16.007715Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1371980, 2), (1371980, 2))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.shape, sample_submission.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T07:14:22.133714Z",
     "start_time": "2022-05-14T07:14:22.123740Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000dbacae5abe5e23885899a1fa44253a17956c6d1c3...</td>\n",
       "      <td>0568601043 0924243001 0918522001 0924243002 09...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000423b00ade91418cceaf3b26c6af3dd342b51fd051e...</td>\n",
       "      <td>0918522001 0924243001 0918292001 0915529003 07...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...</td>\n",
       "      <td>0794321007 0924243001 0924243002 0918522001 09...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00005ca1c9ed5f5146b52ac8639a40ca9d57aeff4d1bd2...</td>\n",
       "      <td>0924243001 0918522001 0918292001 0924243002 09...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00006413d8573cd20ed7128e53b7b13819fe5cfc2d801f...</td>\n",
       "      <td>0896152002 0924243001 0918522001 0924243002 09...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         customer_id  \\\n",
       "0  00000dbacae5abe5e23885899a1fa44253a17956c6d1c3...   \n",
       "1  0000423b00ade91418cceaf3b26c6af3dd342b51fd051e...   \n",
       "2  000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...   \n",
       "3  00005ca1c9ed5f5146b52ac8639a40ca9d57aeff4d1bd2...   \n",
       "4  00006413d8573cd20ed7128e53b7b13819fe5cfc2d801f...   \n",
       "\n",
       "                                          prediction  \n",
       "0  0568601043 0924243001 0918522001 0924243002 09...  \n",
       "1  0918522001 0924243001 0918292001 0915529003 07...  \n",
       "2  0794321007 0924243001 0924243002 0918522001 09...  \n",
       "3  0924243001 0918522001 0918292001 0924243002 09...  \n",
       "4  0896152002 0924243001 0918522001 0924243002 09...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-14T07:14:31.514777Z",
     "start_time": "2022-05-14T07:14:31.502428Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000dbacae5abe5e23885899a1fa44253a17956c6d1c3...</td>\n",
       "      <td>0706016001 0706016002 0372860001 0610776002 07...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000423b00ade91418cceaf3b26c6af3dd342b51fd051e...</td>\n",
       "      <td>0706016001 0706016002 0372860001 0610776002 07...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...</td>\n",
       "      <td>0706016001 0706016002 0372860001 0610776002 07...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00005ca1c9ed5f5146b52ac8639a40ca9d57aeff4d1bd2...</td>\n",
       "      <td>0706016001 0706016002 0372860001 0610776002 07...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00006413d8573cd20ed7128e53b7b13819fe5cfc2d801f...</td>\n",
       "      <td>0706016001 0706016002 0372860001 0610776002 07...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         customer_id  \\\n",
       "0  00000dbacae5abe5e23885899a1fa44253a17956c6d1c3...   \n",
       "1  0000423b00ade91418cceaf3b26c6af3dd342b51fd051e...   \n",
       "2  000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...   \n",
       "3  00005ca1c9ed5f5146b52ac8639a40ca9d57aeff4d1bd2...   \n",
       "4  00006413d8573cd20ed7128e53b7b13819fe5cfc2d801f...   \n",
       "\n",
       "                                          prediction  \n",
       "0  0706016001 0706016002 0372860001 0610776002 07...  \n",
       "1  0706016001 0706016002 0372860001 0610776002 07...  \n",
       "2  0706016001 0706016002 0372860001 0610776002 07...  \n",
       "3  0706016001 0706016002 0372860001 0610776002 07...  \n",
       "4  0706016001 0706016002 0372860001 0610776002 07...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}