8.Final_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Single XGB Classifier after feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Basic ML modules\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.cross_validation import cross_val_score, KFold\n",
    "\n",
    "from time import time\n",
    "from datetime import datetime, timedelta, date\n",
    "\n",
    "# Calendar modules - needed for holidays\n",
    "from workalendar.usa import UnitedStates\n",
    "from workalendar.europe import France, Germany, Italy, UnitedKingdom, Portugal\n",
    "from workalendar.oceania import Australia\n",
    "\n",
    "# Modules for plots\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Command so that plots appear in the iPython Notebook\n",
    "%matplotlib inline\n",
    "\n",
    "# File management modules\n",
    "from os import makedirs\n",
    "from os.path import exists, join\n",
    "\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "intermediate_path = './intermediate/'\n",
    "output_path = './output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load data\n",
    "df_train = pd.read_csv('./input/train_users_2.csv')\n",
    "df_test = pd.read_csv('./input/test_users.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create labels \n",
    "labels = df_train['country_destination'].values\n",
    "df_train = df_train.drop(['country_destination'], axis=1)\n",
    "id_test = df_test['id']\n",
    "piv_train = df_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date_account_created</th>\n",
       "      <th>timestamp_first_active</th>\n",
       "      <th>date_first_booking</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>2010-06-28</td>\n",
       "      <td>20090319043255</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>2011-05-25</td>\n",
       "      <td>20090523174809</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>2010-09-28</td>\n",
       "      <td>20090609231247</td>\n",
       "      <td>2010-08-02</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>IE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>20091031060129</td>\n",
       "      <td>2012-09-08</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Firefox</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>2010-09-14</td>\n",
       "      <td>20091208061105</td>\n",
       "      <td>2010-02-18</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id date_account_created  timestamp_first_active date_first_booking  \\\n",
       "0  gxn3p5htnn           2010-06-28          20090319043255                NaN   \n",
       "1  820tgsjxq7           2011-05-25          20090523174809                NaN   \n",
       "2  4ft3gnwmtx           2010-09-28          20090609231247         2010-08-02   \n",
       "3  bjjt8pjhuk           2011-12-05          20091031060129         2012-09-08   \n",
       "4  87mebub9p4           2010-09-14          20091208061105         2010-02-18   \n",
       "\n",
       "      gender  age signup_method  signup_flow language affiliate_channel  \\\n",
       "0  -unknown-  NaN      facebook            0       en            direct   \n",
       "1       MALE   38      facebook            0       en               seo   \n",
       "2     FEMALE   56         basic            3       en            direct   \n",
       "3     FEMALE   42      facebook            0       en            direct   \n",
       "4  -unknown-   41         basic            0       en            direct   \n",
       "\n",
       "  affiliate_provider first_affiliate_tracked signup_app first_device_type  \\\n",
       "0             direct               untracked        Web       Mac Desktop   \n",
       "1             google               untracked        Web       Mac Desktop   \n",
       "2             direct               untracked        Web   Windows Desktop   \n",
       "3             direct               untracked        Web       Mac Desktop   \n",
       "4             direct               untracked        Web       Mac Desktop   \n",
       "\n",
       "  first_browser  \n",
       "0        Chrome  \n",
       "1        Chrome  \n",
       "2            IE  \n",
       "3       Firefox  \n",
       "4        Chrome  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date_account_created</th>\n",
       "      <th>timestamp_first_active</th>\n",
       "      <th>date_first_booking</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5uwns89zht</td>\n",
       "      <td>2014-07-01</td>\n",
       "      <td>20140701000006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>35</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Moweb</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>Mobile Safari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jtl0dijy2j</td>\n",
       "      <td>2014-07-01</td>\n",
       "      <td>20140701000051</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Moweb</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>Mobile Safari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>xx0ulgorjt</td>\n",
       "      <td>2014-07-01</td>\n",
       "      <td>20140701000148</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>linked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c6puo6ix0</td>\n",
       "      <td>2014-07-01</td>\n",
       "      <td>20140701000215</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>linked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>IE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>czqhjk3yfe</td>\n",
       "      <td>2014-07-01</td>\n",
       "      <td>20140701000305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Safari</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id date_account_created  timestamp_first_active  \\\n",
       "0  5uwns89zht           2014-07-01          20140701000006   \n",
       "1  jtl0dijy2j           2014-07-01          20140701000051   \n",
       "2  xx0ulgorjt           2014-07-01          20140701000148   \n",
       "3  6c6puo6ix0           2014-07-01          20140701000215   \n",
       "4  czqhjk3yfe           2014-07-01          20140701000305   \n",
       "\n",
       "   date_first_booking     gender  age signup_method  signup_flow language  \\\n",
       "0                 NaN     FEMALE   35      facebook            0       en   \n",
       "1                 NaN  -unknown-  NaN         basic            0       en   \n",
       "2                 NaN  -unknown-  NaN         basic            0       en   \n",
       "3                 NaN  -unknown-  NaN         basic            0       en   \n",
       "4                 NaN  -unknown-  NaN         basic            0       en   \n",
       "\n",
       "  affiliate_channel affiliate_provider first_affiliate_tracked signup_app  \\\n",
       "0            direct             direct               untracked      Moweb   \n",
       "1            direct             direct               untracked      Moweb   \n",
       "2            direct             direct                  linked        Web   \n",
       "3            direct             direct                  linked        Web   \n",
       "4            direct             direct               untracked        Web   \n",
       "\n",
       "  first_device_type  first_browser  \n",
       "0            iPhone  Mobile Safari  \n",
       "1            iPhone  Mobile Safari  \n",
       "2   Windows Desktop         Chrome  \n",
       "3   Windows Desktop             IE  \n",
       "4       Mac Desktop         Safari  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(213451, 15)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Creating and Encoding y\n",
    "le_y = LabelEncoder()\n",
    "y = le_y.fit_transform(labels) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Creating a DataFrame with train+test data\n",
    "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
    "\n",
    "# dropping column not available in test data\n",
    "df_all = df_all.drop('date_first_booking', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date_account_created</th>\n",
       "      <th>timestamp_first_active</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>2010-06-28</td>\n",
       "      <td>20090319043255</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>2011-05-25</td>\n",
       "      <td>20090523174809</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>2010-09-28</td>\n",
       "      <td>20090609231247</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>IE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>20091031060129</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Firefox</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>2010-09-14</td>\n",
       "      <td>20091208061105</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id date_account_created  timestamp_first_active     gender  age  \\\n",
       "0  gxn3p5htnn           2010-06-28          20090319043255  -unknown-  NaN   \n",
       "1  820tgsjxq7           2011-05-25          20090523174809       MALE   38   \n",
       "2  4ft3gnwmtx           2010-09-28          20090609231247     FEMALE   56   \n",
       "3  bjjt8pjhuk           2011-12-05          20091031060129     FEMALE   42   \n",
       "4  87mebub9p4           2010-09-14          20091208061105  -unknown-   41   \n",
       "\n",
       "  signup_method  signup_flow language affiliate_channel affiliate_provider  \\\n",
       "0      facebook            0       en            direct             direct   \n",
       "1      facebook            0       en               seo             google   \n",
       "2         basic            3       en            direct             direct   \n",
       "3      facebook            0       en            direct             direct   \n",
       "4         basic            0       en            direct             direct   \n",
       "\n",
       "  first_affiliate_tracked signup_app first_device_type first_browser  \n",
       "0               untracked        Web       Mac Desktop        Chrome  \n",
       "1               untracked        Web       Mac Desktop        Chrome  \n",
       "2               untracked        Web   Windows Desktop            IE  \n",
       "3               untracked        Web       Mac Desktop       Firefox  \n",
       "4               untracked        Web       Mac Desktop        Chrome  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reformatting columns with time data and some feature engineering\n",
    "\n",
    "# from string to datetime format\n",
    "df_all['datehour_timestamp_first_active'] = df_all['timestamp_first_active'].astype(str)\n",
    "df_all['date_timestamp_first_active'] = [ d[:8] for d in df_all['datehour_timestamp_first_active'] ]\n",
    "df_all['hour_timestamp_first_active'] = [ d[8:] for d in df_all['datehour_timestamp_first_active'] ]\n",
    "\n",
    "df_all['date_timestamp_first_active'] = [ datetime.strptime(d, '%Y%m%d') for d in df_all['date_timestamp_first_active']]\n",
    "df_all['hour_timestamp_first_active'] = [ datetime.strptime(h, '%H%M%S') for h in df_all['hour_timestamp_first_active']]\n",
    "\n",
    "df_all['date_account_created'] = [ datetime.strptime(d, '%Y-%m-%d') for d in df_all['date_account_created']]\n",
    "\n",
    " \n",
    "# create d_ac-fa: number of days between account creation and first access\n",
    "df_all['d_ac-fa'] = df_all['date_account_created'] - df_all['date_timestamp_first_active']\n",
    "df_all['d_ac-fa'] = df_all['d_ac-fa'].astype('int')/8.64e13  #  CHECK!!!!\n",
    "\n",
    "# create mo_first_active: month of the first active\n",
    "df_all['mo_first_active'] = [ d.month if not pd.isnull(d) else 0 for d in df_all['date_timestamp_first_active'] ]\n",
    "df_all['ho_first_active'] = [ d.hour if not pd.isnull(d) else 0 for d in df_all['hour_timestamp_first_active'] ]\n",
    "\n",
    "# create ye_first_active: year of the first active\n",
    "# NOT USED\n",
    "#df_all['ye_first_active'] = [ d.year if not pd.isnull(d) else 0 for d in df_all['date_timestamp_first_active'] ]\n",
    "\n",
    "# create weekday_first_active: day of week (categorical variable)\n",
    "df_all['weekday_first_active'] = [ datetime.strftime(d,'%a') if not pd.isnull(d) else d for d in df_all['date_timestamp_first_active']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create features\n",
    "# d_ac : day of the month account creation \n",
    "# d_fa : day of the month first access\n",
    "df_all['d_ac'] = [ d.day if not pd.isnull(d) else 0 for d in df_all['date_account_created']  ]    \n",
    "df_all['d_fa'] = [ d.day if not pd.isnull(d) else 0 for d in df_all['date_timestamp_first_active']  ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Transform timestamp into date format\n",
    "df_all['date_timestamp_first_active'] = [ d.date() for d in df_all.date_timestamp_first_active]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date_account_created</th>\n",
       "      <th>timestamp_first_active</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>...</th>\n",
       "      <th>first_browser</th>\n",
       "      <th>datehour_timestamp_first_active</th>\n",
       "      <th>date_timestamp_first_active</th>\n",
       "      <th>hour_timestamp_first_active</th>\n",
       "      <th>d_ac-fa</th>\n",
       "      <th>mo_first_active</th>\n",
       "      <th>ho_first_active</th>\n",
       "      <th>weekday_first_active</th>\n",
       "      <th>d_ac</th>\n",
       "      <th>d_fa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>2010-06-28</td>\n",
       "      <td>20090319043255</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>20090319043255</td>\n",
       "      <td>2009-03-19</td>\n",
       "      <td>1900-01-01 04:32:55</td>\n",
       "      <td>466</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Thu</td>\n",
       "      <td>28</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>2011-05-25</td>\n",
       "      <td>20090523174809</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>...</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>20090523174809</td>\n",
       "      <td>2009-05-23</td>\n",
       "      <td>1900-01-01 17:48:09</td>\n",
       "      <td>732</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>Sat</td>\n",
       "      <td>25</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>2010-09-28</td>\n",
       "      <td>20090609231247</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>IE</td>\n",
       "      <td>20090609231247</td>\n",
       "      <td>2009-06-09</td>\n",
       "      <td>1900-01-01 23:12:47</td>\n",
       "      <td>476</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>Tue</td>\n",
       "      <td>28</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>20091031060129</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>Firefox</td>\n",
       "      <td>20091031060129</td>\n",
       "      <td>2009-10-31</td>\n",
       "      <td>1900-01-01 06:01:29</td>\n",
       "      <td>765</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>Sat</td>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>2010-09-14</td>\n",
       "      <td>20091208061105</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>20091208061105</td>\n",
       "      <td>2009-12-08</td>\n",
       "      <td>1900-01-01 06:11:05</td>\n",
       "      <td>280</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>Tue</td>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id date_account_created  timestamp_first_active     gender  age  \\\n",
       "0  gxn3p5htnn           2010-06-28          20090319043255  -unknown-  NaN   \n",
       "1  820tgsjxq7           2011-05-25          20090523174809       MALE   38   \n",
       "2  4ft3gnwmtx           2010-09-28          20090609231247     FEMALE   56   \n",
       "3  bjjt8pjhuk           2011-12-05          20091031060129     FEMALE   42   \n",
       "4  87mebub9p4           2010-09-14          20091208061105  -unknown-   41   \n",
       "\n",
       "  signup_method  signup_flow language affiliate_channel affiliate_provider  \\\n",
       "0      facebook            0       en            direct             direct   \n",
       "1      facebook            0       en               seo             google   \n",
       "2         basic            3       en            direct             direct   \n",
       "3      facebook            0       en            direct             direct   \n",
       "4         basic            0       en            direct             direct   \n",
       "\n",
       "   ...  first_browser datehour_timestamp_first_active  \\\n",
       "0  ...         Chrome                  20090319043255   \n",
       "1  ...         Chrome                  20090523174809   \n",
       "2  ...             IE                  20090609231247   \n",
       "3  ...        Firefox                  20091031060129   \n",
       "4  ...         Chrome                  20091208061105   \n",
       "\n",
       "  date_timestamp_first_active hour_timestamp_first_active d_ac-fa  \\\n",
       "0                  2009-03-19         1900-01-01 04:32:55     466   \n",
       "1                  2009-05-23         1900-01-01 17:48:09     732   \n",
       "2                  2009-06-09         1900-01-01 23:12:47     476   \n",
       "3                  2009-10-31         1900-01-01 06:01:29     765   \n",
       "4                  2009-12-08         1900-01-01 06:11:05     280   \n",
       "\n",
       "  mo_first_active ho_first_active  weekday_first_active  d_ac  d_fa  \n",
       "0               3               4                   Thu    28    19  \n",
       "1               5              17                   Sat    25    23  \n",
       "2               6              23                   Tue    28     9  \n",
       "3              10               6                   Sat     5    31  \n",
       "4              12               6                   Tue    14     8  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# checkpoint 0\n",
    "df_all_ck0 = df_all.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# revert to checkpoint 0\n",
    "#df_all = df_all_ck0.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Engineering: Including Holidays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def daterange(start_date, end_date):\n",
    "    for n in range(int ((end_date - start_date).days)):\n",
    "        yield start_date + timedelta(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create days_to_next_CC_hol \n",
    "# CC is the country code\n",
    "# \n",
    "\n",
    "country_dict = { 'NDF': 'None', 'US': UnitedStates(), 'other': 'None', 'FR': France(), \n",
    " 'IT': Italy(), 'GB': UnitedKingdom(), 'ES': 'None', 'CA': 'None', 'DE': Germany(), \n",
    " 'NL' : 'None', 'AU' : Australia(), 'PT' : Portugal() }\n",
    "\n",
    "# only end up using holidays in USA\n",
    "country_dict = { 'NDF': 'None', 'US': UnitedStates(), 'other': 'None', 'FR': 'None', \n",
    " 'IT': 'None', 'GB': 'None', 'ES': 'None', 'CA': 'None', 'DE': 'None', \n",
    " 'NL' : 'None', 'AU' : 'None', 'PT' : 'None' }\n",
    "\n",
    "for c in country_dict:\n",
    "    if country_dict[c] != 'None':\n",
    "        hol = country_dict[c].holidays(2008)\n",
    "        for year in range(2009, 2016, 1):\n",
    "            hol += country_dict[c].holidays(year)\n",
    "        \n",
    "        hol = pd.Series([h[0] for h in hol ])\n",
    "        \n",
    "        hol_dict = {}\n",
    "        for d in daterange( date(2009, 1, 1), date(2014, 12, 31) ):\n",
    "            delta_list = hol - d\n",
    "            delta_list = [dd.days for dd in delta_list]\n",
    "            next_holiday_delta = min([n for n in delta_list if n >= 0])\n",
    "            nex2_holiday_delta = min([n for n in delta_list if n > next_holiday_delta])\n",
    "            hol_dict[d] = (next_holiday_delta, nex2_holiday_delta)\n",
    "            \n",
    "            column_name = 'days_to_next_' + c + '_hol'\n",
    "        df_all[column_name] = [ hol_dict[d][0] for d in df_all['date_timestamp_first_active'] ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(275547, 24)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# creating country ranking columns\n",
    "\n",
    "for c in country_list:\n",
    "    df_all[c + '_rank'] = [ c_am[a, g][c]  if (not pd.isnull(a)) & (a < 105) else -999\n",
    "                           for (a, g) in df_all[['age', 'gender']].itertuples(index=False) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date_account_created</th>\n",
       "      <th>timestamp_first_active</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>...</th>\n",
       "      <th>AU_rank</th>\n",
       "      <th>CA_rank</th>\n",
       "      <th>DE_rank</th>\n",
       "      <th>ES_rank</th>\n",
       "      <th>FR_rank</th>\n",
       "      <th>GB_rank</th>\n",
       "      <th>IT_rank</th>\n",
       "      <th>NL_rank</th>\n",
       "      <th>PT_rank</th>\n",
       "      <th>US_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>2010-06-28</td>\n",
       "      <td>20090319043255</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>2011-05-25</td>\n",
       "      <td>20090523174809</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>45</td>\n",
       "      <td>68</td>\n",
       "      <td>44</td>\n",
       "      <td>47</td>\n",
       "      <td>53</td>\n",
       "      <td>44</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>2010-09-28</td>\n",
       "      <td>20090609231247</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>49</td>\n",
       "      <td>51</td>\n",
       "      <td>50</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>20091031060129</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>49</td>\n",
       "      <td>46</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>58</td>\n",
       "      <td>52</td>\n",
       "      <td>57</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>2010-09-14</td>\n",
       "      <td>20091208061105</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>50</td>\n",
       "      <td>49</td>\n",
       "      <td>66</td>\n",
       "      <td>53</td>\n",
       "      <td>50</td>\n",
       "      <td>61</td>\n",
       "      <td>52</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id date_account_created  timestamp_first_active     gender  age  \\\n",
       "0  gxn3p5htnn           2010-06-28          20090319043255  -unknown-  NaN   \n",
       "1  820tgsjxq7           2011-05-25          20090523174809       MALE   38   \n",
       "2  4ft3gnwmtx           2010-09-28          20090609231247     FEMALE   56   \n",
       "3  bjjt8pjhuk           2011-12-05          20091031060129     FEMALE   42   \n",
       "4  87mebub9p4           2010-09-14          20091208061105  -unknown-   41   \n",
       "\n",
       "  signup_method  signup_flow language affiliate_channel affiliate_provider  \\\n",
       "0      facebook            0       en            direct             direct   \n",
       "1      facebook            0       en               seo             google   \n",
       "2         basic            3       en            direct             direct   \n",
       "3      facebook            0       en            direct             direct   \n",
       "4         basic            0       en            direct             direct   \n",
       "\n",
       "    ...    AU_rank CA_rank DE_rank ES_rank FR_rank GB_rank IT_rank  NL_rank  \\\n",
       "0   ...       -999    -999    -999    -999    -999    -999    -999     -999   \n",
       "1   ...         50      50      45      68      44      47      53       44   \n",
       "2   ...         46      54      54      48      48      46      49       51   \n",
       "3   ...         52      49      46      60      50      50      58       52   \n",
       "4   ...         52      50      49      66      53      50      61       52   \n",
       "\n",
       "   PT_rank  US_rank  \n",
       "0     -999     -999  \n",
       "1       61       48  \n",
       "2       50       51  \n",
       "3       57       47  \n",
       "4       61       48  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# removing features not going to be used\n",
    "features = list(df_all.columns)\n",
    "\n",
    "remove = ['date_account_created',\n",
    " 'timestamp_first_active',\n",
    " 'datehour_timestamp_first_active',\n",
    " 'date_timestamp_first_active',\n",
    " 'hour_timestamp_first_active']\n",
    "\n",
    "for rf in remove:\n",
    "    features.remove(rf)\n",
    "\n",
    "df_all = df_all[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>...</th>\n",
       "      <th>AU_rank</th>\n",
       "      <th>CA_rank</th>\n",
       "      <th>DE_rank</th>\n",
       "      <th>ES_rank</th>\n",
       "      <th>FR_rank</th>\n",
       "      <th>GB_rank</th>\n",
       "      <th>IT_rank</th>\n",
       "      <th>NL_rank</th>\n",
       "      <th>PT_rank</th>\n",
       "      <th>US_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>45</td>\n",
       "      <td>68</td>\n",
       "      <td>44</td>\n",
       "      <td>47</td>\n",
       "      <td>53</td>\n",
       "      <td>44</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>49</td>\n",
       "      <td>51</td>\n",
       "      <td>50</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>49</td>\n",
       "      <td>46</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>58</td>\n",
       "      <td>52</td>\n",
       "      <td>57</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>50</td>\n",
       "      <td>49</td>\n",
       "      <td>66</td>\n",
       "      <td>53</td>\n",
       "      <td>50</td>\n",
       "      <td>61</td>\n",
       "      <td>52</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id     gender  age signup_method  signup_flow language  \\\n",
       "0  gxn3p5htnn  -unknown-  NaN      facebook            0       en   \n",
       "1  820tgsjxq7       MALE   38      facebook            0       en   \n",
       "2  4ft3gnwmtx     FEMALE   56         basic            3       en   \n",
       "3  bjjt8pjhuk     FEMALE   42      facebook            0       en   \n",
       "4  87mebub9p4  -unknown-   41         basic            0       en   \n",
       "\n",
       "  affiliate_channel affiliate_provider first_affiliate_tracked signup_app  \\\n",
       "0            direct             direct               untracked        Web   \n",
       "1               seo             google               untracked        Web   \n",
       "2            direct             direct               untracked        Web   \n",
       "3            direct             direct               untracked        Web   \n",
       "4            direct             direct               untracked        Web   \n",
       "\n",
       "    ...    AU_rank CA_rank  DE_rank  ES_rank  FR_rank GB_rank  IT_rank  \\\n",
       "0   ...       -999    -999     -999     -999     -999    -999     -999   \n",
       "1   ...         50      50       45       68       44      47       53   \n",
       "2   ...         46      54       54       48       48      46       49   \n",
       "3   ...         52      49       46       60       50      50       58   \n",
       "4   ...         52      50       49       66       53      50       61   \n",
       "\n",
       "   NL_rank  PT_rank  US_rank  \n",
       "0     -999     -999     -999  \n",
       "1       44       61       48  \n",
       "2       51       50       51  \n",
       "3       52       57       47  \n",
       "4       52       61       48  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(275547, 29)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# checking age distribution\n",
    "if False:\n",
    "    sns.set_context(\"talk\")\n",
    "    box = sns.distplot(df_all.age.dropna(), hist_kws={\"range\" : (0, 100)}, bins = 50)\n",
    "    box.set(xlim=(0, 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# checkpoint 1\n",
    "df_all_ck1 = df_all.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# revert to checkpoint\n",
    "#df_all = df_all_ck1.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                         275547\n",
       "gender                     275547\n",
       "age                        158681\n",
       "signup_method              275547\n",
       "signup_flow                275547\n",
       "language                   275547\n",
       "affiliate_channel          275547\n",
       "affiliate_provider         275547\n",
       "first_affiliate_tracked    269462\n",
       "signup_app                 275547\n",
       "first_device_type          275547\n",
       "first_browser              275547\n",
       "d_ac-fa                    275547\n",
       "mo_first_active            275547\n",
       "ho_first_active            275547\n",
       "weekday_first_active       275547\n",
       "d_ac                       275547\n",
       "d_fa                       275547\n",
       "days_to_next_US_hol        275547\n",
       "AU_rank                    275547\n",
       "CA_rank                    275547\n",
       "DE_rank                    275547\n",
       "ES_rank                    275547\n",
       "FR_rank                    275547\n",
       "GB_rank                    275547\n",
       "IT_rank                    275547\n",
       "NL_rank                    275547\n",
       "PT_rank                    275547\n",
       "US_rank                    275547\n",
       "dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note age has a lot of NaN\n",
    "# and first_affiliate_tracked has few\n",
    "df_all.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# imputation -1 for ages NaN\n",
    "df_all['age'].fillna(-999, inplace=True)\n",
    "\n",
    "# imputation most frequent 'untracked' for first_affiliate_tracked\n",
    "df_all['first_affiliate_tracked'].fillna(df_all.first_affiliate_tracked.value_counts().index[0], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# checkpoint 2\n",
    "df_all_ck2 = df_all.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# revert to checkpoint\n",
    "#df_all = df_all_ck2.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>...</th>\n",
       "      <th>AU_rank</th>\n",
       "      <th>CA_rank</th>\n",
       "      <th>DE_rank</th>\n",
       "      <th>ES_rank</th>\n",
       "      <th>FR_rank</th>\n",
       "      <th>GB_rank</th>\n",
       "      <th>IT_rank</th>\n",
       "      <th>NL_rank</th>\n",
       "      <th>PT_rank</th>\n",
       "      <th>US_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>-999</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>45</td>\n",
       "      <td>68</td>\n",
       "      <td>44</td>\n",
       "      <td>47</td>\n",
       "      <td>53</td>\n",
       "      <td>44</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>49</td>\n",
       "      <td>51</td>\n",
       "      <td>50</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>49</td>\n",
       "      <td>46</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>58</td>\n",
       "      <td>52</td>\n",
       "      <td>57</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>50</td>\n",
       "      <td>49</td>\n",
       "      <td>66</td>\n",
       "      <td>53</td>\n",
       "      <td>50</td>\n",
       "      <td>61</td>\n",
       "      <td>52</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id     gender  age signup_method  signup_flow language  \\\n",
       "0  gxn3p5htnn  -unknown- -999      facebook            0       en   \n",
       "1  820tgsjxq7       MALE   38      facebook            0       en   \n",
       "2  4ft3gnwmtx     FEMALE   56         basic            3       en   \n",
       "3  bjjt8pjhuk     FEMALE   42      facebook            0       en   \n",
       "4  87mebub9p4  -unknown-   41         basic            0       en   \n",
       "\n",
       "  affiliate_channel affiliate_provider first_affiliate_tracked signup_app  \\\n",
       "0            direct             direct               untracked        Web   \n",
       "1               seo             google               untracked        Web   \n",
       "2            direct             direct               untracked        Web   \n",
       "3            direct             direct               untracked        Web   \n",
       "4            direct             direct               untracked        Web   \n",
       "\n",
       "    ...    AU_rank CA_rank  DE_rank  ES_rank  FR_rank GB_rank  IT_rank  \\\n",
       "0   ...       -999    -999     -999     -999     -999    -999     -999   \n",
       "1   ...         50      50       45       68       44      47       53   \n",
       "2   ...         46      54       54       48       48      46       49   \n",
       "3   ...         52      49       46       60       50      50       58   \n",
       "4   ...         52      50       49       66       53      50       61   \n",
       "\n",
       "   NL_rank  PT_rank  US_rank  \n",
       "0     -999     -999     -999  \n",
       "1       44       61       48  \n",
       "2       51       50       51  \n",
       "3       52       57       47  \n",
       "4       52       61       48  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Remove age outliers\n",
    "\n",
    "av = df_all.age.values\n",
    "df_all['age'] = np.where(np.logical_or(av<14, av>104), -999, av)\n",
    "\n",
    "# reduce cardinality of feature\n",
    "df_all['days_to_next_US_hol'] = np.ceil((df_all.days_to_next_US_hol / 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>...</th>\n",
       "      <th>AU_rank</th>\n",
       "      <th>CA_rank</th>\n",
       "      <th>DE_rank</th>\n",
       "      <th>ES_rank</th>\n",
       "      <th>FR_rank</th>\n",
       "      <th>GB_rank</th>\n",
       "      <th>IT_rank</th>\n",
       "      <th>NL_rank</th>\n",
       "      <th>PT_rank</th>\n",
       "      <th>US_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>-999</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>MALE</td>\n",
       "      <td>38</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>45</td>\n",
       "      <td>68</td>\n",
       "      <td>44</td>\n",
       "      <td>47</td>\n",
       "      <td>53</td>\n",
       "      <td>44</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>56</td>\n",
       "      <td>basic</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>49</td>\n",
       "      <td>51</td>\n",
       "      <td>50</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>42</td>\n",
       "      <td>facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>49</td>\n",
       "      <td>46</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>58</td>\n",
       "      <td>52</td>\n",
       "      <td>57</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>-unknown-</td>\n",
       "      <td>41</td>\n",
       "      <td>basic</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>50</td>\n",
       "      <td>49</td>\n",
       "      <td>66</td>\n",
       "      <td>53</td>\n",
       "      <td>50</td>\n",
       "      <td>61</td>\n",
       "      <td>52</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id     gender  age signup_method  signup_flow language  \\\n",
       "0  gxn3p5htnn  -unknown- -999      facebook            0       en   \n",
       "1  820tgsjxq7       MALE   38      facebook            0       en   \n",
       "2  4ft3gnwmtx     FEMALE   56         basic            3       en   \n",
       "3  bjjt8pjhuk     FEMALE   42      facebook            0       en   \n",
       "4  87mebub9p4  -unknown-   41         basic            0       en   \n",
       "\n",
       "  affiliate_channel affiliate_provider first_affiliate_tracked signup_app  \\\n",
       "0            direct             direct               untracked        Web   \n",
       "1               seo             google               untracked        Web   \n",
       "2            direct             direct               untracked        Web   \n",
       "3            direct             direct               untracked        Web   \n",
       "4            direct             direct               untracked        Web   \n",
       "\n",
       "    ...    AU_rank CA_rank  DE_rank  ES_rank  FR_rank GB_rank  IT_rank  \\\n",
       "0   ...       -999    -999     -999     -999     -999    -999     -999   \n",
       "1   ...         50      50       45       68       44      47       53   \n",
       "2   ...         46      54       54       48       48      46       49   \n",
       "3   ...         52      49       46       60       50      50       58   \n",
       "4   ...         52      50       49       66       53      50       61   \n",
       "\n",
       "   NL_rank  PT_rank  US_rank  \n",
       "0     -999     -999     -999  \n",
       "1       44       61       48  \n",
       "2       51       50       51  \n",
       "3       52       57       47  \n",
       "4       52       61       48  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# removing features based on feature_importances_\n",
    "features = list(df_all.columns)\n",
    "\n",
    "remove = ['d_ac-fa','signup_app', 'd_ac']\n",
    "\n",
    "for rf in remove:\n",
    "    features.remove(rf)\n",
    "\n",
    "df_all = df_all[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 275547 entries, 0 to 275546\n",
      "Columns: 1028 entries, gender to 999\n",
      "dtypes: float64(1005), int64(23)\n",
      "memory usage: 2.1 GB\n"
     ]
    }
   ],
   "source": [
    "# checking any NaN value left\n",
    "df_all.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# building a list of categorical variables to be LabelEncoded \n",
    "to_le = [ i for i, t in zip(df_all.dtypes.index, df_all.dtypes.values)  if t == 'O']\n",
    "to_le.remove('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['-unknown-' 'FEMALE' 'MALE' 'OTHER']\n",
      "[0 1 2 3]\n",
      "------\n",
      "\n",
      "['basic' 'facebook' 'google' 'weibo']\n",
      "[0 1 2 3]\n",
      "------\n",
      "\n",
      "['-unknown-' 'ca' 'cs' 'da' 'de' 'el' 'en' 'es' 'fi' 'fr' 'hr' 'hu' 'id'\n",
      " 'is' 'it' 'ja' 'ko' 'nl' 'no' 'pl' 'pt' 'ru' 'sv' 'th' 'tr' 'zh']\n",
      "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n",
      " 25]\n",
      "------\n",
      "\n",
      "['api' 'content' 'direct' 'other' 'remarketing' 'sem-brand' 'sem-non-brand'\n",
      " 'seo']\n",
      "[0 1 2 3 4 5 6 7]\n",
      "------\n",
      "\n",
      "['baidu' 'bing' 'craigslist' 'daum' 'direct' 'email-marketing' 'facebook'\n",
      " 'facebook-open-graph' 'google' 'gsp' 'meetup' 'naver' 'other' 'padmapper'\n",
      " 'vast' 'wayn' 'yahoo' 'yandex']\n",
      "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17]\n",
      "------\n",
      "\n",
      "['linked' 'local ops' 'marketing' 'omg' 'product' 'tracked-other'\n",
      " 'untracked']\n",
      "[0 1 2 3 4 5 6]\n",
      "------\n",
      "\n",
      "['Android Phone' 'Android Tablet' 'Desktop (Other)' 'Mac Desktop'\n",
      " 'Other/Unknown' 'SmartPhone (Other)' 'Windows Desktop' 'iPad' 'iPhone']\n",
      "[0 1 2 3 4 5 6 7 8]\n",
      "------\n",
      "\n",
      "['-unknown-' 'AOL Explorer' 'Android Browser' 'Apple Mail' 'Arora'\n",
      " 'Avant Browser' 'BlackBerry Browser' 'Camino' 'Chrome' 'Chrome Mobile'\n",
      " 'Chromium' 'CometBird' 'Comodo Dragon' 'Conkeror' 'CoolNovo'\n",
      " 'Crazy Browser' 'Epic' 'Firefox' 'Flock' 'Google Earth' 'Googlebot'\n",
      " 'IBrowse' 'IE' 'IE Mobile' 'IceDragon' 'IceWeasel' 'Iron' 'Kindle Browser'\n",
      " 'Maxthon' 'Mobile Firefox' 'Mobile Safari' 'Mozilla' 'NetNewsWire'\n",
      " 'Nintendo Browser' 'OmniWeb' 'Opera' 'Opera Mini' 'Opera Mobile'\n",
      " 'Outlook 2007' 'PS Vita browser' 'Pale Moon' 'Palm Pre web browser'\n",
      " 'RockMelt' 'Safari' 'SeaMonkey' 'Silk' 'SiteKiosk' 'SlimBrowser'\n",
      " 'Sogou Explorer' 'Stainless' 'TenFourFox' 'TheWorld Browser' 'UC Browser'\n",
      " 'Yandex.Browser' 'wOSBrowser']\n",
      "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n",
      " 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49\n",
      " 50 51 52 53 54]\n",
      "------\n",
      "\n",
      "['Fri' 'Mon' 'Sat' 'Sun' 'Thu' 'Tue' 'Wed']\n",
      "[0 1 2 3 4 5 6]\n",
      "------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# LabelEncode variables\n",
    "\n",
    "le = LabelEncoder()\n",
    "for ri in to_le: # ri: replaced by integer\n",
    "    le.fit(df_all[ri])   # takes labels and assigns each to a integer,\n",
    "    df_all[ri] = le.transform(df_all[ri])\n",
    "    print le.classes_\n",
    "    print le.transform(le.classes_)\n",
    "    print \"------\"\n",
    "    print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>...</th>\n",
       "      <th>AU_rank</th>\n",
       "      <th>CA_rank</th>\n",
       "      <th>DE_rank</th>\n",
       "      <th>ES_rank</th>\n",
       "      <th>FR_rank</th>\n",
       "      <th>GB_rank</th>\n",
       "      <th>IT_rank</th>\n",
       "      <th>NL_rank</th>\n",
       "      <th>PT_rank</th>\n",
       "      <th>US_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>0</td>\n",
       "      <td>-999</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>45</td>\n",
       "      <td>68</td>\n",
       "      <td>44</td>\n",
       "      <td>47</td>\n",
       "      <td>53</td>\n",
       "      <td>44</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>1</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>49</td>\n",
       "      <td>51</td>\n",
       "      <td>50</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>49</td>\n",
       "      <td>46</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>58</td>\n",
       "      <td>52</td>\n",
       "      <td>57</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>52</td>\n",
       "      <td>50</td>\n",
       "      <td>49</td>\n",
       "      <td>66</td>\n",
       "      <td>53</td>\n",
       "      <td>50</td>\n",
       "      <td>61</td>\n",
       "      <td>52</td>\n",
       "      <td>61</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id  gender  age  signup_method  signup_flow  language  \\\n",
       "0  gxn3p5htnn       0 -999              1            0         6   \n",
       "1  820tgsjxq7       2   38              1            0         6   \n",
       "2  4ft3gnwmtx       1   56              0            3         6   \n",
       "3  bjjt8pjhuk       1   42              1            0         6   \n",
       "4  87mebub9p4       0   41              0            0         6   \n",
       "\n",
       "   affiliate_channel  affiliate_provider  first_affiliate_tracked  \\\n",
       "0                  2                   4                        6   \n",
       "1                  7                   8                        6   \n",
       "2                  2                   4                        6   \n",
       "3                  2                   4                        6   \n",
       "4                  2                   4                        6   \n",
       "\n",
       "   first_device_type   ...     AU_rank  CA_rank  DE_rank  ES_rank  FR_rank  \\\n",
       "0                  3   ...        -999     -999     -999     -999     -999   \n",
       "1                  3   ...          50       50       45       68       44   \n",
       "2                  6   ...          46       54       54       48       48   \n",
       "3                  3   ...          52       49       46       60       50   \n",
       "4                  3   ...          52       50       49       66       53   \n",
       "\n",
       "   GB_rank  IT_rank  NL_rank  PT_rank  US_rank  \n",
       "0     -999     -999     -999     -999     -999  \n",
       "1       47       53       44       61       48  \n",
       "2       46       49       51       50       51  \n",
       "3       50       58       52       57       47  \n",
       "4       50       61       52       61       48  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(213451,)\n",
      "(275547, 26)\n"
     ]
    }
   ],
   "source": [
    "print y.shape\n",
    "print df_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# checkpoint 3\n",
    "df_all_ck3 = df_all.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# revert to checkpoint 3\n",
    "df_all = df_all_ck3.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge with session data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load session features processed from sessions.csv in\n",
    "# Extracting features from sessions.csv\n",
    "\n",
    "df_s = pd.read_csv(join(intermediate_path, 'session_features_1000.csv'))\n",
    "df_s = df_s.rename(columns = {'user_id': 'id'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>unique_devices</th>\n",
       "      <th>sum</th>\n",
       "      <th>session_id_30</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>...</th>\n",
       "      <th>990</th>\n",
       "      <th>991</th>\n",
       "      <th>992</th>\n",
       "      <th>993</th>\n",
       "      <th>994</th>\n",
       "      <th>995</th>\n",
       "      <th>996</th>\n",
       "      <th>997</th>\n",
       "      <th>998</th>\n",
       "      <th>999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d1mm9tcy42</td>\n",
       "      <td>2</td>\n",
       "      <td>3427529</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yo8nz8bqcq</td>\n",
       "      <td>1</td>\n",
       "      <td>207842</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4grx6yxeby</td>\n",
       "      <td>2</td>\n",
       "      <td>1135444</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ncf87guaf0</td>\n",
       "      <td>3</td>\n",
       "      <td>3755100</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4rvqpxoh3h</td>\n",
       "      <td>1</td>\n",
       "      <td>2555</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1004 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id  unique_devices      sum  session_id_30  0  1  2  3  4  5 ...   \\\n",
       "0  d1mm9tcy42               2  3427529             48  0  0  0  0  0  0 ...    \n",
       "1  yo8nz8bqcq               1   207842              7  0  0  0  0  0  0 ...    \n",
       "2  4grx6yxeby               2  1135444             10  1  0  0  0  0  0 ...    \n",
       "3  ncf87guaf0               3  3755100             72  0  0  0  0  0  0 ...    \n",
       "4  4rvqpxoh3h               1     2555              1  0  0  0  0  0  0 ...    \n",
       "\n",
       "   990  991  992  993  994  995  996  997  998  999  \n",
       "0    0    0    0    0    0    0    0    0    0    0  \n",
       "1    0    0    0    0    0    0    0    0    0    0  \n",
       "2    0    0    0    0    0    0    0    0    0    0  \n",
       "3    0    0    0    0    0    0    0    0    0    0  \n",
       "4    0    0    0    0    0    0    0    0    0    0  \n",
       "\n",
       "[5 rows x 1004 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_s.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# merging \n",
    "# df_all (original users data) with \n",
    "# df_s (processed session data)\n",
    "df_all = pd.merge(df_all, df_s, how = 'left', on = 'id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>...</th>\n",
       "      <th>990</th>\n",
       "      <th>991</th>\n",
       "      <th>992</th>\n",
       "      <th>993</th>\n",
       "      <th>994</th>\n",
       "      <th>995</th>\n",
       "      <th>996</th>\n",
       "      <th>997</th>\n",
       "      <th>998</th>\n",
       "      <th>999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gxn3p5htnn</td>\n",
       "      <td>0</td>\n",
       "      <td>-999</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>820tgsjxq7</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ft3gnwmtx</td>\n",
       "      <td>1</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bjjt8pjhuk</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87mebub9p4</td>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1029 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id  gender  age  signup_method  signup_flow  language  \\\n",
       "0  gxn3p5htnn       0 -999              1            0         6   \n",
       "1  820tgsjxq7       2   38              1            0         6   \n",
       "2  4ft3gnwmtx       1   56              0            3         6   \n",
       "3  bjjt8pjhuk       1   42              1            0         6   \n",
       "4  87mebub9p4       0   41              0            0         6   \n",
       "\n",
       "   affiliate_channel  affiliate_provider  first_affiliate_tracked  \\\n",
       "0                  2                   4                        6   \n",
       "1                  7                   8                        6   \n",
       "2                  2                   4                        6   \n",
       "3                  2                   4                        6   \n",
       "4                  2                   4                        6   \n",
       "\n",
       "   first_device_type ...   990  991  992  993  994  995  996  997  998  999  \n",
       "0                  3 ...   NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "1                  3 ...   NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "2                  6 ...   NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "3                  3 ...   NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "4                  3 ...   NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "\n",
       "[5 rows x 1029 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all = df_all.drop('id', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(275547, 1028)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# filling NaN with -1\n",
    "df_all.fillna(-999, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>signup_flow</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "      <th>...</th>\n",
       "      <th>990</th>\n",
       "      <th>991</th>\n",
       "      <th>992</th>\n",
       "      <th>993</th>\n",
       "      <th>994</th>\n",
       "      <th>995</th>\n",
       "      <th>996</th>\n",
       "      <th>997</th>\n",
       "      <th>998</th>\n",
       "      <th>999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-999</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "      <td>-999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1028 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender  age  signup_method  signup_flow  language  affiliate_channel  \\\n",
       "0       0 -999              1            0         6                  2   \n",
       "1       2   38              1            0         6                  7   \n",
       "2       1   56              0            3         6                  2   \n",
       "3       1   42              1            0         6                  2   \n",
       "4       0   41              0            0         6                  2   \n",
       "\n",
       "   affiliate_provider  first_affiliate_tracked  first_device_type  \\\n",
       "0                   4                        6                  3   \n",
       "1                   8                        6                  3   \n",
       "2                   4                        6                  6   \n",
       "3                   4                        6                  3   \n",
       "4                   4                        6                  3   \n",
       "\n",
       "   first_browser ...   990  991  992  993  994  995  996  997  998  999  \n",
       "0              8 ...  -999 -999 -999 -999 -999 -999 -999 -999 -999 -999  \n",
       "1              8 ...  -999 -999 -999 -999 -999 -999 -999 -999 -999 -999  \n",
       "2             22 ...  -999 -999 -999 -999 -999 -999 -999 -999 -999 -999  \n",
       "3             17 ...  -999 -999 -999 -999 -999 -999 -999 -999 -999 -999  \n",
       "4              8 ...  -999 -999 -999 -999 -999 -999 -999 -999 -999 -999  \n",
       "\n",
       "[5 rows x 1028 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Splitting train and test\n",
    "vals = df_all.values\n",
    "X = vals[:piv_train]\n",
    "\n",
    "X_test = vals[piv_train:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scaling features\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "\n",
    "#scaler = MinMaxScaler()\n",
    "scaler = StandardScaler()\n",
    "X = scaler.fit_transform(X)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning and predicting, and generating submission file\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning and predicting \n",
    "# Generating submission file\n",
    "\n",
    "clf = XGBClassifier(max_depth=6, learning_rate=0.2, n_estimators=25, #reg_lambda=3.33,\n",
    "                    objective='multi:softprob', subsample=0.75, colsample_bytree=0.5, seed=0)\n",
    "\n",
    "# Fit and predict\n",
    "clf.fit(X, y)\n",
    "y_pred = clf.predict_proba(X_test)  \n",
    "\n",
    "# Building submission files\n",
    "\n",
    "# Taking the 5 classes with highest probabilities\n",
    "ids = []  #list of ids\n",
    "cts = []  #list of countries\n",
    "for i in range(len(id_test)):\n",
    "    idx = id_test[i]\n",
    "    ids += [idx] * 5\n",
    "    cts += le_y.inverse_transform(np.argsort(y_pred[i])[::-1])[:5].tolist()\n",
    "\n",
    "#Generate submission\n",
    "if not exists(output_path):\n",
    "    makedirs(output_path)\n",
    "\n",
    "sub = pd.DataFrame(np.column_stack((ids, cts)), columns=['id', 'country'])\n",
    "f = 'sub.' + datetime.now().strftime(\"%d%m%y_%H%M%S\") + '.csv'\n",
    "sub.to_csv(join(output_path, f), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def scorer(clf, X_trn, y_trn, X_val, y_val):\n",
    "    clf.fit(X_trn, y_trn)\n",
    "    y_pred = clf.predict_proba(X_val)  \n",
    "    \n",
    "    cts = []  #list of countries\n",
    "    for i in range(len(y_val)):\n",
    "        cts.append( np.argsort(y_pred[i])[::-1][:5].tolist() )\n",
    "\n",
    "    cts = pd.DataFrame(cts)\n",
    "    y_val = pd.Series(y_val)\n",
    "    return score_predictions(cts, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predictions:\n",
      "    0   1\n",
      "0  US  FR\n",
      "1  FR  US\n",
      "2  FR  FR\n",
      "truth:\n",
      "0    US\n",
      "1    US\n",
      "2    FR\n",
      "dtype: object\n",
      "scores:\n",
      "0    1.00000\n",
      "1    0.63093\n",
      "2    1.00000\n",
      "Name: score, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def dcg_at_k(r, k, method=1):\n",
    "    r = np.asfarray(r)[:k]\n",
    "    if r.size:\n",
    "        if method == 0:\n",
    "            return r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1)))\n",
    "        elif method == 1:\n",
    "            return np.sum(r / np.log2(np.arange(2, r.size + 2)))\n",
    "        else:\n",
    "            raise ValueError('method must be 0 or 1.')\n",
    "    return 0.\n",
    "\n",
    "\n",
    "def ndcg_at_k(r, k=5, method=1):\n",
    "    dcg_max = dcg_at_k(sorted(r, reverse=True), k, method)\n",
    "    if not dcg_max:\n",
    "        return 0.\n",
    "    return dcg_at_k(r, k, method) / dcg_max\n",
    "\n",
    "\n",
    "def score_predictions(preds, truth, n_modes=5):\n",
    "    \"\"\"\n",
    "    preds: pd.DataFrame\n",
    "      one row for each observation, one column for each prediction.\n",
    "      Columns are sorted from left to right descending in order of likelihood.\n",
    "    truth: pd.Series\n",
    "      one row for each obeservation.\n",
    "    \"\"\"\n",
    "    assert(len(preds)==len(truth))\n",
    "    r = pd.DataFrame(0, index=preds.index, columns=preds.columns, dtype=np.float64)\n",
    "    for col in preds.columns:\n",
    "        r[col] = (preds[col] == truth) * 1.0\n",
    "\n",
    "    score = pd.Series(r.apply(ndcg_at_k, axis=1, reduce=True), name='score')\n",
    "    return score\n",
    "    \n",
    "    \n",
    "    \n",
    "preds = pd.DataFrame([['US','FR'],['FR','US'],['FR','FR']])\n",
    "truth = pd.Series(['US','US','FR'])\n",
    "print'predictions:' \n",
    "print preds\n",
    "print'truth:'\n",
    "print truth\n",
    "print'scores:'\n",
    "print score_predictions(preds, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i3 = 87628\n",
    "i4 = 113088\n",
    "i5 = 170158\n",
    "i6 = len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fold1: same quarter one year before\n",
    "# fold2: one quarter before the test data\n",
    "\n",
    "fold1_train = np.array(range(0, i3) + range(i4, i6))\n",
    "fold1_test  = np.array(range(i3, i4))\n",
    "fold2_train = np.array(range(0, i5))\n",
    "fold2_test  = np.array(range(i5, i6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_folds = 2\n",
    "kf = [(fold1_train, fold1_test), (fold2_train, fold2_test)]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "learning rate: 0.2\n",
      "n_estimators: 23\n",
      "subsample: 0.75\n",
      "\n",
      "2 folds NDCG scores:  ['0.8336', '0.8509']\n",
      "mean: 0.842244749997\n",
      "\n",
      "Classifier:\n",
      "XGBClassifier(base_score=0.5, colsample_bylevel=1, colsample_bytree=0.5,\n",
      "       gamma=0, learning_rate=0.2, max_delta_step=0, max_depth=6,\n",
      "       min_child_weight=1, missing=None, n_estimators=23, nthread=-1,\n",
      "       objective='multi:softprob', reg_alpha=0, reg_lambda=10,\n",
      "       scale_pos_weight=1, seed=0, silent=True, subsample=0.75)\n",
      "\n",
      "Date: Wednesday, 23 March 2016, 09:18AM\n",
      "\n",
      "Running time: 4492.24\n",
      "******************************************************************************\n"
     ]
    }
   ],
   "source": [
    "\n",
    "means = {}\n",
    "\n",
    "for n_estimators in [23]:\n",
    "    for subsample in [0.75]:\n",
    "        for learning_rate in [0.2]:\n",
    "            t0 = time()\n",
    "            \n",
    "            clf = XGBClassifier(max_depth=6, learning_rate=learning_rate, n_estimators=n_estimators, reg_lambda=10,\n",
    "                        objective='multi:softprob', subsample=subsample, colsample_bytree=0.5, seed=0)  \n",
    "            \n",
    "                    \n",
    "            ndcg_scores = []\n",
    "            \n",
    "            for train, test in kf:\n",
    "                cv_pred = scorer(clf, X[train], y[train], X[test], y[test])\n",
    "                ndcg_scores.append( np.mean(cv_pred) )\n",
    "            \n",
    "            print\n",
    "            print \"learning rate:\", learning_rate\n",
    "            print \"n_estimators:\", n_estimators\n",
    "            print \"subsample:\", subsample\n",
    "            print\n",
    "            print n_folds, \"folds NDCG scores: \", [ \"%.4f\" % s for s in ndcg_scores]\n",
    "            print \"mean:\", np.mean(ndcg_scores)\n",
    "            means[(n_estimators, subsample, learning_rate)] = ndcg_scores \n",
    "            print\n",
    "            print \"Classifier:\"\n",
    "            print clf\n",
    "            print\n",
    "            print \"Running time:\", \"%.2f\" % (time() - t0)\n",
    "            print \"******************************************************************************\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 0.75, 0.2)"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(means, key = lambda x: means.get(x) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(17, 0.75, 0.16): [0.83377493817166592, 0.85090935500605025],\n",
       " (17, 0.75, 0.2): [0.8335042303834892, 0.85062305350791889],\n",
       " (17, 0.75, 0.24): [0.83344848334104216, 0.85036823690486718],\n",
       " (20, 0.75, 0.16): [0.83371650880330062, 0.85131632434344051],\n",
       " (20, 0.75, 0.2): [0.833688144124065, 0.85086141386216418],\n",
       " (20, 0.75, 0.24): [0.83336274055005644, 0.85053709389292753],\n",
       " (23, 0.75, 0.16): [0.83378450358593914, 0.85159895061716673],\n",
       " (23, 0.75, 0.2): [0.83372656076927343, 0.85093847752662988],\n",
       " (23, 0.75, 0.24): [0.83343776135317649, 0.85070021889391045]}"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>average_ndcg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">17</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0.75</th>\n",
       "      <th>0.16</th>\n",
       "      <td>0.833775</td>\n",
       "      <td>0.850909</td>\n",
       "      <td>0.842342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>0.833504</td>\n",
       "      <td>0.850623</td>\n",
       "      <td>0.842064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.24</th>\n",
       "      <td>0.833448</td>\n",
       "      <td>0.850368</td>\n",
       "      <td>0.841908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">20</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0.75</th>\n",
       "      <th>0.16</th>\n",
       "      <td>0.833717</td>\n",
       "      <td>0.851316</td>\n",
       "      <td>0.842516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>0.833688</td>\n",
       "      <td>0.850861</td>\n",
       "      <td>0.842275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.24</th>\n",
       "      <td>0.833363</td>\n",
       "      <td>0.850537</td>\n",
       "      <td>0.841950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">23</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0.75</th>\n",
       "      <th>0.16</th>\n",
       "      <td>0.833785</td>\n",
       "      <td>0.851599</td>\n",
       "      <td>0.842692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>0.833727</td>\n",
       "      <td>0.850938</td>\n",
       "      <td>0.842333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.24</th>\n",
       "      <td>0.833438</td>\n",
       "      <td>0.850700</td>\n",
       "      <td>0.842069</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0         1  average_ndcg\n",
       "17 0.75 0.16  0.833775  0.850909      0.842342\n",
       "        0.20  0.833504  0.850623      0.842064\n",
       "        0.24  0.833448  0.850368      0.841908\n",
       "20 0.75 0.16  0.833717  0.851316      0.842516\n",
       "        0.20  0.833688  0.850861      0.842275\n",
       "        0.24  0.833363  0.850537      0.841950\n",
       "23 0.75 0.16  0.833785  0.851599      0.842692\n",
       "        0.20  0.833727  0.850938      0.842333\n",
       "        0.24  0.833438  0.850700      0.842069"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_means['average_ndcg'] = df_means.mean(axis=1)\n",
    "df_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>average_ndcg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">20</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.75</th>\n",
       "      <th>0.16</th>\n",
       "      <td>0.833538</td>\n",
       "      <td>0.851061</td>\n",
       "      <td>0.842299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>0.833529</td>\n",
       "      <td>0.850604</td>\n",
       "      <td>0.842066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">23</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.75</th>\n",
       "      <th>0.16</th>\n",
       "      <td>0.833594</td>\n",
       "      <td>0.851394</td>\n",
       "      <td>0.842494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>0.833703</td>\n",
       "      <td>0.850890</td>\n",
       "      <td>0.842297</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0         1  average_ndcg\n",
       "20 0.75 0.16  0.833538  0.851061      0.842299\n",
       "        0.20  0.833529  0.850604      0.842066\n",
       "23 0.75 0.16  0.833594  0.851394      0.842494\n",
       "        0.20  0.833703  0.850890      0.842297"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_means = pd.DataFrame(means).T\n",
    "df_means['average_ndcg'] = df_means.mean()\n",
    "df_means['average_ndcg'] = df_means.mean(axis=1)\n",
    "df_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>average_ndcg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">23</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.75</th>\n",
       "      <th>0.16</th>\n",
       "      <td>0.833634</td>\n",
       "      <td>0.850666</td>\n",
       "      <td>0.84215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>0.833678</td>\n",
       "      <td>0.851042</td>\n",
       "      <td>0.84236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0         1  average_ndcg\n",
       "23 0.75 0.16  0.833634  0.850666       0.84215\n",
       "        0.20  0.833678  0.851042       0.84236"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_means = pd.DataFrame(means).T\n",
    "df_means['average_ndcg'] = df_means.mean()\n",
    "df_means['average_ndcg'] = df_means.mean(axis=1)\n",
    "df_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>average_ndcg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <th>0.75</th>\n",
       "      <th>0.15</th>\n",
       "      <td>0.833604</td>\n",
       "      <td>0.851393</td>\n",
       "      <td>0.842499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0         1  average_ndcg\n",
       "50 0.75 0.15  0.833604  0.851393      0.842499"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_means = pd.DataFrame(means).T\n",
    "df_means['average_ndcg'] = df_means.mean()\n",
    "df_means['average_ndcg'] = df_means.mean(axis=1)\n",
    "df_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>average_ndcg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <th>0.75</th>\n",
       "      <th>0.15</th>\n",
       "      <td>0.8341</td>\n",
       "      <td>0.852147</td>\n",
       "      <td>0.843124</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   0         1  average_ndcg\n",
       "50 0.75 0.15  0.8341  0.852147      0.843124"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_means = pd.DataFrame(means).T\n",
    "df_means['average_ndcg'] = df_means.mean()\n",
    "df_means['average_ndcg'] = df_means.mean(axis=1)\n",
    "df_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>average_ndcg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <th>0.75</th>\n",
       "      <th>0.2</th>\n",
       "      <td>0.833642</td>\n",
       "      <td>0.851399</td>\n",
       "      <td>0.842521</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0         1  average_ndcg\n",
       "23 0.75 0.2  0.833642  0.851399      0.842521"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# FS 1000\n",
    "# lambda_reg = 10\n",
    "df_means = pd.DataFrame(means).T\n",
    "df_means['average_ndcg'] = df_means.mean()\n",
    "df_means['average_ndcg'] = df_means.mean(axis=1)\n",
    "df_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The scripts give a NDCG score of about 0.8845 , On the kaggle submissions . which is around 140 position in this competition ."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}