feature_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import sklearn\n",
    "import gc\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import RFECV\n",
    "sys.path.insert(0, 'helpers')\n",
    "from prep import train_prep, test_prep\n",
    "from scipy.sparse import csr_matrix, coo_matrix, vstack, hstack, save_npz\n",
    "import time\n",
    "import warnings\n",
    "from sklearn.exceptions import ConvergenceWarning, DataConversionWarning\n",
    "from IPython.display import clear_output\n",
    "from sklearn.decomposition import TruncatedSVD, IncrementalPCA\n",
    "from category_encoders import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'C:\\\\Users\\\\zking\\\\Documents\\\\Kaggle\\\\MS Malware\\\\data\\\\'\n",
    "model_path = 'C:\\\\Users\\\\zking\\\\Documents\\\\Kaggle\\\\MS Malware\\\\MS-Malware-Kaggle\\\\models\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(data_path+'x_df', 'rb') as f:\n",
    "    x_df = pickle.load(f)\n",
    "with open(data_path+'y_df', 'rb') as f:\n",
    "    y_df = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split x and y data into train/test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(x_df,\n",
    "                                  y_df,\n",
    "                                  test_size=0.2,\n",
    "                                  random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_df, y_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get important features w/ iterative RFECV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iters = 1\n",
    "final_cols = []\n",
    "score = 'roc_auc'\n",
    "all_times = []\n",
    "clf = RFECV(LogisticRegression(solver = 'saga',\n",
    "                               max_iter=100),\n",
    "                              scoring = score,\n",
    "                              n_jobs = -1,\n",
    "                              cv = 3,\n",
    "                              step = 50)\n",
    "warnings.filterwarnings(action='ignore', category=ConvergenceWarning)\n",
    "warnings.filterwarnings(action='ignore', category=DataConversionWarning)\n",
    "for i in range(0,iters,1):\n",
    "    start = time.time()\n",
    "    iter_count = i + 1\n",
    "    print(f'Starting iter {iter_count} of {iters}')\n",
    "    x_df, y_df = train_prep(data_path=data_path, model_path= model_path, sample = True, size = .005, narrow = True)\n",
    "    sparse_X_train = csr_matrix(x_df)\n",
    "    x_cols = x_df.columns\n",
    "    del x_df\n",
    "    gc.collect()\n",
    "    print('Starting fit...')\n",
    "    clf.fit(sparse_X_train, y_df)\n",
    "    used_cols = list(x_cols[clf.support_])\n",
    "    final_cols = list(set(final_cols + used_cols))\n",
    "    end = time.time()\n",
    "    final_time = [end - start]\n",
    "    all_times = all_times + final_time\n",
    "    minutes_elapsed = np.sum(all_times)/60\n",
    "    mean_time = np.mean(all_times)\n",
    "    minutes_left = ((iters - iter_count)*mean_time)/60\n",
    "    clear_output()\n",
    "    print(f'Iter {iter_count} took {final_time[0]} seconds')\n",
    "    print(f'Average iter is {mean_time} seconds')\n",
    "    print(f'Time elapsed is {minutes_elapsed} minutes')\n",
    "    print(f'Expected time left is {minutes_left} minutes')\n",
    "    print('\\n')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(data_path + 'final_cols', 'wb') as c:\n",
    "#     pickle.dump(final_cols, c)\n",
    "with open(data_path + 'sample_cols_' + str(iters), 'wb') as c:\n",
    "    pickle.dump(final_cols, c)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hashing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 100% of data\n",
      "Finished reading CSV\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\data.py:625: DataConversionWarning: Data with input dtype float32, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\base.py:462: DataConversionWarning: Data with input dtype float32, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished scaling\n",
      "Finished imputation\n"
     ]
    }
   ],
   "source": [
    "x_df = train_prep(data_path=data_path, model_path = model_path, sample = False, size = .3, df_only = True, x_only = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished encoding cols: ['Census_IsPenCapable', 'Census_IsAlwaysOnAlwaysConnectedCapable', 'Wdft_IsGamer']\n",
      "Iter 18 of 18\n"
     ]
    }
   ],
   "source": [
    "col_chunksize = 20\n",
    "x_cols = list(x_df.columns)\n",
    "n_col_chunks = round(len(x_cols)/col_chunksize)\n",
    "col_chunks = [x_cols[i:i + n_col_chunks] for i in range(0, len(x_cols), n_col_chunks)]\n",
    "xformer_dict = {}\n",
    "csr_dict = {}\n",
    "for i, c_chunk  in enumerate(col_chunks):\n",
    "    enc_bin = BinaryEncoder(cols=None, return_df = True) # cols=None, all string columns encoded\n",
    "    df_trans = enc_bin.fit_transform(x_df.loc[:,c_chunk])\n",
    "    csr_trans = csr_matrix(df_trans)\n",
    "    del df_trans\n",
    "    csr_dict[i] = csr_trans\n",
    "    xformer_dict[i] = enc_bin\n",
    "    del enc_bin\n",
    "    gc.collect()\n",
    "    clear_output()\n",
    "    print(f'Finished encoding cols: {c_chunk}\\nIter {i+1} of {len(col_chunks)}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del x_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished location 1\n",
      "Finished location 2\n",
      "Finished location 3\n",
      "Finished location 4\n",
      "Finished location 5\n",
      "Finished location 6\n",
      "Finished location 7\n",
      "Finished location 8\n",
      "Finished location 9\n",
      "Finished location 10\n",
      "Finished location 11\n",
      "Finished location 12\n",
      "Finished location 13\n",
      "Finished location 14\n",
      "Finished location 15\n",
      "Finished location 16\n",
      "Finished location 17\n"
     ]
    }
   ],
   "source": [
    "x_csr_bin = csr_dict[0]\n",
    "del csr_dict[0]\n",
    "gc.collect()\n",
    "csr_locs = range(1,len(csr_dict)+1)\n",
    "for loc in csr_locs:\n",
    "    if len(csr_locs) == 0:\n",
    "        break\n",
    "    else:\n",
    "        x_csr_bin = hstack([x_csr_bin, csr_dict[loc]], format = 'csr')\n",
    "        del csr_dict[loc]\n",
    "        gc.collect()\n",
    "        print(f'Finished location {loc}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<8921483x240 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 576885772 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_csr_bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del csr_dict\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_npz(data_path + 'x_csr_bin.npz', x_csr_bin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path + 'bin_xformer_dict', 'wb') as c:\n",
    "    pickle.dump(xformer_dict, c)\n",
    "with open(data_path + 'bin_col_chunks', 'wb') as c:\n",
    "    pickle.dump(col_chunks, c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del x_csr_bin\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting CSV read...\n",
      "Finished reading CSV\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "helpers\\prep.py:385: DataConversionWarning: Data with input dtype float32, float64 were all converted to float64 by StandardScaler.\n",
      "  scaled_num_df = pd.DataFrame(scalar.transform(num_df))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished scaling\n",
      "Finished imputation\n"
     ]
    }
   ],
   "source": [
    "test_df = test_prep(data_path, model_path, narrow = True, df_only = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path + 'bin_xformer_dict', 'rb') as c:\n",
    "    xformer_dict = pickle.load(c)\n",
    "with open(data_path + 'bin_col_chunks', 'rb') as c:\n",
    "    col_chunks = pickle.load(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished encoding cols: ['Census_IsPenCapable', 'Census_IsAlwaysOnAlwaysConnectedCapable', 'Wdft_IsGamer']\n",
      "Iter 18 of 18\n"
     ]
    }
   ],
   "source": [
    "csr_dict = {}\n",
    "for i, c_chunk  in enumerate(col_chunks):\n",
    "    enc_bin = xformer_dict[i]\n",
    "    df_trans = enc_bin.transform(test_df.loc[:,c_chunk])\n",
    "    csr_trans = csr_matrix(df_trans)\n",
    "    del df_trans\n",
    "    csr_dict[i] = csr_trans\n",
    "    del enc_bin\n",
    "    gc.collect()\n",
    "    clear_output()\n",
    "    print(f'Finished encoding cols: {c_chunk}\\nIter {i+1} of {len(col_chunks)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished location 1\n",
      "Finished location 2\n",
      "Finished location 3\n",
      "Finished location 4\n",
      "Finished location 5\n",
      "Finished location 6\n",
      "Finished location 7\n",
      "Finished location 8\n",
      "Finished location 9\n",
      "Finished location 10\n",
      "Finished location 11\n",
      "Finished location 12\n",
      "Finished location 13\n",
      "Finished location 14\n",
      "Finished location 15\n",
      "Finished location 16\n",
      "Finished location 17\n"
     ]
    }
   ],
   "source": [
    "x_test_bin = csr_dict[0]\n",
    "del csr_dict[0]\n",
    "csr_locs = range(1,len(csr_dict)+1)\n",
    "for loc in csr_locs:\n",
    "    if len(csr_locs) == 0:\n",
    "        break\n",
    "    else:\n",
    "        x_test_bin = hstack([x_test_bin, csr_dict[loc]], format = 'csr')\n",
    "        del csr_dict[loc]\n",
    "        gc.collect()\n",
    "        print(f'Finished location {loc}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_npz(data_path + 'x_test_bin', x_test_bin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "548px",
    "left": "374.962px",
    "right": "20px",
    "top": "157.963px",
    "width": "800px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}