AAIA17.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "from pandas.tools.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import svm\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import (RandomTreesEmbedding, IsolationForest, RandomForestClassifier, GradientBoostingClassifier)\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "import pickle\n",
    "from sklearn import metrics\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.preprocessing import OneHotEncoder    \n",
    "import json\n",
    "from sklearn import cross_validation, metrics   #Additional scklearn functions\n",
    "from sklearn.grid_search import GridSearchCV   #Perforing grid search\n",
    "from sklearn.model_selection import train_test_split\n",
    "import random\n",
    "from sklearn import preprocessing\n",
    "from random import randint\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn import decomposition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Definisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataPrefix = '/home/data/aaia17/'\n",
    "dataFeaturesOutput = '/home/data/aaia17/features/'\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "def printify(t):\n",
    "    for x in t:\n",
    "        print(x)\n",
    "def readDataAsFrame(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        frame.append(pandas.read_csv(fData))\n",
    "    return pandas.concat(frame)    \n",
    "def readJsonData(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        with open(fData, 'r') as f:\n",
    "            frame.append(json.load(f))\n",
    "        f.close()\n",
    "    return frame  \n",
    "def getNewForIndexes(dataFrame, names):\n",
    "    X_selectedFeature = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        X_selectedFeature = np.column_stack([X_selectedFeature, dataFrame[name].values])\n",
    "    return X_selectedFeature\n",
    "def dataScaling(data, scaler):\n",
    "    data = data/(data+1)\n",
    "    return scaler.fit_transform(data)\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "def readFileByLine(fileName):\n",
    "    arr = []\n",
    "    with open(fileName, 'r') as f:        \n",
    "        for line in f:\n",
    "            arr.append(line.rstrip())\n",
    "        f.close()\n",
    "    return arr\n",
    "def converseVec2Matrix(data):\n",
    "    arr = np.empty((len(data), 1), float)\n",
    "    for i in range(0, len(data)):\n",
    "        arr[i] = data[i]\n",
    "    return arr   \n",
    "def read_from_pickle(path):\n",
    "    with open(path, 'rb') as file:\n",
    "        try:\n",
    "            return pickle.load(file)\n",
    "        except EOFError:\n",
    "            pass     \n",
    "def meanVector(filenames):\n",
    "    sumPredicted = np.zeros(750000)\n",
    "    for filename in filenames:\n",
    "        with open(averageDir+filename) as file:\n",
    "            w = np.array(map(lambda x: float(x), file.readlines()))\n",
    "            sumPredicted += w     \n",
    "        file.close()\n",
    "    return sumPredicted/len(filenames)\n",
    "def replaceNan(data, item):\n",
    "    return [x if x is not None else item for x in data]\n",
    "\n",
    "def getDenseFeatures(dataFrame, names, dictionary):\n",
    "    X_dense = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        values = dataFrame[name].values\n",
    "        X_dense = np.column_stack([X_dense, replaceNan([dictionary[name].get(value) for value in values], 0.5)])\n",
    "    return X_dense\n",
    "def addNewFeature(data, vectors):\n",
    "    return np.column_stack([data, vectors])\n",
    "def getNewForIndexes(dataFrame, names):\n",
    "    X_selectedFeature = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        X_selectedFeature = np.column_stack([X_selectedFeature, dataFrame[name].values])\n",
    "    return X_selectedFeature\n",
    "def computeFractionsForAll(dataFrame, names):\n",
    "    changeDict = dict()\n",
    "    for name in names:\n",
    "        changeDict[name] = _computeFractionForAttribute(dataFrame, name)\n",
    "    return changeDict\n",
    "def dropColumnsFromDataFrame(df, columns):\n",
    "    for column in columns:\n",
    "        df = df.drop(column, 1)\n",
    "    return df\n",
    "def readJsonData(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        with open(fData, 'r') as f:\n",
    "            frame.append(json.load(f))\n",
    "        f.close()\n",
    "    return frame  \n",
    "def readJsonDatabyLine(prefix, fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = prefix+fileName\n",
    "        frame.extend(readFileByLine(fData))\n",
    "    return frame\n",
    "def readJsonFromString(data):\n",
    "    result = []\n",
    "    nulls = []\n",
    "    for i in range(0,len(data)):\n",
    "        result.append(json.loads(data[i]))\n",
    "                    #nulls.append(i)\n",
    "    return result\n",
    "def weighting(x):\n",
    "    return np.exp(-(np.power(xaxis,4))/2.0)       \n",
    "def frange(x0, x1, dx, fract):\n",
    "    return np.array(map(lambda x: x / float(fract), range(int(fract*x0), int(fract*x1), int(fract*(dx)))))\n",
    "def meanVectorW(pref, ids, weights):\n",
    "    sumPredicted = np.zeros(750000)\n",
    "    i = 0\n",
    "    for id in ids:\n",
    "        with open(dataPrefix+\"%s%d.txt\"%(pref, id)) as file:\n",
    "            w = weights[i]*np.array(map(lambda x: float(x), file.readlines()))\n",
    "            sumPredicted += w     \n",
    "        file.close()\n",
    "        i+=1\n",
    "    return sumPredicted/np.sum(weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getHandHpHistogram(state):\n",
    "    handHp = np.zeros(3)\n",
    "    # maxval is 8, we divide by 3\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp'], 8, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getHandDurability(state):\n",
    "    handDur = np.zeros(3)\n",
    "    # maxval is 4, we divide by 3\n",
    "    for hand in state['player']['hand']:\n",
    "        if hand['type'] == \"WEAPON\":\n",
    "            try:\n",
    "                handDur[getBin(hand['durability'], 4, 3)] +=1\n",
    "            except KeyError:\n",
    "                pass\n",
    "    return handDur\n",
    "def getHandAttackHistogram(state, type):\n",
    "    # maxval is 8, we divide by 3 folds\n",
    "    maxval = 8\n",
    "    div = 3\n",
    "    handAttack = np.zeros(div)\n",
    "    if type == \"WEAPON\":\n",
    "        # maxval is 5, we divide by 3\n",
    "        maxval = 5\n",
    "        handAttack = np.zeros(div)\n",
    "    for hand in state['player']['hand']:\n",
    "        if hand['type'] == type:\n",
    "            try:\n",
    "                handAttack[getBin(hand['attack'], maxval, div)] +=1\n",
    "            except KeyError:\n",
    "                pass\n",
    "    return handAttack\n",
    "def getHandCrystalCostHistogram(state, type):\n",
    "    # maxval is 8\n",
    "    maxval = 8\n",
    "    div = 3\n",
    "    handCrystalCost = np.zeros(div) # we divide to 4 folds\n",
    "    \n",
    "    if type == \"WEAPON\":\n",
    "        # maxval is 5\n",
    "        handCrystalCost = np.zeros(div) # we divide to 3 folds\n",
    "        maxval = 5\n",
    "    if type == \"SPELL\":\n",
    "        # maxval is 7\n",
    "        handCrystalCost = np.zeros(div) # we divide to 3 folds\n",
    "        maxval = 7\n",
    "    for hand in state['player']['hand']:\n",
    "        if hand['type'] == type:\n",
    "            try:\n",
    "                handCrystalCost[getBin(hand['crystals_cost'], maxval, div)] +=1\n",
    "            except KeyError:\n",
    "                pass\n",
    "    return handCrystalCost\n",
    "\n",
    "def getHandSumBySpecific(state, attr, specific):\n",
    "    sum = 0\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            if hand[specific] is True and hand['type'] == \"MINION\":\n",
    "                sum+=hand[attr]\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return sum\n",
    "\n",
    "def getPlayedCardsSumBySpecific(state, attr, specific, type):\n",
    "    sum = 0\n",
    "    for hand in state[type]['played_cards']:\n",
    "        try:\n",
    "            if hand[specific] is True:\n",
    "                sum+=hand[attr]\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return sum\n",
    "\n",
    "def getPlayedCardsAllSpecifics(state, type):\n",
    "    attrs = ['hp_max', 'hp_current', 'attack', 'crystals_cost']\n",
    "    specifs = ['taunt', 'charge', 'freezing', 'frozen', 'can_attack']\n",
    "    specs = np.zeros(20)\n",
    "    i = 0\n",
    "    for specif in specifs:\n",
    "        for attr in attrs:\n",
    "            specs[i] = getPlayedCardsSumBySpecific(state, attr, specif, type)\n",
    "            i+=1\n",
    "    return specs\n",
    "\n",
    "def getHandAllSpecifics(state):\n",
    "    attrs = ['hp', 'attack', 'crystals_cost']\n",
    "    specifs = ['taunt', 'charge', 'freezing']\n",
    "    specs = np.zeros(9)\n",
    "    i = 0\n",
    "    for specif in specifs:\n",
    "        for attr in attrs:\n",
    "            specs[i] = getHandSumBySpecific(state, attr, specif)\n",
    "            i+=1\n",
    "    return specs\n",
    "\n",
    "def getHandCountsOfTypes(state):\n",
    "    counts = np.zeros(3)\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            if hand['type']==\"MINION\":\n",
    "                counts[0]+=1\n",
    "            elif hand['type']==\"WEAPON\":\n",
    "                counts[1]+=1    \n",
    "            elif hand['type']==\"SPELL\":\n",
    "                counts[2]+=1    \n",
    "        except KeyError:\n",
    "            pass\n",
    "    return counts\n",
    "\n",
    "def getBin(val, max, nb):\n",
    "    if val > max:\n",
    "        val = max\n",
    "    dx = max/nb\n",
    "    if max % nb >0:\n",
    "        dx+=1\n",
    "    r = val/dx\n",
    "    if val == max and max % nb == 0:\n",
    "        r-=1\n",
    "    return r\n",
    "\n",
    "def getPlayerStats(state, player):\n",
    "    stats = np.zeros(6)\n",
    "    i = 0\n",
    "    data = state[player]['stats']\n",
    "    stats[0] = data['crystals_all']\n",
    "    stats[1] = data['crystals_current']\n",
    "    stats[2] = data['deck_count']\n",
    "    stats[3] = data['fatigue_damage']\n",
    "    stats[4] = data['hand_count']\n",
    "    stats[5] = data['played_minions_count']\n",
    "    \n",
    "    return stats\n",
    "\n",
    "def getHeroStats(state, player):\n",
    "    stats = np.zeros(5)\n",
    "    data = state[player]['hero']\n",
    "    stats[0] = data['armor']\n",
    "    stats[1] = data['attack']\n",
    "    stats[2] = data['hp']\n",
    "    stats[3] = data['special_skill_used'] == True\n",
    "    stats[4] = data['weapon_durability']\n",
    "    \n",
    "    return stats\n",
    "\n",
    "def getPlayerPlayedCardsHpCurrentHistogram(state):\n",
    "    #maxval is 24, we divide to 4 folds\n",
    "    handHp = np.zeros(4)\n",
    "    for hand in state['player']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_current'], 24, 4)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getOpponentPlayedCardsHpCurrentHistogram(state):\n",
    "    #maxval is 32, we divide to 5 folds\n",
    "    handHp = np.zeros(5)\n",
    "    for hand in state['opponent']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_current'], 32, 5)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getPlayerPlayedCardsHpMaxHistogram(state):\n",
    "    #maxval is 30, we divide to 3 folds\n",
    "    handHp = np.zeros(3)\n",
    "    for hand in state['player']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_max'], 30, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getOpponentPlayedCardsHpMaxHistogram(state):\n",
    "    #maxval is 37, we divide to 4 folds\n",
    "    handHp = np.zeros(4)\n",
    "    for hand in state['opponent']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_max'], 37, 4)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getPlayerPlayedCardsAttackHistogram(state):\n",
    "    #maxval is 23, we divide to 4 folds\n",
    "    handHp = np.zeros(4)\n",
    "    for hand in state['player']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['attack'], 23, 4)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getOpponentPlayedCardsAttackHistogram(state):\n",
    "    #maxval is 19, we divide to 3 folds\n",
    "    handHp = np.zeros(3)\n",
    "    for hand in state['opponent']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['attack'], 19, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getPlayedCardsCrystalsCostHistogram(state, type):\n",
    "    #maxval is 8, we divide to 3 folds\n",
    "    handHp = np.zeros(3)\n",
    "    for hand in state[type]['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['crystals_cost'], 8, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getAllMinMax(state):\n",
    "    return np.hstack([getMinMaxBySpecific(state,'hp'), getMinMaxBySpecific(state,'crystals_cost'),\n",
    "           getMinMaxBySpecific(state,'attack')])\n",
    "\n",
    "def craftAllFeatures(state):\n",
    "    return np.hstack([state['gamestate_id'], getHandHpHistogram(state), getHandAttackHistogram(state),\n",
    "                              getHandCrystalCostHistogram(state), getAllSpecifics(state),\n",
    "                              getAllMinMax(state)])\n",
    "\n",
    "def read_from_pickle(path):\n",
    "    with open(path, 'rb') as file:\n",
    "        try:\n",
    "            return pickle.load(file)\n",
    "        except EOFError:\n",
    "            pass    \n",
    "\n",
    "def craftAllFeatures(state):\n",
    "    return np.hstack([\n",
    "            getHandHpHistogram(state), getHandDurability(state), getHandAttackHistogram(state, \"MINION\"),\n",
    "            getHandAttackHistogram(state, \"WEAPON\"), getHandCrystalCostHistogram(state, \"MINION\"),\n",
    "            getHandCrystalCostHistogram(state, \"WEAPON\"), getHandCrystalCostHistogram(state, \"SPELL\"),\n",
    "            getHandAllSpecifics(state), getPlayedCardsAllSpecifics(state, 'player'), getHandCountsOfTypes(state),\n",
    "            getPlayerStats(state, 'player'), getHeroStats(state, 'player'), \n",
    "            getPlayerPlayedCardsHpCurrentHistogram(state), getPlayerPlayedCardsHpMaxHistogram(state),\n",
    "            getPlayerPlayedCardsAttackHistogram(state), getPlayedCardsCrystalsCostHistogram(state, 'player'),\n",
    "            getOpponentPlayedCardsHpCurrentHistogram(state), getOpponentPlayedCardsHpMaxHistogram(state),\n",
    "            getOpponentPlayedCardsAttackHistogram(state), getPlayedCardsCrystalsCostHistogram(state, 'opponent'),\n",
    "            getPlayedCardsAllSpecifics(state, 'opponent')\n",
    "        ])\n",
    "\n",
    "def getAllJsonTraining(data, id):\n",
    "    handsHist = np.empty((0, 113), float)\n",
    "    for state in data:\n",
    "        handsHist = np.vstack([handsHist, craftAllFeatures(state)])\n",
    "    pickle.dump(handsHist, open(dataFeaturesOutput+'handsHist-%d.store'%id,\"wb\"))\n",
    "\n",
    "i = 5\n",
    "trainChunk = read_from_pickle(dataFeaturesOutput+'featureChunk-%d.store'%i)\n",
    "getAllJsonFeatures(trainChunk, i)\n",
    "    \n",
    "    \n",
    "# for random feature selection\n",
    "dataFeaturesPrefix = dataPrefix+\"features/\"\n",
    "filepathToModels = dataPrefix+'dims-heavy-79.store'\n",
    "\n",
    "def stripDim(data, indexes):\n",
    "    indexes.sort(reverse=True)\n",
    "    for i in indexes:\n",
    "        data = np.concatenate((data[:,0:i], data[:,i+1:len(data[1,:])]), axis = 1)\n",
    "    return data\n",
    "def takeSampleDim(train):\n",
    "    dim = train.shape[1]\n",
    "    range2strip = dim/5\n",
    "    nb2strip = randint(1, range2strip)\n",
    "    return random.sample(range(1, dim), nb2strip)\n",
    "def persistSample(train, sample, sampleSet):\n",
    "    while frozenset(sample) in sampleSet:\n",
    "        sample = takeSampleDim(train)\n",
    "    sampleSet.add(frozenset(sample))\n",
    "    pickle.dump(sampleSet, open(filepathToModels,\"wb\"))\n",
    "    return sample\n",
    "def stripDimSets(X_train, X_test, dim2strip):\n",
    "    X_train_str = stripDim(X_train, dim2strip)\n",
    "    X_test_str  = stripDim(X_test, dim2strip)\n",
    "    return (X_train_str, X_test_str)\n",
    "def trainClassifier(clf, X_train, y_train, X_test, y_test):\n",
    "    clf = clf.fit(X_train, y_train)\n",
    "    predicted = clf.predict(X_test)\n",
    "    return metrics.roc_auc_score(y_test, predicted)\n",
    "def writeSolution(sample, auc):\n",
    "    filepath = dataPrefix+\"chosen-heavy-79.store\"\n",
    "    if os.path.isfile(filepath):\n",
    "        sampleF, aucF = read_from_pickle(filepath)\n",
    "        if sampleF is not None:\n",
    "            if auc > float(aucF):\n",
    "                pickle.dump((sample, auc), open(filepath,\"wb\"))\n",
    "    else:\n",
    "        pickle.dump((sample, auc), open(filepath,\"wb\"))\n",
    "def makeOneStep(clf, filepathToModels, data):\n",
    "    xtrain, ytrain, xtest, ytest = data\n",
    "    sampleSet = read_from_pickle(filepathToModels)\n",
    "    if sampleSet is None:\n",
    "        sampleSet = set()\n",
    "    sample = takeSampleDim(xtrain)\n",
    "    sample = persistSample(xtrain, sample, sampleSet)\n",
    "    xtrain, xtest = stripDimSets(xtrain, xtest, sample)\n",
    "    auc = trainClassifier(clf, xtrain, ytrain, xtest, ytest)\n",
    "    writeSolution(sample, auc)\n",
    "    return (auc, sample)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Yet another Feature Computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "from pandas.tools.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import svm\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import (RandomTreesEmbedding, IsolationForest, RandomForestClassifier, GradientBoostingClassifier)\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "import pickle\n",
    "from sklearn import metrics\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.preprocessing import OneHotEncoder    \n",
    "import json\n",
    "\n",
    "def replaceNan(data, item):\n",
    "    return [x if x is not None else item for x in data]\n",
    "\n",
    "def getDenseFeatures(dataFrame, names, dictionary):\n",
    "    X_dense = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        values = dataFrame[name].values\n",
    "        X_dense = np.column_stack([X_dense, replaceNan([dictionary[name].get(value) for value in values], 0.5)])\n",
    "    return X_dense\n",
    "def addNewFeature(data, vectors):\n",
    "    return np.column_stack([data, vectors])\n",
    "def getNewForIndexes(dataFrame, names):\n",
    "    X_selectedFeature = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        X_selectedFeature = np.column_stack([X_selectedFeature, dataFrame[name].values])\n",
    "    return X_selectedFeature\n",
    "def dataScaling(data, scaler):\n",
    "    data = data/(data+1)\n",
    "    return scaler.fit_transform(data)\n",
    "\n",
    "def _computeFractionForAttribute(dataFrame, name):\n",
    "    dictionary = dict()\n",
    "    values = set(dataFrame[name].values)\n",
    "    for value in values:\n",
    "        ind = dataFrame[name].values==value\n",
    "        subset = dataFrame['decision'][ind].values\n",
    "        fraction = 1.0*len(filter(lambda x: x==1, subset)) / len(subset)\n",
    "        dictionary[value] = fraction \n",
    "    return dictionary\n",
    "def computeFractionsForAll(dataFrame, names):\n",
    "    changeDict = dict()\n",
    "    for name in names:\n",
    "        changeDict[name] = _computeFractionForAttribute(dataFrame, name)\n",
    "    return changeDict\n",
    "def dropColumnsFromDataFrame(df, columns):\n",
    "    for column in columns:\n",
    "        df = df.drop(column, 1)\n",
    "    return df\n",
    "\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "def printify(t):\n",
    "    for x in t:\n",
    "        print(x)\n",
    "def readDataAsFrame(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        frame.append(pandas.read_csv(fData))\n",
    "    return pandas.concat(frame)\n",
    "\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "\n",
    "dataPrefix = '/home/data/aaia17/'\n",
    "dataDepreciatedPrefix = dataPrefix+\"depreciated/\"\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "def printify(t):\n",
    "    for x in t:\n",
    "        print(x)\n",
    "def readDataAsFrame(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        frame.append(pandas.read_csv(fData))\n",
    "    return pandas.concat(frame)    \n",
    "def readJsonData(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        with open(fData, 'r') as f:\n",
    "            frame.append(json.load(f))\n",
    "        f.close()\n",
    "    return frame  \n",
    "def readJsonDatabyLine(prefix, fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = prefix+fileName\n",
    "        frame.extend(readFileByLine(fData))\n",
    "    return frame\n",
    "def readJsonFromString(data):\n",
    "    result = []\n",
    "    nulls = []\n",
    "    for i in range(0,len(data)):\n",
    "        result.append(json.loads(data[i]))\n",
    "                    #nulls.append(i)\n",
    "    return result\n",
    "def getNewForIndexes(dataFrame, names):\n",
    "    X_selectedFeature = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        X_selectedFeature = np.column_stack([X_selectedFeature, dataFrame[name].values])\n",
    "    return X_selectedFeature\n",
    "\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "def readFileByLine(fileName):\n",
    "    arr = []\n",
    "    with open(fileName, 'r') as f:        \n",
    "        for line in f:\n",
    "            arr.append(line.rstrip())\n",
    "        f.close()\n",
    "    return arr\n",
    "\n",
    "def getHandHpHistogram(state):\n",
    "    handHp = np.zeros(3)\n",
    "    # maxval is 8, we divide by 3\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp'], 8, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getHandDurability(state):\n",
    "    handDur = np.zeros(3)\n",
    "    # maxval is 4, we divide by 3\n",
    "    for hand in state['player']['hand']:\n",
    "        if hand['type'] == \"WEAPON\":\n",
    "            try:\n",
    "                handDur[getBin(hand['durability'], 4, 3)] +=1\n",
    "            except KeyError:\n",
    "                pass\n",
    "    return handDur\n",
    "def getHandAttackHistogram(state, type):\n",
    "    # maxval is 8, we divide by 3 folds\n",
    "    maxval = 8\n",
    "    div = 3\n",
    "    handAttack = np.zeros(div)\n",
    "    if type == \"WEAPON\":\n",
    "        # maxval is 5, we divide by 3\n",
    "        maxval = 5\n",
    "        handAttack = np.zeros(div)\n",
    "    for hand in state['player']['hand']:\n",
    "        if hand['type'] == type:\n",
    "            try:\n",
    "                handAttack[getBin(hand['attack'], maxval, div)] +=1\n",
    "            except KeyError:\n",
    "                pass\n",
    "    return handAttack\n",
    "def getHandCrystalCostHistogram(state, type):\n",
    "    # maxval is 8\n",
    "    maxval = 8\n",
    "    div = 3\n",
    "    handCrystalCost = np.zeros(div) # we divide to 4 folds\n",
    "    \n",
    "    if type == \"WEAPON\":\n",
    "        # maxval is 5\n",
    "        handCrystalCost = np.zeros(div) # we divide to 3 folds\n",
    "        maxval = 5\n",
    "    if type == \"SPELL\":\n",
    "        # maxval is 7\n",
    "        handCrystalCost = np.zeros(div) # we divide to 3 folds\n",
    "        maxval = 7\n",
    "    for hand in state['player']['hand']:\n",
    "        if hand['type'] == type:\n",
    "            try:\n",
    "                handCrystalCost[getBin(hand['crystals_cost'], maxval, div)] +=1\n",
    "            except KeyError:\n",
    "                pass\n",
    "    return handCrystalCost\n",
    "\n",
    "def getHandSumBySpecific(state, attr, specific):\n",
    "    sum = 0\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            if hand[specific] is True and hand['type'] == \"MINION\":\n",
    "                sum+=hand[attr]\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return sum\n",
    "\n",
    "def getPlayedCardsSumBySpecific(state, attr, specific, type):\n",
    "    sum = 0\n",
    "    for hand in state[type]['played_cards']:\n",
    "        try:\n",
    "            if hand[specific] is True:\n",
    "                sum+=hand[attr]\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return sum\n",
    "\n",
    "def getPlayedCardsAllSpecifics(state, type):\n",
    "    attrs = ['hp_max', 'hp_current', 'attack', 'crystals_cost']\n",
    "    specifs = ['taunt', 'charge', 'freezing', 'frozen', 'can_attack']\n",
    "    specs = np.zeros(20)\n",
    "    i = 0\n",
    "    for specif in specifs:\n",
    "        for attr in attrs:\n",
    "            specs[i] = getPlayedCardsSumBySpecific(state, attr, specif, type)\n",
    "            i+=1\n",
    "    return specs\n",
    "\n",
    "def getHandAllSpecifics(state):\n",
    "    attrs = ['hp', 'attack', 'crystals_cost']\n",
    "    specifs = ['taunt', 'charge', 'freezing']\n",
    "    specs = np.zeros(9)\n",
    "    i = 0\n",
    "    for specif in specifs:\n",
    "        for attr in attrs:\n",
    "            specs[i] = getHandSumBySpecific(state, attr, specif)\n",
    "            i+=1\n",
    "    return specs\n",
    "\n",
    "def getHandCountsOfTypes(state):\n",
    "    counts = np.zeros(3)\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            if hand['type']==\"MINION\":\n",
    "                counts[0]+=1\n",
    "            elif hand['type']==\"WEAPON\":\n",
    "                counts[1]+=1    \n",
    "            elif hand['type']==\"SPELL\":\n",
    "                counts[2]+=1    \n",
    "        except KeyError:\n",
    "            pass\n",
    "    return counts\n",
    "\n",
    "def getBin(val, max, nb):\n",
    "    if val > max:\n",
    "        val = max\n",
    "    dx = max/nb\n",
    "    if max % nb >0:\n",
    "        dx+=1\n",
    "    r = val/dx\n",
    "    if val == max and max % nb == 0:\n",
    "        r-=1\n",
    "    return r\n",
    "\n",
    "def getPlayerStats(state, player):\n",
    "    stats = np.zeros(6)\n",
    "    i = 0\n",
    "    data = state[player]['stats']\n",
    "    stats[0] = data['crystals_all']\n",
    "    stats[1] = data['crystals_current']\n",
    "    stats[2] = data['deck_count']\n",
    "    stats[3] = data['fatigue_damage']\n",
    "    stats[4] = data['hand_count']\n",
    "    stats[5] = data['played_minions_count']\n",
    "    \n",
    "    return stats\n",
    "\n",
    "def getHeroStats(state, player):\n",
    "    stats = np.zeros(5)\n",
    "    data = state[player]['hero']\n",
    "    stats[0] = data['armor']\n",
    "    stats[1] = data['attack']\n",
    "    stats[2] = data['hp']\n",
    "    stats[3] = data['special_skill_used'] == True\n",
    "    stats[4] = data['weapon_durability']\n",
    "    \n",
    "    return stats\n",
    "\n",
    "def getPlayerPlayedCardsHpCurrentHistogram(state):\n",
    "    #maxval is 24, we divide to 4 folds\n",
    "    handHp = np.zeros(4)\n",
    "    for hand in state['player']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_current'], 24, 4)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getOpponentPlayedCardsHpCurrentHistogram(state):\n",
    "    #maxval is 32, we divide to 5 folds\n",
    "    handHp = np.zeros(5)\n",
    "    for hand in state['opponent']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_current'], 32, 5)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getPlayerPlayedCardsHpMaxHistogram(state):\n",
    "    #maxval is 30, we divide to 3 folds\n",
    "    handHp = np.zeros(3)\n",
    "    for hand in state['player']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_max'], 30, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getOpponentPlayedCardsHpMaxHistogram(state):\n",
    "    #maxval is 37, we divide to 4 folds\n",
    "    handHp = np.zeros(4)\n",
    "    for hand in state['opponent']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['hp_max'], 37, 4)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getPlayerPlayedCardsAttackHistogram(state):\n",
    "    #maxval is 23, we divide to 4 folds\n",
    "    handHp = np.zeros(4)\n",
    "    for hand in state['player']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['attack'], 23, 4)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getOpponentPlayedCardsAttackHistogram(state):\n",
    "    #maxval is 19, we divide to 3 folds\n",
    "    handHp = np.zeros(3)\n",
    "    for hand in state['opponent']['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['attack'], 19, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "def getPlayedCardsCrystalsCostHistogram(state, type):\n",
    "    #maxval is 8, we divide to 3 folds\n",
    "    handHp = np.zeros(3)\n",
    "    for hand in state[type]['played_cards']:\n",
    "        try:\n",
    "            handHp[getBin(hand['crystals_cost'], 8, 3)] +=1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return handHp\n",
    "\n",
    "def read_from_pickle(path):\n",
    "    with open(path, 'rb') as file:\n",
    "        try:\n",
    "            return pickle.load(file)\n",
    "        except EOFError:\n",
    "            pass    \n",
    "\n",
    "def craftAllFeatures(state):\n",
    "    return np.hstack([\n",
    "            getHandHpHistogram(state), getHandDurability(state), getHandAttackHistogram(state, \"MINION\"),\n",
    "            getHandAttackHistogram(state, \"WEAPON\"), getHandCrystalCostHistogram(state, \"MINION\"),\n",
    "            getHandCrystalCostHistogram(state, \"WEAPON\"), getHandCrystalCostHistogram(state, \"SPELL\"),\n",
    "            getHandAllSpecifics(state), getPlayedCardsAllSpecifics(state, 'player'), getHandCountsOfTypes(state),\n",
    "            getPlayerStats(state, 'player'), getHeroStats(state, 'player'), \n",
    "            getPlayerPlayedCardsHpCurrentHistogram(state), getPlayerPlayedCardsHpMaxHistogram(state),\n",
    "            getPlayerPlayedCardsAttackHistogram(state), getPlayedCardsCrystalsCostHistogram(state, 'player'),\n",
    "            getOpponentPlayedCardsHpCurrentHistogram(state), getOpponentPlayedCardsHpMaxHistogram(state),\n",
    "            getOpponentPlayedCardsAttackHistogram(state), getPlayedCardsCrystalsCostHistogram(state, 'opponent'),\n",
    "            getPlayedCardsAllSpecifics(state, 'opponent')\n",
    "        ])\n",
    "\n",
    "dataFeaturesOutput = '/home/data/aaia17/features/'\n",
    "\n",
    "def getAllJsonTraining(data, id):\n",
    "    handsHist = np.empty((0, 113), float)\n",
    "    for state in data:\n",
    "        handsHist = np.vstack([handsHist, craftAllFeatures(state)])\n",
    "    pickle.dump(handsHist, open(dataFeaturesOutput+'handsHist-%d.store'%id,\"wb\"))\n",
    "    #return handsHist\n",
    "\n",
    "i = 0\n",
    "#hunk = read_from_pickle(dataFeaturesOutput+'featureChunk-%d.store'%i)\n",
    "#getAllJsonTraining(chunk, i)                         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainingFiles = ['trainingData_tabular_chunk1.csv', 'trainingData_tabular_chunk2.csv',\n",
    "                 'trainingData_tabular_chunk3.csv', 'trainingData_tabular_chunk4.csv']\n",
    "testFiles = ['testData_tabular.csv']\n",
    "rawTrainingDataset = readDataAsFrame(trainingFiles)\n",
    "rawTestDataset = readDataAsFrame(testFiles)\n",
    "\n",
    "exclude_features = ['opponent.hero_card_id', 'player.hero_card_id']\n",
    "rawTrainingDataset = rawTrainingDataset.drop(exclude_features, axis = 1)\n",
    "rawTestDataset = rawTestDataset.drop(exclude_features, axis = 1)\n",
    "\n",
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "XRawTrainingDataset = rawTrainingDataset.values[:,2:43]\n",
    "XRawTestDataset = rawTestDataset.values[:,2:43]\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "XTrainingDatasetScaled = dataScaling(XRawTrainingDataset, min_max_scaler)\n",
    "XTestDatasetScaled = dataScaling(XRawTestDataset, min_max_scaler)\n",
    "\n",
    "dtrain = readJsonDatabyLine(dataPrefix, [\"trainingData_JSON_chunk1.json\", \"trainingData_JSON_chunk2.json\", \n",
    "                                                \"trainingData_JSON_chunk3.json\", \"trainingData_JSON_chunk4.json\"])\n",
    "jtrain = readJsonFromString(dtrain)\n",
    "dtest = readJsonDatabyLine(dataDepreciatedPrefix, [\"testData_JSON_chunk5.json\", \"testData_JSON_chunk6.json\", \n",
    "                                                \"testData_JSON_chunk7.json\"])\n",
    "jtest = readJsonFromString(dtest)\n",
    "ftrain = getAllJsonFeatures(jtrain)\n",
    "fest = getAllJsonFeatures(jtest)\n",
    "\n",
    "i = 0\n",
    "for state in jtrain[:100]:\n",
    "    for hand in state['player']['hand']:\n",
    "        try:\n",
    "            if hand['type'] == u'SPELL':\n",
    "                    print hand\n",
    "                    i+=1\n",
    "        except:\n",
    "            pass\n",
    "    if i==10:\n",
    "        break\n",
    "state = jtrain[0]\n",
    "np.hstack([getHandHpHistogram(state), getHandAttackHistogram(state, \"MINION\"), \n",
    "                              getHandCrystalCostHistogram(state, \"MINION\"), getAllSpecifics(state),\n",
    "                              getHandAttackHistogram(state, \"WEAPON\"), getHandCrystalCostHistogram(state, \"WEAPON\"),\n",
    "                              getHandDurability(state), getHandCrystalCostHistogram(state, \"SPELL\")]).shape        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataTrain = np.empty((0, 109), int)\n",
    "for i in range(0,10):\n",
    "    dataTrain = np.vstack([dataTrain, read_from_pickle(dataFeaturesPrefix+'handsHist-exp-%d.store'%i)])\n",
    "dataTrain = dataTrain[:,1:]    \n",
    "\n",
    "dataTest = np.empty((0, 109), int)\n",
    "for i in range(0,5):\n",
    "    dataTest = np.vstack([dataTest, read_from_pickle(dataFeaturesPrefix+'handsHistTest-exp2-%d.store'%i)])\n",
    "dataTest = dataTest[:,1:]       \n",
    "\n",
    "np.save(dataFeaturesOutput+\"trainSetScaled-old.dat\", XTrainingDatasetScaledWithOutliersNew)\n",
    "np.save(dataFeaturesOutput+\"testSetScaled-old.dat\", XTestDatasetScaledNew)\n",
    "\n",
    "r = np.zeros(750000)\n",
    "for i in range(0,750000):\n",
    "    r[i] = np.all(XRawTest[i,:25] == test_sparse[i,:25])\n",
    "\n",
    "np.all(r == True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainingFiles = ['trainingData_tabular_chunk1.csv', 'trainingData_tabular_chunk2.csv',\n",
    "                 'trainingData_tabular_chunk3.csv', 'trainingData_tabular_chunk4.csv']\n",
    "trainingFilesDepr = ['testData_tabular_chunk5.csv', 'testData_tabular_chunk6.csv',\n",
    "                 'testData_tabular_chunk7.csv']\n",
    "testFiles = ['testData_tabular.csv']\n",
    "rawTraining = readDataAsFrame(dataPrefix, trainingFiles)\n",
    "rawTest     = readDataAsFrame(dataPrefix, testFiles)\n",
    "\n",
    "rawTrainingDepr = readDataAsFrame(dataDepreciatedPrefix, trainingFilesDepr)\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "exclude_features = ['opponent.hero_card_id', 'player.hero_card_id']\n",
    "rawTrainingDataset = rawTraining.drop(exclude_features, axis = 1)\n",
    "rawTrainingDeprDataset = rawTrainingDepr.drop(exclude_features, axis = 1)\n",
    "rawTestDataset = rawTest.drop(exclude_features, axis = 1)\n",
    "\n",
    "XRawTrain = rawTrainingDataset.values[:,2:43]\n",
    "XRawTrainDepr = rawTrainingDeprDataset.values[:,2:43]\n",
    "XRawTest_  = rawTestDataset.values[:,2:43]\n",
    "\n",
    "XRawTrainAll_ = np.vstack([XRawTrain, XRawTrainDepr])\n",
    "XRawTrainAll  = np.hstack([XRawTrainAll_, dataTrain])\n",
    "XRawTest      = np.hstack([XRawTest_, dataTest])\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "XTrain_ = dataScaling(XRawTrainAll, min_max_scaler)\n",
    "XTest_  = dataScaling(XRawTest, min_max_scaler)\n",
    "\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "dimToRemove = getDimToRemove(XTrain_, XTest_)\n",
    "XTrain = stripDim(XTrain_, dimToRemove)\n",
    "XTest  = stripDim(XTest_, dimToRemove)\n",
    "\n",
    "dataFeaturesOutput = '/home/data/aaia17/features/'\n",
    "np.save(dataFeaturesOutput+\"train-features-146-bins.dat\", XTrain)\n",
    "np.save(dataFeaturesOutput+\"test-features-146-bins.dat\", XTest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pca = decomposition.PCA(svd_solver='randomized', iterated_power=7, n_components=63)\n",
    "pca.fit(XTraining)\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "XTrainingPCA = pca.transform(XTraining) \n",
    "XTestPCA = pca.transform(XTest) \n",
    "XTrainingPCA = dataScaling(XTrainingPCA, min_max_scaler)\n",
    "XTestPCA = dataScaling(XTestPCA, min_max_scaler)\n",
    "pickle.dump(XTrainingPCA, open(dataPrefix+'XTraining63PCA.store',\"wb\"))\n",
    "pickle.dump(XTestPCA, open(dataPrefix+'XTest63PCA.store',\"wb\"))\n",
    "print(pca.explained_variance_ratio_) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data reading from all chunks and making one  dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainingFiles = ['trainingData_tabular_chunk1.csv', 'trainingData_tabular_chunk2.csv',\n",
    "                 'trainingData_tabular_chunk3.csv', 'trainingData_tabular_chunk4.csv']\n",
    "rawTrainingDataset = readDataAsFrame(trainingFiles)\n",
    "\n",
    "testFiles = ['testData_tabular.csv']\n",
    "rawTestDataset = readDataAsFrame(testFiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "decision\n",
      "0     990401\n",
      "1    1009599\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(rawTrainingDataset.groupby('decision').size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rawTrainingDataset[]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take All Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XRawTrainingDataset = rawTrainingDataset.values[:,2:45]\n",
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "XRawTestDataset = rawTestDataset.values[:,2:45]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hand chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Actually, we don't use it !\n",
    "\n",
    "names = ['opponent.hp', 'opponent.played_minions_count', 'player.armor','player.hp', 'player.hand_count', \n",
    "         'player.played_minions_count', 'opponent.played.nOfCards', 'opponent.played.attack', 'opponent.played.crystals_cost'\n",
    "         ,'opponent.played.hp_current', 'opponent.played.hp_max', 'player.played.nOfCards', 'player.played.attack',\n",
    "        'player.played.crystals_cost', 'player.played.hp_current', 'player.played.hp_max', 'player.hand.nOfCards']\n",
    "X_train_handChosen = getNewForIndexes(rawTrainingDataset, names)\n",
    "X_test_handChosen  = getNewForIndexes(rawTestDataset, names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Outliers detection and removing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Actually, we don't use it !\n",
    "\n",
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "XRawTrainingDataset = rawTrainingDataset.values[:,2:45]\n",
    "\n",
    "rng = np.random.RandomState(42)\n",
    "clf = IsolationForest(max_samples=10000, random_state=rng, n_jobs=-1)\n",
    "clf.fit(XRawTrainingDataset)\n",
    "\n",
    "trainingOutliers = clf.predict(XRawTrainingDataset)\n",
    "\n",
    "XTrainigDatasetWithoutOutliers = XRawTrainingDataset[trainingOutliers>0]\n",
    "YTrainingDatasetWithoutOutliers = YRawTrainingDataset[trainingOutliers>0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automatic feature selection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Actually, we don't use it !\n",
    "\n",
    "estimator = LogisticRegression(random_state=rng)\n",
    "selector = RFECV(estimator, step=1, cv=10, scoring='roc_auc', n_jobs=-1)\n",
    "XTrainingSelected = selector.fit_transform(Xtry, Ytry)\n",
    "pickle.dump(XTrainingSelected, open(dataPrefix+'XTrainingSelected.store',\"wb\"))\n",
    "\n",
    "selector.ranking_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "XTrainingDatasetScaledWithOutliers = dataScaling(X_train_handChosen, min_max_scaler)\n",
    "XTestDatasetScaled = dataScaling(X_test_handChosen, min_max_scaler)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating train and validation probes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "validation_size = 0.30\n",
    "seed = 7\n",
    "X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(XTrainingDatasetScaledWithOutliers, \n",
    "        YRawTrainingDataset, test_size=validation_size, random_state=seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models probing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Actually, we don't use it !\n",
    "%%time\n",
    "\n",
    "seed = 7\n",
    "scoring = 'roc_auc'\n",
    "models = []\n",
    "models.append(('RFC', RandomForestClassifier(n_jobs=-1, n_estimators=40)))\n",
    "\n",
    "results = []\n",
    "names = []\n",
    "for name, model in models:\n",
    "    kfold = model_selection.KFold(n_splits=10, random_state=seed)\n",
    "    cv_results = model_selection.cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)\n",
    "    results.append(cv_results)\n",
    "    names.append(name)\n",
    "    msg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n",
    "    print(msg)\n",
    "    \n",
    "# Prediction on Test Set\n",
    "validation_size = 0.30\n",
    "seed = 7\n",
    "X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(XTrainingDatasetScaledWithOutliers, \n",
    "        YRawTrainingDataset, test_size=validation_size, random_state=seed)\n",
    "predicted = clf.predict(X_validation)\n",
    "predictedTestWon = predicted[:,1]\n",
    "print(metrics.classification_report(Y_validation, predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_test1 = {'n_estimators':range(20,101,10)}\n",
    "gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=1.9503, min_samples_split=500,min_samples_leaf=50,max_depth=4,max_features='sqrt',subsample=0.8,random_state=10), \n",
    "param_grid = param_test1, scoring='roc_auc',n_jobs=40,iid=False, cv=5)\n",
    "gsearch1.fit(XTraining, YTraining)\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n",
    "\n",
    "param_test2 = {'max_depth':range(4,12,2), 'min_samples_split':range(100,1101,200)}\n",
    "gsearch2 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=1.9503, min_samples_leaf=50,\n",
    "                                                               max_features='sqrt',subsample=0.8,random_state=10), \n",
    "param_grid = param_test2, scoring='roc_auc',n_jobs=40,iid=False, cv=5)\n",
    "gsearch2.fit(XTraining, YTraining)\n",
    "gsearch2.grid_scores_, gsearch2.best_params_, gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Averaging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allFiles = [f for f in os.listdir(averageDir)]\n",
    "mean = meanVector(allFiles)\n",
    "writeToFile(mean, dataPrefix+'final.txt')    \n",
    "# with weights\n",
    "weightsHand = [0.7961,0.7954,0.7954,0.7954,0.7954,0.7953,0.7953,0.7951,0.795,0.795,0.795,0.7949,0.7946,0.7944,0.7944,0.7943,0.7942,0.794,0.794,0.7939,0.7939,0.7938,0.7938,0.7938,0.7936,0.7936,0.7936,0.7935,0.7935,0.7935,0.7935,0.7934,0.7934,0.7933,0.793,0.7929]\n",
    "files=[1000,78,74,66,55,92,51,59,63,96,52,95,62,77,61,84,65,81,80,64,91,82,88,93,37,86,39,58,89,53,34,40,85,38,87,79]\n",
    "meanPredictedW = meanVectorW(\"predicted\", files[0:17], weightsHand[0:17])\n",
    "writeToFile(meanPredictedW, dataPrefix+'mean17.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allFiles = [f for f in os.listdir(modelsDir)]\n",
    "models = []\n",
    "modelsList = []\n",
    "for f in allFiles:\n",
    "    models.append((f, read_from_pickle(modelsDir+f)))\n",
    "    modelsList.append(read_from_pickle(modelsDir+f))\n",
    "\n",
    "map(lambda x: x.n_features, modelsList)    \n",
    "\n",
    "eclf = VotingClassifier(estimators=models, voting='hard', n_jobs=40)\n",
    "eclf = eclf.fit(XTraining, YTraining)\n",
    "\n",
    "eclf = VotingClassifier(estimators=models, voting='soft', n_jobs=20)\n",
    "eclf.estimators_ = modelsList\n",
    "#eclf = eclf.fit(XTraining, YTraining)\n",
    "predictedTestWon = eclf.predict_proba(XTest)[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'voting4.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Staging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-79-old-approach.dat.npy')[:,:-6]\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-79-old-approach.dat.npy')[:,:-6]\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(XTrain, labels, test_size=0.4, random_state=0)\n",
    "\n",
    "print XTrain.shape, XTest.shape\n",
    "\n",
    "i = 1\n",
    "est = 450\n",
    "md  = 5\n",
    "mss = 50\n",
    "msl = 5\n",
    "\n",
    "\n",
    "clf = GradientBoostingClassifier(n_estimators=est, max_depth=md, min_samples_split = mss, min_samples_leaf = msl, warm_start=True)\n",
    "clf = clf.fit(X_train, y_train)\n",
    "#predictedTestWon = clf.predict_proba(XTest)[:,1]\n",
    "#writeToFile(predictedTestWon, dataPrefix+'fb%d.txt'%i)\n",
    "pickle.dump(clf, open(dataPrefix+'overfit-gbc-old-6-%d.model'%i,\"wb\"))\n",
    "\n",
    "\n",
    "test_score = []\n",
    "train_score = []\n",
    "for i, pred in enumerate(clf.staged_decision_function(X_test)):\n",
    "    test_score.append((i,metrics.roc_auc_score(y_test, pred)))\n",
    "for i, pred in enumerate(clf.staged_decision_function(X_train)):\n",
    "    train_score.append((i, metrics.roc_auc_score(y_train, pred)))\n",
    "plt.plot(*zip(*test_score))\n",
    "plt.plot(*zip(*train_score))\n",
    "plt.legend(['test score', 'train score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-79-old-approach.dat.npy')\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-79-old-approach.dat.npy')\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(XTrain, labels, test_size=0.4, random_state=0)\n",
    "data = (X_train, y_train, X_test, y_test)\n",
    "clf = GradientBoostingClassifier(n_estimators=150, max_depth=4, min_samples_split = 50, min_samples_leaf = 5)\n",
    "\n",
    "for i in range(0,1000):\n",
    "    r = makeOneStep(clf, filepathToModels, data)\n",
    "    print i, r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gaussian Mixture Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allFiles = [f for f in os.listdir(averageDir)]\n",
    "models = map(lambda x: converseVec2Matrix(readFileByLine(averageDir+x)), allFiles)\n",
    "X_train = np.hstack(models)\n",
    "f1 = converseVec2Matrix(readFileByLine(averageDir+\"predicted78.txt\"))\n",
    "f2 = converseVec2Matrix(readFileByLine(averageDir+\"predicted74.txt\"))\n",
    "f3 = converseVec2Matrix(readFileByLine(averageDir+\"predicted66.txt\"))\n",
    "f4 = converseVec2Matrix(readFileByLine(averageDir+\"predicted55.txt\"))\n",
    "f5 = converseVec2Matrix(readFileByLine(averageDir+\"predicted1000.txt\"))\n",
    "f6 = converseVec2Matrix(readFileByLine(dataPrefix+\"mean15.txt\"))\n",
    "f7 = converseVec2Matrix(readFileByLine(dataPrefix+\"mean12.txt\"))\n",
    "f8 = converseVec2Matrix(readFileByLine(dataPrefix+\"mean14.txt\"))\n",
    "\n",
    "X_train = np.hstack([f1, f2, f3, f4, f5, f6, f7, f8])\n",
    "\n",
    "clf = mixture.GaussianMixture(n_components=2, covariance_type='full')\n",
    "clf.fit(X_train)\n",
    "\n",
    "arr = np.empty((len(preds), 1), float)\n",
    "for i in range(0, len(preds)):\n",
    "    arr[i] = np.dot(preds[i, :], X_train[i, :]) # ?????????????????"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-79-old-approach.dat.npy')[:-1250000]\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-79-old-approach.dat.npy')\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')[:-1250000]\n",
    "\n",
    "print XTrain.shape, XTest.shape, labels.shape\n",
    "\n",
    "i = 1\n",
    "est = 150\n",
    "md  = 5\n",
    "mss = 50\n",
    "msl = 5\n",
    "\n",
    "clf = GradientBoostingClassifier(n_estimators=est, max_depth=md, min_samples_split = mss, min_samples_leaf = msl)\n",
    "clf = clf.fit(XTrain, labels)\n",
    "predictedTestWon = clf.predict_proba(XTest)[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'f-1mln%d.txt'%i)\n",
    "pickle.dump(clf, open(dataPrefix+'gbc-f79-1mln-%d.model'%i,\"wb\"))\n",
    "\n",
    "clf.set_params(n_estimators=130, warm_start=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-64-bins.dat.npy')\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-64-bins.dat.npy')\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "i = 6\n",
    "est = 100\n",
    "md  = 5\n",
    "mss = 100\n",
    "msl = 10\n",
    "\n",
    "clf = GradientBoostingClassifier(n_estimators=est, max_depth=md, min_samples_split = mss, min_samples_leaf = msl, warm_start=True)\n",
    "clf = clf.fit(XTrain, labels)\n",
    "predictedTestWon = clf.predict_proba(XTest)[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'fb%d.txt'%i)\n",
    "pickle.dump(clf, open(dataPrefix+'gbc-bins-64-%d.model'%i,\"wb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample = XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-79-old-approach.dat.npy')[:,:-6]\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-79-old-approach.dat.npy')[:,:-6]\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "print XTrain.shape, XTest.shape\n",
    "\n",
    "sample = [71, 70, 59, 58, 32, 27, 22, 19, 17, 13, 6]\n",
    "xtrain, xtest = stripDimSets(XTrain, XTest, sample)\n",
    "print xtrain.shape, xtest.shape\n",
    "\n",
    "\n",
    "i = 10\n",
    "est = 150\n",
    "md  = 5\n",
    "mss = 100\n",
    "msl = 10\n",
    "\n",
    "clf = GradientBoostingClassifier(n_estimators=est, max_depth=md, min_samples_split = mss, min_samples_leaf = msl, warm_start=True)\n",
    "clf = clf.fit(xtrain, labels)\n",
    "predictedTestWon = clf.predict_proba(xtest)[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'f79-6-s%d.txt'%i)\n",
    "pickle.dump(clf, open(dataPrefix+'gbc-old-79-6-s%d.model'%i,\"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.optimizers import SGD\n",
    "import numpy as np\n",
    "from sklearn import model_selection\n",
    "import pickle\n",
    "\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "\n",
    "dataPrefix = '/home/data/aaia17/'\n",
    "dataFeaturesPrefix = dataPrefix+\"features/\"\n",
    "\n",
    "XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-79-old-approach.dat.npy')\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-79-old-approach.dat.npy')\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "labelsKeras = keras.utils.to_categorical(labels, num_classes=2)\n",
    "\n",
    "def initializeNN(frameSize):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(units=81, input_dim=frameSize, activation=K.tanh))\n",
    "    model.add(Activation(K.tanh))\n",
    "    model.add(Dropout(0.5))\n",
    "\n",
    "    model.add(Dense(units=60, activation=K.tanh))\n",
    "    model.add(Activation(K.tanh))\n",
    "    model.add(Dropout(0.5))\n",
    "\n",
    "    model.add(Dense(units=30, activation=K.tanh))\n",
    "    model.add(Activation(K.tanh))\n",
    "    model.add(Dropout(0.5))\n",
    "\n",
    "    model.add(Dense(units=2, activation=K.softmax))\n",
    "    model.add(Activation(K.softmax))\n",
    "\n",
    "    sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "\n",
    "model = initializeNN(XTrain.shape[1])\n",
    "\n",
    "model.fit(XTrain, labelsKeras, nb_epoch=200, verbose=1, callbacks=[], validation_data=None, shuffle=True,\n",
    "          class_weight=None,\n",
    "          sample_weight=None)\n",
    "\n",
    "i = 5\n",
    "\n",
    "y_score = model.predict(XTest)\n",
    "predictedTestWon = y_score[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'nn%d.txt'%i)\n",
    "pickle.dump(model, open(dataPrefix+'n110-%d.model'%i,\"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "from pandas.tools.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import svm\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "import pickle\n",
    "from sklearn import metrics\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import (RandomTreesEmbedding, IsolationForest,VotingClassifier, RandomForestClassifier, GradientBoostingClassifier)\n",
    "\n",
    "def replaceNan(data, item):\n",
    "    return [x if x is not None else item for x in data]\n",
    "\n",
    "def getDenseFeatures(dataFrame, names, dictionary):\n",
    "    X_dense = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        values = dataFrame[name].values\n",
    "        X_dense = np.column_stack([X_dense, replaceNan([dictionary[name].get(value) for value in values], 0.5)])\n",
    "    return X_dense\n",
    "def addNewFeature(data, vectors):\n",
    "    return np.column_stack([data, vectors])\n",
    "def getNewForIndexes(dataFrame, names):\n",
    "    X_selectedFeature = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        X_selectedFeature = np.column_stack([X_selectedFeature, dataFrame[name].values])\n",
    "    return X_selectedFeature\n",
    "def dataScaling(data, scaler):\n",
    "    data = data/(data+1)\n",
    "    return scaler.fit_transform(data)\n",
    "\n",
    "def _computeFractionForAttribute(dataFrame, name):\n",
    "    dictionary = dict()\n",
    "    values = set(dataFrame[name].values)\n",
    "    for value in values:\n",
    "        ind = dataFrame[name].values==value\n",
    "        subset = dataFrame['decision'][ind].values\n",
    "        fraction = 1.0*len(filter(lambda x: x==1, subset)) / len(subset)\n",
    "        dictionary[value] = fraction \n",
    "    return dictionary\n",
    "def computeFractionsForAll(dataFrame, names):\n",
    "    changeDict = dict()\n",
    "    for name in names:\n",
    "        changeDict[name] = _computeFractionForAttribute(dataFrame, name)\n",
    "    return changeDict\n",
    "def dropColumnsFromDataFrame(df, columns):\n",
    "    for column in columns:\n",
    "        df = df.drop(column, 1)\n",
    "    return df\n",
    "\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "def printify(t):\n",
    "    for x in t:\n",
    "        print(x)\n",
    "dataPrefix = '/home/data/aaia17/'\n",
    "def readDataAsFrame(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        frame.append(pandas.read_csv(fData))\n",
    "    return pandas.concat(frame)\n",
    "\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "\n",
    "def dataScaling(data, scaler):\n",
    "    return scaler.fit_transform(data)\n",
    "\n",
    "def stripDim(data, indexes):\n",
    "    indexes.sort(reverse=True)\n",
    "    for i in indexes:\n",
    "        data = np.concatenate((data[:,0:i], data[:,i+1:len(data[1,:])]), axis = 1)\n",
    "    return data\n",
    "\n",
    "def getDimToRemove(data1, data2):\n",
    "    dimToRemove = []\n",
    "    for i in range(0,len(data1[0,:])):\n",
    "        if np.max(data1[:,i]) == np.max(data2[:,i]) and np.max(data2[:,i]) == 0:\n",
    "            dimToRemove.append(i)\n",
    "    return dimToRemove\n",
    "\n",
    "trainingFiles = ['trainingData_tabular_chunk1.csv', 'trainingData_tabular_chunk2.csv',\n",
    "                 'trainingData_tabular_chunk3.csv', 'trainingData_tabular_chunk4.csv']\n",
    "rawTrainingDataset = readDataAsFrame(trainingFiles)\n",
    "testFiles = ['testData_tabular.csv']\n",
    "rawTestDataset = readDataAsFrame(testFiles)\n",
    "\n",
    "training_sparse = read_sparse_binary_set(dataPrefix + '/m.npy')\n",
    "test_sparse = read_sparse_binary_set(dataPrefix + '/m_test.npy')\n",
    "\n",
    "training_sparse_New = read_sparse_binary_set(dataPrefix + 'new/m_training.npy')\n",
    "test_sparse_New = read_sparse_binary_set(dataPrefix + 'new/m_test.npy')\n",
    "# dense features\n",
    "columnsToDensify = ['opponent.hero_card_id', 'player.hero_card_id']\n",
    "chd = computeFractionsForAll(rawTrainingDataset, columnsToDensify)\n",
    "X_train_denseFeatures = getDenseFeatures(rawTrainingDataset, columnsToDensify, chd)\n",
    "X_test_denseFeatures  = getDenseFeatures(rawTestDataset, columnsToDensify, chd)\n",
    "# drop categorial or unimportant\n",
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "columnsToDrop = ['decision','gamestate_id', 'opponent.hero_card_id', 'player.hero_card_id']\n",
    "rawDroppedTrainingDataset = dropColumnsFromDataFrame(rawTrainingDataset, columnsToDrop)\n",
    "rawDroppedTestDataset = dropColumnsFromDataFrame(rawTestDataset, columnsToDrop)\n",
    "# add some features before scaling\n",
    "XRawTrainingDataset = rawDroppedTrainingDataset.values.astype(float)\n",
    "XRawTestDataset     = rawDroppedTestDataset.values.astype(float)\n",
    "XRawTrainingDataset = np.concatenate((XRawTrainingDataset, training_sparse[:,0:78]), axis = 1)\n",
    "XRawTestDataset = np.concatenate((XRawTestDataset, test_sparse[:,0:78]), axis = 1)\n",
    "XRawTrainingDataset = np.concatenate((XRawTrainingDataset, training_sparse_New[:,1:40]), axis = 1)\n",
    "XRawTestDataset = np.concatenate((XRawTestDataset, test_sparse_New[:,1:40]), axis = 1)\n",
    "assert np.all(training_sparse[:,78] == rawTrainingDataset.values[:,0]) and  np.all(test_sparse[:,78] == rawTestDataset.values[:,0])\n",
    "assert np.all(training_sparse_New[:,0] == rawTrainingDataset.values[:,0]) and  np.all(test_sparse_New[:,0] == rawTestDataset.values[:,0])\n",
    "# data scaling\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "XTrainingDatasetScaledWithOutliers = dataScaling(XRawTrainingDataset, min_max_scaler)\n",
    "XTestDatasetScaled = dataScaling(XRawTestDataset, min_max_scaler)\n",
    "\n",
    "\n",
    "clf = GradientBoostingClassifier(n_estimators=500, max_depth=5, min_samples_split = 100, min_samples_leaf = 10)\n",
    "clf = clf.fit(XTrainingDatasetScaledWithOutliersNew, YRawTrainingDataset)\n",
    "predictedTestWon = clf.predict_proba(XTestDatasetScaledNew)[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'predicted67.txt')\n",
    "# add new features after scaling\n",
    "#XTrainingDatasetScaledWithOutliers = addNewFeature(XTrainingDatasetScaledWithOutliers, X_train_denseFeatures)\n",
    "#XTestDatasetScaled = addNewFeature(XTestDatasetScaled, X_test_denseFeatures)\n",
    "\n",
    "\n",
    "\n",
    "import pandas\n",
    "from pandas.tools.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import svm\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import (RandomTreesEmbedding, IsolationForest, RandomForestClassifier, GradientBoostingClassifier)\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "import pickle\n",
    "from sklearn import metrics\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import numpy as np\n",
    "import json\n",
    "\n",
    "dataPrefix = '/home/data/aaia17/'\n",
    "\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "\n",
    "def printify(t):\n",
    "    for x in t:\n",
    "        print(x)\n",
    "\n",
    "def readDataAsFrame(fNames):\n",
    "    frame = []\n",
    "    for fileName in fNames:\n",
    "        fData = dataPrefix+fileName\n",
    "        frame.append(pandas.read_csv(fData))\n",
    "    return pandas.concat(frame)\n",
    "\n",
    "def getNewForIndexes(dataFrame, names):\n",
    "    X_selectedFeature = np.empty((len(dataFrame), 0), float)\n",
    "    for name in names:\n",
    "        X_selectedFeature = np.column_stack([X_selectedFeature, dataFrame[name].values])\n",
    "    return X_selectedFeature\n",
    "\n",
    "def dataScaling(data, scaler):\n",
    "    data = data/(data+1)\n",
    "    return scaler.fit_transform(data)\n",
    "\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "\n",
    "def stripDim(data, indexes):\n",
    "    indexes.sort(reverse=True)\n",
    "    for i in indexes:\n",
    "        data = np.concatenate((data[:,0:i], data[:,i+1:len(data[1,:])]), axis = 1)\n",
    "    return data\n",
    "\n",
    "def getDimToRemove(data1, data2):\n",
    "    dimToRemove = []\n",
    "    for i in range(0,len(data1[0,:])):\n",
    "        if np.max(data1[:,i]) == np.max(data2[:,i]) and np.max(data2[:,i]) == 0:\n",
    "            dimToRemove.append(i)\n",
    "    return dimToRemove\n",
    "\n",
    "exclude_features = ['opponent.hero_card_id', 'player.hero_card_id']\n",
    "\n",
    "trainingFiles = ['trainingData_tabular_chunk1.csv', 'trainingData_tabular_chunk2.csv',\n",
    "                 'trainingData_tabular_chunk3.csv', 'trainingData_tabular_chunk4.csv']\n",
    "rawTrainingDataset = readDataAsFrame(trainingFiles)\n",
    "#rawTrainingDataset = rawTrainingDataset.drop(exclude_features, axis = 1)\n",
    "\n",
    "testFiles = ['testData_tabular.csv']\n",
    "rawTestDataset = readDataAsFrame(testFiles)\n",
    "#rawTestDataset = rawTestDataset.drop(exclude_features, axis = 1)\n",
    "\n",
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "XRawTrainingDataset = rawTrainingDataset.values[:,2:45]\n",
    "XRawTestDataset = rawTestDataset.values[:,2:45]\n",
    "\n",
    "training_sparse = read_sparse_binary_set(dataPrefix + 'new/m_training.npy')\n",
    "# training_sparse2 = read_sparse_binary_set(dataPrefix + 'm_test2_tzw_nowe_cechy.npy')\n",
    "\n",
    "test_sparse = read_sparse_binary_set(dataPrefix + 'new/m_test.npy')\n",
    "\n",
    "cards_types_training = np.load(dataPrefix + 'new2/card_details_ready_matrix_training.npy')\n",
    "# cards_types_training2 = np.load(dataPrefix + 'card_details_ready_matrix_test2.npy')\n",
    "\n",
    "# training_sparse = np.concatenate((training_sparse1, training_sparse2), axis = 0)\n",
    "#  cards_types_training = np.concatenate((cards_types_training1, cards_types_training2), axis = 0)\n",
    "\n",
    "cards_types_test = np.load(dataPrefix + 'new2/card_details_ready_matrix_test.npy')\n",
    "\n",
    "#XRawTestDataset = np.concatenate((XRawTestDataset, test_sparse[:,1:40], cards_types_test), axis = 1)\n",
    "#XRawTrainingDataset = np.concatenate((XRawTrainingDataset, training_sparse[:,1:40], cards_types_training), axis = 1)\n",
    "\n",
    "XRawTestDataset = np.concatenate((XRawTestDataset, test_sparse[:,1:40]), axis = 1)\n",
    "XRawTrainingDataset = np.concatenate((XRawTrainingDataset, training_sparse[:,1:40]), axis = 1)\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "#XTrainingDatasetScaledWithOutliers = dataScaling(XRawTrainingDataset, min_max_scaler)\n",
    "#XTestDatasetScaled = dataScaling(XRawTestDataset, min_max_scaler)\n",
    "\n",
    "\n",
    "\n",
    "tuplesTrain = set()\n",
    "XtrainMap=dict()\n",
    "YtrainMap=dict()\n",
    "XtestMap=dict()\n",
    "i = 0\n",
    "for entry in XRawTrainingDataset:\n",
    "    tuplesTrain.add((entry[3], entry[15]))\n",
    "    toAdd = res = np.hstack([entry[0:3],entry[4:15],entry[16:len(entry)]])\n",
    "    if (entry[3], entry[15]) in XtrainMap:\n",
    "        XtrainMap[(entry[3], entry[15])].append(toAdd)\n",
    "        YtrainMap[(entry[3], entry[15])].append(YRawTrainingDataset[i])\n",
    "    else:\n",
    "        XtrainMap[(entry[3], entry[15])] = [toAdd]\n",
    "        YtrainMap[(entry[3], entry[15])] = [YRawTrainingDataset[i]]\n",
    "    i+=1\n",
    "tuplesTest = set()\n",
    "i = 0\n",
    "for entry in XRawTestDataset:\n",
    "    tuplesTest.add((entry[3], entry[15]))  \n",
    "    toAdd = res = np.hstack([entry[0:3],entry[4:15],entry[16:len(entry)]])\n",
    "    if (entry[3], entry[15]) in XtestMap:\n",
    "        XtestMap[(entry[3], entry[15])].append((toAdd,i))\n",
    "    else:\n",
    "        XtestMap[(entry[3], entry[15])] = [(toAdd,i)]\n",
    "    i+=1\n",
    "m1 = []\n",
    "m2 = []\n",
    "m3 = []\n",
    "for (k1,k2) in XtrainMap:\n",
    "    if len(m1)<27:\n",
    "        m1.append((k1,k2))\n",
    "    elif len(m2)<27:\n",
    "        m2.append((k1,k2))\n",
    "    else:\n",
    "        m3.append((k1,k2))\n",
    "def partialPrediction(tuple):\n",
    "    clf = GradientBoostingClassifier(n_estimators=150, max_depth=5, min_samples_split = 50, min_samples_leaf = 5)\n",
    "    clf = clf.fit(dataScaling(np.array(XtrainMap[tuple]),min_max_scaler), YtrainMap[tuple])\n",
    "    predictedTestWon = clf.predict_proba(dataScaling(np.array(map(lambda x: x[0], XtestMap[tuple])), min_max_scaler))[:,1]\n",
    "    return zip(map(lambda x: x[1], XtestMap[tuple]), predictedTestWon)\n",
    "partialRes3 = []\n",
    "for tuple in m3:\n",
    "    partialRes3.append(partialPrediction(tuple))\n",
    "    \n",
    "for (k1,k2) in XtrainMap:\n",
    "    print k1,k2,len(XtrainMap[(k1, k2)]), len(YtrainMap[(k1, k2)])    \n",
    "    \n",
    "from sklearn.ensemble import (RandomTreesEmbedding, IsolationForest, RandomForestClassifier, GradientBoostingClassifier)\n",
    "dataFeaturesPrefix = '/home/data/aaia17/features/'\n",
    "dataPrefix = '/home/data/aaia17/'\n",
    "\n",
    "def read_sparse_binary_set(matrix_path):\n",
    "    return np.load(matrix_path)\n",
    "def writeToFile(data, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for item in data:\n",
    "            f.write(\"%s\\n\" % item)\n",
    "    f.close()\n",
    "XTrain = read_sparse_binary_set(dataFeaturesPrefix + 'train-features-79-old-approach.dat.npy')\n",
    "XTest  = read_sparse_binary_set(dataFeaturesPrefix + 'test-features-79-old-approach.dat.npy')\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "clf = GradientBoostingClassifier(n_estimators=150, max_depth=5, min_samples_split = 50, min_samples_leaf = 5)\n",
    "clf = clf.fit(XTrain, labels)\n",
    "predictedTestWon = clf.predict_proba(XTest)[:,1]\n",
    "writeToFile(predictedTestWon, dataPrefix+'f1.txt')\n",
    "\n",
    "\n",
    "trainData = np.empty((0, 113), float)\n",
    "for i in range(0,10):\n",
    "    trainData = np.vstack([trainData, read_from_pickle(dataFeaturesOutput+'handsHist-%d.store'%i)])\n",
    "    \n",
    "testData = np.empty((0, 113), float)\n",
    "for i in range(0,5):\n",
    "    testData = np.vstack([testData, read_from_pickle(dataFeaturesOutput+'handsHistTest-%d.store'%i)])    \n",
    "    \n",
    "np.save(dataFeaturesOutput+\"trainSet.dat\", trainData)\n",
    "np.save(dataFeaturesOutput+\"testSet.dat\", testData)\n",
    "np.save(dataFeaturesOutput+\"labels.dat\", labels)   \n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "XTrain_t = dataScaling(trainData, min_max_scaler)\n",
    "XTest_t = dataScaling(testData, min_max_scaler)\n",
    "\n",
    "dimToRemove = getDimToRemove(XTrain_t, XTest_t)\n",
    "XTrain = stripDim(XTrain_t, dimToRemove)\n",
    "XTest = stripDim(XTest_t, dimToRemove)\n",
    "\n",
    "np.save(dataFeaturesOutput+\"trainSetScaled.dat\", XTrain)\n",
    "np.save(dataFeaturesOutput+\"testSetScaled.dat\", XTest)\n",
    "\n",
    "\n",
    "chosen = read_from_pickle(dataPrefix+\"chosen-64.store\")\n",
    "chosen\n",
    "\n",
    "\n",
    "trainingFiles = ['trainingData_tabular_chunk1.csv', 'trainingData_tabular_chunk2.csv',\n",
    "                 'trainingData_tabular_chunk3.csv', 'trainingData_tabular_chunk4.csv']\n",
    "trainingFilesDepr = ['testData_tabular_chunk5.csv', 'testData_tabular_chunk6.csv',\n",
    "                 'testData_tabular_chunk7.csv']\n",
    "testFiles = ['testData_tabular.csv']\n",
    "rawTraining = readDataAsFrame(dataPrefix, trainingFiles)\n",
    "rawTest     = readDataAsFrame(dataPrefix, testFiles)\n",
    "\n",
    "rawTrainingDepr = readDataAsFrame(dataDepreciatedPrefix, trainingFilesDepr)\n",
    "labels = read_sparse_binary_set(dataFeaturesPrefix + 'labels.dat.npy')\n",
    "\n",
    "exclude_features = ['opponent.hero_card_id', 'player.hero_card_id']\n",
    "rawTrainingDataset = rawTraining.drop(exclude_features, axis = 1)\n",
    "rawTrainingDeprDataset = rawTrainingDepr.drop(exclude_features, axis = 1)\n",
    "rawTestDataset = rawTest.drop(exclude_features, axis = 1)\n",
    "\n",
    "XRawTrain = rawTrainingDataset.values[:,2:43]\n",
    "XRawTrainDepr = rawTrainingDeprDataset.values[:,2:43]\n",
    "XRawTest_  = rawTestDataset.values[:,2:43]\n",
    "\n",
    "XRawTrainAll_ = np.vstack([XRawTrain, XRawTrainDepr])\n",
    "XRawTrainAll  = np.hstack([XRawTrainAll_, dataTrain])\n",
    "XRawTest      = np.hstack([XRawTest_, dataTest])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make dense features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columnsToDensify = ['opponent.hero_card_id', 'player.hero_card_id']\n",
    "chd = computeFractionsForAll(rawTrainingDataset, columnsToDensify)\n",
    "X_train_denseFeatures = getDenseFeatures(rawTrainingDataset, columnsToDensify, chd)\n",
    "X_test_denseFeatures  = getDenseFeatures(rawTestDataset, columnsToDensify, chd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Drop categorical or unnecessary features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "columnsToDrop = ['decision','gamestate_id', 'opponent.hero_card_id', 'player.hero_card_id']\n",
    "rawDroppedTrainingDataset = dropColumnsFromDataFrame(rawTrainingDataset, columnsToDrop)\n",
    "rawDroppedTestDataset = dropColumnsFromDataFrame(rawTestDataset, columnsToDrop)\n",
    "\n",
    "YRawTrainingDataset = rawTrainingDataset.values[:,1]\n",
    "columnsToDrop = ['decision','gamestate_id']\n",
    "rawDroppedTrainingDataset = dropColumnsFromDataFrame(rawTrainingDataset, columnsToDrop)\n",
    "rawDroppedTestDataset = dropColumnsFromDataFrame(rawTestDataset, columnsToDrop)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add some features before scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XRawTrainingDataset = rawDroppedTrainingDataset.values.astype(float)\n",
    "XRawTestDataset     = rawDroppedTestDataset.values.astype(float)\n",
    "\n",
    "#XRawTrainingDataset = np.concatenate((XRawTrainingDataset, training_sparse[:,0:78]), axis = 1)\n",
    "#XRawTestDataset = np.concatenate((XRawTestDataset, test_sparse[:,0:78]), axis = 1)\n",
    "XRawTrainingDataset = np.concatenate((XRawTrainingDataset, training_sparse_New[:,1:40]), axis = 1)\n",
    "XRawTestDataset = np.concatenate((XRawTestDataset, test_sparse_New[:,1:40]), axis = 1)\n",
    "#assert np.all(training_sparse[:,78] == rawTrainingDataset.values[:,0]) and  np.all(test_sparse[:,78] == rawTestDataset.values[:,0])\n",
    "assert np.all(training_sparse_New[:,0] == rawTrainingDataset.values[:,0]) and  np.all(test_sparse_New[:,0] == rawTestDataset.values[:,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add new features after scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XTrainingDatasetScaledWithOutliers = addNewFeature(XTrainingDatasetScaledWithOutliers, X_train_denseFeatures)\n",
    "XTestDatasetScaled = addNewFeature(XTestDatasetScaled, X_test_denseFeatures)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}