statoli_iceberg_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model from https://www.kaggle.com/sinkie/keras-data-augmentation-with-multiple-inputs/notebookm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(656)\n",
    "import os\n",
    "import keras\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from IPython.display import FileLink\n",
    "from keras.preprocessing import image\n",
    "from subprocess import check_output\n",
    "from matplotlib import pyplot\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten\n",
    "from keras.layers import GlobalMaxPooling2D, merge, BatchNormalization, Permute\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Load data\n",
    "train = pd.read_json(\"/home/odenigborig/Data/kaggle/statoli_iceberg/data/processed/train.json\")\n",
    "test = pd.read_json(\"/home/odenigborig/Data/kaggle/statoli_iceberg/data/processed/test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#replace na with zeros in inc_angle (incline) columne\n",
    "train.inc_angle = train.inc_angle.replace('na', 0) \n",
    "train.inc_angle = train.inc_angle.astype(float).fillna(0.0)\n",
    "\n",
    "test.inc_angle = test.inc_angle.replace('na', 0) \n",
    "test.inc_angle = test.inc_angle.astype(float).fillna(0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Prepare training data\n",
    "x_band1 = np.array([np.array(band).astype(np.float32).reshape(75,75) for band in train[\"band_1\"]])\n",
    "x_band2 = np.array([np.array(band).astype(np.float32).reshape(75,75) for band in train[\"band_2\"]])\n",
    "X_train = np.concatenate([x_band1[:, :, :, np.newaxis],\n",
    "                          x_band2[:, :, :, np.newaxis],\n",
    "                          ((x_band1 + x_band2)/2)[:,:,:, np.newaxis]], axis=-1)\n",
    "X_angle_train = np.array(train.inc_angle)\n",
    "y_train = np.array(train[\"is_iceberg\"])\n",
    "\n",
    "#Prepare test data\n",
    "x_band1 = np.array([np.array(band).astype(np.float32).reshape(75,75) for band in test[\"band_1\"]])\n",
    "x_band2 = np.array([np.array(band).astype(np.float32).reshape(75,75) for band in test[\"band_2\"]])\n",
    "X_test = np.concatenate([x_band1[:, :, :, np.newaxis],\n",
    "                          x_band2[:, :, :, np.newaxis],\n",
    "                          ((x_band1 + x_band2)/2)[:,:,:, np.newaxis]], axis=-1)\n",
    "X_angle_test = np.array(test.inc_angle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare validation set\n",
    "X_train, X_valid, X_angle_train, X_angle_valid, y_train, y_valid = train_test_split(X_train, \n",
    "            X_angle_train,y_train, random_state=123, train_size=0.75)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#functions\n",
    "def get_callbacks(filepath, patience=2):\n",
    "    #callback to save model after each epoch\n",
    "    es = EarlyStopping('val_loss', patience=patience, mode=\"min\")\n",
    "    msave = ModelCheckpoint(filepath, save_best_only=True)\n",
    "    return [es, msave]\n",
    "\n",
    "def get_model(p_dropout=0):\n",
    "    p_activation = \"relu\"\n",
    "        \n",
    "    img_input = Input(shape=(75,75,3), name=\"image_input\")\n",
    "    angle_input = Input(shape=(1,), name=\"input_angle\")\n",
    "\n",
    "    cnn_input = Permute([3,1,2])(img_input)\n",
    "    cnn = BatchNormalization()(cnn_input)\n",
    "    \n",
    "    for i in range(4):\n",
    "        cnn = Conv2D(8*2**i, 3, 3, activation=p_activation, name='conv_' + str(i))(cnn)\n",
    "        #print(cnn._keras_shape)\n",
    "        cnn = MaxPooling2D((2,2), name='max_pool_' + str(i))(cnn)\n",
    "        cnn = Dropout(p_dropout)(cnn)\n",
    "        #print(cnn._keras_shape)\n",
    "\n",
    "    cnn = GlobalMaxPooling2D()(cnn)\n",
    "\n",
    "    #use lower case merge for concatenation of TensorVariables, uppercase Merge is for layers\n",
    "    cnn = merge([cnn, angle_input], mode='concat',concat_axis=1)\n",
    "\n",
    "    cnn = Dense(32,activation=p_activation) (cnn)\n",
    "    cnn_output = Dense(1, activation='sigmoid')(cnn)\n",
    "\n",
    "    model = Model(input=[img_input, angle_input], output=cnn_output,name='CNN')\n",
    "    model.compile(optimizer='adam', loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = os.getcwd()\n",
    "model_path = path + 'models/'\n",
    "file_path = model_path + \".model2_weights.hdf5\"\n",
    "callbacks = get_callbacks(filepath=file_path, patience=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1203 samples, validate on 401 samples\n",
      "Epoch 1/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1595 - acc: 0.9335 - val_loss: 0.2457 - val_acc: 0.9052\n",
      "Epoch 2/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1488 - acc: 0.9451 - val_loss: 0.2490 - val_acc: 0.8928\n",
      "Epoch 3/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1388 - acc: 0.9485 - val_loss: 0.2441 - val_acc: 0.9052\n",
      "Epoch 4/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1584 - acc: 0.9277 - val_loss: 0.2479 - val_acc: 0.8953\n",
      "Epoch 5/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1273 - acc: 0.9593 - val_loss: 0.2538 - val_acc: 0.9027\n",
      "Epoch 6/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1350 - acc: 0.9426 - val_loss: 0.2665 - val_acc: 0.9077\n",
      "Epoch 7/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1235 - acc: 0.9518 - val_loss: 0.3037 - val_acc: 0.8853\n",
      "Epoch 8/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1201 - acc: 0.9534 - val_loss: 0.2971 - val_acc: 0.8903\n",
      "Epoch 9/25\n",
      "1203/1203 [==============================] - 0s - loss: 0.1118 - acc: 0.9584 - val_loss: 0.2937 - val_acc: 0.8878\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bd3905750>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([X_train, X_angle_train], y_train, nb_epoch=25, \n",
    "          validation_data=([X_valid, X_angle_valid], y_valid), \n",
    "          batch_size=64, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8352/8424 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='mod2_subm1.csv' target='_blank'>mod2_subm1.csv</a><br>"
      ],
      "text/plain": [
       "/home/odenigborig/Github/Kaggle/statoli_iceberg/mod2_subm1.csv"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_weights(model_path+'model2_2.h5')\n",
    "prediction = model.predict([X_test, X_angle_test], verbose=1)\n",
    "submission = pd.DataFrame({'id': test[\"id\"], 'is_iceberg': prediction.reshape((prediction.shape[0]))})\n",
    "submission.to_csv(\"mod2_subm2.csv\", index=False)\n",
    "\n",
    "#create downloadable submission file to upload\n",
    "submission_filename = 'mod2_subm2.csv'\n",
    "\n",
    "FileLink(submission_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#specify data augmentation types\n",
    "batch_size = 64\n",
    "\n",
    "#define generator and image transformations\n",
    "gen = image.ImageDataGenerator(rotation_range=40, width_shift_range=0.1,\n",
    "                               height_shift_range=0.1, zoom_range=0.1, \n",
    "                               vertical_flip = True, horizontal_flip=True,\n",
    "                               dim_ordering = 'tf')\n",
    "\n",
    "#Merge two generators with the same random seed\n",
    "def gen_flow_two_inputs(img_data, angle_data, y_label):\n",
    "    gen1 = gen.flow(X=img_data,y=y_label, batch_size=batch_size, seed=656)\n",
    "    gen2 = gen.flow(X=img_data,y=angle_data, batch_size=batch_size, seed=656)\n",
    "    \n",
    "    while True:\n",
    "        X1 = gen1.next()\n",
    "        X2 = gen2.next()\n",
    "        \n",
    "        #return batches\n",
    "        yield [X1[0], X2[1]], X1[1]\n",
    "\n",
    "#create generator\n",
    "gen_flow = gen_flow_two_inputs(X_train, X_angle_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 75, 75, 3)\n",
      "(64,)\n",
      "(64,)\n"
     ]
    }
   ],
   "source": [
    "gen1 = gen.flow(X=X_train,y=y_train, batch_size=batch_size, seed=656)\n",
    "gen2 = gen.flow(X=X_train,y=X_angle_train, batch_size=batch_size, seed=656)\n",
    "X1 = gen1.next()\n",
    "X2 = gen2.next()\n",
    "\n",
    "print(X1[0].shape) \n",
    "print(X1[1].shape)\n",
    "print(X2[1].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit Model output with generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "64/64 [==============================] - 0s - loss: 0.6137 - acc: 0.6406 - val_loss: 1.4423 - val_acc: 0.5187\n",
      "Epoch 2/25\n",
      "64/64 [==============================] - 0s - loss: 0.7751 - acc: 0.4688 - val_loss: 2.8410 - val_acc: 0.5187\n",
      "Epoch 3/25\n",
      "64/64 [==============================] - 0s - loss: 0.7125 - acc: 0.3594 - val_loss: 4.1891 - val_acc: 0.5187\n",
      "Epoch 4/25\n",
      "64/64 [==============================] - 0s - loss: 0.6953 - acc: 0.5938 - val_loss: 3.8804 - val_acc: 0.5187\n",
      "Epoch 5/25\n",
      "64/64 [==============================] - 0s - loss: 0.7099 - acc: 0.5312 - val_loss: 3.1858 - val_acc: 0.5187\n",
      "Epoch 6/25\n",
      "64/64 [==============================] - 0s - loss: 0.6743 - acc: 0.6719 - val_loss: 2.4731 - val_acc: 0.5810\n",
      "Epoch 7/25\n",
      "64/64 [==============================] - 0s - loss: 0.7073 - acc: 0.5781 - val_loss: 1.7408 - val_acc: 0.5835\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bd2323150>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fit model with generator batches\n",
    "model.fit_generator(gen_flow,samples_per_epoch= batch_size,\n",
    "                    validation_data=([X_valid, X_angle_valid], y_valid),\n",
    "                    nb_epoch=25, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(filepath=file_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4068 - acc: 0.8438 - val_loss: 0.2499 - val_acc: 0.8928\n",
      "Epoch 2/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4174 - acc: 0.8906 - val_loss: 0.2703 - val_acc: 0.8903\n",
      "Epoch 3/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3562 - acc: 0.8594 - val_loss: 0.2659 - val_acc: 0.8878\n",
      "Epoch 4/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3248 - acc: 0.8594 - val_loss: 0.2485 - val_acc: 0.8953\n",
      "Epoch 5/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2832 - acc: 0.9219 - val_loss: 0.2396 - val_acc: 0.9002\n",
      "Epoch 6/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.5045 - acc: 0.7500 - val_loss: 0.2389 - val_acc: 0.9052\n",
      "Epoch 7/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3824 - acc: 0.8281 - val_loss: 0.2511 - val_acc: 0.8978\n",
      "Epoch 8/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3926 - acc: 0.7969 - val_loss: 0.2443 - val_acc: 0.8928\n",
      "Epoch 9/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4197 - acc: 0.8125 - val_loss: 0.2477 - val_acc: 0.8953\n",
      "Epoch 10/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4014 - acc: 0.8438 - val_loss: 0.2474 - val_acc: 0.9027\n",
      "Epoch 11/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3873 - acc: 0.8594 - val_loss: 0.2512 - val_acc: 0.8928\n",
      "Epoch 12/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2485 - acc: 0.8906 - val_loss: 0.2554 - val_acc: 0.8978\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bd2851310>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(gen_flow,samples_per_epoch= len(X_train)/batch_size,\n",
    "                    validation_data=([X_valid, X_angle_valid], y_valid),\n",
    "                    nb_epoch=25, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8352/8424 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='mod2_subm3.csv' target='_blank'>mod2_subm3.csv</a><br>"
      ],
      "text/plain": [
       "/home/odenigborig/Github/Kaggle/statoli_iceberg/mod2_subm3.csv"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_weights(model_path+'model2_3.h5')\n",
    "prediction = model.predict([X_test, X_angle_test], verbose=1)\n",
    "submission = pd.DataFrame({'id': test[\"id\"], 'is_iceberg': prediction.reshape((prediction.shape[0]))})\n",
    "submission.to_csv(\"mod2_subm3.csv\", index=False)\n",
    "\n",
    "#create downloadable submission file to upload\n",
    "submission_filename = 'mod2_subm3.csv'\n",
    "\n",
    "FileLink(submission_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = Adam(lr=1e-4)\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4688 - acc: 0.7812 - val_loss: 0.2525 - val_acc: 0.9002\n",
      "Epoch 2/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4089 - acc: 0.8281 - val_loss: 0.2509 - val_acc: 0.9002\n",
      "Epoch 3/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4948 - acc: 0.8125 - val_loss: 0.2500 - val_acc: 0.9027\n",
      "Epoch 4/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.5399 - acc: 0.7656 - val_loss: 0.2492 - val_acc: 0.9002\n",
      "Epoch 5/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4610 - acc: 0.7969 - val_loss: 0.2483 - val_acc: 0.9052\n",
      "Epoch 6/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4881 - acc: 0.7969 - val_loss: 0.2478 - val_acc: 0.9052\n",
      "Epoch 7/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4616 - acc: 0.7812 - val_loss: 0.2471 - val_acc: 0.9102\n",
      "Epoch 8/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2256 - acc: 0.9062 - val_loss: 0.2460 - val_acc: 0.9077\n",
      "Epoch 9/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.4360 - acc: 0.7812 - val_loss: 0.2445 - val_acc: 0.9077\n",
      "Epoch 10/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3582 - acc: 0.8281 - val_loss: 0.2435 - val_acc: 0.9077\n",
      "Epoch 11/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3150 - acc: 0.9062 - val_loss: 0.2430 - val_acc: 0.9102\n",
      "Epoch 12/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2196 - acc: 0.9062 - val_loss: 0.2426 - val_acc: 0.9102\n",
      "Epoch 13/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2939 - acc: 0.8438 - val_loss: 0.2423 - val_acc: 0.9102\n",
      "Epoch 14/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2064 - acc: 0.9375 - val_loss: 0.2421 - val_acc: 0.9077\n",
      "Epoch 15/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2770 - acc: 0.8750 - val_loss: 0.2418 - val_acc: 0.9052\n",
      "Epoch 16/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3191 - acc: 0.8594 - val_loss: 0.2417 - val_acc: 0.9052\n",
      "Epoch 17/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3728 - acc: 0.8438 - val_loss: 0.2415 - val_acc: 0.9052\n",
      "Epoch 18/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2777 - acc: 0.8750 - val_loss: 0.2414 - val_acc: 0.9052\n",
      "Epoch 19/25\n",
      "51/18 [=====================================================================================] - 0s - loss: 0.3667 - acc: 0.8431 - val_loss: 0.2409 - val_acc: 0.9077\n",
      "Epoch 20/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3992 - acc: 0.8594 - val_loss: 0.2401 - val_acc: 0.9077\n",
      "Epoch 21/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3489 - acc: 0.8281 - val_loss: 0.2397 - val_acc: 0.9077\n",
      "Epoch 22/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2671 - acc: 0.8438 - val_loss: 0.2394 - val_acc: 0.9102\n",
      "Epoch 23/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2478 - acc: 0.8750 - val_loss: 0.2390 - val_acc: 0.9102\n",
      "Epoch 24/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.3798 - acc: 0.8125 - val_loss: 0.2386 - val_acc: 0.9102\n",
      "Epoch 25/25\n",
      "64/18 [==========================================================================================================] - 0s - loss: 0.2527 - acc: 0.8750 - val_loss: 0.2384 - val_acc: 0.9027\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bc385a1d0>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(gen_flow,samples_per_epoch= len(X_train)/batch_size,\n",
    "                    validation_data=([X_valid, X_angle_valid], y_valid),\n",
    "                    nb_epoch=25, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      "64/64 [==============================] - 0s - loss: 0.3775 - acc: 0.7969 - val_loss: 0.2381 - val_acc: 0.9077\n",
      "Epoch 2/40\n",
      "64/64 [==============================] - 0s - loss: 0.1831 - acc: 0.9531 - val_loss: 0.2379 - val_acc: 0.9077\n",
      "Epoch 3/40\n",
      "64/64 [==============================] - 0s - loss: 0.4738 - acc: 0.7969 - val_loss: 0.2377 - val_acc: 0.9077\n",
      "Epoch 4/40\n",
      "64/64 [==============================] - 0s - loss: 0.3556 - acc: 0.8750 - val_loss: 0.2375 - val_acc: 0.9077\n",
      "Epoch 5/40\n",
      "64/64 [==============================] - 0s - loss: 0.4887 - acc: 0.8281 - val_loss: 0.2375 - val_acc: 0.9052\n",
      "Epoch 6/40\n",
      "64/64 [==============================] - 0s - loss: 0.3701 - acc: 0.8125 - val_loss: 0.2377 - val_acc: 0.9052\n",
      "Epoch 7/40\n",
      "64/64 [==============================] - 0s - loss: 0.3490 - acc: 0.8906 - val_loss: 0.2380 - val_acc: 0.9052\n",
      "Epoch 8/40\n",
      "64/64 [==============================] - 0s - loss: 0.2217 - acc: 0.9219 - val_loss: 0.2382 - val_acc: 0.9102\n",
      "Epoch 9/40\n",
      "64/64 [==============================] - 0s - loss: 0.4408 - acc: 0.7969 - val_loss: 0.2384 - val_acc: 0.9102\n",
      "Epoch 10/40\n",
      "64/64 [==============================] - 0s - loss: 0.2752 - acc: 0.8594 - val_loss: 0.2385 - val_acc: 0.9077\n",
      "Epoch 11/40\n",
      "64/64 [==============================] - 0s - loss: 0.3522 - acc: 0.8281 - val_loss: 0.2387 - val_acc: 0.9077\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bc3870090>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(gen_flow,samples_per_epoch= len(X_train)/batch_size,\n",
    "                    validation_data=([X_valid, X_angle_valid], y_valid),\n",
    "                    nb_epoch=40, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8352/8424 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='mod2_subm4.csv' target='_blank'>mod2_subm4.csv</a><br>"
      ],
      "text/plain": [
       "/home/odenigborig/Github/Kaggle/statoli_iceberg/mod2_subm4.csv"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_weights(model_path+'model2_4.h5')\n",
    "prediction = model.predict([X_test, X_angle_test], verbose=1)\n",
    "submission = pd.DataFrame({'id': test[\"id\"], 'is_iceberg': prediction.reshape((prediction.shape[0]))})\n",
    "submission.to_csv(\"mod2_subm4.csv\", index=False)\n",
    "\n",
    "#create downloadable submission file to upload\n",
    "submission_filename = 'mod2_subm4.csv'\n",
    "\n",
    "FileLink(submission_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#specify data augmentation types\n",
    "batch_size = 64\n",
    "\n",
    "#define generator and image transformations\n",
    "gen = image.ImageDataGenerator(rotation_range=10, width_shift_range=0.1,\n",
    "                               height_shift_range=0.1, zoom_range=0.1, \n",
    "                               vertical_flip = True, horizontal_flip=True,\n",
    "                               dim_ordering = 'tf')\n",
    "#create generator\n",
    "gen_flow = gen_flow_two_inputs(X_train, X_angle_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_model()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "64/32 [============================================================] - 0s - loss: 0.1889 - acc: 0.8750 - val_loss: 0.2254 - val_acc: 0.9202\n",
      "Epoch 2/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2402 - acc: 0.8906 - val_loss: 0.2254 - val_acc: 0.9202\n",
      "Epoch 3/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2625 - acc: 0.9375 - val_loss: 0.2254 - val_acc: 0.9202\n",
      "Epoch 4/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2759 - acc: 0.8750 - val_loss: 0.2253 - val_acc: 0.9202\n",
      "Epoch 5/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3091 - acc: 0.8438 - val_loss: 0.2253 - val_acc: 0.9202\n",
      "Epoch 6/50\n",
      "51/32 [===============================================] - 0s - loss: 0.3377 - acc: 0.8431 - val_loss: 0.2252 - val_acc: 0.9202\n",
      "Epoch 7/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2985 - acc: 0.8594 - val_loss: 0.2252 - val_acc: 0.9202\n",
      "Epoch 8/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2180 - acc: 0.9062 - val_loss: 0.2252 - val_acc: 0.9202\n",
      "Epoch 9/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2009 - acc: 0.9219 - val_loss: 0.2252 - val_acc: 0.9202\n",
      "Epoch 10/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2817 - acc: 0.8594 - val_loss: 0.2252 - val_acc: 0.9202\n",
      "Epoch 11/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3033 - acc: 0.8750 - val_loss: 0.2252 - val_acc: 0.9202\n",
      "Epoch 12/50\n",
      "64/32 [============================================================] - 0s - loss: 0.2099 - acc: 0.9062 - val_loss: 0.2252 - val_acc: 0.9177\n",
      "Epoch 13/50\n",
      "64/32 [============================================================] - 0s - loss: 0.1424 - acc: 0.9375 - val_loss: 0.2253 - val_acc: 0.9177\n",
      "Epoch 14/50\n",
      "64/32 [============================================================] - 0s - loss: 0.4757 - acc: 0.7812 - val_loss: 0.2253 - val_acc: 0.9177\n",
      "Epoch 15/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3149 - acc: 0.8594 - val_loss: 0.2253 - val_acc: 0.9177\n",
      "Epoch 16/50\n",
      "64/32 [============================================================] - 0s - loss: 0.1891 - acc: 0.8906 - val_loss: 0.2254 - val_acc: 0.9177\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bbcf0add0>"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_weights(filepath=file_path)\n",
    "optimizer = Adam(lr=1e-5)\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\",\n",
    "              metrics=[\"accuracy\"])\n",
    "\n",
    "model.fit_generator(gen_flow,samples_per_epoch= batch_size/2,\n",
    "                    validation_data=([X_valid, X_angle_valid], y_valid),\n",
    "                    nb_epoch=50, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8416/8424 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='mod2_subm5.csv' target='_blank'>mod2_subm5.csv</a><br>"
      ],
      "text/plain": [
       "/home/odenigborig/Github/Kaggle/statoli_iceberg/mod2_subm5.csv"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_weights(model_path+'model2_5.h5')\n",
    "prediction = model.predict([X_test, X_angle_test], verbose=1)\n",
    "submission = pd.DataFrame({'id': test[\"id\"], 'is_iceberg': prediction.reshape((prediction.shape[0]))})\n",
    "submission.to_csv(\"mod2_subm5.csv\", index=False)\n",
    "\n",
    "#create downloadable submission file to upload\n",
    "submission_filename = 'mod2_subm5.csv'\n",
    "\n",
    "FileLink(submission_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_model(p_dropout=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1203 samples, validate on 401 samples\n",
      "Epoch 1/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1266 - acc: 0.9526 - val_loss: 0.3331 - val_acc: 0.8603\n",
      "Epoch 2/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1219 - acc: 0.9551 - val_loss: 0.3305 - val_acc: 0.8628\n",
      "Epoch 3/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1261 - acc: 0.9460 - val_loss: 0.3335 - val_acc: 0.8603\n",
      "Epoch 4/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1066 - acc: 0.9601 - val_loss: 0.3354 - val_acc: 0.8603\n",
      "Epoch 5/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1132 - acc: 0.9609 - val_loss: 0.3339 - val_acc: 0.8603\n",
      "Epoch 6/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1106 - acc: 0.9609 - val_loss: 0.3336 - val_acc: 0.8603\n",
      "Epoch 7/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1195 - acc: 0.9543 - val_loss: 0.3337 - val_acc: 0.8603\n",
      "Epoch 8/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1198 - acc: 0.9534 - val_loss: 0.3299 - val_acc: 0.8603\n",
      "Epoch 9/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1060 - acc: 0.9609 - val_loss: 0.3269 - val_acc: 0.8603\n",
      "Epoch 10/10\n",
      "1203/1203 [==============================] - 0s - loss: 0.1323 - acc: 0.9510 - val_loss: 0.3274 - val_acc: 0.8603\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bb5ec2050>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer = Adam(lr=1e-4)\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\",\n",
    "              metrics=[\"accuracy\"])\n",
    "\n",
    "model.fit([X_train, X_angle_train], y_train, nb_epoch=10, \n",
    "          validation_data=([X_valid, X_angle_valid], y_valid), \n",
    "          batch_size=batch_size*2, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_path2 = model_path+'model2_dropout.h5'\n",
    "model.save_weights(file_path2)\n",
    "prediction = model.predict([X_test, X_angle_test], verbose=1)\n",
    "submission = pd.DataFrame({'id': test[\"id\"], 'is_iceberg': prediction.reshape((prediction.shape[0]))})\n",
    "submission.to_csv(\"mod2_subm_drop1.csv\", index=False)\n",
    "\n",
    "#create downloadable submission file to upload\n",
    "submission_filename = 'mod2_subm_drop1.csv'\n",
    "\n",
    "FileLink(submission_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3832 - acc: 0.9062 - val_loss: 0.3292 - val_acc: 0.8603\n",
      "Epoch 2/50\n",
      "64/32 [============================================================] - 0s - loss: 0.6037 - acc: 0.7656 - val_loss: 0.3308 - val_acc: 0.8603\n",
      "Epoch 3/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3498 - acc: 0.9062 - val_loss: 0.3326 - val_acc: 0.8579\n",
      "Epoch 4/50\n",
      "64/32 [============================================================] - 0s - loss: 0.4342 - acc: 0.8594 - val_loss: 0.3342 - val_acc: 0.8579\n",
      "Epoch 5/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3935 - acc: 0.8594 - val_loss: 0.3358 - val_acc: 0.8579\n",
      "Epoch 6/50\n",
      "64/32 [============================================================] - 0s - loss: 0.4429 - acc: 0.8750 - val_loss: 0.3373 - val_acc: 0.8579\n",
      "Epoch 7/50\n",
      "64/32 [============================================================] - 0s - loss: 0.3497 - acc: 0.8906 - val_loss: 0.3387 - val_acc: 0.8603\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0bb1c15210>"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model.load_weights(filepath=file_path2)\n",
    "optimizer = Adam(lr=1e-5)\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\",\n",
    "              metrics=[\"accuracy\"])\n",
    "\n",
    "model.fit_generator(gen_flow,samples_per_epoch= batch_size/2,\n",
    "                    validation_data=([X_valid, X_angle_valid], y_valid),\n",
    "                    nb_epoch=50, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}