Final_Refactored_Convolutional_105x105.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# from matplotlib.pyplot import imread\n",
    "from skimage.io import imread\n",
    "from skimage.transform import resize\n",
    "import pickle\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import cv2\n",
    "from time import time\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.optimizers import Adam\n",
    "from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.layers.core import Lambda, Flatten, Dense\n",
    "from keras.initializers import glorot_uniform\n",
    "\n",
    "from keras.engine.topology import Layer\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "import numpy.random as rng\n",
    "from keras.utils import plot_model\n",
    "from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask\n",
    "from keras.callbacks import ModelCheckpoint, TensorBoard, LearningRateScheduler\n",
    "from time import time\n",
    "import glob\n",
    "from IPython.display import Image\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Please give below paths as per the paths in your system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_folder = \"data/images_background/\"\n",
    "val_folder = 'data/images_evaluation/'\n",
    "save_path = 'data/'\n",
    "train_type = 'convolutional' #CONVOLUTIONAL or CAPSULE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadimgs(path,n = 0):\n",
    "    '''\n",
    "    path => Path of train directory or test directory\n",
    "    '''\n",
    "    X=[]\n",
    "    y = []\n",
    "    cat_dict = {}\n",
    "    lang_dict = {}\n",
    "    curr_y = n\n",
    "    # we load every alphabet seperately so we can isolate them later\n",
    "    for alphabet in [x for x in os.listdir(path) if not x.startswith('.')]:\n",
    "        print(\"loading alphabet: \" + alphabet)\n",
    "        lang_dict[alphabet] = [curr_y,None]\n",
    "        alphabet_path = os.path.join(path,alphabet)\n",
    "        # every letter/category has it's own column in the array, so  load seperately\n",
    "        for letter in [x for x in os.listdir(alphabet_path) if not x.startswith('.')]:\n",
    "            cat_dict[curr_y] = (alphabet, letter)\n",
    "            category_images=[]\n",
    "            letter_path = os.path.join(alphabet_path, letter)\n",
    "            # read all the images in the current category\n",
    "            for filename in [x for x in os.listdir(letter_path) if not x.startswith('.')]:\n",
    "                image_path = os.path.join(letter_path, filename)\n",
    "                image = imread(image_path)\n",
    "#                 image = resize(image, (52, 52))\n",
    "                category_images.append(image)\n",
    "                y.append(curr_y)\n",
    "            try:\n",
    "                X.append(np.stack(category_images))\n",
    "            # edge case  - last one\n",
    "            except ValueError as e:\n",
    "                print(e)\n",
    "                print(\"error - category_images:\", category_images)\n",
    "            curr_y += 1\n",
    "            lang_dict[alphabet][1] = curr_y - 1\n",
    "    y = np.vstack(y)\n",
    "    X = np.stack(X)\n",
    "    return X,y,lang_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading images into tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X,y,c=loadimgs(train_folder)\n",
    "# Xval,yval,cval=loadimgs(val_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the tensors on disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(os.path.join(save_path,\"train.pickle\"), \"wb\") as f:\n",
    "#     pickle.dump((X,c),f)\n",
    "\n",
    "# with open(os.path.join(save_path,\"val.pickle\"), \"wb\") as f:\n",
    "#     pickle.dump((Xval,cval),f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter initializers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_weights(shape, name=None):\n",
    "    \"\"\"\n",
    "        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "        suggests to initialize CNN layer weights with mean as 0.0 and standard deviation of 0.01\n",
    "    \"\"\"\n",
    "    return np.random.normal(loc = 0.0, scale = 1e-2, size = shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_bias(shape, name=None):\n",
    "    \"\"\"\n",
    "        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "        suggests to initialize CNN layer bias with mean as 0.5 and standard deviation of 0.01\n",
    "    \"\"\"\n",
    "    return np.random.normal(loc = 0.5, scale = 1e-2, size = shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training alphabets: \n",
      "\n",
      "['Gujarati', 'Korean', 'Arcadian', 'Malay_(Jawi_-_Arabic)', 'Grantha', 'Blackfoot_(Canadian_Aboriginal_Syllabics)', 'Balinese', 'Futurama', 'N_Ko', 'Burmese_(Myanmar)', 'Anglo-Saxon_Futhorc', 'Mkhedruli_(Georgian)', 'Latin', 'Braille', 'Sanskrit', 'Japanese_(hiragana)', 'Tagalog', 'Greek', 'Ojibwe_(Canadian_Aboriginal_Syllabics)', 'Japanese_(katakana)', 'Early_Aramaic', 'Hebrew', 'Tifinagh', 'Asomtavruli_(Georgian)', 'Armenian', 'Syriac_(Estrangelo)', 'Alphabet_of_the_Magi', 'Cyrillic', 'Bengali', 'Inuktitut_(Canadian_Aboriginal_Syllabics)']\n",
      "Validation alphabets:\n",
      "\n",
      "['Oriya', 'ULOG', 'Tengwar', 'Malayalam', 'Atlantean', 'Keble', 'Manipuri', 'Gurmukhi', 'Tibetan', 'Aurek-Besh', 'Ge_ez', 'Angelic', 'Old_Church_Slavonic_(Cyrillic)', 'Kannada', 'Avesta', 'Mongolian', 'Syriac_(Serto)', 'Atemayar_Qelisayer', 'Sylheti', 'Glagolitic']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(save_path, \"train.pickle\"), \"rb\") as f:\n",
    "    (Xtrain, train_classes) = pickle.load(f)\n",
    "    \n",
    "print(\"Training alphabets: \\n\")\n",
    "print(list(train_classes.keys()))\n",
    "\n",
    "with open(os.path.join(save_path, \"val.pickle\"), \"rb\") as f:\n",
    "    (Xval, val_classes) = pickle.load(f)\n",
    "\n",
    "print(\"Validation alphabets:\", end=\"\\n\\n\")\n",
    "print(list(val_classes.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and test util functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(batch_size,s=\"train\"):\n",
    "    \"\"\"Create batch of n pairs, half same class, half different class\"\"\"\n",
    "    if s == 'train':\n",
    "        X = Xtrain\n",
    "        categories = train_classes\n",
    "    else:\n",
    "        X = Xval\n",
    "        categories = val_classes\n",
    "    n_classes, n_examples, w, h = X.shape\n",
    "\n",
    "    # randomly sample several classes to use in the batch\n",
    "    categories = rng.choice(n_classes,size=(batch_size,),replace=False)\n",
    "    \n",
    "    # initialize 2 empty arrays for the input image batch\n",
    "    pairs=[np.zeros((batch_size, h, w,1)) for i in range(2)]\n",
    "    \n",
    "    # initialize vector for the targets\n",
    "    targets=np.zeros((batch_size,))\n",
    "    \n",
    "    # make one half of it '1's, so 2nd half of batch has same class\n",
    "    targets[batch_size//2:] = 1\n",
    "    for i in range(batch_size):\n",
    "        category = categories[i]\n",
    "        idx_1 = rng.randint(0, n_examples)\n",
    "        pairs[0][i,:,:,:] = X[category, idx_1].reshape(w, h, 1)\n",
    "        idx_2 = rng.randint(0, n_examples)\n",
    "        \n",
    "        # pick images of same class for 1st half, different for 2nd\n",
    "        if i >= batch_size // 2:\n",
    "            category_2 = category  \n",
    "        else: \n",
    "            # add a random number to the category modulo n classes to ensure 2nd image has a different category\n",
    "            category_2 = (category + rng.randint(1,n_classes)) % n_classes\n",
    "        \n",
    "        pairs[1][i,:,:,:] = X[category_2,idx_2].reshape(w, h,1)\n",
    "    \n",
    "    return pairs, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate(batch_size, s=\"train\"):\n",
    "    \"\"\"a generator for batches, so model.fit_generator can be used. \"\"\"\n",
    "    while True:\n",
    "        pairs, targets = get_batch(batch_size,s)\n",
    "        yield (pairs, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_oneshot_task(N, s=\"val\", language=None):\n",
    "    \"\"\"Create pairs of test image, support set for testing N way one-shot learning. \"\"\"\n",
    "    if s == 'train':\n",
    "        X = Xtrain\n",
    "        categories = train_classes\n",
    "    else:\n",
    "        X = Xval\n",
    "        categories = val_classes\n",
    "    n_classes, n_examples, w, h = X.shape\n",
    "    \n",
    "    indices = rng.randint(0, n_examples,size=(N,))\n",
    "    if language is not None: # if language is specified, select characters for that language\n",
    "        low, high = categories[language]\n",
    "        if N > high - low:\n",
    "            raise ValueError(\"This language ({}) has less than {} letters\".format(language, N))\n",
    "        categories = rng.choice(range(low,high),size=(N,),replace=False)\n",
    "\n",
    "    else: # if no language specified just pick a bunch of random letters\n",
    "        categories = rng.choice(range(n_classes),size=(N,),replace=False)            \n",
    "    true_category = categories[0]\n",
    "    ex1, ex2 = rng.choice(n_examples,replace=False,size=(2,))\n",
    "    test_image = np.asarray([X[true_category,ex1,:,:]]*N).reshape(N, w, h,1)\n",
    "    support_set = X[categories,indices,:,:]\n",
    "    support_set[0,:,:] = X[true_category,ex2]\n",
    "    support_set = support_set.reshape(N, w, h,1)\n",
    "    targets = np.zeros((N,))\n",
    "    targets[0] = 1\n",
    "    targets, test_image, support_set = shuffle(targets, test_image, support_set)\n",
    "    pairs = [test_image,support_set]\n",
    "\n",
    "    return pairs, targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_siamese_model(input_shape):\n",
    "    \"\"\"\n",
    "        Model architecture based on the one provided in: http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "    \"\"\"\n",
    "    \n",
    "    # Define the tensors for the two input images\n",
    "    left_input = Input(input_shape)\n",
    "    right_input = Input(input_shape)\n",
    "    \n",
    "    # Convolutional Neural Network\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(64, (10,10), activation='relu', input_shape=input_shape,\n",
    "                   kernel_initializer=initialize_weights, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(MaxPooling2D())\n",
    "    model.add(Conv2D(128, (7,7), activation='relu',\n",
    "                     kernel_initializer=initialize_weights,\n",
    "                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(MaxPooling2D())\n",
    "    model.add(Conv2D(128, (4,4), activation='relu', kernel_initializer=initialize_weights,\n",
    "                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(MaxPooling2D())\n",
    "    model.add(Conv2D(256, (4,4), activation='relu', kernel_initializer=initialize_weights,\n",
    "                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(4096, activation='sigmoid',\n",
    "                   kernel_regularizer=l2(1e-3),\n",
    "                   kernel_initializer=initialize_weights,bias_initializer=initialize_bias))\n",
    "    \n",
    "    # Generate the encodings (feature vectors) for the two images\n",
    "    encoded_l = model(left_input)\n",
    "    encoded_r = model(right_input)\n",
    "    \n",
    "    # Add a customized layer to compute the absolute difference between the encodings\n",
    "    L1_layer = Lambda(lambda tensors:K.abs(tensors[0] - tensors[1]))\n",
    "    L1_distance = L1_layer([encoded_l, encoded_r])\n",
    "    \n",
    "    # Add a dense layer with a sigmoid unit to generate the similarity score\n",
    "    prediction = Dense(1,activation='sigmoid',bias_initializer=initialize_bias)(L1_distance)\n",
    "    \n",
    "    # Connect the inputs with the outputs\n",
    "    siamese_net = Model(inputs=[left_input,right_input],outputs=prediction)\n",
    "    \n",
    "    # return the model\n",
    "    return siamese_net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_capsnet_model(input_shape, n_class, routings):\n",
    "\n",
    "    left_input = Input(shape=input_shape)\n",
    "    right_input = Input(shape=input_shape)\n",
    "\n",
    "    input = Input(shape=input_shape)\n",
    "    # Layer 1: Just a conventional Conv2D layer\n",
    "    conv1 = Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1',\n",
    "                  kernel_initializer=initialize_weights, bias_initializer=initialize_bias,\n",
    "                   kernel_regularizer=l2(2e-4),)(input)\n",
    "\n",
    "    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]\n",
    "    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')\n",
    "\n",
    "    # Layer 3: Capsule layer. Routing algorithm works here.\n",
    "    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,\n",
    "                             name='digitcaps')(primarycaps)\n",
    "\n",
    "    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.\n",
    "    # If using tensorflow, this will not be necessary. :)\n",
    "    out_caps = Length(name='capsnet')(digitcaps)\n",
    "\n",
    "    tunnel = Model(input, out_caps)\n",
    "    tunnel.summary()\n",
    "\n",
    "    encoded_l = tunnel(left_input)\n",
    "    encoded_r = tunnel(right_input)\n",
    "\n",
    "    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))\n",
    "    L1_distance = L1_layer([encoded_l, encoded_r])\n",
    "\n",
    "    prediction = Dense(1, activation='sigmoid', bias_initializer=initialize_bias)(L1_distance)\n",
    "\n",
    "    train_model = Model(inputs=[left_input, right_input], outputs=prediction)\n",
    "\n",
    "    return train_model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper parameters\n",
    "evaluate_every = 200 # interval for evaluating on one-shot tasks\n",
    "batch_size = 32\n",
    "n_iter = 20000 # No. of training iterations\n",
    "N_way = 20 # how many classes for testing one-shot tasks\n",
    "n_val = 250 # how many one-shot tasks to validate on\n",
    "best = -1\n",
    "\n",
    "model_path = \"checkpoints_{0}\".format(train_type)\n",
    "\n",
    "os.makedirs(\"checkpoints_{0}\".format(train_type), exist_ok=True)\n",
    "file_path = \"checkpoints_{0}\".format(train_type)+\"/siamese-epoch-{epoch:05d}-lr-\" + \"-train_loss-{loss:.4f}-val_loss-{val_loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(file_path,\n",
    "                             monitor=['loss', 'val_loss'],\n",
    "                             verbose=1,\n",
    "                             save_best_only=False,\n",
    "                             save_weights_only=False,\n",
    "                             mode='min',\n",
    "                             period=20)\n",
    "tensorboard = TensorBoard(log_dir=\"logs_cnn/{}\".format(time()), histogram_freq=0)\n",
    "lr_decay = LearningRateScheduler(schedule=lambda epoch: 0.001 * (0.9 ** epoch))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***New model for convolutional network loaded***\n",
      "loading from checkpoint:  checkpoints_convolutional/siamese-epoch-00222-lr--train_loss-2.8576-val_loss-2.4838.hdf5\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 105, 105, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            (None, 105, 105, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sequential_3 (Sequential)       (None, 4096)         38947648    input_5[0][0]                    \n",
      "                                                                 input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 4096)         0           sequential_3[1][0]               \n",
      "                                                                 sequential_3[2][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 1)            4097        lambda_3[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 38,951,745\n",
      "Trainable params: 38,951,745\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "ckpts = glob.glob(\"checkpoints_{0}\".format(train_type)+\"/*.hdf5\")\n",
    "initial_epoch = 0\n",
    "\n",
    "if train_type == 'capsule':\n",
    "    print(\"***New model for capsule network loaded***\")\n",
    "    model = get_capsnet_model(input_shape=[28, 28, 1], n_class=10, routings=3)\n",
    "elif train_type == 'convolutional':\n",
    "    print(\"***New model for convolutional network loaded***\")\n",
    "    model = get_siamese_model((105, 105, 1))\n",
    "\n",
    "if len(ckpts) != 0:\n",
    "    latest_ckpt = max(ckpts, key=os.path.getctime)\n",
    "    print(\"loading from checkpoint: \", latest_ckpt)\n",
    "    initial_epoch = int(latest_ckpt[latest_ckpt.find(\"-epoch-\") + len(\"-epoch-\"):latest_ckpt.rfind(\"-lr-\")])\n",
    "    model.load_weights(latest_ckpt)\n",
    "\n",
    "    \n",
    "model.summary()\n",
    "plot_model(model, to_file='model.png')\n",
    "Image(retina=True, filename='model.png')\n",
    "\n",
    "optimizer = Adam(lr = 0.00006)\n",
    "model.compile(loss=\"binary_crossentropy\",optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 223/1000\n",
      " 15/100 [===>..........................] - ETA: 2:27 - loss: 2.5093"
     ]
    }
   ],
   "source": [
    "model.fit_generator(generator=generate(16), \n",
    "                    steps_per_epoch=100, \n",
    "                    epochs=1000,\n",
    "                    initial_epoch=initial_epoch,\n",
    "                    validation_data=generate(32, 'val'), \n",
    "                    validation_steps=2,\n",
    "                    callbacks=[checkpoint, tensorboard],\n",
    "                    use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test, y_test = get_batch(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_name = 'lambda_1'\n",
    "intermediate_layer_model = Model(inputs=model.input,\n",
    "                                 outputs=model.get_layer(layer_name).output)\n",
    "intermediate_output = intermediate_layer_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_name = 'model_1'\n",
    "intermediate_layer_model = Model(inputs=model.input,\n",
    "                                 outputs=model.get_layer(layer_name).get_output_at(-1))\n",
    "intermediate_output = intermediate_layer_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "intermediate_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Starting training process!\")\n",
    "print(\"-------------------------------------\")\n",
    "t_start = time()\n",
    "for i in range(1, n_iter+1):\n",
    "    (inputs,targets) = get_batch(batch_size)\n",
    "    loss = model.train_on_batch(inputs, targets)\n",
    "    if i % evaluate_every == 0:\n",
    "        print(\"\\n ------------- \\n\")\n",
    "        print(\"Time for {0} iterations: {1} mins\".format(i, (time()-t_start)/60.0))\n",
    "        print(\"Train Loss: {0}\".format(loss)) \n",
    "        val_acc = test_oneshot(model, N_way, n_val, verbose=True)\n",
    "        model.save_weights(os.path.join(model_path, 'weights.{}.h5'.format(i)))\n",
    "        if val_acc >= best:\n",
    "            print(\"Current best: {0}, previous best: {1}\".format(val_acc, best))\n",
    "            best = val_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nearest_neighbour_correct(pairs,targets):\n",
    "    \"\"\"returns 1 if nearest neighbour gets the correct answer for a one-shot task\n",
    "        given by (pairs, targets)\"\"\"\n",
    "    L2_distances = np.zeros_like(targets)\n",
    "    for i in range(len(targets)):\n",
    "        L2_distances[i] = np.sum(np.sqrt(pairs[0][i]**2 - pairs[1][i]**2))\n",
    "    if np.argmin(L2_distances) == np.argmax(targets):\n",
    "        return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_nn_accuracy(N_ways,n_trials):\n",
    "    \"\"\"Returns accuracy of NN approach \"\"\"\n",
    "    print(\"Evaluating nearest neighbour on {} unique {} way one-shot learning tasks ...\".format(n_trials,N_ways))\n",
    "\n",
    "    n_right = 0\n",
    "    \n",
    "    for i in range(n_trials):\n",
    "        pairs,targets = make_oneshot_task(N_ways,\"val\")\n",
    "        correct = nearest_neighbour_correct(pairs,targets)\n",
    "        n_right += correct\n",
    "    return 100.0 * n_right / n_trials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_oneshot(model, N, k, s = \"val\", verbose = 0):\n",
    "    \"\"\"Test average N way oneshot learning accuracy of a siamese neural net over k one-shot tasks\"\"\"\n",
    "    n_correct = 0\n",
    "    if verbose:\n",
    "        print(\"Evaluating model on {} random {} way one-shot learning tasks ... \\n\".format(k,N))\n",
    "    for i in range(k):\n",
    "        inputs, targets = make_oneshot_task(N,s)\n",
    "        probs = model.predict(inputs)\n",
    "        if np.argmax(probs) == np.argmax(targets):\n",
    "            n_correct+=1\n",
    "    percent_correct = (100.0 * n_correct / k)\n",
    "    if verbose:\n",
    "        print(\"Got an average of {}% {} way one-shot learning accuracy \\n\".format(percent_correct,N))\n",
    "    return percent_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ways = np.arange(1,20,2)\n",
    "resume =  False\n",
    "trials = 50\n",
    "val_accs, train_accs,nn_accs = [], [], []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for N in ways:    \n",
    "    val_accs.append(test_oneshot(model, N, trials, \"val\", verbose=True))\n",
    "#     train_accs.append(test_oneshot(model, N, trials, \"train\", verbose=True))\n",
    "    nn_acc = test_nn_accuracy(N, trials)\n",
    "    nn_accs.append(nn_acc)\n",
    "    print (\"NN Accuracy = \", nn_acc)\n",
    "    print(\"---------------------------------------------------------------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(save_path,\"accuracies.pickle\"), \"wb\") as f:\n",
    "    pickle.dump((val_accs,train_accs,nn_accs),f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Result visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_images(X):\n",
    "    \"\"\"Concatenates a bunch of images into a big matrix for plotting purposes.\"\"\"\n",
    "    nc, h , w, _ = X.shape\n",
    "    X = X.reshape(nc, h, w)\n",
    "    n = np.ceil(np.sqrt(nc)).astype(\"int8\")\n",
    "    img = np.zeros((n*w,n*h))\n",
    "    x = 0\n",
    "    y = 0\n",
    "    for example in range(nc):\n",
    "        img[x*w:(x+1)*w,y*h:(y+1)*h] = X[example]\n",
    "        y += 1\n",
    "        if y >= n:\n",
    "            y = 0\n",
    "            x += 1\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_oneshot_task(pairs):\n",
    "    fig,(ax1,ax2) = plt.subplots(nrows=1, ncols=2)\n",
    "    ax1.matshow(pairs[0][0].reshape(105,105), cmap='gray')\n",
    "    img = concat_images(pairs[1])\n",
    "    ax1.get_yaxis().set_visible(False)\n",
    "    ax1.get_xaxis().set_visible(False)\n",
    "    ax2.matshow(img,cmap='gray')\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of concat image visualization\n",
    "pairs, targets = make_oneshot_task(16,\"train\",\"Sanskrit\")\n",
    "plot_oneshot_task(pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1)\n",
    "ax.plot(ways, val_accs, \"m\", label=\"Siamese(val set)\")\n",
    "ax.plot(ways, train_accs, \"y\", label=\"Siamese(train set)\")\n",
    "plt.plot(ways, nn_accs, label=\"Nearest neighbour\")\n",
    "\n",
    "ax.plot(ways, 100.0/ways, \"g\", label=\"Random guessing\")\n",
    "plt.xlabel(\"Number of possible classes in one-shot tasks\")\n",
    "plt.ylabel(\"% Accuracy\")\n",
    "plt.title(\"Omiglot One-Shot Learning Performance of a Siamese Network\")\n",
    "box = ax.get_position()\n",
    "ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])\n",
    "ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "inputs,targets = make_oneshot_task(20, \"val\", 'Oriya')\n",
    "plt.show()\n",
    "\n",
    "plot_oneshot_task(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}