Final_Refactored_Capsule_105x105.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# from matplotlib.pyplot import imread\n",
    "from skimage.io import imread\n",
    "from skimage.transform import resize\n",
    "import pickle\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import cv2\n",
    "from time import time\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.optimizers import Adam\n",
    "from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.layers.core import Lambda, Flatten, Dense\n",
    "from keras.initializers import glorot_uniform\n",
    "\n",
    "from keras.engine.topology import Layer\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "import numpy.random as rng\n",
    "from keras.utils import plot_model\n",
    "from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask\n",
    "from keras.callbacks import ModelCheckpoint, TensorBoard, LearningRateScheduler\n",
    "from time import time\n",
    "import glob\n",
    "from IPython.display import Image\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Please give below paths as per the paths in your system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_folder = \"data/images_background/\"\n",
    "val_folder = 'data/images_evaluation/'\n",
    "save_path = 'data/'\n",
    "train_type = 'capsule' #CONVOLUTIONAL or CAPSULE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadimgs(path,n = 0):\n",
    "    '''\n",
    "    path => Path of train directory or test directory\n",
    "    '''\n",
    "    X=[]\n",
    "    y = []\n",
    "    cat_dict = {}\n",
    "    lang_dict = {}\n",
    "    curr_y = n\n",
    "    # we load every alphabet seperately so we can isolate them later\n",
    "    for alphabet in [x for x in os.listdir(path) if not x.startswith('.')]:\n",
    "        print(\"loading alphabet: \" + alphabet)\n",
    "        lang_dict[alphabet] = [curr_y,None]\n",
    "        alphabet_path = os.path.join(path,alphabet)\n",
    "        # every letter/category has it's own column in the array, so  load seperately\n",
    "        for letter in [x for x in os.listdir(alphabet_path) if not x.startswith('.')]:\n",
    "            cat_dict[curr_y] = (alphabet, letter)\n",
    "            category_images=[]\n",
    "            letter_path = os.path.join(alphabet_path, letter)\n",
    "            # read all the images in the current category\n",
    "            for filename in [x for x in os.listdir(letter_path) if not x.startswith('.')]:\n",
    "                image_path = os.path.join(letter_path, filename)\n",
    "                image = imread(image_path)\n",
    "#                 image = resize(image, (52, 52))\n",
    "                category_images.append(image)\n",
    "                y.append(curr_y)\n",
    "            try:\n",
    "                X.append(np.stack(category_images))\n",
    "            # edge case  - last one\n",
    "            except ValueError as e:\n",
    "                print(e)\n",
    "                print(\"error - category_images:\", category_images)\n",
    "            curr_y += 1\n",
    "            lang_dict[alphabet][1] = curr_y - 1\n",
    "    y = np.vstack(y)\n",
    "    X = np.stack(X)\n",
    "    return X,y,lang_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading images into tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X,y,c=loadimgs(train_folder)\n",
    "# Xval,yval,cval=loadimgs(val_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the tensors on disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(os.path.join(save_path,\"train.pickle\"), \"wb\") as f:\n",
    "#     pickle.dump((X,c),f)\n",
    "\n",
    "# with open(os.path.join(save_path,\"val.pickle\"), \"wb\") as f:\n",
    "#     pickle.dump((Xval,cval),f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter initializers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_weights(shape, name=None):\n",
    "    \"\"\"\n",
    "        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "        suggests to initialize CNN layer weights with mean as 0.0 and standard deviation of 0.01\n",
    "    \"\"\"\n",
    "    return np.random.normal(loc = 0.0, scale = 1e-2, size = shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_bias(shape, name=None):\n",
    "    \"\"\"\n",
    "        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "        suggests to initialize CNN layer bias with mean as 0.5 and standard deviation of 0.01\n",
    "    \"\"\"\n",
    "    return np.random.normal(loc = 0.5, scale = 1e-2, size = shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training alphabets: \n",
      "\n",
      "['Burmese_(Myanmar)', 'Grantha', 'Alphabet_of_the_Magi', 'Armenian', 'Arcadian', 'Bengali', 'Japanese_(katakana)', 'Japanese_(hiragana)', 'Futurama', 'Tagalog', 'N_Ko', 'Mkhedruli_(Georgian)', 'Gujarati', 'Sanskrit', 'Syriac_(Estrangelo)', 'Latin', 'Malay_(Jawi_-_Arabic)', 'Korean', 'Greek', 'Blackfoot_(Canadian_Aboriginal_Syllabics)', 'Anglo-Saxon_Futhorc', 'Tifinagh', 'Braille', 'Asomtavruli_(Georgian)', 'Inuktitut_(Canadian_Aboriginal_Syllabics)', 'Balinese', 'Cyrillic', 'Hebrew', 'Ojibwe_(Canadian_Aboriginal_Syllabics)', 'Early_Aramaic']\n",
      "Validation alphabets:\n",
      "\n",
      "['Keble', 'Tengwar', 'Atlantean', 'Tibetan', 'Sylheti', 'Oriya', 'Malayalam', 'Mongolian', 'Atemayar_Qelisayer', 'Aurek-Besh', 'Gurmukhi', 'ULOG', 'Glagolitic', 'Avesta', 'Old_Church_Slavonic_(Cyrillic)', 'Kannada', 'Syriac_(Serto)', 'Ge_ez', 'Angelic', 'Manipuri']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(save_path, \"train.pickle\"), \"rb\") as f:\n",
    "    (Xtrain, train_classes) = pickle.load(f)\n",
    "    \n",
    "print(\"Training alphabets: \\n\")\n",
    "print(list(train_classes.keys()))\n",
    "\n",
    "with open(os.path.join(save_path, \"val.pickle\"), \"rb\") as f:\n",
    "    (Xval, val_classes) = pickle.load(f)\n",
    "\n",
    "print(\"Validation alphabets:\", end=\"\\n\\n\")\n",
    "print(list(val_classes.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and test util functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(batch_size,s=\"train\"):\n",
    "    \"\"\"Create batch of n pairs, half same class, half different class\"\"\"\n",
    "    if s == 'train':\n",
    "        X = Xtrain\n",
    "        categories = train_classes\n",
    "    else:\n",
    "        X = Xval\n",
    "        categories = val_classes\n",
    "    n_classes, n_examples, w, h = X.shape\n",
    "\n",
    "    # randomly sample several classes to use in the batch\n",
    "    categories = rng.choice(n_classes,size=(batch_size,),replace=False)\n",
    "    \n",
    "    # initialize 2 empty arrays for the input image batch\n",
    "    pairs=[np.zeros((batch_size, h, w,1)) for i in range(2)]\n",
    "    \n",
    "    # initialize vector for the targets\n",
    "    targets=np.zeros((batch_size,))\n",
    "    \n",
    "    # make one half of it '1's, so 2nd half of batch has same class\n",
    "    targets[batch_size//2:] = 1\n",
    "    for i in range(batch_size):\n",
    "        category = categories[i]\n",
    "        idx_1 = rng.randint(0, n_examples)\n",
    "        pairs[0][i,:,:,:] = X[category, idx_1].reshape(w, h, 1)\n",
    "        idx_2 = rng.randint(0, n_examples)\n",
    "        \n",
    "        # pick images of same class for 1st half, different for 2nd\n",
    "        if i >= batch_size // 2:\n",
    "            category_2 = category  \n",
    "        else: \n",
    "            # add a random number to the category modulo n classes to ensure 2nd image has a different category\n",
    "            category_2 = (category + rng.randint(1,n_classes)) % n_classes\n",
    "        \n",
    "        pairs[1][i,:,:,:] = X[category_2,idx_2].reshape(w, h,1)\n",
    "    \n",
    "    return pairs, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate(batch_size, s=\"train\"):\n",
    "    \"\"\"a generator for batches, so model.fit_generator can be used. \"\"\"\n",
    "    while True:\n",
    "        pairs, targets = get_batch(batch_size,s)\n",
    "        yield (pairs, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_oneshot_task(N, s=\"val\", language=None):\n",
    "    \"\"\"Create pairs of test image, support set for testing N way one-shot learning. \"\"\"\n",
    "    if s == 'train':\n",
    "        X = Xtrain\n",
    "        categories = train_classes\n",
    "    else:\n",
    "        X = Xval\n",
    "        categories = val_classes\n",
    "    n_classes, n_examples, w, h = X.shape\n",
    "    \n",
    "    indices = rng.randint(0, n_examples,size=(N,))\n",
    "    if language is not None: # if language is specified, select characters for that language\n",
    "        low, high = categories[language]\n",
    "        if N > high - low:\n",
    "            raise ValueError(\"This language ({}) has less than {} letters\".format(language, N))\n",
    "        categories = rng.choice(range(low,high),size=(N,),replace=False)\n",
    "\n",
    "    else: # if no language specified just pick a bunch of random letters\n",
    "        categories = rng.choice(range(n_classes),size=(N,),replace=False)            \n",
    "    true_category = categories[0]\n",
    "    ex1, ex2 = rng.choice(n_examples,replace=False,size=(2,))\n",
    "    test_image = np.asarray([X[true_category,ex1,:,:]]*N).reshape(N, w, h,1)\n",
    "    support_set = X[categories,indices,:,:]\n",
    "    support_set[0,:,:] = X[true_category,ex2]\n",
    "    support_set = support_set.reshape(N, w, h,1)\n",
    "    targets = np.zeros((N,))\n",
    "    targets[0] = 1\n",
    "    targets, test_image, support_set = shuffle(targets, test_image, support_set)\n",
    "    pairs = [test_image,support_set]\n",
    "\n",
    "    return pairs, targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_siamese_model(input_shape):\n",
    "    \"\"\"\n",
    "        Model architecture based on the one provided in: http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "    \"\"\"\n",
    "    \n",
    "    # Define the tensors for the two input images\n",
    "    left_input = Input(input_shape)\n",
    "    right_input = Input(input_shape)\n",
    "    \n",
    "    # Convolutional Neural Network\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(64, (10,10), activation='relu', input_shape=input_shape,\n",
    "                   kernel_initializer=initialize_weights, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(MaxPooling2D())\n",
    "    model.add(Conv2D(128, (7,7), activation='relu',\n",
    "                     kernel_initializer=initialize_weights,\n",
    "                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(MaxPooling2D())\n",
    "    model.add(Conv2D(128, (4,4), activation='relu', kernel_initializer=initialize_weights,\n",
    "                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(MaxPooling2D())\n",
    "    model.add(Conv2D(256, (4,4), activation='relu', kernel_initializer=initialize_weights,\n",
    "                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(4096, activation='sigmoid',\n",
    "                   kernel_regularizer=l2(1e-3),\n",
    "                   kernel_initializer=initialize_weights,bias_initializer=initialize_bias))\n",
    "    \n",
    "    # Generate the encodings (feature vectors) for the two images\n",
    "    encoded_l = model(left_input)\n",
    "    encoded_r = model(right_input)\n",
    "    \n",
    "    # Add a customized layer to compute the absolute difference between the encodings\n",
    "    L1_layer = Lambda(lambda tensors:K.abs(tensors[0] - tensors[1]))\n",
    "    L1_distance = L1_layer([encoded_l, encoded_r])\n",
    "    \n",
    "    # Add a dense layer with a sigmoid unit to generate the similarity score\n",
    "    prediction = Dense(1,activation='sigmoid',bias_initializer=initialize_bias)(L1_distance)\n",
    "    \n",
    "    # Connect the inputs with the outputs\n",
    "    siamese_net = Model(inputs=[left_input,right_input],outputs=prediction)\n",
    "    \n",
    "    # return the model\n",
    "    return siamese_net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper parameters\n",
    "evaluate_every = 200 # interval for evaluating on one-shot tasks\n",
    "batch_size = 32\n",
    "n_iter = 20000 # No. of training iterations\n",
    "N_way = 20 # how many classes for testing one-shot tasks\n",
    "n_val = 250 # how many one-shot tasks to validate on\n",
    "best = -1\n",
    "\n",
    "model_path = \"checkpoints_{0}\".format(train_type)\n",
    "\n",
    "os.makedirs(\"checkpoints_{0}\".format(train_type), exist_ok=True)\n",
    "file_path = \"checkpoints_{0}\".format(train_type)+\"/siamese-epoch-{epoch:05d}-lr-\" + \"-train_loss-{loss:.4f}-val_loss-{val_loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(file_path,\n",
    "                             monitor=['loss', 'val_loss'],\n",
    "                             verbose=1,\n",
    "                             save_best_only=False,\n",
    "                             save_weights_only=False,\n",
    "                             mode='min',\n",
    "                             period=2)\n",
    "tensorboard = TensorBoard(log_dir=\"logs/{}\".format(time()), histogram_freq=0)\n",
    "lr_decay = LearningRateScheduler(schedule=lambda epoch: 0.001 * (0.9 ** epoch))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_capsnet_model(input_shape, n_class, routings):\n",
    "\n",
    "    left_input = Input(shape=input_shape)\n",
    "    right_input = Input(shape=input_shape)\n",
    "\n",
    "    input = Input(shape=input_shape)\n",
    "    # Layer 1: Just a conventional Conv2D layer\n",
    "    conv1 = Conv2D(filters=128, kernel_size=36, strides=1, padding='valid', activation='relu', name='conv1',\n",
    "                  kernel_initializer=initialize_weights, bias_initializer=initialize_bias,\n",
    "                   kernel_regularizer=l2(2e-4),)(input)\n",
    "\n",
    "    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]\n",
    "    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=16, kernel_size=36, strides=2, padding='valid')\n",
    "\n",
    "    # Layer 3: Capsule layer. Routing algorithm works here.\n",
    "    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,\n",
    "                             name='digitcaps')(primarycaps)\n",
    "\n",
    "    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.\n",
    "    # If using tensorflow, this will not be necessary. :)\n",
    "    out_caps = Length(name='capsnet')(digitcaps)\n",
    "\n",
    "    tunnel = Model(input, out_caps)\n",
    "    tunnel.summary()\n",
    "\n",
    "    encoded_l = tunnel(left_input)\n",
    "    encoded_r = tunnel(right_input)\n",
    "\n",
    "    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))\n",
    "    L1_distance = L1_layer([encoded_l, encoded_r])\n",
    "\n",
    "    prediction = Dense(1, activation='sigmoid', bias_initializer=initialize_bias)(L1_distance)\n",
    "\n",
    "    train_model = Model(inputs=[left_input, right_input], outputs=prediction)\n",
    "\n",
    "    return train_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ckpts = glob.glob(\"checkpoints_{0}\".format(train_type)+\"/*.hdf5\")\n",
    "initial_epoch = 0\n",
    "\n",
    "model = get_capsnet_model(input_shape=[105, 105, 1], n_class=10, routings=3)\n",
    "\n",
    "if len(ckpts) != 0:\n",
    "    latest_ckpt = max(ckpts, key=os.path.getctime)\n",
    "    print(\"loading from checkpoint: \", latest_ckpt)\n",
    "    initial_epoch = int(latest_ckpt[latest_ckpt.find(\"-epoch-\") + len(\"-epoch-\"):latest_ckpt.rfind(\"-lr-\")])\n",
    "    model.load_weights(latest_ckpt)\n",
    "\n",
    "model.summary()\n",
    "plot_model(model, to_file='model.png')\n",
    "Image(retina=True, filename='model.png')\n",
    "\n",
    "optimizer = Adam(lr = 0.00001)\n",
    "model.compile(loss=\"binary_crossentropy\",optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0704 16:44:04.965363 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0704 16:44:04.969850 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***New model for capsule network loaded***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0704 16:44:06.379992 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0704 16:44:06.454232 140485856409344 deprecation.py:506] From /home/ubuntu/Projects/capsule_simaese_network/capsulelayers.py:163: calling softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 105, 105, 1)       0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 70, 70, 128)       166016    \n",
      "_________________________________________________________________\n",
      "primarycap_conv2d (Conv2D)   (None, 18, 18, 128)       21233792  \n",
      "_________________________________________________________________\n",
      "primarycap_reshape (Reshape) (None, 5184, 8)           0         \n",
      "_________________________________________________________________\n",
      "primarycap_squash (Lambda)   (None, 5184, 8)           0         \n",
      "_________________________________________________________________\n",
      "digitcaps (CapsuleLayer)     (None, 10, 16)            6635520   \n",
      "_________________________________________________________________\n",
      "capsnet (Length)             (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 28,035,328\n",
      "Trainable params: 28,035,328\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0704 16:44:07.227631 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0704 16:44:07.234039 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:3376: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n",
      "W0704 16:44:07.238971 140485856409344 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 105, 105, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 105, 105, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Model)                 (None, 10)           28035328    input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 10)           0           model_1[1][0]                    \n",
      "                                                                 model_1[2][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            11          lambda_1[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 28,035,339\n",
      "Trainable params: 28,035,339\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "ckpts = glob.glob(\"checkpoints_{0}\".format(train_type)+\"/*.hdf5\")\n",
    "initial_epoch = 0\n",
    "if len(ckpts) != 0 and False:\n",
    "    latest_ckpt = max(ckpts, key=os.path.getctime)\n",
    "    print(\"loading from checkpoint: \", latest_ckpt)\n",
    "    initial_epoch = int(latest_ckpt[latest_ckpt.find(\"-epoch-\") + len(\"-epoch-\"):latest_ckpt.rfind(\"-lr-\")])\n",
    "    model = load_model(latest_ckpt, custom_objects={'CapsuleLayer': CapsuleLayer, 'Length': Length, 'PrimaryCap':PrimaryCap })\n",
    "else:\n",
    "    if train_type == 'capsule':\n",
    "        print(\"***New model for capsule network loaded***\")\n",
    "        model = get_capsnet_model(input_shape=[105, 105, 1], n_class=10, routings=3)\n",
    "    elif train_type == 'convolutional':\n",
    "        print(\"***New model for convolutional network loaded***\")\n",
    "        model = get_siamese_model((105, 105, 1))\n",
    "model.summary()\n",
    "plot_model(model, to_file='model.png')\n",
    "Image(retina=True, filename='model.png')\n",
    "\n",
    "optimizer = Adam(lr = 0.001)\n",
    "model.compile(loss=\"binary_crossentropy\",optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0704 16:44:14.966843 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "W0704 16:44:21.169281 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/callbacks.py:850: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
      "\n",
      "W0704 16:44:21.170348 140485856409344 deprecation_wrapper.py:119] From /usr/local/lib/python3.5/dist-packages/keras/callbacks.py:853: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "100/100 [==============================] - 75s 751ms/step - loss: 0.7752 - val_loss: 0.7174\n",
      "Epoch 2/1000\n",
      "100/100 [==============================] - 68s 678ms/step - loss: 0.7164 - val_loss: 0.7151\n",
      "\n",
      "Epoch 00002: saving model to checkpoints_capsule/siamese-epoch-00002-lr--train_loss-0.7164-val_loss-0.7151.hdf5\n",
      "Epoch 3/1000\n",
      "100/100 [==============================] - 68s 677ms/step - loss: 0.7142 - val_loss: 0.7074\n",
      "Epoch 4/1000\n",
      "100/100 [==============================] - 67s 674ms/step - loss: 0.7054 - val_loss: 0.6995\n",
      "\n",
      "Epoch 00004: saving model to checkpoints_capsule/siamese-epoch-00004-lr--train_loss-0.7054-val_loss-0.6995.hdf5\n",
      "Epoch 5/1000\n",
      "100/100 [==============================] - 68s 679ms/step - loss: 0.7202 - val_loss: 0.7106\n",
      "Epoch 6/1000\n",
      "100/100 [==============================] - 68s 679ms/step - loss: 0.7024 - val_loss: 0.6989\n",
      "\n",
      "Epoch 00006: saving model to checkpoints_capsule/siamese-epoch-00006-lr--train_loss-0.7024-val_loss-0.6989.hdf5\n",
      "Epoch 7/1000\n",
      "100/100 [==============================] - 68s 680ms/step - loss: 0.6970 - val_loss: 0.6949\n",
      "Epoch 8/1000\n",
      "100/100 [==============================] - 68s 675ms/step - loss: 0.6965 - val_loss: 0.6971\n",
      "\n",
      "Epoch 00008: saving model to checkpoints_capsule/siamese-epoch-00008-lr--train_loss-0.6965-val_loss-0.6971.hdf5\n",
      "Epoch 9/1000\n",
      "100/100 [==============================] - 68s 680ms/step - loss: 0.6952 - val_loss: 0.6943\n",
      "Epoch 10/1000\n",
      " 36/100 [=========>....................] - ETA: 43s - loss: 0.6995"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process ForkPoolWorker-2:\n",
      "Process ForkPoolWorker-1:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-b3ce0d8798e9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m                     \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                     \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr_decay\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m                     use_multiprocessing=True)\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m     90\u001b[0m                               'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1416\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1417\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1420\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m    215\u001b[0m                 outs = model.train_on_batch(x, y,\n\u001b[1;32m    216\u001b[0m                                             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m                                             class_weight=class_weight)\n\u001b[0m\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1215\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1216\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1217\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1218\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1456\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1457\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1458\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1459\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1460\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit_generator(generator=generate(32), \n",
    "                    steps_per_epoch=100, \n",
    "                    epochs=1000,\n",
    "                    initial_epoch=initial_epoch,\n",
    "                    validation_data=generate(32, 'val'), \n",
    "                    validation_steps=2,\n",
    "                    callbacks=[checkpoint, tensorboard, lr_decay],\n",
    "                    use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test, y_test = get_batch(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_name = 'lambda_1'\n",
    "intermediate_layer_model = Model(inputs=model.input,\n",
    "                                 outputs=model.get_layer(layer_name).output)\n",
    "intermediate_output = intermediate_layer_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_name = 'model_1'\n",
    "intermediate_layer_model = Model(inputs=model.input,\n",
    "                                 outputs=model.get_layer(layer_name).get_output_at(-1))\n",
    "intermediate_output = intermediate_layer_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "intermediate_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training process!\n",
      "-------------------------------------\n",
      "\n",
      " ------------- \n",
      "\n",
      "Time for 200 iterations: 2.2417839407920837 mins\n",
      "Train Loss: 0.697585940361023\n",
      "Evaluating model on 250 random 20 way one-shot learning tasks ... \n",
      "\n",
      "Got an average of 13.6% 20 way one-shot learning accuracy \n",
      "\n",
      "Current best: 13.6, previous best: -1\n",
      "\n",
      " ------------- \n",
      "\n",
      "Time for 400 iterations: 5.2440164883931475 mins\n",
      "Train Loss: 0.6942368745803833\n",
      "Evaluating model on 250 random 20 way one-shot learning tasks ... \n",
      "\n",
      "Got an average of 12.0% 20 way one-shot learning accuracy \n",
      "\n",
      "\n",
      " ------------- \n",
      "\n",
      "Time for 600 iterations: 8.240609582265218 mins\n",
      "Train Loss: 0.6934194564819336\n",
      "Evaluating model on 250 random 20 way one-shot learning tasks ... \n",
      "\n",
      "Got an average of 9.6% 20 way one-shot learning accuracy \n",
      "\n",
      "\n",
      " ------------- \n",
      "\n",
      "Time for 800 iterations: 11.228420158227285 mins\n",
      "Train Loss: 0.6990328431129456\n",
      "Evaluating model on 250 random 20 way one-shot learning tasks ... \n",
      "\n",
      "Got an average of 10.4% 20 way one-shot learning accuracy \n",
      "\n",
      "\n",
      " ------------- \n",
      "\n",
      "Time for 1000 iterations: 14.205393501122792 mins\n",
      "Train Loss: 0.6945714354515076\n",
      "Evaluating model on 250 random 20 way one-shot learning tasks ... \n",
      "\n",
      "Got an average of 18.4% 20 way one-shot learning accuracy \n",
      "\n",
      "Current best: 18.4, previous best: 13.6\n",
      "\n",
      " ------------- \n",
      "\n",
      "Time for 1200 iterations: 17.178325474262238 mins\n",
      "Train Loss: 0.6937480568885803\n",
      "Evaluating model on 250 random 20 way one-shot learning tasks ... \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Starting training process!\")\n",
    "print(\"-------------------------------------\")\n",
    "t_start = time()\n",
    "for i in range(1, n_iter+1):\n",
    "    (inputs,targets) = get_batch(batch_size)\n",
    "    loss = model.train_on_batch(inputs, targets)\n",
    "    if i % evaluate_every == 0:\n",
    "        print(\"\\n ------------- \\n\")\n",
    "        print(\"Time for {0} iterations: {1} mins\".format(i, (time()-t_start)/60.0))\n",
    "        print(\"Train Loss: {0}\".format(loss)) \n",
    "        val_acc = test_oneshot(model, N_way, n_val, verbose=True)\n",
    "        model.save_weights(os.path.join(model_path, 'weights.{}.h5'.format(i)))\n",
    "        if val_acc >= best:\n",
    "            print(\"Current best: {0}, previous best: {1}\".format(val_acc, best))\n",
    "            best = val_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nearest_neighbour_correct(pairs,targets):\n",
    "    \"\"\"returns 1 if nearest neighbour gets the correct answer for a one-shot task\n",
    "        given by (pairs, targets)\"\"\"\n",
    "    L2_distances = np.zeros_like(targets)\n",
    "    for i in range(len(targets)):\n",
    "        L2_distances[i] = np.sum(np.sqrt(pairs[0][i]**2 - pairs[1][i]**2))\n",
    "    if np.argmin(L2_distances) == np.argmax(targets):\n",
    "        return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_nn_accuracy(N_ways,n_trials):\n",
    "    \"\"\"Returns accuracy of NN approach \"\"\"\n",
    "    print(\"Evaluating nearest neighbour on {} unique {} way one-shot learning tasks ...\".format(n_trials,N_ways))\n",
    "\n",
    "    n_right = 0\n",
    "    \n",
    "    for i in range(n_trials):\n",
    "        pairs,targets = make_oneshot_task(N_ways,\"val\")\n",
    "        correct = nearest_neighbour_correct(pairs,targets)\n",
    "        n_right += correct\n",
    "    return 100.0 * n_right / n_trials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_oneshot(model, N, k, s = \"val\", verbose = 0):\n",
    "    \"\"\"Test average N way oneshot learning accuracy of a siamese neural net over k one-shot tasks\"\"\"\n",
    "    n_correct = 0\n",
    "    if verbose:\n",
    "        print(\"Evaluating model on {} random {} way one-shot learning tasks ... \\n\".format(k,N))\n",
    "    for i in range(k):\n",
    "        inputs, targets = make_oneshot_task(N,s)\n",
    "        probs = model.predict(inputs)\n",
    "        if np.argmax(probs) == np.argmax(targets):\n",
    "            n_correct+=1\n",
    "    percent_correct = (100.0 * n_correct / k)\n",
    "    if verbose:\n",
    "        print(\"Got an average of {}% {} way one-shot learning accuracy \\n\".format(percent_correct,N))\n",
    "    return percent_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ways = np.arange(1,20,2)\n",
    "resume =  False\n",
    "trials = 50\n",
    "val_accs, train_accs,nn_accs = [], [], []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for N in ways:    \n",
    "    val_accs.append(test_oneshot(model, N, trials, \"val\", verbose=True))\n",
    "#     train_accs.append(test_oneshot(model, N, trials, \"train\", verbose=True))\n",
    "    nn_acc = test_nn_accuracy(N, trials)\n",
    "    nn_accs.append(nn_acc)\n",
    "    print (\"NN Accuracy = \", nn_acc)\n",
    "    print(\"---------------------------------------------------------------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(save_path,\"accuracies.pickle\"), \"wb\") as f:\n",
    "    pickle.dump((val_accs,train_accs,nn_accs),f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Result visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_images(X):\n",
    "    \"\"\"Concatenates a bunch of images into a big matrix for plotting purposes.\"\"\"\n",
    "    nc, h , w, _ = X.shape\n",
    "    X = X.reshape(nc, h, w)\n",
    "    n = np.ceil(np.sqrt(nc)).astype(\"int8\")\n",
    "    img = np.zeros((n*w,n*h))\n",
    "    x = 0\n",
    "    y = 0\n",
    "    for example in range(nc):\n",
    "        img[x*w:(x+1)*w,y*h:(y+1)*h] = X[example]\n",
    "        y += 1\n",
    "        if y >= n:\n",
    "            y = 0\n",
    "            x += 1\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_oneshot_task(pairs):\n",
    "    fig,(ax1,ax2) = plt.subplots(nrows=1, ncols=2)\n",
    "    ax1.matshow(pairs[0][0].reshape(105,105), cmap='gray')\n",
    "    img = concat_images(pairs[1])\n",
    "    ax1.get_yaxis().set_visible(False)\n",
    "    ax1.get_xaxis().set_visible(False)\n",
    "    ax2.matshow(img,cmap='gray')\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of concat image visualization\n",
    "pairs, targets = make_oneshot_task(16,\"train\",\"Sanskrit\")\n",
    "plot_oneshot_task(pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1)\n",
    "ax.plot(ways, val_accs, \"m\", label=\"Siamese(val set)\")\n",
    "ax.plot(ways, train_accs, \"y\", label=\"Siamese(train set)\")\n",
    "plt.plot(ways, nn_accs, label=\"Nearest neighbour\")\n",
    "\n",
    "ax.plot(ways, 100.0/ways, \"g\", label=\"Random guessing\")\n",
    "plt.xlabel(\"Number of possible classes in one-shot tasks\")\n",
    "plt.ylabel(\"% Accuracy\")\n",
    "plt.title(\"Omiglot One-Shot Learning Performance of a Siamese Network\")\n",
    "box = ax.get_position()\n",
    "ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])\n",
    "ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "inputs,targets = make_oneshot_task(20, \"val\", 'Oriya')\n",
    "plt.show()\n",
    "\n",
    "plot_oneshot_task(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}