character_wise_RNN.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "colab": {
   "name": "character_wise_RNN.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "accelerator": "GPU"
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "sby3EvKFHscQ"
   },
   "source": [
    "# character-wise RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "DDzwBY87Hscd"
   },
   "source": [
    "![Overview](https://github.com/udacity/deep-learning/raw/78c91a5607ecfdc29b762e45c082d7ca5047c8a1/intro-to-rnns/assets/charseq.jpeg)"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "JG4ymNGZHsce"
   },
   "source": [
    "\n",
    "import time\n",
    "from collections import namedtuple\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "torch.manual_seed(0)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "np.random.seed(0)"
   ],
   "execution_count": 19,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "u7IQJtsLHscg"
   },
   "source": [
    "## Loading data"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "P3SKxnVwHsch",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "outputId": "d2cd7611-8709-4f0e-dd45-81ee88e56476"
   },
   "source": [
    "!curl http://www.gutenberg.org/files/2591/2591-0.txt --create-dirs -o .pytorch/trialReport/trial.txt\n",
    "with open('.pytorch/trialReport/trial.txt') as r:\n",
    "    reports = r.read()\n",
    "reports[:100]"
   ],
   "execution_count": 2,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  547k  100  547k    0     0   568k      0 --:--:-- --:--:-- --:--:--  568k\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "execute_result",
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'\\ufeffThe Project Gutenberg EBook of Grimms’ Fairy Tales, by The Brothers Grimm\\n\\nThis eBook is for the us'"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 2
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "r4evrtXuFkz-",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "335d8837-5b14-4fd5-c39c-931f1dc7fd95"
   },
   "source": [
    "import re\n",
    "words = re.split(r'\\W+', reports)\n",
    "words = [word.lower() for word in words]\n",
    "print(words[:100])"
   ],
   "execution_count": 3,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "['', 'the', 'project', 'gutenberg', 'ebook', 'of', 'grimms', 'fairy', 'tales', 'by', 'the', 'brothers', 'grimm', 'this', 'ebook', 'is', 'for', 'the', 'use', 'of', 'anyone', 'anywhere', 'at', 'no', 'cost', 'and', 'with', 'almost', 'no', 'restrictions', 'whatsoever', 'you', 'may', 'copy', 'it', 'give', 'it', 'away', 'or', 're', 'use', 'it', 'under', 'the', 'terms', 'of', 'the', 'project', 'gutenberg', 'license', 'included', 'with', 'this', 'ebook', 'or', 'online', 'at', 'www', 'gutenberg', 'org', 'title', 'grimms', 'fairy', 'tales', 'author', 'the', 'brothers', 'grimm', 'translator', 'edgar', 'taylor', 'and', 'marian', 'edwardes', 'posting', 'date', 'december', '14', '2008', 'ebook', '2591', 'release', 'date', 'april', '2001', 'last', 'updated', 'november', '7', '2016', 'language', 'english', 'character', 'set', 'encoding', 'utf', '8', 'start', 'of', 'this']\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "SBvMLLuvFko2"
   },
   "source": [
    "reports = ' '.join(words)"
   ],
   "execution_count": 4,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "rm5g3PVPHscj"
   },
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "eK0cXQ7DHsck",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "6f40fafa-a35e-41c8-b7f4-345c5c27f170"
   },
   "source": [
    "vocab = tuple(set(reports))\n",
    "print('vocabs:\\n', vocab)\n",
    "int_to_vocab = dict(enumerate(vocab))\n",
    "print('int to vocab:\\n', int_to_vocab)\n",
    "vocab_to_int = {v: i for i,v in int_to_vocab.items()}\n",
    "encoded = np.array([vocab_to_int[v] for v in reports], dtype=np.int32)"
   ],
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "vocabs:\n",
      " ('0', '6', 'h', '1', '5', 'v', 's', '4', '8', '_', '7', 'c', 'l', 'r', 'd', 'x', 'i', 'm', '9', 'z', '3', 'n', 'f', 'b', 'e', 'p', 'u', ' ', 'g', 'w', 'o', 'y', '2', 'q', 'k', 'j', 't', 'a')\n",
      "int to vocab:\n",
      " {0: '0', 1: '6', 2: 'h', 3: '1', 4: '5', 5: 'v', 6: 's', 7: '4', 8: '8', 9: '_', 10: '7', 11: 'c', 12: 'l', 13: 'r', 14: 'd', 15: 'x', 16: 'i', 17: 'm', 18: '9', 19: 'z', 20: '3', 21: 'n', 22: 'f', 23: 'b', 24: 'e', 25: 'p', 26: 'u', 27: ' ', 28: 'g', 29: 'w', 30: 'o', 31: 'y', 32: '2', 33: 'q', 34: 'k', 35: 'j', 36: 't', 37: 'a'}\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "iN-rQg0pHsck",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "00f55d8b-932c-4f2b-ccec-034395b6db18"
   },
   "source": [
    "encoded[:100]"
   ],
   "execution_count": 6,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([27, 36,  2, 24, 27, 25, 13, 30, 35, 24, 11, 36, 27, 28, 26, 36, 24,\n",
       "       21, 23, 24, 13, 28, 27, 24, 23, 30, 30, 34, 27, 30, 22, 27, 28, 13,\n",
       "       16, 17, 17,  6, 27, 22, 37, 16, 13, 31, 27, 36, 37, 12, 24,  6, 27,\n",
       "       23, 31, 27, 36,  2, 24, 27, 23, 13, 30, 36,  2, 24, 13,  6, 27, 28,\n",
       "       13, 16, 17, 17, 27, 36,  2, 16,  6, 27, 24, 23, 30, 30, 34, 27, 16,\n",
       "        6, 27, 22, 30, 13, 27, 36,  2, 24, 27, 26,  6, 24, 27, 30],\n",
       "      dtype=int32)"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 6
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "8eBm9UNSHscl"
   },
   "source": [
    "## One-hot Encoding"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "NNbhQvcMHscm",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "5aeb78be-29f8-4493-a691-6d81970e7795"
   },
   "source": [
    "def one_hot_encode(arr, n_labels):\n",
    "\n",
    "    return np.eye(n_labels,n_labels,  dtype=np.float32)[arr]\n",
    "# check that the function works as expected\n",
    "test_seq = np.array([[3, 5, 1]])\n",
    "one_hot = one_hot_encode(test_seq, 8)\n",
    "\n",
    "print(one_hot)"
   ],
   "execution_count": 7,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "[[[0. 0. 0. 1. 0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0. 0. 1. 0. 0.]\n",
      "  [0. 1. 0. 0. 0. 0. 0. 0.]]]\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "_4ecvZkFHscm"
   },
   "source": [
    "## Batching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "QnwHn58kHscn"
   },
   "source": [
    "```\n",
    "                M steps ( seq length )\n",
    "               xxx                 xxx\n",
    "               x                     x\n",
    "               x                     x                                Starting sequence:\n",
    "               x                     x                                [1 2 3 4 5 6 7 8 9 10 11 12]\n",
    "               x                     x\n",
    "N batch size   x                     x                                Batch size = 2\n",
    "(No. of steps) x                     x                                [1 2 3 4 5 6]\n",
    "               x                     x                                [7 8 9 10 11 12]\n",
    "               x                     x\n",
    "               x                     x                                Seq length = 3\n",
    "               x                     x\n",
    "               x                     x                                  ┌─────┐\n",
    "               x                     x                                [ │1 2 3│ 4 5 6]\n",
    "               x                     x                                [ │7 8 9│ 10 11 12]\n",
    "               x                     x                                  └─────┘\n",
    "               xxx                 xxx\n",
    "\n",
    "            xxxxxxxxxxxxxx   xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n",
    "                         xxxxx\n",
    "                          xx\n",
    "\n",
    "                          k= No. of batches = total chars/ N.M\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "Jq-s8InRHsco"
   },
   "source": [
    "def create_batches(arr, batch_size, seq_length):\n",
    "    batch_size_total = batch_size * seq_length\n",
    "    n_batches = len(arr) // batch_size_total\n",
    "    arr = arr[:n_batches*batch_size_total]\n",
    "    arr = arr.reshape((batch_size,-1))\n",
    "\n",
    "    for n in range(0, arr.shape[1], seq_length):\n",
    "        x = arr[:, n:n+seq_length]\n",
    "        y = np.zeros_like(x)\n",
    "        try:\n",
    "            y[:, :-1], y[:, -1] = x[:, 1:], arr[:, n+seq_length]\n",
    "        except IndexError:\n",
    "            y[:, :-1], y[:, -1] = x[:, 1:], arr[:, 0]\n",
    "        yield x, y"
   ],
   "execution_count": 8,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "KOuURObaHscp",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "35fd5b90-dec6-48d7-cb3a-21280827f66d"
   },
   "source": [
    "batches = create_batches(encoded, 8, 50)\n",
    "x, y= next(batches)\n",
    "print('x:\\n', x[:10, :10])\n",
    "print('\\ny:\\n', y[:10, :10])"
   ],
   "execution_count": 9,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "x:\n",
      " [[27 36  2 24 27 25 13 30 35 24]\n",
      " [ 6 27 37 21 14 27 36  2 24 27]\n",
      " [23 26 13 14 24 21 27 25 37 13]\n",
      " [ 6 34 27 36  2 24 27 13 16 21]\n",
      " [ 2 27 31 30 26 27 29 30 26 12]\n",
      " [ 6 36 13 37 21 28 24 27 36 30]\n",
      " [30 21 27 36  2 24 16 13 27 22]\n",
      " [13 14 27 37 27 28 13 24 37 36]]\n",
      "\n",
      "y:\n",
      " [[36  2 24 27 25 13 30 35 24 11]\n",
      " [27 37 21 14 27 36  2 24 27  6]\n",
      " [26 13 14 24 21 27 25 37 13 36]\n",
      " [34 27 36  2 24 27 13 16 21 28]\n",
      " [27 31 30 26 27 29 30 26 12 14]\n",
      " [36 13 37 21 28 24 27 36 30 29]\n",
      " [21 27 36  2 24 16 13 27 22 37]\n",
      " [14 27 37 27 28 13 24 37 36 27]]\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "2uvL80mYHscp"
   },
   "source": [
    "## Defining model"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "05xGLCSRHscq",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "ecae3d2c-8c73-4621-c583-f9566a4eb433"
   },
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ],
   "execution_count": 10,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "cuda\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "Pnq5Gc04Hscq"
   },
   "source": [
    "class textgenRNN(nn.Module):\n",
    "    def __init__(self,n_output, n_hidden=256, n_layers=2, drop_prob=0.5, lr=0.001):\n",
    "        super().__init__()\n",
    "        self.drop_prob = drop_prob\n",
    "        self.n_layers = n_layers\n",
    "        self.n_hidden = n_hidden\n",
    "        self.lr = lr\n",
    "        self.n_output = n_output\n",
    "\n",
    "        self.lstm = nn.LSTM(n_output, n_hidden, n_layers, dropout=drop_prob, batch_first=True)\n",
    "        self.dropout1 = nn.Dropout(drop_prob)\n",
    "        self.fc1 = nn.Linear(n_hidden, n_output)\n",
    "\n",
    "    def forward(self, x, hidden):\n",
    "        out, hid = self.lstm(x, hidden)\n",
    "        out = self.dropout1(out)\n",
    "\n",
    "        out = out.contiguous().view(-1, self.n_hidden)\n",
    "\n",
    "        out = self.fc1(out)\n",
    "        return out, hid\n",
    "\n",
    "    def init_hidden(self, batch_size):\n",
    "        ''' Initializes hidden state '''\n",
    "        # Create two new tensors with sizes n_layers x batch_size x n_hidden,\n",
    "        # initialized to zero, for hidden state and cell state of LSTM\n",
    "        weight = next(self.parameters()).data\n",
    "\n",
    "        hidden = (torch.zeros(self.n_layers, batch_size, self.n_hidden).to(device),\n",
    "                  torch.zeros(self.n_layers, batch_size, self.n_hidden).to(device))\n",
    "\n",
    "        return hidden"
   ],
   "execution_count": 11,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "rLVfosNFHscr",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "36e789d4-b4fc-4fbb-91d7-73ded0d0236f"
   },
   "source": [
    "from torchsummary import summary\n",
    "n_hidden=512\n",
    "n_layers=2\n",
    "model = textgenRNN(len(vocab), n_hidden=n_hidden, n_layers=n_layers)\n",
    "model.to(device)\n",
    "print(model)"
   ],
   "execution_count": 12,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "textgenRNN(\n",
      "  (lstm): LSTM(38, 512, num_layers=2, batch_first=True, dropout=0.5)\n",
      "  (dropout1): Dropout(p=0.5, inplace=False)\n",
      "  (fc1): Linear(in_features=512, out_features=38, bias=True)\n",
      ")\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "J1EYiUkOHscs"
   },
   "source": [
    "import os\n",
    "if os.path.isdir('checkpoint') and os.path.isfile('./checkpoint/trialckpt.t7'):\n",
    "    model.load_state_dict(torch.load('./checkpoint/trialckpt.t7', map_location=torch.device(device)))"
   ],
   "execution_count": 13,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "oL--pDxPHsct"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "id": "t0HSH7QxHsct"
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "epochs=100\n",
    "batch_size=32\n",
    "seq_length=64\n",
    "lr=0.01\n",
    "clip=5\n",
    "test_portion=0.1\n",
    "\n",
    "# change model mode to train\n",
    "model.train()\n",
    "\n",
    "optim = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# create training and validation data\n",
    "data, test_data = train_test_split(encoded, test_size=test_portion, shuffle=False, random_state=0)"
   ],
   "execution_count": 14,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    },
    "id": "i0ItBh6hHscv",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "fa379266-8007-40b5-c3f3-4845d32b615e"
   },
   "source": [
    "counter = 0\n",
    "n_vocab = len(vocab)\n",
    "best_val_loss = 1000\n",
    "for e in range(epochs):\n",
    "    # initialize hidden state\n",
    "    h = model.init_hidden(batch_size)\n",
    "\n",
    "    for x, y in create_batches(data, batch_size, seq_length):\n",
    "        counter += 1\n",
    "\n",
    "        # One-hot encode our data and make them Torch tensors\n",
    "        x = one_hot_encode(x, n_vocab)\n",
    "        inputs, targets = torch.from_numpy(x), torch.from_numpy(y)\n",
    "\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "\n",
    "        # Creating new variables for the hidden state, otherwise\n",
    "        # we'd backprop through the entire training history\n",
    "        h = tuple([each.data for each in h])\n",
    "\n",
    "        # zero accumulated gradients\n",
    "        model.zero_grad()\n",
    "\n",
    "        # get the output from the model\n",
    "        output, h = model(inputs, h)\n",
    "\n",
    "        # calculate the loss and perform backprop\n",
    "        loss = criterion(output, targets.view(batch_size*seq_length).long())\n",
    "        loss.backward()\n",
    "        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        optim.step()\n",
    "\n",
    "        # loss stats\n",
    "        if counter % 100 == 0:\n",
    "            # Get validation loss\n",
    "            val_h = model.init_hidden(batch_size)\n",
    "            val_losses = []\n",
    "            model.eval()\n",
    "            for x, y in create_batches(test_data, batch_size, seq_length):\n",
    "                # One-hot encode our data and make them Torch tensors\n",
    "                x = one_hot_encode(x, n_vocab)\n",
    "                x, y = torch.from_numpy(x), torch.from_numpy(y)\n",
    "\n",
    "                # Creating new variables for the hidden state, otherwise\n",
    "                # we'd backprop through the entire training history\n",
    "                val_h = tuple([each.data for each in val_h])\n",
    "\n",
    "                inputs, targets = x, y\n",
    "                inputs, targets = inputs.to(device), targets.to(device)\n",
    "\n",
    "                output, val_h = model(inputs, val_h)\n",
    "                val_loss = criterion(output, targets.view(batch_size*seq_length).long())\n",
    "\n",
    "                val_losses.append(val_loss.item())\n",
    "\n",
    "            print('==> Saving model ...')\n",
    "            best_val_loss = min(np.mean(val_losses), best_val_loss)\n",
    "            if not os.path.isdir('checkpoint'):\n",
    "                os.mkdir('checkpoint')\n",
    "            if best_val_loss == np.mean(val_losses):\n",
    "              torch.save(model.state_dict(), './checkpoint/trialckpt.t7')\n",
    "\n",
    "            model.train() # reset to train mode after iterationg through validation data\n",
    "\n",
    "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "                  \"Step: {}...\".format(counter),\n",
    "                  \"Loss: {:.4f}...\".format(loss.item()),\n",
    "                  \"Val Loss: {:.4f}\".format(np.mean(val_losses)))"
   ],
   "execution_count": 15,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "==> Saving model ...\n",
      "Epoch: 1/100... Step: 100... Loss: 2.6166... Val Loss: 2.6582\n",
      "==> Saving model ...\n",
      "Epoch: 1/100... Step: 200... Loss: 2.2398... Val Loss: 2.3429\n",
      "==> Saving model ...\n",
      "Epoch: 2/100... Step: 300... Loss: 2.0099... Val Loss: 2.1482\n",
      "==> Saving model ...\n",
      "Epoch: 2/100... Step: 400... Loss: 1.8993... Val Loss: 2.0162\n",
      "==> Saving model ...\n",
      "Epoch: 3/100... Step: 500... Loss: 1.7479... Val Loss: 1.9247\n",
      "==> Saving model ...\n",
      "Epoch: 3/100... Step: 600... Loss: 1.5934... Val Loss: 1.8656\n",
      "==> Saving model ...\n",
      "Epoch: 4/100... Step: 700... Loss: 1.6504... Val Loss: 1.8033\n",
      "==> Saving model ...\n",
      "Epoch: 4/100... Step: 800... Loss: 1.5091... Val Loss: 1.7713\n",
      "==> Saving model ...\n",
      "Epoch: 4/100... Step: 900... Loss: 1.5150... Val Loss: 1.7537\n",
      "==> Saving model ...\n",
      "Epoch: 5/100... Step: 1000... Loss: 1.4988... Val Loss: 1.7170\n",
      "==> Saving model ...\n",
      "Epoch: 5/100... Step: 1100... Loss: 1.4187... Val Loss: 1.7102\n",
      "==> Saving model ...\n",
      "Epoch: 6/100... Step: 1200... Loss: 1.3841... Val Loss: 1.6771\n",
      "==> Saving model ...\n",
      "Epoch: 6/100... Step: 1300... Loss: 1.4016... Val Loss: 1.6869\n",
      "==> Saving model ...\n",
      "Epoch: 7/100... Step: 1400... Loss: 1.3724... Val Loss: 1.6676\n",
      "==> Saving model ...\n",
      "Epoch: 7/100... Step: 1500... Loss: 1.3746... Val Loss: 1.6699\n",
      "==> Saving model ...\n",
      "Epoch: 8/100... Step: 1600... Loss: 1.3966... Val Loss: 1.6325\n",
      "==> Saving model ...\n",
      "Epoch: 8/100... Step: 1700... Loss: 1.3895... Val Loss: 1.6290\n",
      "==> Saving model ...\n",
      "Epoch: 8/100... Step: 1800... Loss: 1.3864... Val Loss: 1.6389\n",
      "==> Saving model ...\n",
      "Epoch: 9/100... Step: 1900... Loss: 1.3167... Val Loss: 1.6347\n",
      "==> Saving model ...\n",
      "Epoch: 9/100... Step: 2000... Loss: 1.3221... Val Loss: 1.6252\n",
      "==> Saving model ...\n",
      "Epoch: 10/100... Step: 2100... Loss: 1.2819... Val Loss: 1.6046\n",
      "==> Saving model ...\n",
      "Epoch: 10/100... Step: 2200... Loss: 1.2292... Val Loss: 1.6154\n",
      "==> Saving model ...\n",
      "Epoch: 11/100... Step: 2300... Loss: 1.2446... Val Loss: 1.5955\n",
      "==> Saving model ...\n",
      "Epoch: 11/100... Step: 2400... Loss: 1.2870... Val Loss: 1.6071\n",
      "==> Saving model ...\n",
      "Epoch: 12/100... Step: 2500... Loss: 1.2690... Val Loss: 1.5988\n",
      "==> Saving model ...\n",
      "Epoch: 12/100... Step: 2600... Loss: 1.3093... Val Loss: 1.6018\n",
      "==> Saving model ...\n",
      "Epoch: 12/100... Step: 2700... Loss: 1.2602... Val Loss: 1.6162\n",
      "==> Saving model ...\n",
      "Epoch: 13/100... Step: 2800... Loss: 1.2712... Val Loss: 1.5782\n",
      "==> Saving model ...\n",
      "Epoch: 13/100... Step: 2900... Loss: 1.1975... Val Loss: 1.5980\n",
      "==> Saving model ...\n",
      "Epoch: 14/100... Step: 3000... Loss: 1.2739... Val Loss: 1.5845\n",
      "==> Saving model ...\n",
      "Epoch: 14/100... Step: 3100... Loss: 1.1876... Val Loss: 1.5862\n",
      "==> Saving model ...\n",
      "Epoch: 15/100... Step: 3200... Loss: 1.1904... Val Loss: 1.5775\n",
      "==> Saving model ...\n",
      "Epoch: 15/100... Step: 3300... Loss: 1.2567... Val Loss: 1.5807\n",
      "==> Saving model ...\n",
      "Epoch: 15/100... Step: 3400... Loss: 1.1839... Val Loss: 1.6019\n",
      "==> Saving model ...\n",
      "Epoch: 16/100... Step: 3500... Loss: 1.2186... Val Loss: 1.5820\n",
      "==> Saving model ...\n",
      "Epoch: 16/100... Step: 3600... Loss: 1.2102... Val Loss: 1.5947\n",
      "==> Saving model ...\n",
      "Epoch: 17/100... Step: 3700... Loss: 1.2101... Val Loss: 1.5801\n",
      "==> Saving model ...\n",
      "Epoch: 17/100... Step: 3800... Loss: 1.1587... Val Loss: 1.5932\n",
      "==> Saving model ...\n",
      "Epoch: 18/100... Step: 3900... Loss: 1.2371... Val Loss: 1.5814\n",
      "==> Saving model ...\n",
      "Epoch: 18/100... Step: 4000... Loss: 1.2323... Val Loss: 1.5871\n",
      "==> Saving model ...\n",
      "Epoch: 19/100... Step: 4100... Loss: 1.2242... Val Loss: 1.5657\n",
      "==> Saving model ...\n",
      "Epoch: 19/100... Step: 4200... Loss: 1.2040... Val Loss: 1.5780\n",
      "==> Saving model ...\n",
      "Epoch: 19/100... Step: 4300... Loss: 1.1775... Val Loss: 1.5861\n",
      "==> Saving model ...\n",
      "Epoch: 20/100... Step: 4400... Loss: 1.1660... Val Loss: 1.5870\n",
      "==> Saving model ...\n",
      "Epoch: 20/100... Step: 4500... Loss: 1.1569... Val Loss: 1.5986\n",
      "==> Saving model ...\n",
      "Epoch: 21/100... Step: 4600... Loss: 1.1850... Val Loss: 1.5804\n",
      "==> Saving model ...\n",
      "Epoch: 21/100... Step: 4700... Loss: 1.1854... Val Loss: 1.5875\n",
      "==> Saving model ...\n",
      "Epoch: 22/100... Step: 4800... Loss: 1.1704... Val Loss: 1.5838\n",
      "==> Saving model ...\n",
      "Epoch: 22/100... Step: 4900... Loss: 1.1952... Val Loss: 1.5919\n",
      "==> Saving model ...\n",
      "Epoch: 23/100... Step: 5000... Loss: 1.1733... Val Loss: 1.5807\n",
      "==> Saving model ...\n",
      "Epoch: 23/100... Step: 5100... Loss: 1.1720... Val Loss: 1.5758\n",
      "==> Saving model ...\n",
      "Epoch: 23/100... Step: 5200... Loss: 1.1927... Val Loss: 1.5833\n",
      "==> Saving model ...\n",
      "Epoch: 24/100... Step: 5300... Loss: 1.1185... Val Loss: 1.5665\n",
      "==> Saving model ...\n",
      "Epoch: 24/100... Step: 5400... Loss: 1.1170... Val Loss: 1.5877\n",
      "==> Saving model ...\n",
      "Epoch: 25/100... Step: 5500... Loss: 1.1878... Val Loss: 1.5776\n",
      "==> Saving model ...\n",
      "Epoch: 25/100... Step: 5600... Loss: 1.1958... Val Loss: 1.5937\n",
      "==> Saving model ...\n",
      "Epoch: 26/100... Step: 5700... Loss: 1.1600... Val Loss: 1.5725\n",
      "==> Saving model ...\n",
      "Epoch: 26/100... Step: 5800... Loss: 1.1584... Val Loss: 1.5867\n",
      "==> Saving model ...\n",
      "Epoch: 26/100... Step: 5900... Loss: 1.1309... Val Loss: 1.6075\n",
      "==> Saving model ...\n",
      "Epoch: 27/100... Step: 6000... Loss: 1.1114... Val Loss: 1.5846\n",
      "==> Saving model ...\n",
      "Epoch: 27/100... Step: 6100... Loss: 1.1362... Val Loss: 1.5928\n",
      "==> Saving model ...\n",
      "Epoch: 28/100... Step: 6200... Loss: 1.1723... Val Loss: 1.5807\n",
      "==> Saving model ...\n",
      "Epoch: 28/100... Step: 6300... Loss: 1.1547... Val Loss: 1.5897\n",
      "==> Saving model ...\n",
      "Epoch: 29/100... Step: 6400... Loss: 1.1706... Val Loss: 1.5762\n",
      "==> Saving model ...\n",
      "Epoch: 29/100... Step: 6500... Loss: 1.1743... Val Loss: 1.5949\n",
      "==> Saving model ...\n",
      "Epoch: 30/100... Step: 6600... Loss: 1.0998... Val Loss: 1.5759\n",
      "==> Saving model ...\n",
      "Epoch: 30/100... Step: 6700... Loss: 1.1097... Val Loss: 1.5895\n",
      "==> Saving model ...\n",
      "Epoch: 30/100... Step: 6800... Loss: 1.0780... Val Loss: 1.5992\n",
      "==> Saving model ...\n",
      "Epoch: 31/100... Step: 6900... Loss: 1.1475... Val Loss: 1.6029\n",
      "==> Saving model ...\n",
      "Epoch: 31/100... Step: 7000... Loss: 1.0870... Val Loss: 1.6087\n",
      "==> Saving model ...\n",
      "Epoch: 32/100... Step: 7100... Loss: 1.0959... Val Loss: 1.5988\n",
      "==> Saving model ...\n",
      "Epoch: 32/100... Step: 7200... Loss: 1.0654... Val Loss: 1.5923\n",
      "==> Saving model ...\n",
      "Epoch: 33/100... Step: 7300... Loss: 1.1321... Val Loss: 1.5913\n",
      "==> Saving model ...\n",
      "Epoch: 33/100... Step: 7400... Loss: 1.0934... Val Loss: 1.6053\n",
      "==> Saving model ...\n",
      "Epoch: 34/100... Step: 7500... Loss: 1.1625... Val Loss: 1.5825\n",
      "==> Saving model ...\n",
      "Epoch: 34/100... Step: 7600... Loss: 1.1147... Val Loss: 1.5826\n",
      "==> Saving model ...\n",
      "Epoch: 34/100... Step: 7700... Loss: 1.1190... Val Loss: 1.6133\n",
      "==> Saving model ...\n",
      "Epoch: 35/100... Step: 7800... Loss: 1.0789... Val Loss: 1.5923\n",
      "==> Saving model ...\n",
      "Epoch: 35/100... Step: 7900... Loss: 1.0855... Val Loss: 1.6217\n",
      "==> Saving model ...\n",
      "Epoch: 36/100... Step: 8000... Loss: 1.1315... Val Loss: 1.5866\n",
      "==> Saving model ...\n",
      "Epoch: 36/100... Step: 8100... Loss: 1.0992... Val Loss: 1.5991\n",
      "==> Saving model ...\n",
      "Epoch: 37/100... Step: 8200... Loss: 1.0491... Val Loss: 1.5770\n",
      "==> Saving model ...\n",
      "Epoch: 37/100... Step: 8300... Loss: 1.1028... Val Loss: 1.6020\n",
      "==> Saving model ...\n",
      "Epoch: 38/100... Step: 8400... Loss: 1.0838... Val Loss: 1.5977\n",
      "==> Saving model ...\n",
      "Epoch: 38/100... Step: 8500... Loss: 1.0878... Val Loss: 1.6005\n",
      "==> Saving model ...\n",
      "Epoch: 38/100... Step: 8600... Loss: 1.1185... Val Loss: 1.5995\n",
      "==> Saving model ...\n",
      "Epoch: 39/100... Step: 8700... Loss: 1.1159... Val Loss: 1.5888\n",
      "==> Saving model ...\n",
      "Epoch: 39/100... Step: 8800... Loss: 1.1078... Val Loss: 1.6061\n",
      "==> Saving model ...\n",
      "Epoch: 40/100... Step: 8900... Loss: 1.0875... Val Loss: 1.5930\n",
      "==> Saving model ...\n",
      "Epoch: 40/100... Step: 9000... Loss: 1.1182... Val Loss: 1.6075\n",
      "==> Saving model ...\n",
      "Epoch: 41/100... Step: 9100... Loss: 1.0958... Val Loss: 1.5858\n",
      "==> Saving model ...\n",
      "Epoch: 41/100... Step: 9200... Loss: 1.1083... Val Loss: 1.6129\n",
      "==> Saving model ...\n",
      "Epoch: 41/100... Step: 9300... Loss: 1.0745... Val Loss: 1.6098\n",
      "==> Saving model ...\n",
      "Epoch: 42/100... Step: 9400... Loss: 1.1277... Val Loss: 1.6014\n",
      "==> Saving model ...\n",
      "Epoch: 42/100... Step: 9500... Loss: 1.1236... Val Loss: 1.6326\n",
      "==> Saving model ...\n",
      "Epoch: 43/100... Step: 9600... Loss: 1.1048... Val Loss: 1.6088\n",
      "==> Saving model ...\n",
      "Epoch: 43/100... Step: 9700... Loss: 1.1217... Val Loss: 1.6180\n",
      "==> Saving model ...\n",
      "Epoch: 44/100... Step: 9800... Loss: 1.0674... Val Loss: 1.5991\n",
      "==> Saving model ...\n",
      "Epoch: 44/100... Step: 9900... Loss: 1.0828... Val Loss: 1.6145\n",
      "==> Saving model ...\n",
      "Epoch: 45/100... Step: 10000... Loss: 1.1533... Val Loss: 1.5952\n",
      "==> Saving model ...\n",
      "Epoch: 45/100... Step: 10100... Loss: 1.0719... Val Loss: 1.6197\n",
      "==> Saving model ...\n",
      "Epoch: 45/100... Step: 10200... Loss: 1.0505... Val Loss: 1.6219\n",
      "==> Saving model ...\n",
      "Epoch: 46/100... Step: 10300... Loss: 1.0737... Val Loss: 1.6123\n",
      "==> Saving model ...\n",
      "Epoch: 46/100... Step: 10400... Loss: 1.0710... Val Loss: 1.6263\n",
      "==> Saving model ...\n",
      "Epoch: 47/100... Step: 10500... Loss: 1.0709... Val Loss: 1.6094\n",
      "==> Saving model ...\n",
      "Epoch: 47/100... Step: 10600... Loss: 1.1089... Val Loss: 1.6156\n",
      "==> Saving model ...\n",
      "Epoch: 48/100... Step: 10700... Loss: 1.1062... Val Loss: 1.6079\n",
      "==> Saving model ...\n",
      "Epoch: 48/100... Step: 10800... Loss: 1.0571... Val Loss: 1.6211\n",
      "==> Saving model ...\n",
      "Epoch: 49/100... Step: 10900... Loss: 1.1154... Val Loss: 1.6275\n",
      "==> Saving model ...\n",
      "Epoch: 49/100... Step: 11000... Loss: 1.0847... Val Loss: 1.6033\n",
      "==> Saving model ...\n",
      "Epoch: 49/100... Step: 11100... Loss: 1.0874... Val Loss: 1.6203\n",
      "==> Saving model ...\n",
      "Epoch: 50/100... Step: 11200... Loss: 1.0420... Val Loss: 1.6128\n",
      "==> Saving model ...\n",
      "Epoch: 50/100... Step: 11300... Loss: 1.0985... Val Loss: 1.6288\n",
      "==> Saving model ...\n",
      "Epoch: 51/100... Step: 11400... Loss: 1.0811... Val Loss: 1.6243\n",
      "==> Saving model ...\n",
      "Epoch: 51/100... Step: 11500... Loss: 1.0926... Val Loss: 1.6284\n",
      "==> Saving model ...\n",
      "Epoch: 52/100... Step: 11600... Loss: 1.1106... Val Loss: 1.5962\n",
      "==> Saving model ...\n",
      "Epoch: 52/100... Step: 11700... Loss: 1.0864... Val Loss: 1.6209\n",
      "==> Saving model ...\n",
      "Epoch: 52/100... Step: 11800... Loss: 1.0686... Val Loss: 1.6335\n",
      "==> Saving model ...\n",
      "Epoch: 53/100... Step: 11900... Loss: 1.0967... Val Loss: 1.6224\n",
      "==> Saving model ...\n",
      "Epoch: 53/100... Step: 12000... Loss: 1.1048... Val Loss: 1.6165\n",
      "==> Saving model ...\n",
      "Epoch: 54/100... Step: 12100... Loss: 1.0699... Val Loss: 1.6162\n",
      "==> Saving model ...\n",
      "Epoch: 54/100... Step: 12200... Loss: 1.1172... Val Loss: 1.6226\n",
      "==> Saving model ...\n",
      "Epoch: 55/100... Step: 12300... Loss: 1.0938... Val Loss: 1.6291\n",
      "==> Saving model ...\n",
      "Epoch: 55/100... Step: 12400... Loss: 1.0814... Val Loss: 1.6211\n",
      "==> Saving model ...\n",
      "Epoch: 56/100... Step: 12500... Loss: 1.1120... Val Loss: 1.6029\n",
      "==> Saving model ...\n",
      "Epoch: 56/100... Step: 12600... Loss: 1.0976... Val Loss: 1.6177\n",
      "==> Saving model ...\n",
      "Epoch: 56/100... Step: 12700... Loss: 1.0250... Val Loss: 1.6283\n",
      "==> Saving model ...\n",
      "Epoch: 57/100... Step: 12800... Loss: 1.0834... Val Loss: 1.6264\n",
      "==> Saving model ...\n",
      "Epoch: 57/100... Step: 12900... Loss: 1.0471... Val Loss: 1.6283\n",
      "==> Saving model ...\n",
      "Epoch: 58/100... Step: 13000... Loss: 1.0653... Val Loss: 1.6051\n",
      "==> Saving model ...\n",
      "Epoch: 58/100... Step: 13100... Loss: 1.0222... Val Loss: 1.6339\n",
      "==> Saving model ...\n",
      "Epoch: 59/100... Step: 13200... Loss: 1.0540... Val Loss: 1.6144\n",
      "==> Saving model ...\n",
      "Epoch: 59/100... Step: 13300... Loss: 1.0812... Val Loss: 1.6301\n",
      "==> Saving model ...\n",
      "Epoch: 60/100... Step: 13400... Loss: 1.0556... Val Loss: 1.6331\n",
      "==> Saving model ...\n",
      "Epoch: 60/100... Step: 13500... Loss: 1.0925... Val Loss: 1.6304\n",
      "==> Saving model ...\n",
      "Epoch: 60/100... Step: 13600... Loss: 1.0598... Val Loss: 1.6565\n",
      "==> Saving model ...\n",
      "Epoch: 61/100... Step: 13700... Loss: 1.0365... Val Loss: 1.6263\n",
      "==> Saving model ...\n",
      "Epoch: 61/100... Step: 13800... Loss: 1.0345... Val Loss: 1.6404\n",
      "==> Saving model ...\n",
      "Epoch: 62/100... Step: 13900... Loss: 1.0990... Val Loss: 1.6259\n",
      "==> Saving model ...\n",
      "Epoch: 62/100... Step: 14000... Loss: 1.0647... Val Loss: 1.6444\n",
      "==> Saving model ...\n",
      "Epoch: 63/100... Step: 14100... Loss: 1.0482... Val Loss: 1.6289\n",
      "==> Saving model ...\n",
      "Epoch: 63/100... Step: 14200... Loss: 1.0659... Val Loss: 1.6328\n",
      "==> Saving model ...\n",
      "Epoch: 63/100... Step: 14300... Loss: 1.0696... Val Loss: 1.6415\n",
      "==> Saving model ...\n",
      "Epoch: 64/100... Step: 14400... Loss: 1.0918... Val Loss: 1.6495\n",
      "==> Saving model ...\n",
      "Epoch: 64/100... Step: 14500... Loss: 1.0600... Val Loss: 1.6445\n",
      "==> Saving model ...\n",
      "Epoch: 65/100... Step: 14600... Loss: 1.0768... Val Loss: 1.6518\n",
      "==> Saving model ...\n",
      "Epoch: 65/100... Step: 14700... Loss: 1.0518... Val Loss: 1.6635\n",
      "==> Saving model ...\n",
      "Epoch: 66/100... Step: 14800... Loss: 1.0897... Val Loss: 1.6444\n",
      "==> Saving model ...\n",
      "Epoch: 66/100... Step: 14900... Loss: 1.1085... Val Loss: 1.6451\n",
      "==> Saving model ...\n",
      "Epoch: 67/100... Step: 15000... Loss: 1.0943... Val Loss: 1.6291\n",
      "==> Saving model ...\n",
      "Epoch: 67/100... Step: 15100... Loss: 1.0122... Val Loss: 1.6470\n",
      "==> Saving model ...\n",
      "Epoch: 67/100... Step: 15200... Loss: 1.0708... Val Loss: 1.6528\n",
      "==> Saving model ...\n",
      "Epoch: 68/100... Step: 15300... Loss: 1.0259... Val Loss: 1.6493\n",
      "==> Saving model ...\n",
      "Epoch: 68/100... Step: 15400... Loss: 1.0667... Val Loss: 1.6190\n",
      "==> Saving model ...\n",
      "Epoch: 69/100... Step: 15500... Loss: 1.0684... Val Loss: 1.6338\n",
      "==> Saving model ...\n",
      "Epoch: 69/100... Step: 15600... Loss: 1.0418... Val Loss: 1.6424\n",
      "==> Saving model ...\n",
      "Epoch: 70/100... Step: 15700... Loss: 1.0783... Val Loss: 1.6151\n",
      "==> Saving model ...\n",
      "Epoch: 70/100... Step: 15800... Loss: 1.0447... Val Loss: 1.6479\n",
      "==> Saving model ...\n",
      "Epoch: 71/100... Step: 15900... Loss: 1.0855... Val Loss: 1.6287\n",
      "==> Saving model ...\n",
      "Epoch: 71/100... Step: 16000... Loss: 1.0277... Val Loss: 1.6301\n",
      "==> Saving model ...\n",
      "Epoch: 71/100... Step: 16100... Loss: 1.0388... Val Loss: 1.6409\n",
      "==> Saving model ...\n",
      "Epoch: 72/100... Step: 16200... Loss: 1.0619... Val Loss: 1.6386\n",
      "==> Saving model ...\n",
      "Epoch: 72/100... Step: 16300... Loss: 1.0441... Val Loss: 1.6401\n",
      "==> Saving model ...\n",
      "Epoch: 73/100... Step: 16400... Loss: 1.0991... Val Loss: 1.6331\n",
      "==> Saving model ...\n",
      "Epoch: 73/100... Step: 16500... Loss: 1.0534... Val Loss: 1.6464\n",
      "==> Saving model ...\n",
      "Epoch: 74/100... Step: 16600... Loss: 1.0508... Val Loss: 1.6402\n",
      "==> Saving model ...\n",
      "Epoch: 74/100... Step: 16700... Loss: 1.0734... Val Loss: 1.6677\n",
      "==> Saving model ...\n",
      "Epoch: 75/100... Step: 16800... Loss: 1.1247... Val Loss: 1.6564\n",
      "==> Saving model ...\n",
      "Epoch: 75/100... Step: 16900... Loss: 1.0261... Val Loss: 1.6466\n",
      "==> Saving model ...\n",
      "Epoch: 75/100... Step: 17000... Loss: 1.0138... Val Loss: 1.6541\n",
      "==> Saving model ...\n",
      "Epoch: 76/100... Step: 17100... Loss: 1.0810... Val Loss: 1.6386\n",
      "==> Saving model ...\n",
      "Epoch: 76/100... Step: 17200... Loss: 1.0750... Val Loss: 1.6512\n",
      "==> Saving model ...\n",
      "Epoch: 77/100... Step: 17300... Loss: 1.0387... Val Loss: 1.6426\n",
      "==> Saving model ...\n",
      "Epoch: 77/100... Step: 17400... Loss: 1.0661... Val Loss: 1.6323\n",
      "==> Saving model ...\n",
      "Epoch: 78/100... Step: 17500... Loss: 1.0877... Val Loss: 1.6355\n",
      "==> Saving model ...\n",
      "Epoch: 78/100... Step: 17600... Loss: 1.0717... Val Loss: 1.6546\n",
      "==> Saving model ...\n",
      "Epoch: 78/100... Step: 17700... Loss: 1.0007... Val Loss: 1.6552\n",
      "==> Saving model ...\n",
      "Epoch: 79/100... Step: 17800... Loss: 1.0531... Val Loss: 1.6430\n",
      "==> Saving model ...\n",
      "Epoch: 79/100... Step: 17900... Loss: 1.0909... Val Loss: 1.6570\n",
      "==> Saving model ...\n",
      "Epoch: 80/100... Step: 18000... Loss: 1.0437... Val Loss: 1.6448\n",
      "==> Saving model ...\n",
      "Epoch: 80/100... Step: 18100... Loss: 1.0692... Val Loss: 1.6536\n",
      "==> Saving model ...\n",
      "Epoch: 81/100... Step: 18200... Loss: 1.0471... Val Loss: 1.6469\n",
      "==> Saving model ...\n",
      "Epoch: 81/100... Step: 18300... Loss: 1.0831... Val Loss: 1.6431\n",
      "==> Saving model ...\n",
      "Epoch: 82/100... Step: 18400... Loss: 1.1114... Val Loss: 1.6394\n",
      "==> Saving model ...\n",
      "Epoch: 82/100... Step: 18500... Loss: 1.0507... Val Loss: 1.6462\n",
      "==> Saving model ...\n",
      "Epoch: 82/100... Step: 18600... Loss: 1.0846... Val Loss: 1.6614\n",
      "==> Saving model ...\n",
      "Epoch: 83/100... Step: 18700... Loss: 1.0149... Val Loss: 1.6668\n",
      "==> Saving model ...\n",
      "Epoch: 83/100... Step: 18800... Loss: 1.0385... Val Loss: 1.6695\n",
      "==> Saving model ...\n",
      "Epoch: 84/100... Step: 18900... Loss: 1.0580... Val Loss: 1.6348\n",
      "==> Saving model ...\n",
      "Epoch: 84/100... Step: 19000... Loss: 1.0393... Val Loss: 1.6568\n",
      "==> Saving model ...\n",
      "Epoch: 85/100... Step: 19100... Loss: 1.0880... Val Loss: 1.6427\n",
      "==> Saving model ...\n",
      "Epoch: 85/100... Step: 19200... Loss: 1.0344... Val Loss: 1.6550\n",
      "==> Saving model ...\n",
      "Epoch: 86/100... Step: 19300... Loss: 1.1015... Val Loss: 1.6658\n",
      "==> Saving model ...\n",
      "Epoch: 86/100... Step: 19400... Loss: 1.0533... Val Loss: 1.6562\n",
      "==> Saving model ...\n",
      "Epoch: 86/100... Step: 19500... Loss: 1.0981... Val Loss: 1.6612\n",
      "==> Saving model ...\n",
      "Epoch: 87/100... Step: 19600... Loss: 1.0522... Val Loss: 1.6521\n",
      "==> Saving model ...\n",
      "Epoch: 87/100... Step: 19700... Loss: 1.0710... Val Loss: 1.6577\n",
      "==> Saving model ...\n",
      "Epoch: 88/100... Step: 19800... Loss: 1.0584... Val Loss: 1.6483\n",
      "==> Saving model ...\n",
      "Epoch: 88/100... Step: 19900... Loss: 1.0351... Val Loss: 1.6630\n",
      "==> Saving model ...\n",
      "Epoch: 89/100... Step: 20000... Loss: 1.0575... Val Loss: 1.6437\n",
      "==> Saving model ...\n",
      "Epoch: 89/100... Step: 20100... Loss: 1.0202... Val Loss: 1.6447\n",
      "==> Saving model ...\n",
      "Epoch: 89/100... Step: 20200... Loss: 1.0300... Val Loss: 1.6561\n",
      "==> Saving model ...\n",
      "Epoch: 90/100... Step: 20300... Loss: 1.0159... Val Loss: 1.6568\n",
      "==> Saving model ...\n",
      "Epoch: 90/100... Step: 20400... Loss: 1.0333... Val Loss: 1.6785\n",
      "==> Saving model ...\n",
      "Epoch: 91/100... Step: 20500... Loss: 1.0511... Val Loss: 1.6514\n",
      "==> Saving model ...\n",
      "Epoch: 91/100... Step: 20600... Loss: 1.0445... Val Loss: 1.6660\n",
      "==> Saving model ...\n",
      "Epoch: 92/100... Step: 20700... Loss: 1.0435... Val Loss: 1.6395\n",
      "==> Saving model ...\n",
      "Epoch: 92/100... Step: 20800... Loss: 1.0274... Val Loss: 1.6680\n",
      "==> Saving model ...\n",
      "Epoch: 93/100... Step: 20900... Loss: 1.0322... Val Loss: 1.6468\n",
      "==> Saving model ...\n",
      "Epoch: 93/100... Step: 21000... Loss: 1.0488... Val Loss: 1.6542\n",
      "==> Saving model ...\n",
      "Epoch: 93/100... Step: 21100... Loss: 1.0548... Val Loss: 1.6640\n",
      "==> Saving model ...\n",
      "Epoch: 94/100... Step: 21200... Loss: 1.0286... Val Loss: 1.6706\n",
      "==> Saving model ...\n",
      "Epoch: 94/100... Step: 21300... Loss: 1.0322... Val Loss: 1.6730\n",
      "==> Saving model ...\n",
      "Epoch: 95/100... Step: 21400... Loss: 1.0569... Val Loss: 1.6564\n",
      "==> Saving model ...\n",
      "Epoch: 95/100... Step: 21500... Loss: 1.0222... Val Loss: 1.6722\n",
      "==> Saving model ...\n",
      "Epoch: 96/100... Step: 21600... Loss: 1.0308... Val Loss: 1.6561\n",
      "==> Saving model ...\n",
      "Epoch: 96/100... Step: 21700... Loss: 1.0258... Val Loss: 1.6650\n",
      "==> Saving model ...\n",
      "Epoch: 97/100... Step: 21800... Loss: 1.0896... Val Loss: 1.6595\n",
      "==> Saving model ...\n",
      "Epoch: 97/100... Step: 21900... Loss: 1.0781... Val Loss: 1.6679\n",
      "==> Saving model ...\n",
      "Epoch: 97/100... Step: 22000... Loss: 1.0562... Val Loss: 1.6716\n",
      "==> Saving model ...\n",
      "Epoch: 98/100... Step: 22100... Loss: 1.0334... Val Loss: 1.6712\n",
      "==> Saving model ...\n",
      "Epoch: 98/100... Step: 22200... Loss: 1.0295... Val Loss: 1.6717\n",
      "==> Saving model ...\n",
      "Epoch: 99/100... Step: 22300... Loss: 1.1214... Val Loss: 1.6659\n",
      "==> Saving model ...\n",
      "Epoch: 99/100... Step: 22400... Loss: 1.0562... Val Loss: 1.6607\n",
      "==> Saving model ...\n",
      "Epoch: 100/100... Step: 22500... Loss: 1.0303... Val Loss: 1.6519\n",
      "==> Saving model ...\n",
      "Epoch: 100/100... Step: 22600... Loss: 1.0342... Val Loss: 1.6575\n",
      "==> Saving model ...\n",
      "Epoch: 100/100... Step: 22700... Loss: 1.1012... Val Loss: 1.6646\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "OJWkbwyKHsc5"
   },
   "source": [
    "# Predicting next character"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    },
    "id": "0a1qYFEEHsc-"
   },
   "source": [
    "def predict(model, inputs, h=None, top_k= None):\n",
    "    x = np.array([[vocab_to_int[inputs]]])\n",
    "    x = one_hot_encode(x, len(vocab))\n",
    "    inp = torch.from_numpy(x).to(device)\n",
    "\n",
    "    # get access to hidden state\n",
    "    h = tuple([i.data for i in h])\n",
    "    # pass inputs and hidden state to model\n",
    "    out, h = model(inp, h)\n",
    "\n",
    "    # get prob of the char\n",
    "    p = F.softmax(out, dim=1).data\n",
    "    p = p.to('cpu')\n",
    "\n",
    "    if top_k is None:\n",
    "        top_ch = np.arange(len(vocab))\n",
    "    else:\n",
    "        p, top_ch = p.topk(top_k)\n",
    "        top_ch = top_ch.numpy().squeeze()\n",
    "\n",
    "    p = p.numpy().squeeze()\n",
    "    v = np.random.choice(top_ch, p=p/p.sum())\n",
    "\n",
    "    return int_to_vocab[v], h\n"
   ],
   "execution_count": 16,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "O_gxWBx9Hsc_"
   },
   "source": [
    "# Sample text"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    },
    "id": "exOuMPOxHsc_",
    "outputId": "24a08d6d-6873-4b44-847d-61a759d46986",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "source": [
    "def sample(model, size, prime='the ', top_k=None):\n",
    "\n",
    "    model.to(device)\n",
    "\n",
    "    model.eval() # eval mode\n",
    "\n",
    "    # First off, run through the prime characters\n",
    "    chars = [ch for ch in prime]\n",
    "    h = model.init_hidden(1)\n",
    "    for ch in prime:\n",
    "        char, h = predict(model, ch, h, top_k=top_k)\n",
    "\n",
    "    chars.append(char)\n",
    "\n",
    "    # Now pass in the previous character and get a new one\n",
    "    for ii in range(size):\n",
    "        char, h = predict(model, chars[-1], h, top_k=top_k)\n",
    "        chars.append(char)\n",
    "\n",
    "    return ''.join(chars)\n",
    "print(sample(model, 2000, prime='the ', top_k=26))"
   ],
   "execution_count": 17,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "the monster threw off she contented he ought to be fools on the window lid down the door oh not so good robbery dreading out of the house it came into the way and curling a fiddle and screamed to him the fish corn the forest all but it struck him into what s prince for and gone but when the sausages rose away in the meantime i am marleen and the bough straight him some wheel make carried and travelling his horse and amy to your golden he liked cut pearls it have he were rowend and asked him that they burnt as if the wide way of the died why did the horses about moying looking in the old cog why all his sister he ventures to her pain i never once more would save you along and looked down the poor world and the branches now she fell till there in the sausage en show carried back and she got on the ant take them before it shook so much by a cunning so batter at first as she dressed them up the sun who when he had eaten the servant who had way and how it was telling him crept down then he brought that the wild fabour very know the parson put his pitched and threw the grandmother that his white dove said she held rushed down her with her hands and looked into the saed looking we are come much horse for that pieces and the huntsmen are such a men to clurs it and forgot three hold and unseen your country blow it eating her beautiful bride on the day step of no doubt all with their dugn there were done that he wept and had passed that an hour liager when he looked at my master old man look heinel have however therefore will fall by wimb time little tailor and can i jed tell him it would have done you and if one did not dinnershelman the ass he had done and as he made all the house fast asleep on the letters and when i am too smoled there but whistled in the roof he stopped no passy and after it ashes sirst on the gallows s brother go over the friend and there were silent his did they brought and the youth went up and told him i can quick that once more only find her to forkeat a\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "AfLZRqjSwa03",
    "outputId": "82185bf2-2ce3-490f-df81-03c891d6b31e",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "source": [
    "!ls -la ."
   ],
   "execution_count": 18,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "total 24\n",
      "drwxr-xr-x 1 root root 4096 Apr  2 12:58 .\n",
      "drwxr-xr-x 1 root root 4096 Apr  2 12:30 ..\n",
      "drwxr-xr-x 2 root root 4096 Apr  2 12:58 checkpoint\n",
      "drwxr-xr-x 4 root root 4096 Mar 25 13:38 .config\n",
      "drwxr-x--- 3 root root 4096 Apr  2 12:58 .pytorch\n",
      "drwxr-xr-x 1 root root 4096 Mar 25 13:38 sample_data\n"
     ],
     "name": "stdout"
    }
   ]
  }
 ]
}