run_bilstm_crf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ktrain\n",
    "from ktrain import text as txt\n",
    "\n",
    "import os\n",
    "os.environ['DISABLE_V2_BEHAVIOR'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "TDATA = 'dataset/train.txt'\n",
    "VDATA = 'dataset/test.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "detected encoding: utf-8 (if wrong, set manually)\n",
      "Number of sentences:  6719\n",
      "Number of words in the dataset:  11441\n",
      "Tags: ['I-DLEG', 'B-DLEI', 'I-DEC', 'B-LEI', 'O', 'I-LEI', 'B-DEC', 'I-LCOMP', 'B-DLEG', 'I-EC', 'B-LCOMP', 'B-MP', 'I-DLEI', 'I-LDEL', 'I-MP', 'B-LDEL', 'B-EC']\n",
      "Number of Labels:  17\n",
      "Longest sentence: 1557 words\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "(trn, val, preproc) = txt.entities_from_txt('dataset/train.txt', 'dataset/test.txt',                   \n",
    "                                            use_char=True) # enable character embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ktrain/text/ner/models.py:108: UserWarning: Falling back to BiLSTM (no CRF) because DISABLE_V2_BEHAVIOR=False\n",
      "  warnings.warn('Falling back to BiLSTM (no CRF) because DISABLE_V2_BEHAVIOR=False')\n",
      "/opt/conda/lib/python3.7/site-packages/ktrain/text/ner/models.py:122: UserWarning: Setting use_char=False:  character embeddings cannot be used in TF2 due to open TensorFlow 2 bug (#33148).\n",
      "Add os.environ[\"DISABLE_V2_BEHAVIOR\"] = \"1\" to the top of script if you really want to use it.\n",
      "  warnings.warn('Setting use_char=False:  character embeddings cannot be used in TF2 due to open TensorFlow 2 bug (#33148).\\n' +\\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "IMPORTANT NOTE: ktrain uses the CRF module from keras_contrib, which is not yet\n",
      "fully compatible with TensorFlow 2. You can still use the BiLSTM-CRF model\n",
      "in ktrain for sequence tagging with TensorFlow 2, but you must add the\n",
      "following to the top of your script or notebook BEFORE you import ktrain:\n",
      "\n",
      "import os\n",
      "os.environ['DISABLE_V2_BEHAVIOR'] = '1'\n",
      "\n",
      "For this run, a vanilla BiLSTM model (with no CRF layer) will be used.\n",
      "\n",
      "embedding schemes employed:\n",
      "\tword embeddings initialized randomly\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# load model\n",
    "model = txt.sequence_tagger('bilstm-crf', preproc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wrap model and data in ktrain.Learner object\n",
    "learner = ktrain.get_learner(model, train_data=trn, val_data=val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preparing training data ...done.\n",
      "preparing validation data ...done.\n",
      "Epoch 1/5\n",
      "210/210 [==============================] - 84s 400ms/step - loss: 0.0342 - val_loss: 0.0018\n",
      "Epoch 2/5\n",
      "210/210 [==============================] - 102s 484ms/step - loss: 0.0010 - val_loss: 9.7614e-04\n",
      "Epoch 3/5\n",
      "210/210 [==============================] - 107s 509ms/step - loss: 4.9171e-04 - val_loss: 9.1732e-04\n",
      "Epoch 4/5\n",
      "210/210 [==============================] - 108s 517ms/step - loss: 2.6435e-04 - val_loss: 7.7684e-04\n",
      "Epoch 5/5\n",
      "210/210 [==============================] - 94s 447ms/step - loss: 1.8801e-04 - val_loss: 7.7093e-04\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f5ca1ad3d90>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# conventional training for 1 epoch using a learning rate of 0.001 (Keras default for Adam optmizer)\n",
    "learner.fit(0.01, 1, cycle_len=5, checkpoint_folder='/tmp/saved_weights')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   F1:  99.70\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         DEC       1.00      1.00      1.00       776\n",
      "        DLEG       1.00      1.00      1.00         8\n",
      "        DLEI       0.99      1.00      1.00       102\n",
      "          EC       1.00      1.00      1.00         7\n",
      "       LCOMP       0.95      0.97      0.96        39\n",
      "        LDEL       0.00      0.00      0.00         1\n",
      "         LEI       1.00      1.00      1.00      1223\n",
      "          MP       0.98      1.00      0.99        40\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      2196\n",
      "   macro avg       0.86      0.87      0.87      2196\n",
      "weighted avg       1.00      1.00      1.00      2196\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9970447829052058"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.validate(class_names=preproc.get_classes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqTklEQVR4nO3deXiV1b328e8vM0NCAkmYZ1BElClAQFEcWkGtWCcEBERkcKitbY/V0/f0dHjb12prnSoKggMKiDPHKmr1KCoECMo8BmQIYwgQAgESyHr/2A9tjElIIDt7uj/Xta+99/OstfdvMeTOMy5zziEiIlJdUYEuQEREQouCQ0REakTBISIiNaLgEBGRGlFwiIhIjSg4RESkRhQcInXIzNqZmTOzmFr6vC1mdmVttxWpioJDIkplPzzNbJCZlZrZYTMrNLP1Zjb2NJ81zszWee33mNk/zCzRf9WLBIda+a1HJEzsdM61MjMDhgBzzWyBc259+YZmdinwJ2Cwc+4bM2sM/KiO6xUJCG1xiJTjfN4H9gMXVtKsD7DQOfeN12e/c+4l51whgJnVM7O/mtlWMyswsy/NrF6Z/iPNbJuZ7TOzX59aaGZRZvagmW0ys3wzm+OF0qn1o7zPzC/bz1v3opn93zLvB5lZbkXFn+57RKqi4BApx/uheh2QCuRU0mwRcJWZ/c7MLjKz+HLr/wL0BgYAjYEHgNIy6y8GzgWuAH5jZud5y+8DrgcuBVoAB4C/e3V1BSYDo7x1TYBWZzjMSr9H5HQUHCL/1sLMDgJHgbeBn5/aoijPOfcFcAPQC/gHkG9mj5lZtJlFAXcAP3XO7XDOnXTOLXDOHS/zEb9zzh11zi0HlgPdveUTgV8753K99r8FbvIOpt8EvOecm++t+y++G0Y1UdX3iFRJ/0hE/u3UMY544GHgcuDxyho75z4APvCC4jLgdWA9vtBJADZV8V27y7wuAhp6r9sCb5tZ2UA4CTTFt2Wwvcz3HzGz/OoN7Xuq+p4dZ/iZEiG0xSFSjvcb+K+AC8zs+mq0L3XOfQJ8CnQD9gHHgI5n8PXbgSHOueQyjwTn3A5gF9D6VEMzq49vd9UpR4D6Zd43O8PvEamSgkMiUayZJZR5fG/L2zlXDPwV+E1FH2BmQ83sVjNLMZ+++I4XZDnnSoHpwGNm1sLbfdW/guMgFXkW+KOZtfW+J83Mhnrr3gCuNbOLzSwO+D3f/T+8DLjazBqbWTPgZ2f4PSJVUnBIJHof33GMU4/fVtJuOtDGzCo6zfYAMB7YCBwCXgEedc696q3/JbASWILv7Kw/U73/b08Ac4GPzKwQyAL6ATjnVgP3ADPxbX0cAMqeNTUD3/GSLcBHwGtn8j0ip2OayElERGpCWxwiIlIjCg4REakRBYeIiNSIgkNERGokIi4ATE1Nde3atQt0GSIiIWXp0qX7nHNp5ZdHRHC0a9eO7OzsQJchIhJSzGxrRcu1q0pERGpEwSEiIjWi4BARkRpRcIiISI0oOEREpEb8GhxmNtjM1ptZjpk9WMF6M7MnvfUrzKxXmXXTzWyvma0q16exmX1sZhu95xR/jkFERL7Lb8FhZtH4pqIcAnQFhntTX5Y1BOjsPSbgmxbzlBeBwRV89IPAJ865zsAn3nsREakj/ryOoy+Q45zbDGBms4GhwJoybYYCLzvfLXqzzCzZzJo753Y55+abWbsKPncoMMh7/RLwGb5Jd2rdJ2v3sCK3gLiYKOKio4iNNmJjomgYH0NK/ThS6seRXD+WlAZxNIyPiEtiRET8GhwtKTPNJb55A8rf77+iNi3xzTVQmabOuV0AzrldZpZeUSMzm4BvK4Y2bdrUrHLPZ+vzmJFV4fUv35OUEEPLlPq0TK5Hq5R6tGlcn3ObJdK5aUPSGsZjZmdUg4hIsPFncFT0k7L85B/VaXNGnHNTgCkAGRkZZ/SZf7i+G78fej4lJx3FJ0spOVFK8clSCo+d4GBRMQeKSjhYVEz+kWJ2HjzKjgNHyT1QRNbmfA4fP/Gvz0muH8s5TRO5oGUjerZJpmebFFo0SlCYiEhI8mdw5FJmfmSgFbDzDNqUt+fU7iwzaw7sPetKq2BmxMUYcTFR4E382TSp6j7OOfYdLmbjnkLW7ylkw57DrN99iFeytjLty28BSE+Mp1ebFAZ0asKAjql0TGugIBGRkODP4FgCdDaz9sAO4FZgRLk2c4F7veMf/YCCU7uhqjAXGAM87D2/W6tV1wIzIy0xnrTEeAZ0Sv3X8pKTpazbVcg32w+wbNtBFm/Zz7zVuwFolpTARZ1SueScVC7rkk5SQmygyhcRqZJfp441s6uBx4FoYLpz7o9mNgnAOfes+X7Ffhrf2VNFwFjnXLbXdxa+g+CpwB7gv51z08ysCTAHaANsA252zu2vqo6MjAwXrDc53JZfxJc5+/gqZx9fbdrHwaISYqON/h1T+WHXpvywa1PSkxICXaaIRCAzW+qcy/je8kiYczyYg6Osk6WOZdsP8OHqPXy4ejdb84sA6NMuhet7tuTaC1rQqL62RESkbig4QiA4ynLOsXHvYT5ctZt3l+8kZ+9h4qKjuLxLOtf3bMnlXdJ9x11ERPxEwRFiwVGWc47VOw/x1tc7mLt8J/sOH6dJgzhu6dOaEX3b0Lpx/UCXKCJhSMERwsFR1omTpXyRs4+Zi7bxydo9OGDQOWncltmWQeemEx2lM7NEpHYoOMIkOMraefAosxZvY/aS7eQVHqddk/qMG9iBm3u3IiE2OtDliUiIU3CEYXCcUnKylHmrdvP8F5tZnltA4wZxjO7fltH929G4QVygyxOREKXgCOPgOMU5x6Jv9zNl/mY+XbeXhNgoRvRty6RBHUhP1Cm9IlIzlQWH7swXRsyMzA5NyOzQhA17Cnnu8828tHALMxdv5bZ+bZl4aUfSEuMDXaaIhDhtcYS5LfuO8OSnG3nnmx3ExUQxun87Jl7SgSYNFSAiUjXtqorQ4Dhlc95hnvo0h3eX7aB+XAyTLu3AuIs7UC9OB9FFpGIKjggPjlNy9h7mkXnr+GjNHpolJfDzH57Djb1a6TReEfmeyoJDlx5HmE7pDZkyOoM5E/vTtFECD7yxgmue/IL5G/ICXZqIhAgFR4Tq274x79w9gKdH9KSo+CSjpy9m4oxscg8UBbo0EQlyCo4IZmZce2ELPv75JTww+Fzmb9jHlY99zlOfbORYyclAlyciQUrBIcTHRHP3oE788xeXcnmXdP768Qauenw+/7vOr3NkiUiIUnDIv7RMrsczI3szY1xfoqOMsS8uYdKMpew9dCzQpYlIEFFwyPcM7JzGvJ9ewn9cdS6frt/LlY99zmtLthEJZ+CJyOkpOKRCcTFR3HNZJ+b9dCBdmifxqzdXMvL5RWzNPxLo0kQkwBQcUqUOaQ2ZPT6TP/64GytzC7jq8flMmb+Jk6Xa+hCJVAoOOa2oKGNkv7Z8/PNLGdg5jT+9v45hzy1kW75O3RWJRAoOqbZmjRKYMqo3j93SnfW7Cxn8xHxmLdaxD5FIo+CQGjEzbujVinn3X0KP1sk89NZK7nhxic68EokgCg45Iy2T6/HKuH7894+6smBTPj98fD4frNwV6LJEpA4oOOSMRUUZYy9qzz/uG0ibxvW569Wv+c+3V+qqc5Ewp+CQs9YpvSFv3jWAiZd2YOaibQx9+is27CkMdFki4icKDqkVsdFRPDTkPF66oy/5R45z3dNfMlsHzkXCkoJDatWl56Tx/k8HktG2MQ++tZJ7Z33DoWMlgS5LRGqRgkNqXXpiAi/f0ZcHBp/LvFW7ufbJL1m9syDQZYlILVFwiF9ERRl3D+rEnIn9KT5Ryg3PLOCNpbmBLktEaoGCQ/yqd9sU3rvvYnq3TeGXry/nobdWcvyEzroSCWUKDvG71IbxvHxHX+4a1JFZi7dx87MLNdOgSAhTcEidiImO4leDuzBlVG++zTvCtU99qXnORUKUX4PDzAab2XozyzGzBytYb2b2pLd+hZn1Ol1fM+thZllmtszMss2srz/HILXrh+c3Y+5PLqZZUgJjXljM5M826ZRdkRDjt+Aws2jg78AQoCsw3My6lms2BOjsPSYAk6vR9xHgd865HsBvvPcSQtqnNuCtuwdwzQXN+fO8dfzstWW62lwkhPhzi6MvkOOc2+ycKwZmA0PLtRkKvOx8soBkM2t+mr4OSPJeNwJ2+nEM4if142J4anhP/uOqc3l32U5ueW4huwt0o0SRUODP4GgJbC/zPtdbVp02VfX9GfComW0H/gI8VNGXm9kEb1dWdl6e9qUHIzPjnss6MWVUbzbtPcx1T3/JN9sOBLosETkNfwaHVbCs/M7sytpU1fcu4H7nXGvgfmBaRV/unJvinMtwzmWkpaVVs2QJhB+e34y37r6I+Ngohk3J4k1d7yES1PwZHLlA6zLvW/H93UqVtamq7xjgLe/16/h2a0mIO7dZInPvuZjebVL4xevL+dP7azU9rUiQ8mdwLAE6m1l7M4sDbgXmlmszFxjtnV2VCRQ453adpu9O4FLv9eXARj+OQepQSoM4Xh7Xl1GZbZkyfzN3v7qUo8U6aC4SbGL89cHOuRNmdi/wIRANTHfOrTazSd76Z4H3gauBHKAIGFtVX++jxwNPmFkMcAzf2VgSJmKjo/jD9d3okNaA37+3hlunLGTqmAzSExMCXZqIeCwSzqHPyMhw2dnZgS5DaujjNXu4b9Y3NG4Qxwtj+3BO08RAlyQSUcxsqXMuo/xyXTkuQesHXZv6bpJ4spQbn1nAlxv3BbokEUHBIUHuglaNeOeei2iZUo/bX1jMa0u2BbokkYin4JCg1zK5Hq9P6k//jk341ZsreWTeOkp1xpVIwCg4JCQkJsQy/fY+DO/bhmc+28QvXl9OycnSQJclEpH8dlaVSG2LjY7iTz/uRsvkBP7y0QbyjxQzeWQvGsTrn7FIXdIWh4QUM+PeyzvzyI0X8lXOPoZPzWLf4eOBLkskoig4JCTd0qc1U0b1ZsOeQm6avIBt+ZoYSqSuKDgkZF1xXlNevTOTg0dLuGHyAlbtKAh0SSIRQcEhIa132xTemNSf+Jgohj23UNd6iNQBBYeEvE7pibx51wBapdRn7IuLmbtcU7SI+JOCQ8JCs0YJzJnUn55tUvjp7G94JWtroEsSCVsKDgkbjerF8vIdfbns3HT+zzureOaznECXJBKWFBwSVhJio3luVG+u696CR+at5+EP1hEJN/IUqUu6ckrCTmx0FH8b1oPEhBie/XwThcdK+P3QbkRHVTSxpIjUlIJDwlJ0lPF/r+9GUr1YJn+2icJjJ/jrLd2JjdZGtsjZUnBI2DIzfjW4C0kJsfx53joOHz/BMyN7kRAbHejSREKafv2SsHfXoI788cfd+N/1exk9fTGFx0oCXZJISFNwSEQY2a8tjw/rwddbDzBi6iIOHCkOdEkiIUvBIRFjaI+WTBndm/V7CnVzRJGzoOCQiHJ5l6a8cHsftuQfYdhzC9lz6FigSxIJOQoOiTgXdUrlpbF92V1wjGHPLWTHwaOBLkkkpCg4JCL169CEGXf2I/9IMcOeW8j2/botu0h1KTgkYvVqk8LMOzMpPHaCW55byOa8w4EuSSQkKDgkol3QqhGzxmdSfKKUYVOy2LinMNAliQQ9BYdEvK4tkpg9IROAYVOyWLPzUIArEgluCg4RoHPTROZM9E0INXxqFityDwa6JJGgpeAQ8bRPbcCcif1JTIhh5NRFLN16INAliQQlBYdIGa0b12fOxP40aRjHmOmLWbp1f6BLEgk6Cg6Rclok12P2hP6kJcYzeprCQ6Q8BYdIBZo1SmDW+EzSkxIUHiLl+DU4zGywma03sxwze7CC9WZmT3rrV5hZr+r0NbOfeOtWm9kj/hyDRK5mjRKYPeHf4ZG9ReEhAn4MDjOLBv4ODAG6AsPNrGu5ZkOAzt5jAjD5dH3N7DJgKHChc+584C/+GoNI0yRfeDRNSmDM9MUsUXiI+HWLoy+Q45zb7JwrBmbj+4Ff1lDgZeeTBSSbWfPT9L0LeNg5dxzAObfXj2MQoWlSArO88Lhd4SHi1+BoCWwv8z7XW1adNlX1PQcYaGaLzOxzM+tT0Zeb2QQzyzaz7Ly8vLMYhsh3w0NbHhLp/BkcVsEyV802VfWNAVKATOA/gDlm9r32zrkpzrkM51xGWlpa9asWqcSp3VbNFB4S4fwZHLlA6zLvWwE7q9mmqr65wFve7q3FQCmQWot1i1Qq/VR4NPKFx+JvFR4SefwZHEuAzmbW3szigFuBueXazAVGe2dXZQIFzrldp+n7DnA5gJmdA8QB+/w4DpHvSE9KYPZ4X3jc/sJiFm3OD3RJInXKb8HhnDsB3At8CKwF5jjnVpvZJDOb5DV7H9gM5ABTgbur6uv1mQ50MLNV+A6aj3HOld8FJuJXp8KjeaMExr64ROEhEcUi4WduRkaGy87ODnQZEob2HjrG8KlZ7Co4xotj+9K3feNAlyRSa8xsqXMuo/xyXTkuchbSvbOtTu220kWCEgkUHCJnKT3Rd3uSpkkJ3P7CEr7eprvqSnhTcIjUgqZJvvBo0jCOMdMWs3z7wUCXJOI3Cg6RWnLqxojJDWIZNW0Rq3YUBLokEb9QcIjUohbJ9Zg1PpPEhFhGPr+I1TsVHhJ+FBwitaxVSn1mjc+kQVw0tz2/iHW7NYe5hBcFh4gftGlSn5njM4mPiWbk1EVs2FMY6JJEao2CQ8RP2qU2YOb4fkRHGSOmZpGzV+Eh4UHBIeJHHdIaMnN8JmAMn7qITXmHA12SyFlTcIj4Waf0hswc34/SUseIqVls2Xck0CWJnBUFh0gdOKdpIq+O70fxiVKGT81iW35RoEsSOWMKDpE60qVZEq/emcnRkpMMn5rF9v0KDwlNCg6ROtS1RRKvjOtH4bESRjyfxY6DRwNdkkiNnTY4zCzKu4W5iNSCbi0b8cqd/ThYVMKIqVnsKlB4SGg5bXA450qB5WbWpg7qEYkIF7ZK5uU7+pJ/uJgRUxex59CxQJckUm3V3VXVHFhtZp+Y2dxTD38WJhLuerZJ4aU7+vxrTo+9hQoPCQ3VmsjJzC6taLlz7vNar8gPNJGTBLMlW/YzZvpiWiTXY/aETFIbxge6JBHgLCdycs59XtGj9ssUiTx92jVm+u19yD1QxMipi9h/pDjQJYlUqcrgMLNCMztUwaPQzHTnNpFaktmhCdPG9GFL/hFGPr+IAwoPCWJVBodzLtE5l1TBI9E5l1RXRYpEgos6pTJ1dAab8g5z27RFFBSVBLokkQrpOg6RIHLJOWk8N6o3G/ccZtT0RRQcVXhI8FFwiASZy85NZ/JtvVi76xCjpyk8JPgoOESC0BXnNWXyyN6s2XWI0dMXc+iYwkOCh4JDJEhd2bUpz4zszZqdBYyetphChYcECQWHSBD7QdemPD2iF6t2FDBmusJDgoOCQyTIXXV+M54e0YsVuQXc/sISDh8/EeiSJMIpOERCwOBuzXhqeE+WbT/I7dMXKzwkoBQcIiFiyAXNeWp4T77ZfpCxLyzmiMJDAkTBIRJCrr6gOU/e2pOvtx1k7AtLFB4SEAoOkRBzzYXNeXxYD7K37ueOF5dQVKzwkLql4BAJQT/q3oK/DevBki0KD6l7fg0OMxtsZuvNLMfMHqxgvZnZk976FWbWqwZ9f2lmzsxS/TkGkWA1tEdL/jasB4u/3c+4F7M5Wnwy0CVJhPBbcJhZNPB3YAjQFRhuZl3LNRsCdPYeE4DJ1elrZq2BHwDb/FW/SCgY2qMlj93Sg0Xf5jPupSUKD6kT/tzi6AvkOOc2O+eKgdnA0HJthgIvO58sINnMmlej79+AB4DTz0IlEuau79mSv9zcnYWb8xn/cjbHShQe4l/+DI6WwPYy73O9ZdVpU2lfM7sO2OGcW17Vl5vZBDPLNrPsvLy8MxuBSIi4oVcrHr2pO19t2qfwEL/zZ3BYBcvKbyFU1qbC5WZWH/g18JvTfblzbopzLsM5l5GWlnbaYkVC3U29W/HIjRfyZc4+JsxYqvAQv/FncOQCrcu8bwXsrGabypZ3BNoDy81si7f8azNrVquVi4SomzNa8+cbL+SLjXna8hC/8WdwLAE6m1l7M4sDbgXmlmszFxjtnV2VCRQ453ZV1tc5t9I5l+6ca+eca4cvYHo553b7cRwiIeUWLzy+zNmnU3XFL/wWHM65E8C9wIfAWmCOc261mU0ys0les/eBzUAOMBW4u6q+/qpVJNzcktGax27pTtbmfG6frhsjSu0y58L/xKSMjAyXnZ0d6DJE6tzc5Tu5/7Vl9GidzItj+5CYEBvokiSEmNlS51xG+eW6clwkjF3XvQVPD+/J8u0HuW3aYk1DK7VCwSES5oZc0JzJt/lmEhz5fBYHi4oDXZKEOAWHSAT4QdemTBmVwYY9hxk+dRH5h48HuiQJYQoOkQhxWZd0po3JYHPeYYZPzSKvUOEhZ0bBIRJBBnZO44Wxfdi+/yi3TlnInkPHAl2ShCAFh0iEGdAxlZfu6MvugmMMe24hOw8eDXRJEmIUHCIRqG/7xrw8rh/5h4sZNmUh2/cXBbokCSEKDpEI1bttCq/c2Y+CohJufnYhOXsPB7okCREKDpEI1r11Mq9N7M+JUsew5xayakdBoEuSEKDgEIlw5zVP4vVJ/UmIjWb41Cyyt+wPdEkS5BQcIkL71AbMmdSf1IbxjJq2mC82ag4bqZyCQ0QAaJlcjzkT+9O2SX3GvZjNvFW66bRUTMEhIv+SlhjPaxP6c37LJO6Z+TVvf5Mb6JIkCCk4ROQ7GtWP5ZVx/ejXvjH3v7acGQu3BLokCTIKDhH5ngbxMUy/vQ9XnteU/3p3Nc98lhPokiSIKDhEpEIJsdFMvq0XQ3u04JF56/nT+2spLQ3/+Xvk9GICXYCIBK/Y6Cj+dksPGtWLZcr8zewrPM6fb7qQ2Gj9zhnJFBwiUqWoKON3151PWsN4/vrxBvYXFfPMyF7Uj9OPj0ilXxtE5LTMjJ9c0Zn/d8MFzN+Qx4ipi9h/RBNCRSoFh4hU2/C+bZh8W2/W7jrETc8uIPeAbo4YiRQcIlIjV53fjBnj+pFXeJwbJy9g/e7CQJckdUzBISI11rd9Y16f1B+Am59dwBLd3yqiKDhE5Ix0aZbEm3cNIDUxntueX6RblEQQBYeInLFWKfV5Y9IAzmuexF2vLmXal9/inK71CHcKDhE5K40bxDF7QiZXdW3GH95bw2/nruakLhQMawoOETlrCbHRPDOyF+MHtuelhVuZOCObouITgS5L/ETBISK1IirK+PU1XfnD0PP5dN1ehj2Xxd5DxwJdlviBgkNEatWo/u14fkwGm/IO8+NndLpuOFJwiEitu7xLU+ZM7E/JyVJumrxAMwqGGQWHiPhFt5aNeOeei2iZUo/bX1jCi1/pjKtw4dfgMLPBZrbezHLM7MEK1puZPemtX2FmvU7X18weNbN1Xvu3zSzZn2MQkTPXIrkeb9w1gMvOTee3/7OG/3x7JcUnSgNdlpwlvwWHmUUDfweGAF2B4WbWtVyzIUBn7zEBmFyNvh8D3ZxzFwIbgIf8NQYROXsN42OYMqo3dw/qyKzF27lt2iLyDx8PdFlyFvy5xdEXyHHObXbOFQOzgaHl2gwFXnY+WUCymTWvqq9z7iPn3Knz/LKAVn4cg4jUgqgo44HBXXji1h4s236QoX//inW7DwW6LDlD/gyOlsD2Mu9zvWXVaVOdvgB3AB9U9OVmNsHMss0sOy9PB+ZEgsHQHi15fWJ/ik+UcuMzC/hotW5TEor8GRxWwbLyR8Yqa3Pavmb2a+AE8GpFX+6cm+Kcy3DOZaSlpVWjXBGpC91bJ/M/P7mYTukNmTBjKU99slFT0oYYfwZHLtC6zPtWwM5qtqmyr5mNAa4FRjqdpiEScpomJfDaxP5c36MFf/14AxNmZFNwtCTQZUk1+TM4lgCdzay9mcUBtwJzy7WZC4z2zq7KBAqcc7uq6mtmg4FfAdc55zSLjEiISoiN5m/DevC7687ns/V5XPf0l6zZqeMeocBvweEdwL4X+BBYC8xxzq02s0lmNslr9j6wGcgBpgJ3V9XX6/M0kAh8bGbLzOxZf41BRPzLzBgzoB2vTczkWMlJbpj8FW99nRvosuQ0LBL29GRkZLjs7OxAlyEiVcgrPM69M79m0bf7GZXZlv+6titxMbpGOZDMbKlzLqP8cv2tiEhQSEuM59U7+zHhkg7MyNrKLc8tZMfBo4EuSyqg4BCRoBETHcV/Xn0ez4zsxcY9hVz9xBd8qFN2g46CQ0SCztUXNOcf9w2kdeN6TJyxlN/OXc2xkpOBLks8Cg4RCUrtUhvw5l0DuOOi9ry4YAs3PLOAzXmHA12WoOAQkSAWHxPNb37UledHZ7Cz4CjXPvWlzroKAgoOEQl6V3Ztygc/HUi3Fo34+Zzl3DfrGwqKdMFgoCg4RCQkNG9Uj5nj+/HzH5zD+yt3cdXj8zVBVIAoOEQkZMRER3HfFZ156+4BNEyIYdS0xfzm3VUUFZ84fWepNQoOEQk5F7ZK5r2fXMwdF7Xn5YVbuebJL/lm24FAlxUxFBwiEpISYn0HzmeO78fxkpPcOHkBf563Tqft1gEFh4iEtAEdU5l3/yXc2KsVkz/bxJAnviBrc36gywprCg4RCXlJCbE8enN3XhnXjxOlpdw6JYuH3lrJoWM688ofFBwiEjYu7pzKhz+7hPED2/Pakm1c+dfPdcsSP1BwiEhYqR8Xw6+v6co791xE4wZxTJyxlDtfWsK2fE3fU1sUHCISli5s5Zui9qEhXViwKZ8r//Y5j328QQfPa4GCQ0TCVmx0FBMv7cinvxjE4POb8eQnG7nyMd/uq0iYi8hfFBwiEvaaNUrgyeE9mT0hkwZxMUycsZTR0xdrqtozpOAQkYiR2aEJ7913Mb+5tisrcgu45qkv+MWc5ezUhFE1oqljRSQiFRSV8MxnObywYAsG3HFxe+4a1JGkhNhAlxY0Kps6VsEhIhEt90ARj320gbeX7SC5XiwTL+3IqMy2NIiPCXRpAafgUHCISBVW7Sjg0Q/X8/mGPBo3iGP8wA6M7h/ZAaLgUHCISDV8ve0AT/xzI59vyCOlfiwTLunIqP5taRiBAaLgUHCISA2UDZDEhBhG9GvD2AHtadYoIdCl1RkFh4JDRM7Asu0HmTp/Mx+s2kWUGdd1b8GdAzvQtUVSoEvzOwWHgkNEzsL2/UVM/+pbXluynaLik/Tv0IQR/dpw1fnNiIsJzysbFBwKDhGpBQVFJcxcvI1XF20l98BRmjSI46aMVgzv04Z2qQ0CXV6tUnAoOESkFpWWOuZvzGPmom18sm4vJ0sdmR0ac32Plgzp1pxG9UP/ehAFh4JDRPxkz6FjvLZkO29/s4Nv9x0hLjqKQeemMbRHS644L52E2OhAl3hGFBwKDhHxM+ccK3cU8O6ynfzP8p3sLTxOQmwUAzun8YPzmnJZl3TSEuMDXWa1KTgUHCJSh06WOrI25/PR6t38c+1edhw8ihn0bJ3MpeekM6BTE7q3Sg7qA+sKDgWHiASIc461uwr559o9/HPtHlbuKMA5qBcbTUa7FPp3bELP1ilc0KpRUF1oGJDgMLPBwBNANPC8c+7hcuvNW381UATc7pz7uqq+ZtYYeA1oB2wBbnHOHaiqDgWHiASTg0XFLPp2Pws35bNg0z427DkMgBl0SmtI99bJdGuRRKf0RDqlN6RpUjy+H5d1q86Dw8yigQ3AD4BcYAkw3Dm3pkybq4Gf4AuOfsATzrl+VfU1s0eA/c65h83sQSDFOferqmpRcIhIMNt/pJgVuQdZvr3A95x7kH2Hi/+1PjE+hg7pDWmdUo8WyfVo3iiB5o3q0axRAsn1YmlUL5bEhBhiomt3t1dlweHPbaK+QI5zbrNXwGxgKLCmTJuhwMvOl15ZZpZsZs3xbU1U1ncoMMjr/xLwGVBlcIiIBLPGDeIYdG46g85NB3y7tvIOHydn7+HvPFbuKOCjNXsoPlFa4ec0jI+hflw0sdFRxEQbsdFR/OnHF9C3feNardefwdES2F7mfS6+rYrTtWl5mr5NnXO7AJxzu8wsvaIvN7MJwASANm3anOEQRETqnpmRnphAemICAzqmfmedc478I8XsOniMPYeOcehYCQVHSzh09ASHjpVQVHyCkpOOkpOlnDjpaBBf+6cC+zM4KtohV36/WGVtqtO3Ss65KcAU8O2qqklfEZFgZWakNowntWE8F9AoIDX48zywXKB1mfetgJ3VbFNV3z3e7iy85721WLOIiJyGP4NjCdDZzNqbWRxwKzC3XJu5wGjzyQQKvN1QVfWdC4zxXo8B3vXjGEREpBy/7apyzp0ws3uBD/GdUjvdObfazCZ5658F3sd3RlUOvtNxx1bV1/voh4E5ZjYO2Abc7K8xiIjI9+kCQBERqVBlp+MG77XuIiISlBQcIiJSIwoOERGpEQWHiIjUSEQcHDezPGDrGXZPBfbVYjnBSuMMLxpneAnUONs659LKL4yI4DgbZpZd0VkF4UbjDC8aZ3gJtnFqV5WIiNSIgkNERGpEwXF6UwJdQB3ROMOLxhlegmqcOsYhIiI1oi0OERGpEQWHiIjUiIKjCmY22MzWm1mON795SDKz1mb2v2a21sxWm9lPveWNzexjM9voPaeU6fOQN+71ZnZV4KqvOTOLNrNvzOw9733YjdObZvkNM1vn/b32D9Nx3u/9m11lZrPMLCFcxmlm081sr5mtKrOsxmMzs95mttJb96SZVTQRXu1yzulRwQPf7dw3AR2AOGA50DXQdZ3hWJoDvbzXicAGoCvwCPCgt/xB4M/e667eeOOB9t6fQ3Sgx1GD8f4cmAm8570Pu3ECLwF3eq/jgORwGye+KaS/Bep57+cAt4fLOIFLgF7AqjLLajw2YDHQH9/MqR8AQ/xdu7Y4KtcXyHHObXbOFQOzgaEBrumMOOd2Oee+9l4XAmvx/acciu8HEN7z9d7rocBs59xx59y3+OZL6VunRZ8hM2sFXAM8X2ZxWI3TzJLw/dCZBuCcK3bOHSTMxumJAeqZWQxQH99MoGExTufcfGB/ucU1Gps3C2qSc26h86XIy2X6+I2Co3Itge1l3ud6y0KambUDegKLgKbON+Mi3nO61yyUx/448ABQWmZZuI2zA5AHvODtknvezBoQZuN0zu0A/oJvwrZd+GYI/YgwG2c5NR1bS+91+eV+peCoXEX7CUP63GUzawi8CfzMOXeoqqYVLAv6sZvZtcBe59zS6napYFnQjxPfb+G9gMnOuZ7AEXy7NSoTkuP09u8PxbdrpgXQwMxuq6pLBcuCfpzVVNnYAjJmBUflcoHWZd63wreZHJLMLBZfaLzqnHvLW7zH29TFe97rLQ/VsV8EXGdmW/DtWrzczF4h/MaZC+Q65xZ579/AFyThNs4rgW+dc3nOuRLgLWAA4TfOsmo6tlzvdfnlfqXgqNwSoLOZtTezOOBWYG6Aazoj3lkW04C1zrnHyqyaC4zxXo8B3i2z/FYzizez9kBnfAfggppz7iHnXCvnXDt8f1+fOuduI/zGuRvYbmbneouuANYQZuPEt4sq08zqe/+Gr8B3fC7cxllWjcbm7c4qNLNM789odJk+/hPoMwuC+QFcje8MpE3ArwNdz1mM42J8m68rgGXe42qgCfAJsNF7blymz6+9ca+nDs7S8MOYB/Hvs6rCbpxADyDb+zt9B0gJ03H+DlgHrAJm4DurKCzGCczCd+ymBN+Ww7gzGRuQ4f35bAKexrsjiD8fuuWIiIjUiHZViYhIjSg4RESkRhQcIiJSIwoOERGpEQWHiIjUiIJDpBrMbIH33M7MRtTyZ/9nRd8lEqx0Oq5IDZjZIOCXzrlra9An2jl3sor1h51zDWuhPJE6oS0OkWows8Pey4eBgWa2zJsrItrMHjWzJWa2wswmeu0HmW8OlJnASm/ZO2a21JtfYoK37GF8d39dZmavlv0u83nUm4tipZkNK/PZn9m/5+N49dQcDGb2sJmt8Wr5S13+GUnkiAl0ASIh5kHKbHF4AVDgnOtjZvHAV2b2kde2L9DN+W6DDXCHc26/mdUDlpjZm865B83sXudcjwq+6wZ8V4h3B1K9PvO9dT2B8/Hdl+gr4CIzWwP8GOjinHNmlly7Qxfx0RaHyNn5ITDazJbhu1V9E3z3EQLfvYS+LdP2PjNbDmThu2FdZ6p2MTDLOXfSObcH+BzoU+azc51zpfhuIdMOOAQcA543sxuAorMcm0iFFBwiZ8eAnzjneniP9s43ZwT4bnfua+Q7NnIl0N851x34BkioxmdX5niZ1yeBGOfcCXxbOW/im8xnXg3GIVJtCg6RminEN/3uKR8Cd3m3rcfMzvEmVSqvEXDAOVdkZl2AzDLrSk71L2c+MMw7jpKGb9a/Su/26s230sg59z7wM3y7uURqnY5xiNTMCuCEt8vpReAJfLuJvvYOUOdR8dSd84BJZrYC391Ns8qsmwKsMLOvnXMjyyx/G99c0svx3d34Aefcbi94KpIIvGtmCfi2Vu4/oxGKnIZOxxURkRrRrioREakRBYeIiNSIgkNERGpEwSEiIjWi4BARkRpRcIiISI0oOEREpEb+PyuYZ+VRwB9CAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.plot('lr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "name": "tf2-2-3-gpu.2-3.m59",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-2-3-gpu.2-3:m59"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}