Deep_Learning_Project_87604_Tung_Son_Tran.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Deep Learning Project - 87604 - Tung Son Tran.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyOyoESEAMcPE3UtEusw+fHt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JohanObluda/DeepLearningProject/blob/master/Deep_Learning_Project_87604_Tung_Son_Tran.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g_AZ31iBKgIv",
        "colab_type": "text"
      },
      "source": [
        "**Link to Google Drive**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_lPd7xx3YSI",
        "colab_type": "code",
        "outputId": "1098ffbe-b234-4e4e-fce8-75c061f6511c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        }
      },
      "source": [
        "!pip install --upgrade pip\n",
        "!pip install -U -q pydrive\n",
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pip\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/54/0c/d01aa759fdc501a58f431eb594a17495f15b88da142ce14b5845662c13f3/pip-20.0.2-py2.py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 3.5MB/s \n",
            "\u001b[?25hInstalling collected packages: pip\n",
            "  Found existing installation: pip 19.3.1\n",
            "    Uninstalling pip-19.3.1:\n",
            "      Successfully uninstalled pip-19.3.1\n",
            "Successfully installed pip-20.0.2\n",
            "E: Package 'python-software-properties' has no installation candidate\n",
            "Selecting previously unselected package google-drive-ocamlfuse.\n",
            "(Reading database ... 134443 files and directories currently installed.)\n",
            "Preparing to unpack .../google-drive-ocamlfuse_0.7.17-0ubuntu2~ubuntu18.04.1_amd64.deb ...\n",
            "Unpacking google-drive-ocamlfuse (0.7.17-0ubuntu2~ubuntu18.04.1) ...\n",
            "Setting up google-drive-ocamlfuse (0.7.17-0ubuntu2~ubuntu18.04.1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gvt-s5DNfrpj",
        "colab_type": "code",
        "outputId": "a2a5c436-78bc-4320-92de-63b3e9f56740",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 214
        }
      },
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "# Generate creds for the Drive FUSE library.\n",
        "from oauth2client.client import GoogleCredentials\n",
        "creds = GoogleCredentials.get_application_default()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1v-s00KZ-hai",
        "colab_type": "code",
        "outputId": "f81ff797-e636-405e-e8bb-a8057002e0ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "import getpass\n",
        "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
        "vcode = getpass.getpass()\n",
        "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "··········\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "Please enter the verification code: Access token retrieved correctly.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NEtjC1GcftYZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir -p drive\n",
        "!google-drive-ocamlfuse drive"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LLmp4X52e0rO",
        "colab_type": "text"
      },
      "source": [
        "**Mount drive**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JxNSZBFke4e-",
        "colab_type": "code",
        "outputId": "8cbdce73-7c37-4d79-fb2e-9d88cac38266",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\", force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YBkLvzrrmAfV",
        "colab_type": "text"
      },
      "source": [
        "**Environment path**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lJNrYz1iOC_d",
        "colab_type": "text"
      },
      "source": [
        "If you want to test on your own drive, please set up your own environment, otherwise please arrange it as follow:\n",
        " \n",
        "0.   **Root** folder: contains notebook file\n",
        "  1.   **Checkpoint** folder: contains checkpoint file (*.hdf5)\n",
        "  2.   **Embeddings** folder: contains embeddings file\n",
        "      * **glove.6B** folder: contains glove (100d) files downloaded from https://www.kaggle.com/anindya2906/glove6b\n",
        "      * **patent-100** folder: contains domain specific (100d) files downloaded from https://hpi.de/naumann/projects/repeatability/text-mining.html\n",
        "  3.   **Model** folder: contains BERT repo and pretrained BERT model \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0FlU_MDRbnOH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# if have time will convert this to os.path.join...\n",
        "ROOT_DIR = \"/content/drive/My Drive/Colab Notebooks/\"\n",
        "EMBED_DIR = ROOT_DIR+\"Embeddings/\"\n",
        "GLOVE_DIR = EMBED_DIR+\"glove.6B/\"\n",
        "KRESTEL_DIR = EMBED_DIR+\"patent-100/\"\n",
        "MODEL_DIR = ROOT_DIR+\"Model/\"\n",
        "BERT_DIR = MODEL_DIR+\"uncased_L-12_H-768_A-12/\"\n",
        "CKPT_DIR = ROOT_DIR+\"Checkpoint/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mYsl3DNtqYb0",
        "colab_type": "code",
        "outputId": "6cd0261b-60df-4b7e-9780-ec609d734177",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "cd $ROOT_DIR"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/Colab Notebooks\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0KxAblWDwz7-",
        "colab_type": "text"
      },
      "source": [
        "**Import**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uJXJqQVtw2cn",
        "colab_type": "code",
        "outputId": "f66ac463-e073-4ed6-94fd-e20f2c8a1a0f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "import re\n",
        "import os\n",
        "import sys\n",
        "import json\n",
        "\n",
        "import string\n",
        "import numpy as np\n",
        "import pickle\n",
        "import logging\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint\n",
        "\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.optimizers import RMSprop\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from google.colab import auth, drive\n",
        "\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.tokenize import word_tokenize"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ofbQB5EkNl-V",
        "colab_type": "text"
      },
      "source": [
        "**Dataset**\n",
        "\n",
        "Import the Patents Public dataset from Google Cloud Big Querry using PatentView API, from 01/01/2013 to 30/06/2013"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GOos4WmJNlpd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "project_id = 'stone-climate-267317'\n",
        "\n",
        "from google.cloud import bigquery\n",
        "client = bigquery.Client(project=project_id)\n",
        "df = pd.io.gbq.read_gbq('''\n",
        "  SELECT STRING_AGG(distinct t2. section_id order by t2.\n",
        "  section_id) AS cpc_ids, t1.id, t1.date, text\n",
        "  FROM `patents-public-data.patentsview.patent` t1,\n",
        "  `patents-public-data.patentsview.cpc_current` t2,\n",
        "  `patents-public-data.patentsview.claim` t3\n",
        "  where t1.id = t2.patent_id\n",
        "  and t1.id = t3.patent_id\n",
        "  and timestamp(t1.date) >= timestamp('2013-01-01')\n",
        "  and timestamp(t1.date) <= timestamp('2013-06-30')\n",
        "  and t3.sequence='1'\n",
        "  and t1.type='utility'\n",
        "  group by t1.id, t1.date, t3.text\n",
        "  ''', project_id=project_id, dialect='standard')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xQxEwdfgnaeh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# interactive dataframe \n",
        "%load_ext google.colab.data_table"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mnwUU5wDncO7",
        "colab_type": "code",
        "outputId": "5b914fa6-5396-46e5-80ef-464dca01f580",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 509
        }
      },
      "source": [
        "# show some samples\n",
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.module+javascript": "\n      import \"https://ssl.gstatic.com/colaboratory/data_table/84ef27dae82052e3/data_table.js\";\n\n      window.createDataTable({\n        data: [[{\n            'v': 0,\n            'f': \"0\",\n        },\n\"A,B\",\n\"8342257\",\n\"2013-01-01\",\n\"1. A track assembly for a vehicle, the track assembly having a longitudinal axis and comprising: a. a single piece endless track defining a ground engaging run, the ground engaging run having: b. a support wheel arrangement engaging the ground engaging run, the support wheel arrangement including: c. the first support wheel and the second support wheel being mounted for pivotal movement via a pivot structure about a third axis which is generally perpendicular to the first axis and to the second axis, the third axis being vertically spaced apart in an inward direction from the tip portion of the track guide.\"],\n [{\n            'v': 1,\n            'f': \"1\",\n        },\n\"A,F\",\n\"8342467\",\n\"2013-01-01\",\n\"1. A force-exerting device comprising: a force-exerting arm having:\"],\n [{\n            'v': 2,\n            'f': \"2\",\n        },\n\"G,H\",\n\"8345971\",\n\"2013-01-01\",\n\"1. A method for spatial-temporal denoising and demosaicking for noisy color filter array (CFA) video, the method comprising: applying spatial-temporal CFA video denoising to the CFA video in order to generate a denoised CFA using a video processing device; applying initial color demosaicking (CDM) to the denoised CFA video in order to generate a demosaicked video using the video processing device; and applying spatial-temporal post-processing to the demosaicked video in order to reduce CDM artifacts and CDM errors and enhance the quality of the video using the video processing device; wherein spatial-temporal CFA video denoising comprises:\"],\n [{\n            'v': 3,\n            'f': \"3\",\n        },\n\"F\",\n\"8342793\",\n\"2013-01-01\",\n\"1. A turbine engine control system, comprising: at least one surge sensor adapted to detect surge events and adapted to be mounted in the air flow path of a turbine engine; a high speed interrogation unit in electronic data communication with the at least one surge sensor and adapted to receive data from the surge sensor; and an information processing module in electronic data communication with the high speed interrogation unit and adapted to determine whether data obtained from the at least one surge sensor indicates a surge event or a pre-surge condition, and the information processing unit being in electronic controlling communication with one or more engine control elements, wherein the engine control elements are adapted to adjust the operation of the engine in response to data received from the information processing module and to maintain engine operation in a non-surge condition.\"],\n [{\n            'v': 4,\n            'f': \"4\",\n        },\n\"H\",\n\"8344640\",\n\"2013-01-01\",\n\"1. A method for operating a fluorescent lamp, which is connected to a series resonant circuit having a resonant circuit inductance and a resonant circuit capacitance, the method comprising: applying an excitation AC voltage at an excitation frequency to the series resonant circuit using a half bridge circuit, which has an output to which the series resonant circuit is coupled, and which has a first and a second switch which are alternately switched on and off on the basis of a frequency signal; monitoring of a current flowing through the resonant circuit, for the presence of a critical operating state; and shortening of the switched-on times of the first and second switches in comparison to switched-on times which are predetermined by the frequency signal, on detection of a critical operating state.\"]],\n        columns: [[\"number\", \"index\"], [\"string\", \"cpc_ids\"], [\"string\", \"id\"], [\"string\", \"date\"], [\"string\", \"text\"]],\n        columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n        rowsPerPage: 25,\n        helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n        suppressOutputScrolling: true,\n      });\n    ",
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cpc_ids</th>\n",
              "      <th>id</th>\n",
              "      <th>date</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A,B</td>\n",
              "      <td>8342257</td>\n",
              "      <td>2013-01-01</td>\n",
              "      <td>1. A track assembly for a vehicle, the track a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A,F</td>\n",
              "      <td>8342467</td>\n",
              "      <td>2013-01-01</td>\n",
              "      <td>1. A force-exerting device comprising: a force...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>G,H</td>\n",
              "      <td>8345971</td>\n",
              "      <td>2013-01-01</td>\n",
              "      <td>1. A method for spatial-temporal denoising and...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>F</td>\n",
              "      <td>8342793</td>\n",
              "      <td>2013-01-01</td>\n",
              "      <td>1. A turbine engine control system, comprising...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>H</td>\n",
              "      <td>8344640</td>\n",
              "      <td>2013-01-01</td>\n",
              "      <td>1. A method for operating a fluorescent lamp, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  cpc_ids  ...                                               text\n",
              "0     A,B  ...  1. A track assembly for a vehicle, the track a...\n",
              "1     A,F  ...  1. A force-exerting device comprising: a force...\n",
              "2     G,H  ...  1. A method for spatial-temporal denoising and...\n",
              "3       F  ...  1. A turbine engine control system, comprising...\n",
              "4       H  ...  1. A method for operating a fluorescent lamp, ...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WRi3rhpUvJvm",
        "colab_type": "text"
      },
      "source": [
        "Extract input samples from dataframe. Input data will be the claim, while input label will be the cpc id at section level. There are 9 labels in total: A, B, C, D, E, F, G, H, Y"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fU62ROjXzFIw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# input data collected from patents's claim\n",
        "texts = np.array(df['text'])\n",
        "\n",
        "cpc_ids = df['cpc_ids']\n",
        "\n",
        "cpc_ids_list = list(cpc_ids)\n",
        "# labels_raw are collected from the df column, need to be cleaned of punctuations\n",
        "labels_raw = []\n",
        "for id in cpc_ids_list:\n",
        "    id = id.replace(\",\",\" \")\n",
        "    labels_raw.append(word_tokenize(id))\n",
        "\n",
        "# labels_set contains all available labels: A, B, C, D, E, F, G, H, Y\n",
        "labels_set = set()\n",
        "unique = [[x for x in y if x not in labels_set and (labels_set.add(x) or True)] for y in labels_raw]\n",
        "labels_set = list(labels_set)\n",
        "labels_set.sort()\n",
        "\n",
        "# labels_list contains binary vectorized label\n",
        "labels_list = []\n",
        "for labels_doc in labels_raw:\n",
        "  labels_vec = np.zeros(9)\n",
        "  for label in labels_doc:\n",
        "    label_idx = labels_set.index(label)\n",
        "    labels_vec[label_idx] = 1\n",
        "  labels_list.append(labels_vec)\n",
        "\n",
        "# standardized to np.array\n",
        "labels = np.array(labels_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Ac89FLgNXBm",
        "colab_type": "text"
      },
      "source": [
        "Confirm labels shape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J56QBBzXXxlw",
        "colab_type": "code",
        "outputId": "dcd9b9b0-d925-4923-f537-ebd3509dcbf0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "labels[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0., 0., 0., 0., 0., 0., 1., 1., 0.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rms2a0dpYKPX",
        "colab_type": "code",
        "outputId": "a2b95829-3e88-4308-bd6f-10c517574385",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "labels_set"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'Y']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tK0AnpXtvEWR",
        "colab_type": "text"
      },
      "source": [
        "Show dataset class distribution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6WpA6w_xp2yC",
        "colab_type": "code",
        "outputId": "cc49b641-bba9-4ff7-f820-2e06978eb7bc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "w = [0]*9\n",
        "for x in labels_set:\n",
        "  for y in labels_raw: \n",
        "    z = y.count(x)\n",
        "    if z > 0:\n",
        "      w[labels_set.index(x)] += 1\n",
        "dict(zip(labels_set, w))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'A': 44441,\n",
              " 'B': 44852,\n",
              " 'C': 34127,\n",
              " 'D': 2145,\n",
              " 'E': 7865,\n",
              " 'F': 23498,\n",
              " 'G': 102425,\n",
              " 'H': 99481,\n",
              " 'Y': 45024}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KgpdxNiz50_G",
        "colab_type": "text"
      },
      "source": [
        "**Note**: *it's imbalance but since the dataset is multi-label, i wonder if i can really seperate the categories then downsampling/oversampling them... For now i will just increase dataset size instead.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IQZNN0YlONMC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trX, tsX, trY, tsY = train_test_split(texts, labels, shuffle=True, test_size=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XOO1NmvEQFkg",
        "colab_type": "text"
      },
      "source": [
        "###**0. Universal function**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kEryco1ipH4L",
        "colab_type": "text"
      },
      "source": [
        "**Create embedded sentences**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wb2wBbrbpLvA",
        "colab_type": "code",
        "outputId": "4bad7d29-7bed-43f1-c9a9-b924a37d07e6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# Convert sentence to numeric form\n",
        "word_tokenizer_tr = Tokenizer()\n",
        "word_tokenizer_tr.fit_on_texts(trX)\n",
        "\n",
        "word_tokenizer_ts = Tokenizer()\n",
        "word_tokenizer_ts.fit_on_texts(tsX)\n",
        "\n",
        "vocab_length = len(word_tokenizer_tr.word_index) + 1\n",
        "\n",
        "embedded_sentences_tr = word_tokenizer_tr.texts_to_sequences(trX)\n",
        "embedded_sentences_ts = word_tokenizer_ts.texts_to_sequences(tsX)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xya3fW-3sI6A",
        "colab_type": "text"
      },
      "source": [
        "**Padding sentence**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V7mvSq8oovKu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### length_long_sentence on this dataset is 9700, too large. \n",
        "### Even with 12GB RAM it's guaranted to crash so i have to limit the sequence length to 2000.\n",
        "\n",
        "length_long_sentence = 2000\n",
        "\n",
        "# word_count = lambda sentence: len(word_tokenize_tr(sentence))\n",
        "# longest_sentence = max(corpus, key=word_count)\n",
        "# length_long_sentence = len(word_tokenize_tr(longest_sentence))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GcuKLy7BmAWe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "padded_sentences_tr = pad_sequences(embedded_sentences_tr, length_long_sentence, padding='post', truncating = 'post')\n",
        "padded_sentences_ts = pad_sequences(embedded_sentences_ts, length_long_sentence, padding='post', truncating = 'post')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "498n29p2R8QC",
        "colab_type": "text"
      },
      "source": [
        "**Create embedding matrix**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nD5vg0KPuOho",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_embedding_matrix(embedding_index, embedding_dimension):\n",
        "  embedding_matrix = np.zeros((vocab_length, embedding_dimension))\n",
        "  for word, index in word_tokenizer_tr.word_index.items():\n",
        "      embedding_vector = embedding_index['embeddings_index'].get(word)\n",
        "      if embedding_vector is not None:\n",
        "          embedding_matrix[index] = embedding_vector\n",
        "  return embedding_matrix"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uLz58lAt8Tsn",
        "colab_type": "text"
      },
      "source": [
        "**Define metrics**\n",
        "\n",
        "Beside accuracy, since the classes are imbalance, we also use precision, recall and F1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgB7YnTB6Eri",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras import backend as K\n",
        "\n",
        "def recall(y_true, y_pred):\n",
        "  \"\"\"Recall metric.\n",
        "\n",
        "  Only computes a batch-wise average of recall.\n",
        "\n",
        "  Computes the recall, a metric for multi-label classification of\n",
        "  how many relevant items are selected.\n",
        "  \"\"\"\n",
        "  true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "  possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "  recall = true_positives / (possible_positives + K.epsilon())\n",
        "  return recall\n",
        "\n",
        "def precision(y_true, y_pred):\n",
        "  \"\"\"Precision metric.\n",
        "\n",
        "  Only computes a batch-wise average of precision.\n",
        "\n",
        "  Computes the precision, a metric for multi-label classification of\n",
        "  how many selected items are relevant.\n",
        "  \"\"\"\n",
        "  true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "  predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "  precision = true_positives / (predicted_positives + K.epsilon())\n",
        "  return precision\n",
        "\n",
        "def f1(y_true, y_pred):\n",
        "  p = precision(y_true, y_pred)\n",
        "  r = recall(y_true, y_pred)\n",
        "  return 2*((p*r)/(p+r+K.epsilon()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MOgh5AFk1cF",
        "colab_type": "text"
      },
      "source": [
        "###**1. Finetuning using GLOVE** "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mzpk3meQs6On",
        "colab_type": "text"
      },
      "source": [
        "**Setup Glove 6B**\n",
        "\n",
        "Glove 6B need to be uploaded to drive and setup accordingly to the path config. I uploaded GLOVE directly to my google drive because it is faster that way than using !wget then unzip it."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6fW6REgJ3OcW",
        "colab_type": "text"
      },
      "source": [
        "**Indexing embeddings**\n",
        "\n",
        "Index the vectors from embeddings files, and save it to the drive for faster loading next time"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-1s5Q3qEHOBH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def embeddings_indexer(filepath):\n",
        "  print('Indexing word vectors.')\n",
        "\n",
        "  embeddings_index = {}\n",
        "  f = open(filepath,'r',encoding='utf-8', errors='ignored')\n",
        "  for line in tqdm(f):\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "  f.close()\n",
        "\n",
        "  print('Found %s word vectors.' % len(embeddings_index))\n",
        "  \n",
        "  pickle.dump({'embeddings_index' : embeddings_index } , open(os.path.splitext(filepath)[0]+'.p', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkLhZS3rjA-K",
        "colab_type": "text"
      },
      "source": [
        "**Note**: *Only run this once for the 1st time*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IdswQoWSHkUU",
        "colab_type": "code",
        "outputId": "9ce7914f-476a-4595-d158-1dd639a1ba51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "embeddings_indexer(GLOVE_DIR+'glove.6B.100d.txt')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Indexing word vectors.\n",
            "Found 400000 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7KAeB9BqLVA0",
        "colab_type": "text"
      },
      "source": [
        "**Load indexed embeddings**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bGCNfcLLtfRA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pickled = open(GLOVE_DIR+'glove.6B.100d.p','rb')\n",
        "glove6b100d = pickle.load(pickled)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y1rXkGpk7SmR",
        "colab_type": "text"
      },
      "source": [
        "**Create GLOVE embedding matrix**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m8nPml790z9h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_matrix = create_embedding_matrix(glove6b100d, 100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WenEFErsSPVt",
        "colab_type": "text"
      },
      "source": [
        "**Define model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dEZIFK2vNgEf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Sequential()\n",
        "embedding_layer = layers.Embedding(vocab_length, 100, weights=[embedding_matrix], input_length=length_long_sentence, trainable=False)\n",
        "model.add(embedding_layer)\n",
        "model.add(layers.Flatten())\n",
        "model.add(layers.Dense(9, activation='sigmoid'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uzPik_w67LDR",
        "colab_type": "code",
        "outputId": "8deb3b91-6cdf-4126-cf0d-15bae6ae09ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277
        }
      },
      "source": [
        "model.compile(\n",
        "      optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5, ),\n",
        "      loss=\"binary_crossentropy\",\n",
        "      metrics=['accuracy',f1,precision,recall])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 2000, 100)         7414900   \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 200000)            0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 9)                 1800009   \n",
            "=================================================================\n",
            "Total params: 9,214,909\n",
            "Trainable params: 1,800,009\n",
            "Non-trainable params: 7,414,900\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mm8wj9vFStGU",
        "colab_type": "text"
      },
      "source": [
        "**Model training**\n",
        "\n",
        "This model was trained for 30 epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZSeKrzI6LYd",
        "colab_type": "code",
        "outputId": "673d88f9-5239-45c9-ff72-332e55c965ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 818
        }
      },
      "source": [
        "# model.fit(padded_sentences, labels, epochs=10, verbose=1)\n",
        "saver = keras.callbacks.ModelCheckpoint(CKPT_DIR+\"glove6b100d.hdf5\")\n",
        "model.fit(padded_sentences_tr, trY, validation_data=[padded_sentences_ts, tsY], batch_size=32, epochs=20, callbacks=[saver])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 105040 samples, validate on 26260 samples\n",
            "Epoch 1/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2367 - acc: 0.9004 - f1: 0.6223 - precision: 0.7990 - recall: 0.5140 - val_loss: 0.2373 - val_acc: 0.9001 - val_f1: 0.6269 - val_precision: 0.7901 - val_recall: 0.5238\n",
            "Epoch 2/20\n",
            "105040/105040 [==============================] - 132s 1ms/sample - loss: 0.2339 - acc: 0.9019 - f1: 0.6295 - precision: 0.8024 - recall: 0.5223 - val_loss: 0.2377 - val_acc: 0.9000 - val_f1: 0.6250 - val_precision: 0.7907 - val_recall: 0.5210\n",
            "Epoch 3/20\n",
            "105040/105040 [==============================] - 133s 1ms/sample - loss: 0.2314 - acc: 0.9029 - f1: 0.6345 - precision: 0.8048 - recall: 0.5281 - val_loss: 0.2381 - val_acc: 0.8997 - val_f1: 0.6240 - val_precision: 0.7894 - val_recall: 0.5202\n",
            "Epoch 4/20\n",
            "105040/105040 [==============================] - 132s 1ms/sample - loss: 0.2290 - acc: 0.9040 - f1: 0.6400 - precision: 0.8066 - recall: 0.5347 - val_loss: 0.2386 - val_acc: 0.8995 - val_f1: 0.6236 - val_precision: 0.7874 - val_recall: 0.5207\n",
            "Epoch 5/20\n",
            "105040/105040 [==============================] - 133s 1ms/sample - loss: 0.2267 - acc: 0.9051 - f1: 0.6452 - precision: 0.8092 - recall: 0.5406 - val_loss: 0.2392 - val_acc: 0.8989 - val_f1: 0.6233 - val_precision: 0.7826 - val_recall: 0.5222\n",
            "Epoch 6/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2246 - acc: 0.9061 - f1: 0.6504 - precision: 0.8119 - recall: 0.5467 - val_loss: 0.2397 - val_acc: 0.8989 - val_f1: 0.6247 - val_precision: 0.7794 - val_recall: 0.5254\n",
            "Epoch 7/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2227 - acc: 0.9070 - f1: 0.6545 - precision: 0.8137 - recall: 0.5516 - val_loss: 0.2403 - val_acc: 0.8983 - val_f1: 0.6209 - val_precision: 0.7802 - val_recall: 0.5198\n",
            "Epoch 8/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2207 - acc: 0.9078 - f1: 0.6585 - precision: 0.8152 - recall: 0.5567 - val_loss: 0.2407 - val_acc: 0.8982 - val_f1: 0.6240 - val_precision: 0.7736 - val_recall: 0.5272\n",
            "Epoch 9/20\n",
            "105040/105040 [==============================] - 133s 1ms/sample - loss: 0.2190 - acc: 0.9087 - f1: 0.6628 - precision: 0.8174 - recall: 0.5616 - val_loss: 0.2413 - val_acc: 0.8977 - val_f1: 0.6179 - val_precision: 0.7781 - val_recall: 0.5167\n",
            "Epoch 10/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2173 - acc: 0.9096 - f1: 0.6666 - precision: 0.8191 - recall: 0.5661 - val_loss: 0.2420 - val_acc: 0.8973 - val_f1: 0.6197 - val_precision: 0.7724 - val_recall: 0.5214\n",
            "Epoch 11/20\n",
            "105040/105040 [==============================] - 132s 1ms/sample - loss: 0.2157 - acc: 0.9104 - f1: 0.6704 - precision: 0.8210 - recall: 0.5705 - val_loss: 0.2425 - val_acc: 0.8972 - val_f1: 0.6220 - val_precision: 0.7660 - val_recall: 0.5275\n",
            "Epoch 12/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2141 - acc: 0.9111 - f1: 0.6738 - precision: 0.8220 - recall: 0.5749 - val_loss: 0.2431 - val_acc: 0.8970 - val_f1: 0.6171 - val_precision: 0.7719 - val_recall: 0.5183\n",
            "Epoch 13/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2126 - acc: 0.9119 - f1: 0.6773 - precision: 0.8241 - recall: 0.5790 - val_loss: 0.2436 - val_acc: 0.8969 - val_f1: 0.6198 - val_precision: 0.7665 - val_recall: 0.5243\n",
            "Epoch 14/20\n",
            "105040/105040 [==============================] - 132s 1ms/sample - loss: 0.2112 - acc: 0.9125 - f1: 0.6801 - precision: 0.8250 - recall: 0.5825 - val_loss: 0.2443 - val_acc: 0.8967 - val_f1: 0.6188 - val_precision: 0.7654 - val_recall: 0.5235\n",
            "Epoch 15/20\n",
            "105040/105040 [==============================] - 131s 1ms/sample - loss: 0.2098 - acc: 0.9132 - f1: 0.6834 - precision: 0.8270 - recall: 0.5863 - val_loss: 0.2450 - val_acc: 0.8962 - val_f1: 0.6179 - val_precision: 0.7625 - val_recall: 0.5236\n",
            "Epoch 16/20\n",
            "105040/105040 [==============================] - 129s 1ms/sample - loss: 0.2085 - acc: 0.9138 - f1: 0.6863 - precision: 0.8277 - recall: 0.5902 - val_loss: 0.2455 - val_acc: 0.8959 - val_f1: 0.6197 - val_precision: 0.7563 - val_recall: 0.5289\n",
            "Epoch 17/20\n",
            "105040/105040 [==============================] - 129s 1ms/sample - loss: 0.2072 - acc: 0.9143 - f1: 0.6884 - precision: 0.8290 - recall: 0.5926 - val_loss: 0.2460 - val_acc: 0.8960 - val_f1: 0.6201 - val_precision: 0.7562 - val_recall: 0.5295\n",
            "Epoch 18/20\n",
            "105040/105040 [==============================] - 130s 1ms/sample - loss: 0.2060 - acc: 0.9149 - f1: 0.6913 - precision: 0.8297 - recall: 0.5966 - val_loss: 0.2466 - val_acc: 0.8954 - val_f1: 0.6159 - val_precision: 0.7566 - val_recall: 0.5235\n",
            "Epoch 19/20\n",
            "105040/105040 [==============================] - 133s 1ms/sample - loss: 0.2048 - acc: 0.9154 - f1: 0.6937 - precision: 0.8313 - recall: 0.5994 - val_loss: 0.2473 - val_acc: 0.8952 - val_f1: 0.6147 - val_precision: 0.7562 - val_recall: 0.5220\n",
            "Epoch 20/20\n",
            "105040/105040 [==============================] - 130s 1ms/sample - loss: 0.2036 - acc: 0.9160 - f1: 0.6963 - precision: 0.8318 - recall: 0.6027 - val_loss: 0.2477 - val_acc: 0.8948 - val_f1: 0.6174 - val_precision: 0.7481 - val_recall: 0.5297\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe7e27200b8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qH1XxcZylF9M",
        "colab_type": "text"
      },
      "source": [
        "###**2. Finetuning using domain specific embedding**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eHQhVOrbPCoY",
        "colab_type": "text"
      },
      "source": [
        "**Setup Patent Domain Embedding**\n",
        "\n",
        "Patent domain specific embeddings need to be uploaded to drive and setup accordingly to the path config. Glove 6B need to be uploaded to drive and setup accordingly to the path config. I uploaded the embedding directly to my google drive because it is faster that way than using !wget then unzip it.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ehPbIT6eTck-",
        "colab_type": "text"
      },
      "source": [
        "**Note**: *The below 2 steps are to checks for vectors indexed correctly or not*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eUH5JancxYit",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### these are to checks for vectors indexed correctly or not since \n",
        "### i can not get pass this step due to limited resource\n",
        "filepath = KRESTEL_DIR+'patent-100.vec'\n",
        "f = open(filepath,'r',encoding='utf-8', errors='ignore')\n",
        "lines = f.readlines()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMrMb6PF1_pg",
        "colab_type": "code",
        "outputId": "8b029a91-3ec8-4e98-fbb4-23387ef719ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        }
      },
      "source": [
        "### these are to checks for vectors indexed correctly or not since \n",
        "### i can not get pass this step due to limited resource\n",
        "embeddings_index = {}\n",
        "values = re.split(\" \",lines[115307])\n",
        "coefs = np.asarray(values[1:-1], dtype='float32')\n",
        "word = values[0].split()\n",
        "word = ' '.join(word)\n",
        "embeddings_index[word] = coefs\n",
        "embeddings_index\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'seq id no: 14': array([-0.73068 ,  0.53468 , -0.032044, -0.64512 , -0.15496 ,  0.38292 ,\n",
              "         0.29444 ,  0.27074 ,  0.33108 , -0.36283 , -0.63907 , -0.66945 ,\n",
              "        -0.35477 , -0.5582  , -0.67749 , -0.98486 , -0.23161 ,  0.72564 ,\n",
              "        -0.11081 , -0.30779 ,  0.22977 , -0.62038 , -0.70511 , -0.78045 ,\n",
              "        -0.51018 , -0.87693 , -0.58377 , -0.1422  , -0.47397 ,  0.16768 ,\n",
              "        -0.011603, -0.33347 , -0.035861, -0.21827 , -0.96943 ,  0.20243 ,\n",
              "        -0.099404,  0.23013 , -0.65855 , -0.1635  , -0.76947 ,  0.17338 ,\n",
              "         1.2006  ,  0.098666,  0.24926 ,  1.2408  , -0.51845 ,  0.016026,\n",
              "         0.004362,  0.083166,  1.1201  ,  0.17461 , -1.079   , -0.64793 ,\n",
              "        -0.97192 ,  0.38055 ,  0.73005 ,  0.57319 , -0.40212 ,  0.043041,\n",
              "        -0.65559 ,  0.24174 , -0.62599 ,  0.32414 ,  0.26648 ,  1.0558  ,\n",
              "        -0.21024 , -0.70518 , -0.84609 , -0.82383 ,  0.41788 , -0.36059 ,\n",
              "        -1.1687  , -0.018692, -0.33578 , -1.1909  , -0.33768 , -0.90154 ,\n",
              "        -0.5453  ,  0.093319, -0.74883 ,  0.97327 , -0.94556 ,  0.21322 ,\n",
              "         0.69287 , -0.13801 ,  0.45993 , -0.38687 , -0.29572 , -0.71922 ,\n",
              "        -0.14698 , -0.93173 , -0.10122 ,  1.1635  ,  0.46576 , -1.0125  ,\n",
              "         0.11923 ,  0.044079,  0.32563 , -0.39085 ], dtype=float32)}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6bKMDRReT7_M",
        "colab_type": "text"
      },
      "source": [
        "**Indexing embeddings**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kA7EJLy5n1nq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### FastText need another implementation of indexer due to different format\n",
        "### Don't run this if you have less than 16GB RAM\n",
        "\n",
        "def embeddings_indexer_fasttext(filepath):\n",
        "  print('Indexing word vectors.')\n",
        "  \n",
        "  embeddings_index = {}\n",
        "  f = open(filepath,'r',encoding='utf-8', errors='ignore')\n",
        "  lines = f.readlines()\n",
        "  for line in lines:\n",
        "    values = re.split(\" \",line)\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:-1], dtype='float32')\n",
        "    word = values[0].split()\n",
        "    word = ' '.join(word)\n",
        "    embeddings_index[word] = coefs\n",
        "  f.close()\n",
        "\n",
        "  print('Found %s word vectors.' % len(embeddings_index))\n",
        "   \n",
        "  pickle.dump({'embeddings_index' : embeddings_index } , open(os.path.splitext(filepath)[0]+'.p', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51TfRSLOjMpe",
        "colab_type": "text"
      },
      "source": [
        "**Note**: *Only run this once for the 1st time*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_KZbEyMzSnTz",
        "colab_type": "code",
        "outputId": "e78b29f6-6205-4a47-bfde-8252730d6ad0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "embeddings_indexer_fasttext(KRESTEL_DIR+'patent-100.vec')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Indexing word vectors.\n",
            "Found 7075697 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UuqxDauDgRBt",
        "colab_type": "text"
      },
      "source": [
        "**Note**: *Could have use Gensim to index the vector but they stopped supporting loading .vec file, only .bin file for now*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pHWC_V3sUGdm",
        "colab_type": "text"
      },
      "source": [
        "**Load indexed embeddings**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EDvEly4RSnY3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pickled = open(KRESTEL_DIR+'patent-100.p','rb')\n",
        "patent100d = pickle.load(pickled)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UYpSfoKYUfu3",
        "colab_type": "text"
      },
      "source": [
        "**Create domain specific embeddings matrix**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3-vAbs-iSnd9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_matrix_ = create_embedding_matrix(patent100d, 100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NsUjdxe1kiPC",
        "colab_type": "code",
        "outputId": "64b9ac73-f3ef-43f6-c845-eedc909d911f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 389
        }
      },
      "source": [
        "embedding_matrix_[1]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-0.33860001,  0.31740001, -0.67751998, -0.65139002,  0.71763003,\n",
              "        0.43919   ,  0.61008   ,  0.38993999,  0.056848  , -0.67866999,\n",
              "        0.33126   , -0.68711001, -0.30983999, -0.51788002, -0.32139999,\n",
              "       -0.18276   , -0.82393998, -0.20269001,  0.58515   ,  0.66766   ,\n",
              "        0.1734    ,  0.19246   , -0.82081997, -0.61110997, -0.67978001,\n",
              "       -0.22826   , -0.36210001,  0.41457999,  0.50678003,  0.97635001,\n",
              "        0.92013001, -0.21087   , -0.48159   , -0.72724998, -1.35880005,\n",
              "        0.92988998, -0.28830001,  0.58445001, -0.12084   , -0.80913001,\n",
              "        0.41358   ,  0.70564997,  0.86321998,  0.34935001,  1.17709994,\n",
              "        0.40610999, -1.11889994,  0.30017999,  0.18553001, -0.6049    ,\n",
              "       -0.73605001, -0.45118001,  0.01649   , -0.30338001, -1.3398    ,\n",
              "        0.28955001,  0.77275997,  1.12020004, -0.56440002, -0.45278001,\n",
              "       -0.33410999,  0.38435999, -0.91064   ,  1.14289999,  1.07980001,\n",
              "        0.022331  , -0.43042001,  0.31935999, -1.06140006,  0.34715   ,\n",
              "        0.92816001,  0.62706   ,  0.29578   ,  0.066978  ,  0.66676003,\n",
              "       -1.57850003, -1.30879998,  0.047214  , -1.12820005,  0.3439    ,\n",
              "       -0.36919001, -0.064143  , -0.24642999,  0.71578997,  1.46689999,\n",
              "       -0.74743998,  0.079957  , -0.094167  , -0.66996002, -0.11417   ,\n",
              "       -0.27537   , -1.0007    , -0.11659   ,  0.5887    ,  0.64894998,\n",
              "        0.28388   ,  0.61194998,  0.55382001,  1.28799999,  0.09057   ])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "trJ3HGawUxW3"
      },
      "source": [
        "**Define model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "buH71JIGU1VY",
        "outputId": "8b970e4a-420b-4f0e-cf3d-97f995d21cb9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 149
        }
      },
      "source": [
        "model = Sequential()\n",
        "embedding_layer = layers.Embedding(vocab_length, 100, weights=[embedding_matrix_], input_length=length_long_sentence, trainable=False)\n",
        "model.add(embedding_layer)\n",
        "model.add(layers.Flatten())\n",
        "model.add(layers.Dense(9, activation='sigmoid'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "a949a6fc-668c-4e93-838e-95a468df83cf",
        "id": "um552uEKU5VQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353
        }
      },
      "source": [
        "model.compile(\n",
        "      optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5, ),\n",
        "      loss=\"binary_crossentropy\",\n",
        "      metrics=['accuracy',f1,precision,recall])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 2000, 100)         6687100   \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 200000)            0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 9)                 1800009   \n",
            "=================================================================\n",
            "Total params: 8,487,109\n",
            "Trainable params: 1,800,009\n",
            "Non-trainable params: 6,687,100\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_b5uo5xlU_BC"
      },
      "source": [
        "**Model training**\n",
        "\n",
        "This model was trained for 70 epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "th4_Ku1iVERE",
        "colab_type": "code",
        "outputId": "5d721de7-f522-4c42-b0da-bd935f8d7699",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 446
        }
      },
      "source": [
        "# model.fit(padded_sentences, labels, epochs=10, verbose=1)\n",
        "saver = keras.callbacks.ModelCheckpoint(CKPT_DIR+\"patent100d.hdf5\")\n",
        "model.fit(padded_sentences_tr, trY, validation_data=[padded_sentences_ts, tsY], batch_size=32, epochs=10, callbacks=[saver])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 105040 samples, validate on 26260 samples\n",
            "Epoch 1/10\n",
            "105040/105040 [==============================] - 29s 277us/sample - loss: 0.1781 - acc: 0.9273 - f1: 0.7457 - precision: 0.8528 - recall: 0.6661 - val_loss: 0.4631 - val_acc: 0.8333 - val_f1: 0.2051 - val_precision: 0.4452 - val_recall: 0.1351\n",
            "Epoch 2/10\n",
            "105040/105040 [==============================] - 29s 277us/sample - loss: 0.1775 - acc: 0.9275 - f1: 0.7470 - precision: 0.8533 - recall: 0.6678 - val_loss: 0.4635 - val_acc: 0.8329 - val_f1: 0.2131 - val_precision: 0.4427 - val_recall: 0.1422\n",
            "Epoch 3/10\n",
            "105040/105040 [==============================] - 29s 274us/sample - loss: 0.1770 - acc: 0.9278 - f1: 0.7481 - precision: 0.8536 - recall: 0.6694 - val_loss: 0.4642 - val_acc: 0.8332 - val_f1: 0.2072 - val_precision: 0.4437 - val_recall: 0.1373\n",
            "Epoch 4/10\n",
            "105040/105040 [==============================] - 30s 286us/sample - loss: 0.1765 - acc: 0.9279 - f1: 0.7487 - precision: 0.8538 - recall: 0.6702 - val_loss: 0.4638 - val_acc: 0.8329 - val_f1: 0.2039 - val_precision: 0.4397 - val_recall: 0.1346\n",
            "Epoch 5/10\n",
            "105040/105040 [==============================] - 29s 280us/sample - loss: 0.1760 - acc: 0.9283 - f1: 0.7500 - precision: 0.8542 - recall: 0.6721 - val_loss: 0.4649 - val_acc: 0.8325 - val_f1: 0.2091 - val_precision: 0.4390 - val_recall: 0.1389\n",
            "Epoch 6/10\n",
            "105040/105040 [==============================] - 29s 274us/sample - loss: 0.1755 - acc: 0.9285 - f1: 0.7510 - precision: 0.8549 - recall: 0.6730 - val_loss: 0.4666 - val_acc: 0.8328 - val_f1: 0.2110 - val_precision: 0.4412 - val_recall: 0.1404\n",
            "Epoch 7/10\n",
            "105040/105040 [==============================] - 30s 281us/sample - loss: 0.1751 - acc: 0.9288 - f1: 0.7520 - precision: 0.8558 - recall: 0.6741 - val_loss: 0.4652 - val_acc: 0.8327 - val_f1: 0.2091 - val_precision: 0.4397 - val_recall: 0.1392\n",
            "Epoch 8/10\n",
            "105040/105040 [==============================] - 29s 273us/sample - loss: 0.1746 - acc: 0.9290 - f1: 0.7530 - precision: 0.8556 - recall: 0.6758 - val_loss: 0.4678 - val_acc: 0.8322 - val_f1: 0.2166 - val_precision: 0.4382 - val_recall: 0.1457\n",
            "Epoch 9/10\n",
            "105040/105040 [==============================] - 29s 277us/sample - loss: 0.1742 - acc: 0.9292 - f1: 0.7539 - precision: 0.8558 - recall: 0.6771 - val_loss: 0.4699 - val_acc: 0.8326 - val_f1: 0.2091 - val_precision: 0.4380 - val_recall: 0.1392\n",
            "Epoch 10/10\n",
            "105040/105040 [==============================] - 29s 279us/sample - loss: 0.1737 - acc: 0.9295 - f1: 0.7552 - precision: 0.8568 - recall: 0.6785 - val_loss: 0.4713 - val_acc: 0.8324 - val_f1: 0.2179 - val_precision: 0.4422 - val_recall: 0.1464\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f009db38d68>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HWSn8dE1kcje",
        "colab_type": "text"
      },
      "source": [
        "###**3. Fine-tuning using pretrained model BERT**\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Y4wKnBxMx39",
        "colab_type": "text"
      },
      "source": [
        "**Install BERT**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6w3YkV-EF0nF",
        "colab_type": "code",
        "outputId": "84c25ead-48d1-445f-d500-194244a65ac6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        }
      },
      "source": [
        "!test -d bert_repo || git clone https://github.com/google-research/bert bert_repo"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'bert_repo'...\n",
            "remote: Enumerating objects: 336, done.\u001b[K\n",
            "Receiving objects:   0% (1/336)   \rReceiving objects:   1% (4/336)   \rReceiving objects:   2% (7/336)   \rReceiving objects:   3% (11/336)   \rReceiving objects:   4% (14/336)   \rReceiving objects:   5% (17/336)   \rReceiving objects:   6% (21/336)   \rReceiving objects:   7% (24/336)   \rReceiving objects:   8% (27/336)   \rReceiving objects:   9% (31/336)   \rReceiving objects:  10% (34/336)   \rReceiving objects:  11% (37/336)   \rReceiving objects:  12% (41/336)   \rReceiving objects:  13% (44/336)   \rReceiving objects:  14% (48/336)   \rReceiving objects:  15% (51/336)   \rReceiving objects:  16% (54/336)   \rReceiving objects:  17% (58/336)   \rReceiving objects:  18% (61/336)   \rReceiving objects:  19% (64/336)   \rReceiving objects:  20% (68/336)   \rReceiving objects:  21% (71/336)   \rReceiving objects:  22% (74/336)   \rReceiving objects:  23% (78/336)   \rReceiving objects:  24% (81/336)   \rReceiving objects:  25% (84/336)   \rReceiving objects:  26% (88/336)   \rReceiving objects:  27% (91/336)   \rReceiving objects:  28% (95/336)   \rReceiving objects:  29% (98/336)   \rReceiving objects:  30% (101/336)   \rReceiving objects:  31% (105/336)   \rReceiving objects:  32% (108/336)   \rReceiving objects:  33% (111/336)   \rReceiving objects:  34% (115/336)   \rReceiving objects:  35% (118/336)   \rReceiving objects:  36% (121/336)   \rReceiving objects:  37% (125/336)   \rReceiving objects:  38% (128/336)   \rReceiving objects:  39% (132/336)   \rReceiving objects:  40% (135/336)   \rReceiving objects:  41% (138/336)   \rReceiving objects:  42% (142/336)   \rReceiving objects:  43% (145/336)   \rReceiving objects:  44% (148/336)   \rReceiving objects:  45% (152/336)   \rReceiving objects:  46% (155/336)   \rReceiving objects:  47% (158/336)   \rReceiving objects:  48% (162/336)   \rReceiving objects:  49% (165/336)   \rReceiving objects:  50% (168/336)   \rReceiving objects:  51% (172/336)   \rReceiving objects:  52% (175/336)   \rremote: Total 336 (delta 0), reused 0 (delta 0), pack-reused 336\u001b[K\n",
            "Receiving objects:  53% (179/336)   \rReceiving objects:  54% (182/336)   \rReceiving objects:  55% (185/336)   \rReceiving objects:  56% (189/336)   \rReceiving objects:  57% (192/336)   \rReceiving objects:  58% (195/336)   \rReceiving objects:  59% (199/336)   \rReceiving objects:  60% (202/336)   \rReceiving objects:  61% (205/336)   \rReceiving objects:  62% (209/336)   \rReceiving objects:  63% (212/336)   \rReceiving objects:  64% (216/336)   \rReceiving objects:  65% (219/336)   \rReceiving objects:  66% (222/336)   \rReceiving objects:  67% (226/336)   \rReceiving objects:  68% (229/336)   \rReceiving objects:  69% (232/336)   \rReceiving objects:  70% (236/336)   \rReceiving objects:  71% (239/336)   \rReceiving objects:  72% (242/336)   \rReceiving objects:  73% (246/336)   \rReceiving objects:  74% (249/336)   \rReceiving objects:  75% (252/336)   \rReceiving objects:  76% (256/336)   \rReceiving objects:  77% (259/336)   \rReceiving objects:  78% (263/336)   \rReceiving objects:  79% (266/336)   \rReceiving objects:  80% (269/336)   \rReceiving objects:  81% (273/336)   \rReceiving objects:  82% (276/336)   \rReceiving objects:  83% (279/336)   \rReceiving objects:  84% (283/336)   \rReceiving objects:  85% (286/336)   \rReceiving objects:  86% (289/336)   \rReceiving objects:  87% (293/336)   \rReceiving objects:  88% (296/336)   \rReceiving objects:  89% (300/336)   \rReceiving objects:  90% (303/336)   \rReceiving objects:  91% (306/336)   \rReceiving objects:  92% (310/336)   \rReceiving objects:  93% (313/336)   \rReceiving objects:  94% (316/336)   \rReceiving objects:  95% (320/336)   \rReceiving objects:  96% (323/336)   \rReceiving objects:  97% (326/336)   \rReceiving objects:  98% (330/336)   \rReceiving objects:  99% (333/336)   \rReceiving objects: 100% (336/336)   \rReceiving objects: 100% (336/336), 290.24 KiB | 4.15 MiB/s, done.\n",
            "Resolving deltas:   0% (0/184)   \rResolving deltas:   2% (4/184)   \rResolving deltas:   3% (6/184)   \rResolving deltas:   5% (11/184)   \rResolving deltas:   7% (13/184)   \rResolving deltas:   8% (16/184)   \rResolving deltas:   9% (18/184)   \rResolving deltas:  11% (22/184)   \rResolving deltas:  13% (24/184)   \rResolving deltas:  14% (26/184)   \rResolving deltas:  20% (37/184)   \rResolving deltas:  26% (49/184)   \rResolving deltas:  35% (66/184)   \rResolving deltas:  36% (68/184)   \rResolving deltas:  37% (69/184)   \rResolving deltas:  41% (76/184)   \rResolving deltas:  56% (104/184)   \rResolving deltas:  62% (115/184)   \rResolving deltas: 100% (184/184)   \rResolving deltas: 100% (184/184), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8cpRoKo0F1E9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if not MODEL_DIR+'bert_repo' in sys.path:\n",
        "    sys.path.insert(0, MODEL_DIR+'bert_repo')\n",
        "from modeling import BertModel, BertConfig\n",
        "from tokenization import FullTokenizer, convert_to_unicode\n",
        "from extract_features import convert_examples_to_features, InputExample"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ExJSD8eh_2ke",
        "colab_type": "text"
      },
      "source": [
        "1.   **Defining tf module**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTZZzS96lpM2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_module_fn(config_path, vocab_path, do_lower_case=True):\n",
        "\n",
        "    def bert_module_fn(is_training):\n",
        "        \"\"\"Spec function for a token embedding module.\"\"\"\n",
        "\n",
        "        input_ids = tf.placeholder(shape=[None, None], dtype=tf.int32, name=\"input_ids\")\n",
        "        input_mask = tf.placeholder(shape=[None, None], dtype=tf.int32, name=\"input_mask\")\n",
        "        token_type = tf.placeholder(shape=[None, None], dtype=tf.int32, name=\"segment_ids\")\n",
        "\n",
        "        config = BertConfig.from_json_file(config_path)\n",
        "        model = BertModel(config=config, is_training=is_training,\n",
        "                          input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type)\n",
        "          \n",
        "        seq_output = model.all_encoder_layers[-1]\n",
        "        pool_output = model.get_pooled_output()\n",
        "\n",
        "        config_file = tf.constant(value=config_path, dtype=tf.string, name=\"config_file\")\n",
        "        vocab_file = tf.constant(value=vocab_path, dtype=tf.string, name=\"vocab_file\")\n",
        "        lower_case = tf.constant(do_lower_case)\n",
        "\n",
        "        tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, config_file)\n",
        "        tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file)\n",
        "        \n",
        "        input_map = {\"input_ids\": input_ids,\n",
        "                     \"input_mask\": input_mask,\n",
        "                     \"segment_ids\": token_type}\n",
        "        \n",
        "        output_map = {\"pooled_output\": pool_output,\n",
        "                      \"sequence_output\": seq_output}\n",
        "\n",
        "        output_info_map = {\"vocab_file\": vocab_file,\n",
        "                           \"do_lower_case\": lower_case}\n",
        "                \n",
        "        hub.add_signature(name=\"tokens\", inputs=input_map, outputs=output_map)\n",
        "        hub.add_signature(name=\"tokenization_info\", inputs={}, outputs=output_info_map)\n",
        "\n",
        "    return bert_module_fn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oikgiiAXpj7A",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "2.   **Exporting the module**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d564X8LBpnPT",
        "colab_type": "code",
        "outputId": "e9fc8f0a-e632-4794-d88f-f206f97f647a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 365
        }
      },
      "source": [
        "config_path = BERT_DIR+\"bert_config.json\"\n",
        "vocab_path = BERT_DIR+\"vocab.txt\"\n",
        "bert_module_path = BERT_DIR+\"bert-module\"\n",
        "\n",
        "tags_and_args = []\n",
        "for is_training in (True, False):\n",
        "  tags = set()\n",
        "  if is_training:\n",
        "    tags.add(\"train\")\n",
        "  tags_and_args.append((tags, dict(is_training=is_training)))\n",
        "\n",
        "module_fn = build_module_fn(config_path, vocab_path)\n",
        "spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)\n",
        "spec.export(bert_module_path, \n",
        "            checkpoint_path = BERT_DIR+\"bert_model.ckpt\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AlreadyExistsError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAlreadyExistsError\u001b[0m                        Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-7ad27aea37f1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_module_spec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags_and_args\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtags_and_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m spec.export(bert_module_path, \n\u001b[0;32m---> 15\u001b[0;31m             checkpoint_path = BERT_DIR+\"bert_model.ckpt\")\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/module_spec.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(self, path, _sentinel, checkpoint_path, name_transform_fn)\u001b[0m\n\u001b[1;32m     78\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Missing mandatory `checkpoint_path` parameter\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m     \u001b[0mname_transform_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_transform_fn\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m     \u001b[0mexport_module_spec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_transform_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mabc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36mexport_module_spec\u001b[0;34m(spec, path, checkpoint_path, name_transform_fn)\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtf_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m       \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit_op\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m       \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(self, path, session)\u001b[0m\n\u001b[1;32m    323\u001b[0m       raise RuntimeError(\"session graph differs from the graph where the \"\n\u001b[1;32m    324\u001b[0m                          \"module was instantiated.\")\n\u001b[0;32m--> 325\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_impl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    326\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    327\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/native_module.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(self, path, session)\u001b[0m\n\u001b[1;32m    633\u001b[0m             write_state=False)\n\u001b[1;32m    634\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 635\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_export\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables_saver\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/native_module.py\u001b[0m in \u001b[0;36m_export\u001b[0;34m(self, path, variables_saver)\u001b[0m\n\u001b[1;32m    394\u001b[0m         \u001b[0mcheckpoint\u001b[0m \u001b[0mon\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mgiven\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    395\u001b[0m     \"\"\"\n\u001b[0;32m--> 396\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_saved_model_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables_saver\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvariables_saver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    397\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    398\u001b[0m     \u001b[0mmodule_def_proto\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule_def_pb2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModuleDef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/saved_model_lib.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(self, path, variables_saver)\u001b[0m\n\u001b[1;32m    402\u001b[0m     \u001b[0massets_map\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_make_assets_key_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 404\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_all_assets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0massets_map\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    405\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables_saver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    406\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_proto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_hub/saved_model_lib.py\u001b[0m in \u001b[0;36m_save_all_assets\u001b[0;34m(self, path, assets_map)\u001b[0m\n\u001b[1;32m    422\u001b[0m     \u001b[0mtf_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMakeDirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massets_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestination\u001b[0m \u001b[0;32min\u001b[0m \u001b[0massets_map\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 424\u001b[0;31m       \u001b[0mtf_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestination\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    426\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_save_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables_saver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/lib/io/file_io.py\u001b[0m in \u001b[0;36mcopy\u001b[0;34m(oldpath, newpath, overwrite)\u001b[0m\n\u001b[1;32m    467\u001b[0m     \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIf\u001b[0m \u001b[0mthe\u001b[0m \u001b[0moperation\u001b[0m \u001b[0mfails\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    468\u001b[0m   \"\"\"\n\u001b[0;32m--> 469\u001b[0;31m   \u001b[0mcopy_v2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moldpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnewpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    470\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/lib/io/file_io.py\u001b[0m in \u001b[0;36mcopy_v2\u001b[0;34m(src, dst, overwrite)\u001b[0m\n\u001b[1;32m    484\u001b[0m   \"\"\"\n\u001b[1;32m    485\u001b[0m   pywrap_tensorflow.CopyFile(\n\u001b[0;32m--> 486\u001b[0;31m       compat.as_bytes(src), compat.as_bytes(dst), overwrite)\n\u001b[0m\u001b[1;32m    487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAlreadyExistsError\u001b[0m: file already exists"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iLHFliNKR_yX",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "3.   **Input preprocessing**\n",
        "\n",
        "This step converts raw input into the corresponding format consists of the 3 required signatures: *input_ids, input_mask* & *segment_ids* \n",
        "\n",
        "Convert raw string into input example by adding unique ID to every string. \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z3UgnFomSAX1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_examples(str_list):\n",
        "    \"\"\"Read a list of `InputExample`s from a list of strings.\"\"\"\n",
        "    unique_id = 0\n",
        "    for s in str_list:\n",
        "        line = convert_to_unicode(s)\n",
        "        if not line:\n",
        "            continue\n",
        "        text_a = line.strip()\n",
        "        text_b = None\n",
        "        \n",
        "        yield InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b)\n",
        "        unique_id += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BqKZCHxmSOzX",
        "colab_type": "text"
      },
      "source": [
        "From InputExample, extract features using *convert_examples_to_features* (BERT's build-in feature extractor).\n",
        "\n",
        "\n",
        "Then arrange features as matrices using *features_to_arrays* "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bo-xccXdSVuE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def features_to_arrays(features):\n",
        "    \"\"\"Convert a list of InputFeatures to np.arrays\"\"\"\n",
        "\n",
        "    all_input_ids = []\n",
        "    all_input_mask = []\n",
        "    all_segment_ids = []\n",
        "\n",
        "    for feature in features:\n",
        "        all_input_ids.append(feature.input_ids)\n",
        "        all_input_mask.append(feature.input_mask)\n",
        "        all_segment_ids.append(feature.input_type_ids)\n",
        "\n",
        "    return (np.array(all_input_ids, dtype='int32'), \n",
        "            np.array(all_input_mask, dtype='int32'), \n",
        "            np.array(all_segment_ids, dtype='int32'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2WV75T0wX6Jn",
        "colab_type": "text"
      },
      "source": [
        "Defining preprocessing pipeline\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Uv9GDc-SaIv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_preprocessor(voc_path, seq_len, lower=True):\n",
        "  \"\"\"\n",
        "  Build a text preprocessing pipeline for BERT\n",
        "  Returns a function which converts a list of strings to a list\n",
        "  of three np.arrays with [input_ids, input_mask, segment_ids]\n",
        "  \"\"\"\n",
        "  tokenizer = FullTokenizer(vocab_file=voc_path, do_lower_case=lower)\n",
        "  \n",
        "  def strings_to_arrays(sents):\n",
        "  \n",
        "      sents = np.atleast_1d(sents).reshape((-1,))\n",
        "\n",
        "      examples = []\n",
        "      for example in read_examples(sents):\n",
        "          examples.append(example)\n",
        "\n",
        "      features = convert_examples_to_features(examples, seq_len, tokenizer)\n",
        "      arrays = features_to_arrays(features)\n",
        "      return arrays\n",
        "  \n",
        "  return strings_to_arrays"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "edVgM5jn2ewt",
        "colab_type": "text"
      },
      "source": [
        "4.   **Defining Bert Layer**\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3JELYNA2dny",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BertLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self, bert_path, seq_len=64, n_tune_layers=3, \n",
        "                 pooling=\"cls\", do_preprocessing=True, verbose=False,\n",
        "                 tune_embeddings=False, trainable=True, **kwargs):\n",
        "\n",
        "        self.trainable = trainable\n",
        "        self.n_tune_layers = n_tune_layers\n",
        "        self.tune_embeddings = tune_embeddings\n",
        "        self.do_preprocessing = do_preprocessing\n",
        "\n",
        "        self.verbose = verbose\n",
        "        self.seq_len = seq_len\n",
        "        self.pooling = pooling\n",
        "        self.bert_path = bert_path\n",
        "\n",
        "        self.var_per_encoder = 16\n",
        "        if self.pooling not in [\"cls\", \"mean\", None]:\n",
        "            raise NameError(\n",
        "                f\"Undefined pooling type (must be either 'cls', 'mean', or None, but is {self.pooling}\"\n",
        "            )\n",
        "\n",
        "        super(BertLayer, self).__init__(**kwargs)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        \n",
        "        # Load module spec, determine trainability, set name\n",
        "        self.bert = hub.Module(self.build_abspath(self.bert_path), \n",
        "                               trainable=self.trainable, name=f\"{self.name}_module\")\n",
        "\n",
        "        trainable_layers = []\n",
        "        if self.tune_embeddings:\n",
        "            trainable_layers.append(\"embeddings\")\n",
        "\n",
        "        if self.pooling == \"cls\":\n",
        "            trainable_layers.append(\"pooler\")\n",
        "\n",
        "        if self.n_tune_layers > 0:\n",
        "            encoder_var_names = [var.name for var in self.bert.variables if 'encoder' in var.name]\n",
        "            n_encoder_layers = int(len(encoder_var_names) / self.var_per_encoder)\n",
        "            for i in range(self.n_tune_layers):\n",
        "                trainable_layers.append(f\"encoder/layer_{str(n_encoder_layers - 1 - i)}/\")\n",
        "        \n",
        "        # Add module variables to layer's trainable weights\n",
        "        for var in self.bert.variables:\n",
        "            if any([l in var.name for l in trainable_layers]):\n",
        "                self._trainable_weights.append(var)\n",
        "            else:\n",
        "                self._non_trainable_weights.append(var)\n",
        "\n",
        "        if self.verbose:\n",
        "            print(\"*** TRAINABLE VARS *** \")\n",
        "            for var in self._trainable_weights:\n",
        "                print(var)\n",
        "\n",
        "        self.build_preprocessor()\n",
        "        self.initialize_module()\n",
        "\n",
        "        super(BertLayer, self).build(input_shape)\n",
        "\n",
        "    # return the path of exported module on web (eg: tf.hub) or on disk\n",
        "    def build_abspath(self, path):\n",
        "        if path.startswith(\"https://\") or path.startswith(\"gs://\"):\n",
        "          return path\n",
        "        else:\n",
        "          return os.path.abspath(path)\n",
        "\n",
        "    def build_preprocessor(self):\n",
        "        sess = tf.keras.backend.get_session()\n",
        "        tokenization_info = self.bert(signature=\"tokenization_info\", as_dict=True)\n",
        "        vocab_file, do_lower_case = sess.run([tokenization_info[\"vocab_file\"],\n",
        "                                              tokenization_info[\"do_lower_case\"]])\n",
        "        self.preprocessor = build_preprocessor(vocab_file, self.seq_len, do_lower_case)\n",
        "\n",
        "    def initialize_module(self):\n",
        "        sess = tf.keras.backend.get_session()\n",
        "        \n",
        "        vars_initialized = sess.run([tf.is_variable_initialized(var) \n",
        "                                     for var in self.bert.variables])\n",
        "\n",
        "        uninitialized = []\n",
        "        for var, is_initialized in zip(self.bert.variables, vars_initialized):\n",
        "            if not is_initialized:\n",
        "                uninitialized.append(var)\n",
        "\n",
        "        if len(uninitialized):\n",
        "            sess.run(tf.variables_initializer(uninitialized))\n",
        "\n",
        "    def call(self, input):\n",
        "\n",
        "        if self.do_preprocessing:\n",
        "          input = tf.numpy_function(self.preprocessor, \n",
        "                                    [input], [tf.int32, tf.int32, tf.int32], \n",
        "                                    name='preprocessor')\n",
        "          for feature in input:\n",
        "            feature.set_shape((None, self.seq_len))\n",
        "        \n",
        "        input_ids, input_mask, segment_ids = input\n",
        "        \n",
        "        bert_inputs = dict(\n",
        "            input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids\n",
        "        )\n",
        "        output = self.bert(inputs=bert_inputs, signature=\"tokens\", as_dict=True)\n",
        "        \n",
        "        if self.pooling == \"cls\":\n",
        "            pooled = output[\"pooled_output\"]\n",
        "        else:\n",
        "            result = output[\"sequence_output\"]\n",
        "            \n",
        "            input_mask = tf.cast(input_mask, tf.float32)\n",
        "            mul_mask = lambda x, m: x * tf.expand_dims(m, axis=-1)\n",
        "            masked_reduce_mean = lambda x, m: tf.reduce_sum(mul_mask(x, m), axis=1) / (\n",
        "                    tf.reduce_sum(m, axis=1, keepdims=True) + 1e-10)\n",
        "            \n",
        "            if self.pooling == \"mean\":\n",
        "              pooled = masked_reduce_mean(result, input_mask)\n",
        "            else:\n",
        "              pooled = mul_mask(result, input_mask)\n",
        "\n",
        "        return pooled\n",
        "\n",
        "    def get_config(self):\n",
        "        config_dict = {\n",
        "            \"bert_path\": self.bert_path, \n",
        "            \"seq_len\": self.seq_len,\n",
        "            \"pooling\": self.pooling,\n",
        "            \"n_tune_layers\": self.n_tune_layers,\n",
        "            \"tune_embeddings\": self.tune_embeddings,\n",
        "            \"do_preprocessing\": self.do_preprocessing,\n",
        "            \"verbose\": self.verbose\n",
        "        }\n",
        "        super(BertLayer, self).get_config()\n",
        "        return config_dict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KUCY_7JpqT8O",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "5.   **Finetuning BERT layer for patent classification**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swJxA3wgx7YZ",
        "colab_type": "code",
        "outputId": "e719e092-933e-4e4e-8e7e-160f65bd2773",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "inp = tf.keras.Input(shape=(1,), dtype=tf.string)\n",
        "encoder = BertLayer(bert_path=bert_module_path, seq_len=192, tune_embeddings=False,\n",
        "                    pooling='cls', n_tune_layers=3, verbose=False)\n",
        "\n",
        "pred = tf.keras.layers.Dense(9, activation='sigmoid')(encoder(inp))\n",
        "\n",
        "model = tf.keras.models.Model(inputs=[inp], outputs=[pred])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a3wBtVleSlGj",
        "colab_type": "code",
        "outputId": "8cfe0219-b70f-4ce0-b1cc-3a44b2262f14",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         [(None, 1)]               0         \n",
            "_________________________________________________________________\n",
            "bert_layer_1 (BertLayer)     (None, 768)               109482240 \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 9)                 6921      \n",
            "=================================================================\n",
            "Total params: 109,489,161\n",
            "Trainable params: 21,861,129\n",
            "Non-trainable params: 87,628,032\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kageCeyXcpkR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(\n",
        "      optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5, ),\n",
        "      loss=\"binary_crossentropy\",\n",
        "      metrics=['accuracy',f1,precision,recall])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "npywFMZNStaB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import logging\n",
        "logging.getLogger(\"tensorflow\").setLevel(logging.WARNING)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TCbal5zKSxBx",
        "colab_type": "code",
        "outputId": "51b7b804-7b3f-4e1a-b830-afd4803fff24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        }
      },
      "source": [
        "saver = keras.callbacks.ModelCheckpoint(CKPT_DIR+\"bert_tuned.hdf5\")\n",
        "\n",
        "model.fit(trX, trY, validation_data=[tsX, tsY], batch_size=32, epochs=5, callbacks=[saver])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 105040 samples, validate on 26260 samples\n",
            "Epoch 1/5\n",
            " 24768/105040 [======>.......................] - ETA: 1:59:15 - loss: 0.2695 - acc: 0.8880 - f1: 0.5589 - precision: 0.7379 - recall: 0.4760"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-45-6f7d461f7c5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModelCheckpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCKPT_DIR\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"bert_tuned.hdf5\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtsX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtsY\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msaver\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    728\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    729\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)\u001b[0m\n\u001b[1;32m    673\u001b[0m         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    674\u001b[0m         \u001b[0mvalidation_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 675\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m    676\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    677\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    392\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 394\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    395\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3474\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3475\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3476\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3477\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3478\u001b[0m     output_structure = nest.pack_sequence_as(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1470\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1471\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1472\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1473\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1474\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L_-qyhHeDno5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}