Md_Hosen_Zisad_Word2Vec_NLP_Task_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fZ3ut8XCatI8"
      },
      "source": [
        "##Import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wn_Izf81tDH9",
        "outputId": "34eac968-a3b5-47ed-8837-24ed88da7f72"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Importing the Keras libraries and packages\n",
        "import keras,os\n",
        "import tensorflow as tf\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "import string"
      ],
      "metadata": {
        "id": "zLuB_k51s58O"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b2ri6RQELPui",
        "outputId": "4866b34f-b8dc-4674-ffa2-0f9e00bc15de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Loading dataset and rename columns\n",
        "\n",
        "df = pd.read_csv(\"/content/drive/MyDrive/Thesis/Dataset/FAQs.csv\")\n",
        "df.columns=[\"questions\",\"answers\"];\n",
        "df"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           questions  \\\n",
              "0                     When was Albert Einstein born?   \n",
              "1                                 Where was he born?   \n",
              "2                                   When did he die?   \n",
              "3                              Who were his parents?   \n",
              "4              Did he have any sisters and brothers?   \n",
              "5                    Did he marry and have children?   \n",
              "6                Where did he receive his education?   \n",
              "7  When was Albert Einstein awarded the Nobel Pri...   \n",
              "8  Did Albert Einstein attend the Nobel Prize Awa...   \n",
              "9           For what did he receive the Nobel Prize?   \n",
              "\n",
              "                                             answers  \n",
              "0         Albert Einstein was born on 14 March 1879.  \n",
              "1                      He was born in Ulm, Germany.   \n",
              "2  He died 18 April 1955 in Princeton, New Jersey...  \n",
              "3  His father was Hermann Einstein and his mother...  \n",
              "4                      He had one sister named Maja.  \n",
              "5  He was married to Mileva Marić between 1903 an...  \n",
              "6  He received his main education at the followin...  \n",
              "7  The Nobel Prize Awarding Institution, the Roya...  \n",
              "8  The Nobel Prize was announced on 9 November 19...  \n",
              "9  Einstein was rewarded for his many contributio...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b5dfa91-35eb-40cf-8740-0df16dbf0b20\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>answers</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>When was Albert Einstein born?</td>\n",
              "      <td>Albert Einstein was born on 14 March 1879.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Where was he born?</td>\n",
              "      <td>He was born in Ulm, Germany.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>When did he die?</td>\n",
              "      <td>He died 18 April 1955 in Princeton, New Jersey...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Who were his parents?</td>\n",
              "      <td>His father was Hermann Einstein and his mother...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Did he have any sisters and brothers?</td>\n",
              "      <td>He had one sister named Maja.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Did he marry and have children?</td>\n",
              "      <td>He was married to Mileva Marić between 1903 an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Where did he receive his education?</td>\n",
              "      <td>He received his main education at the followin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>When was Albert Einstein awarded the Nobel Pri...</td>\n",
              "      <td>The Nobel Prize Awarding Institution, the Roya...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Did Albert Einstein attend the Nobel Prize Awa...</td>\n",
              "      <td>The Nobel Prize was announced on 9 November 19...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>For what did he receive the Nobel Prize?</td>\n",
              "      <td>Einstein was rewarded for his many contributio...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b5dfa91-35eb-40cf-8740-0df16dbf0b20')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4b5dfa91-35eb-40cf-8740-0df16dbf0b20 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4b5dfa91-35eb-40cf-8740-0df16dbf0b20');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jx8GakBta1ba"
      },
      "source": [
        "##Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CKYjAjKIMNuD",
        "outputId": "ead167bd-7bc6-41b4-dfa6-0a02482502b6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import re\n",
        "import gensim \n",
        "from gensim.parsing.preprocessing import remove_stopwords\n",
        "\n",
        "#keeping only words in the sentence and removing punctuations.\n",
        "\n",
        "def clean_sentence(sentence, stopwords=False):\n",
        "    \n",
        "    sentence = sentence.lower().strip()\n",
        "    sentence = re.sub(r'[^a-z0-9\\s]', '', sentence)\n",
        "\n",
        "    \n",
        "    if stopwords:\n",
        "         sentence = remove_stopwords(sentence)\n",
        "    \n",
        "\n",
        "    \n",
        "    return sentence\n",
        "                    \n",
        "def get_cleaned_sentences(df,stopwords=False):    \n",
        "    sents=df[[\"questions\"]];\n",
        "    cleaned_sentences=[]\n",
        "\n",
        "    for index,row in df.iterrows():\n",
        "        #print(index,row)\n",
        "        cleaned=clean_sentence(row[\"questions\"],stopwords);\n",
        "        cleaned_sentences.append(cleaned);\n",
        "    return cleaned_sentences;\n",
        "\n",
        "cleaned_sentences=get_cleaned_sentences(df,stopwords=True)\n",
        "#print(cleaned_sentences);\n",
        "\n",
        "print(\"\\n\")\n",
        "\n",
        "cleaned_sentences_with_stopwords=get_cleaned_sentences(df,stopwords=False)\n",
        "#print(cleaned_sentences_with_stopwords[0]);"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2-xJgtNkva49",
        "outputId": "01a28aa7-bea9-4cfe-dbe1-1c8d47b7759d"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HyFWo8pfveH3",
        "outputId": "f2cabefb-0eeb-4887-f343-a825ac218daf"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['questions', 'answers'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.isnull().sum())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ivD6ksoQvib1",
        "outputId": "528c87d7-1a93-48d3-ed65-03cb44012e03"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "questions    0\n",
            "answers      0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W60k9PEUckz_",
        "outputId": "e0d9d70c-d147-4c82-9899-8fce645e2703",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(cleaned_sentences_with_stopwords[0]);"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "when was albert einstein born\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6c5XPbI0u3uR"
      },
      "source": [
        "##Word2Vec-Skipgram Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "__W7NbDAs6js",
        "outputId": "6bbeb80f-b198-4579-c817-692230d1afa6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from gensim.models import Word2Vec \n",
        "import gensim.downloader as api\n",
        "\n",
        "#loading Word2vec google news pretrained model  \n",
        "v2w_model=None;\n",
        "try:\n",
        "    v2w_model = gensim.models.KeyedVectors.load(\"./w2vecmodel.mod\")\n",
        "    print(\"Loaded saved W2V model\")\n",
        "except:            \n",
        "    v2w_model = api.load('word2vec-google-news-300')\n",
        "    v2w_model.save(\"./w2vecmodel.mod\")\n",
        "    print(\"Downloaded and saved glove model\")\n",
        "\n",
        "w2vec_embedding_size=len(v2w_model['computer']);"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloaded and saved glove model\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I59O3tZTtUdi"
      },
      "source": [
        "#creating vectors\n",
        "def getWordVec(word,model):\n",
        "        samp=model['computer'];\n",
        "        vec=[0]*len(samp);\n",
        "        try:\n",
        "                vec=model[word];\n",
        "        except:\n",
        "                vec=[0]*len(samp);\n",
        "        return (vec)\n",
        "\n",
        "\n",
        "def getPhraseEmbedding(phrase,embeddingmodel):\n",
        "                       \n",
        "        samp=getWordVec('computer', embeddingmodel);\n",
        "        vec=numpy.array([0]*len(samp));\n",
        "        den=0;\n",
        "        for word in phrase.split():\n",
        "            #print(word)\n",
        "            den=den+1;\n",
        "            vec=vec+numpy.array(getWordVec(word,embeddingmodel));\n",
        "\n",
        "        return vec.reshape(1, -1)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HJxAgmWny5uw"
      },
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "#finding cosine similarity from vectors\n",
        "def retrieveAndPrintFAQAnswer(question_embedding,sentence_embeddings,FAQdf,sentences):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings):\n",
        "        #sim=cosine_similarity(embedding.reshape(1, -1),question_embedding.reshape(1, -1))[0][0];\n",
        "        sim=cosine_similarity(faq_embedding,question_embedding)[0][0];\n",
        "        print(index, sim, sentences[index])\n",
        "        if sim>max_sim:\n",
        "            max_sim=sim;\n",
        "            index_sim=index;\n",
        "    print(\"\\n\")\n",
        "    print(\"Question: \",question)\n",
        "    print(\"\\n\");\n",
        "    print(\"Retrieved: \",FAQdf.iloc[index_sim,0]) \n",
        "    print(FAQdf.iloc[index_sim,1]) "
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X3TNh0-Aymx_"
      },
      "source": [
        "question_orig=\"Who was his wife?\"\n",
        "#question_orig=input()\n",
        "question=clean_sentence(question_orig,stopwords=False);"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cb8cXiNutWcf",
        "outputId": "c71953f8-3074-41b4-c3d5-900b813318a6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#With w2Vec\n",
        "import numpy\n",
        "sent_embeddings=[];\n",
        "for sent in cleaned_sentences:\n",
        "    sent_embeddings.append(getPhraseEmbedding(sent,v2w_model));\n",
        "\n",
        "question_embedding=getPhraseEmbedding(question,v2w_model);\n",
        "\n",
        "retrieveAndPrintFAQAnswer(question_embedding,sent_embeddings,df, cleaned_sentences);"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 0.25802384215964247 albert einstein born\n",
            "1 0.32660566691400267 born\n",
            "2 0.1614098551897814 die\n",
            "3 0.3614683752376988 parents\n",
            "4 0.3834036401370407 sisters brothers\n",
            "5 0.3912437999536038 marry children\n",
            "6 0.10785768332422598 receive education\n",
            "7 0.15500725625985434 albert einstein awarded nobel prize physics\n",
            "8 0.2631050850224404 albert einstein attend nobel prize award ceremony\n",
            "9 0.15663458974063318 receive nobel prize\n",
            "\n",
            "\n",
            "Question:  who was his wife\n",
            "\n",
            "\n",
            "Retrieved:  Did he marry and have children?\n",
            "He was married to Mileva Marić between 1903 and 1919. They had three children, Lieserl (born 1902), Hans Albert (born 1904) and Eduard (born 1910). He married Elsa Löwenthal in 1919 and they lived together until her death in 1936.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A7cHSvBHidRO"
      },
      "source": [
        "#Implementing Chatbot using Word2Vec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-2dpBwuLJvwL"
      },
      "source": [
        "#embedding the exsiting dataset\n",
        "sent_embeddings=[];\n",
        "for sent in cleaned_sentences:\n",
        "  sent_embeddings.append(getPhraseEmbedding(sent,v2w_model));"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ar79m_aWT-O-",
        "outputId": "370bba26-3ad0-426b-ecc2-a6f050e3a684",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "import numpy\n",
        "import time\n",
        "\n",
        "print(\"Welcome to the Question Answering System\")      \n",
        "\n",
        "while True:\n",
        "  print(\"Question:\") \n",
        "  \n",
        "  question_orig=input()\n",
        "  \n",
        "  if (question_orig.lower()=='quit'):\n",
        "    break\n",
        "  measure1 = time.time()\n",
        "  question=clean_sentence(question_orig,stopwords=False);\n",
        "  \n",
        "\n",
        "\n",
        "  question_embedding=getPhraseEmbedding(question,v2w_model);  \n",
        "  def retrieveAndPrintFAQAnswer(question_embedding,sentence_embeddings,FAQdf,sentences):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings):\n",
        "      sim=cosine_similarity(faq_embedding,question_embedding)[0][0];\n",
        "      #print(index, sim, sentences[index])\n",
        "      if sim>max_sim:\n",
        "        max_sim=sim;\n",
        "        index_sim=index;\n",
        "    #print(\"\\n\")\n",
        "    #print(\"Question: \",question)\n",
        "    #print(\"\\n\");\n",
        "    #print(\"Retrieved: \",FAQdf.iloc[index_sim,0]) \n",
        "    #print(FAQdf.iloc[index_sim,1]) \n",
        "    print(\"Answer:\",FAQdf.iloc[index_sim,1])\n",
        "    print(\"----------------------\")\n",
        "    measure2 = time.time()\n",
        "    print(\"Response Time\",measure2 - measure1)\n",
        "    print(\"----------------------\")\n",
        "    print(\"\\n\");\n",
        "    print(\"Enter Quit to exit: \")\n",
        "  retrieveAndPrintFAQAnswer(question_embedding,sent_embeddings,df, cleaned_sentences);\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "print(\"----------------------\")\n",
        "print(\"Thank You\")\n",
        "print(\"----------------------\")"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Welcome to the Question Answering System\n",
            "Question:\n",
            "What was Einstein's father's name?\n",
            "Answer: His father was Hermann Einstein and his mother was Pauline Einstein (born Koch).\n",
            "----------------------\n",
            "Response Time 0.0053060054779052734\n",
            "----------------------\n",
            "\n",
            "\n",
            "Enter Quit to exit: \n",
            "Question:\n",
            "At what institutions did he study?\n",
            "Answer: He received his main education at the following schools: Catholic elementary school in Munich, Germany (1885-1888)Luitpold Gymnasium in Munich, Germany (1888-1894) Cantonal school in Aarau, Switzerland (1895-1896) Swiss Federal Institute of Technology in Zurich, Switzerland (1896-1900) Ph.D. from Zurich University, Switzerland (1905)\n",
            "----------------------\n",
            "Response Time 0.004952669143676758\n",
            "----------------------\n",
            "\n",
            "\n",
            "Enter Quit to exit: \n",
            "Question:\n",
            "quit\n",
            "----------------------\n",
            "Thank You\n",
            "----------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QAcXc1ug5LP9",
        "outputId": "b408dfde-e3c8-4d07-f7c1-81d97a7fdc79"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['questions', 'answers'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "B9rUxffl5oII"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**Test Data**"
      ],
      "metadata": {
        "id": "evPfRnrP7YKI"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4R88GCS-0ReG",
        "outputId": "1b64f637-096e-4b0c-d5c2-1d810648ed54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Load dataset and rename columns\n",
        "\n",
        "df_test= pd.read_csv(\"/content/drive/MyDrive/Thesis/Dataset/FAQs_test.csv\")\n",
        "df_test.columns=[\"questions\"];\n",
        "\n",
        "df_test"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            questions\n",
              "0      What is the date of his death?\n",
              "1         Did Einstein have siblings?\n",
              "2                   Who was his wife?\n",
              "3  What was Einstein's father's name?\n",
              "4  At what institutions did he study?"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-12faa371-1093-4b6e-a345-d22082d0a118\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What is the date of his death?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Did Einstein have siblings?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Who was his wife?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What was Einstein's father's name?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>At what institutions did he study?</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-12faa371-1093-4b6e-a345-d22082d0a118')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-12faa371-1093-4b6e-a345-d22082d0a118 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-12faa371-1093-4b6e-a345-d22082d0a118');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q3WNUsUE5Rh1",
        "outputId": "2c24c9ea-06f7-47ff-861b-b7511cc66745"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['questions'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "def retrieveAndPrintFAQAnswer(question_embeddings_test,sentence_embeddings,FAQdf,sentences,FAQdf_test,sentences_test):\n",
        "  for index_test,faq_embedding_test in enumerate(question_embeddings_test):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings):\n",
        "        \n",
        "      sim=cosine_similarity(faq_embedding,faq_embedding_test)[0][0];\n",
        "      #print(index, sim, sentences[index])\n",
        "      if sim>max_sim:\n",
        "        max_sim=sim;\n",
        "        index_sim=index;\n",
        "\n",
        "    FAQdf_test.iloc[index_test,2]=FAQdf.iloc[index_sim,2]\n",
        "    FAQdf_test.iloc[index_test,5]=FAQdf.iloc[index_sim,0]\n",
        "    print(\"\\n\")\n",
        "    print(\"Question: \",sentences_test[index_test])\n",
        "    print(\"\\n\");\n",
        "    print(\"Retrieved: \",FAQdf.iloc[index_sim,0]) \n",
        "    print(\"Answer:\",FAQdf.iloc[index_sim,1])    \n",
        "        "
      ],
      "metadata": {
        "id": "o8rxLRtM33az"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpsU_wT45G39",
        "outputId": "6a1bd387-62d0-4212-c75e-aca0e1cc942a"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['questions', 'answers'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(df_test.shape[0]):\n",
        " if df_test.iloc[i,0]!=df_test.iloc[i,0]:\n",
        "   df_test.iloc[i,0]='InCorrect'\n",
        " else:\n",
        "   df_test.iloc[i,0]='correct'\n",
        "df_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ZeX7NZa84Fid",
        "outputId": "0fdeacec-91f0-42af-c513-f7e06c7e633a"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  questions\n",
              "0   correct\n",
              "1   correct\n",
              "2   correct\n",
              "3   correct\n",
              "4   correct"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-95f4e3fd-ba62-4e11-a830-84a22b78f0f4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>correct</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>correct</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>correct</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>correct</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>correct</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-95f4e3fd-ba62-4e11-a830-84a22b78f0f4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-95f4e3fd-ba62-4e11-a830-84a22b78f0f4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-95f4e3fd-ba62-4e11-a830-84a22b78f0f4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KiYRIGu9zvWO",
        "outputId": "e5c84100-fbb8-4535-8b77-f6c7198e8e9f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(cleaned_sentences_with_stopwords[0]);"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "when was albert einstein born\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list_data_answers=df.questions.values.tolist()\n",
        "list_data_FoundCode=df.answers.tolist()"
      ],
      "metadata": {
        "id": "AJ29eiOq8jo0"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import multilabel_confusion_matrix\n",
        "from sklearn import metrics\n",
        "multilabel_confusion_matrix(list_data_answers, list_data_FoundCode)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BiwfMtrR9PZH",
        "outputId": "d2017ffe-7ea4-4e3d-8a86-3e2361a371d5"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 1.],\n",
              "        [0., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]],\n",
              "\n",
              "       [[9., 0.],\n",
              "        [1., 0.]]])"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "\n",
        "\n",
        "cm = confusion_matrix(list_data_answers, list_data_FoundCode, binary=False)"
      ],
      "metadata": {
        "id": "zDBXluNB9q7S"
      },
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "from mlxtend.plotting import plot_confusion_matrix\n",
        "\n",
        "import pprint\n",
        "#creating confusion matrix\n",
        "fig, ax = plot_confusion_matrix(conf_mat=cm)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 335
        },
        "id": "BI8pGX_w9vG3",
        "outputId": "45671aaf-61d1-42c0-c461-4034cc29fffc"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/mlxtend/plotting/plot_confusion_matrix.py:59: RuntimeWarning: invalid value encountered in true_divide\n",
            "  normed_conf_mat = conf_mat.astype('float') / total_samples\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQkAAAEGCAYAAAB2PmCxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXmElEQVR4nO2dfZBdZX3HPz9iU3yJik1EDEIS3hQUYxJ8qY4FX5iAIthRC7VTrVSohdpqteILEhwdVKp2qlbFTiZ0EBRsUYogaDojNaJkE3mJQgRiUhNekhBNKa7NYn/9456Em83du+e5u2d/53n2+5k5s/fe/ez9PueEPJz77PecmLsjhBBjsV/0AIQQ7UaThBCiL5okhBB90SQhhOiLJgkhRF8eFz2AOsyePdsPPXRe9DCEKJZNmzayfft26/W9LCaJQw+dx6ofDUUPQ4hieemLloz5vSw/btx4w7c59pijOObZh3PxJz/eajc6Pzc3Or9kN+U998Ldp3wDlgLrgXuA88bzFy1a7MMj7sMj7v/zm0d9/oIF/tP19/rOR/7Xn/e8Y33tbT/Z8/3uLdqNzs/Njc4v2R3PW7RosY/192/KzyTMbAbweeAk4GjgDDM7uu7Pr77lFg477HDmL1jAzJkzeeMfnc61//7NVrrR+bm50fkluynvOZqIjxsvBO5x9w3uvgv4KnBq3R++774tHHzws/Y8nzv3YLZs2dJKNzo/Nzc6v2Q35T1HEzFJzAV+0fV8c/XaXpjZWWY2ZGZD27Zvm7LBCSH2prULl+5+ibsvcfclc2bP2fP6M585l82bH5tjtmzZzNy5+8wxrXCj83Nzo/NLdlPecx8CFi1fAtzQ9fz9wPvrLlw+PDzi8+bP9zt/tmHPAsyaW9f1XNSJdqPzc3Oj80t2x/P6LVxGTBKPAzYA84GZwG3AMXUnieER96uv+ZYffsQRPn/BAl/2kY/2PHhtcaPzc3Oj80t2+3n9JgnzgPtJmNnJwD8AM4Dl7v6xfv7ixUtcZSohmuOlL1rCmjVD7Wlcuvt1wHUR2WL6cMBx59Z2f7n6cw2OJG9au3AphGgHWU4S0fXWFDc6Pze3qfwvXvBmNq28iKGrPtDXa3IM0W5WtezUTbXs6eFO9nvuv/CcPdsr3/Zpf/HpF/m6u7fs9frurS3HIOrYtqqWPVGi660pbnR+bm5T+QCr1t7Ljp2/HvP7bTkGTbm51bInRHS9NcWNzs/NbSo/hehj0JSbWy1bCJER2U0S0fXWFDc6Pze3qfwUoo9BU25WteyJLlxG11tT3Oj83NzJfs/Ri5NHnnT+uAuX0ccg6ti2qpY90UlieCS+3priRufn5k7me3ZPAl+7frXft/VXvmvXo775gR1+9rLLek4SbTgGEce2dbXsVFTLFoOgxmV9WlfLFs2ivxwdSt63qSS7hUuIb66luNH5KU3DNow3Or9kV43LHp/Bot2o/EGahtP5eE0HV43LAlpuTbl1m4ZtGG90fsmuGpcFtNzUNozPL9lV41II0RjZTRLRzbUUNzo/lejxRueX7KpxmWnLrSl3kKbhdD5e08FV47LHwWuLG5E/aNNwuh6v6eKqcSn2oDKVSKVf4zK7NQkhxNSiWnaBtOHsQGcz5ZDlmUR0vTXFjc5vg6ub0LbDVS27x0JNtBudH+nqJrTtclXLLqAKW7I73W9C2wZXtewCqrAlu3Vpw1hLdVXLFkI0RnaTRHS9NcWNzm+LW5c2jLVUV7XsTKuwJbu6CW27XNWyexy8trjR+VGubkLbPle1bNEqVKbKC9WyhRADo1q2aASdHZRDlmcS0fXWFDc6Pzc3Or9kV7XsHgs10W50fm5udH7JrmrZBVRh5cbnl+yqll1AFVZufH7JrmrZQojGyG6SiK63prjR+bm50fklu6plZ1qFlduu/JJd1bJ7HLy2uNH5ubnR+SW7qmULIQamXy1bjcsGqHvdglqJIgeyW7gUQkwtWU4S0fXWFFd3ik5zo/NLdrOqZQMbgTuAW4GhlIXL6HprHbfunaLbMNY2udH5Jbu51rJPcPeF7r4k5Yei662pru4UrVp2G1zVsltahU2hDWONdqPzS3ZzrGU7cKOZrTGzs3oJZnaWmQ2Z2dC27dumeHhCiN1ETRIvc/dFwEnAOWb28tGCu1/i7kvcfcmc2XP2vB5db01169KGsUa70fklu1nXsoFlwHvqLlxG11vruHXvFN2GsbbJjc4v2c2qlg08EZjV9fgHwNK6k8TwSHy9dTy37p2i2zDWtrnR+SW72dSyzWwBcHX19HHA5e7+sX4/k1stW41LkRutqmW7+wbg+VOdO5XoL78oiex+BSqEmFqynCSi660pbnR+bm50fsluVrXs1C23WnZb8nNzo/NLdnOtZQ9EdL01xY3Oz82Nzi/ZVS27gCqs3Pj8kt0ca9lCiEzIbpKIrremuNH5ubnR+SW7WdeyUxcuo+utKW50fm5udH7Jbla17IlOEsMj8fXWFDc6Pzc3Or9kN5ta9iDkVssWIjf61bKzW5MQQkwtmiSEEH3JcpKIrremuNH5ubnR+SW7qmX3WKiJdqPzc3Oj80t2VcsuoAorNz6/ZFe17AKqsHLj80t2VcsWQjRGdpNEdL01xY3Oz82Nzi/ZVS070yqs3Hbll+yqlt3j4LXFjc7PzY3OL9lVLVsIMTCqZQshBmbMW+qb2cN0/s1OgN0zjFeP3d2f3PDYhBAtYMwzCXef5e5PrrZZXc9nRU8Q0fXWFDc6Pzc3Or9kt9FaNvAy4M+qx7OB+VELl9H11hQ3Oj83Nzq/ZLfRWraZXQC8D3h/9dJM4LL609DkEl1vTXGj83Nzo/NLdpuuZb8eeB3wCIC73wfMqvXuDRBdb01xo/Nzc6PzS3abrmXv8s7nDAcwsyfWemchRBHUmSSuNLMvAU81s7cD3wW+3Oywxia63priRufn5kbnl+w2XssGXg1cXG2vjmxcRtdbU9zo/Nzc6PyS3cZr2cAz6KxLnAI8I3KSGB6Jr7emuNH5ubnR+SW7jdWyzezPgQ8D/0GnSPUHwEfcfXm9c5WJo1q2EM3Sr5Y9ZuOyi/cCL3D3hwDM7PeAHwBTNkkIIeKos3D5EPBw1/OHq9fCiG6upbjR+bm50fklu5PeuATeXW3/AvwYWAZcAKwFVkStSUQ311Lc6Pzc3Oj8kt2mGpezqu1e4Bs8drHXN4Gf15+GJpfo5lqKG52fmxudX7LbSOPS3S/st9V69waIbq6luNH5ubnR+SW7E2lcjrtwaWZzgL8DjgH23/26u7+iVoIQImvqLFx+BbgLmA9cCGwEVjc4pr5EN9dS3Oj83Nzo/JLdRhuXwJrq6+1dr62OWriMbq6luNH5ubnR+SW7jTYugR9WX28AXgO8ALg3apIYHolvrqW40fm5udH5JbtNNi5fC/wn8Czgs8CTgQvd/Zp65yoTR41LIZplQo1Ld7+2ergTOGEyByaEaD/9boT7WR7rRuyDu7+zkREJIVpFv99uDAFr+mxhRNdbU9zo/Nzc6PyS3UZvhDvIRucCsK3Auq7XngZ8B7i7+npA6sJldL01xY3Oz82Nzi/ZbfRGuBNgBbB01GvnASvd/QhgZfU8ieh6a4obnZ+bG51fstv0jXAHwt1vAnaMevlU4NLq8aXAaanvG11vTXGj83Nzo/NLdpu+Ee5kcqC73189fgA4cIrzhRCJ1Pl3N440s5Vmtq56fqyZfWiiwd5ZpBjztydmdpaZDZnZ0Lbt2/a8Hl1vTXGj83Nzo/NLdpuuZX8PeCHw467X1tVZcATmsffC5XrgoOrxQcD61IXL6Hprihudn5sbnV+y23Qte3X1tXuSuHXASeJi4Lzq8XnAJ1XLltum/JLdJmvZ1wPnAle5+yIzewNwprufNM7PXQEcT+ffDn2Qzl2tvgFcCRwCbALe5O6jFzf3QbVsIZplojfCPQe4BHi2mW2hc1eqPxnvh9z9jDG+9coamUKIllDn2o0NwKuqf95vP3d/eLyfEUKUQ53fbnzYzD4M/C3wrq7nYUTXW1Pc6Pzc3Oj8kt3Gatl0Jofd2weBm4Hlg1S1B91Uy54ebnR+yW6jtWx3/1TX9jE6i5EL6k9Dk0t0vTXFjc7PzY3OL9md6lr2E4CDB/i5SSG63priRufn5kbnl+w2fbfsO3isGTkDmAN8pNa7CyGyp86vQF/b9fhR4EF3f7Sh8YxLdL01xY3Oz82Nzi/ZbayWTefM4a6pXKQcb+Eyut6a4kbn5+ZG55fsNl3L/iZwSFsmieGR+Hprihudn5sbnV+y22Qt+yY6t9G/BXik6wzkdfXOVSaOatlCNMtEa9nnT/J4hBAZUWeSONnd39f9gpl9gs4l5EKIwqnTk3h1j9f6XgHaNNH11hQ3Oj83Nzq/ZHfSa9nAO4A76KxD3N61/Ry4LGrhMrremuJG5+fmRueX7DZVy74cOAW4pvq6e1vs7uNeKt4U0fXWFDc6Pzc3Or9kt5FatrvvdPeN7n6Gu2/q2sa9SUyTRNdbU9zo/Nzc6PyS3Zzuli2EyIzsJonoemuKG52fmxudX7Lb6N2y27Cplj093Oj8kt1Ga9lt2FTLnj5udH7JbmO17DagWrYQzdKvlp3dmoQQYmrRJCGE6EuWk0R0vTXFjc7PzY3OL9lt7G7ZbdhUy54ebnR+yW6jd8tuG9H11hQ3Oj83Nzq/ZHeq75YdSnS9NcWNzs/Njc4v2VUtWwjRGNlNEtH11hQ3Oj83Nzq/ZFe17EyrsHLblV+yq1p2j4PXFjc6Pzc3Or9kV7VsIcTAqJYthBiYOnfLFi3ggOPOre3+cvXnGhyJmG5keSYRXW9NcZvK/+IFb2bTyosYuuoDfb0mx5DT8ZKrWnbPhZpodzLfc/+F5+y1vfJtn/YXn36Rr7t7yz7fa9MxiDpectM81bIzqMKm1mZXrb2XHTt/Peb323IMUtzo/JJd1bILqMJOpDbbj+hjkOJG55fsqpYthGiM7CaJ6HprittUfgrRxyDFjc4v2VUtO9MqbIo7enFy/4Xn+JEnnT/uwmX0MYg6XnLTPNWyW1qFTXFHTwRfu36137f1V75r16O++YEdfvayy3pOEm04BhHHS26ap1p2AahMJZqkXy1bjctM0F/8Dposp57sFi6FEFNLlpNEdL01xY3Oz81VNb05t3W1bGA5sBVY1/XaMmALcGu1nZy6cBldb01xo/Nzc+t4davp3Qu40fvVBrettewVwNIer3/G3RdW23Wpbxpdb01xo/Nzc1VNb85tZS3b3W8Cdkz2+0bXW1Pc6PzcXFXTm3Nzq2Wfa2a3m9lyMztgLMnMzjKzITMb2rZ921SOTwjRxVRPEl8ADgMWAvcDnxpLdPdL3H2Juy+ZM3vOntej660pbnR+bq6q6c25ra1lA/PoWris+71+C5fR9dYUNzo/N7eOV7ea3r1wGb1fbXBbW8sePREAB3U9fhfw1dRJYngkvt6a4kbn5+aO59Wtpo+up0fvVxvc1tWyzewK4HhgNvAgcEH1fCHgwEbgbHe/f7z3Ui1b7EaNy2boV8vO4tqN/Z7wdP/do95Uy9V/GEKko1vqCyEGJrtJIqWKC3lVYeXG55fstq6WPZmbPX5OchV3eKT9VVi57cov2W1rLbsR6lZxIa8qrNz4/JLdVtay20BOVVi58fklu7nVsoUQGVH0JJFTFVZufH7Jbmtr2U0sXNat4g6PtL8KK7dd+SW7ra1lNzFJpFRxh0faXYWV2778kt3W1bInEzUuhWiW7O+W/YLnHMKqH+kvfxPoWggxHlkuXEY311Lc6PwUtw1t1uhjULJbdONSN8Jtzh3kxrLT+Xjl6k6rxmV0cy3Fjc5PdaPbrG04BqW606pxGd1cS3Gj81PdFHS88nLVuBRCNEZ2k0R0cy3Fjc5PdVPQ8crLLb5xqRvhNucOcmPZ6Xy8cnWLb1zqRrjNuYPeWHa6Hq+c3aIbl7oRbnOoTCVA97gUQkyALGrZojlSzg501jE9yfJMIrremuJG5zflNlXhjt6vkl3Vsnss1ES70fmT7TZd4c7hGOTqqpZdQBU2N7eJCncb9qtUV7XsAqqwubkp6HjFu6plCyEaI7tJIrremuJG5zfppqDjFe+qlp1pFTY3t+kKdw7HIFdXteweB68tbnT+ZLpTUeFu+zHI2VUtWzSOylTlolq2EGJgVMsWtWnq7EBnKO0myzOJ6Hprihudn5ubUveOHmturmrZPRZqot3o/FzcQereOexXm1zVsguowsrtULfu3Yax5uSqll1AFVZuGm0Ya06uatlCiMbIbpKIrremuNH5Obp1acNYc3JVy860Cit334XLunXvHParTa5q2T0OXlvc6Pwc3EHr3m3fr7a5qmWLbFGZKh7VsoUQA5PFmYSZbQM2jXp5NrA9YDhTQan7pv1qL4e6+5xe38hikuiFmQ25+5LocTRBqfum/coTfdwQQvRFk4QQoi85TxKXRA+gQUrdN+1XhmS7JiGEmBpyPpMQQkwBmiSEEH3JcpIws6Vmtt7M7jGz86LHM1mY2UYzu8PMbjWzrCumZrbczLaa2bqu155mZt8xs7urrwdEjnEQxtivZWa2pfpzu9XMTo4c42ST3SRhZjOAzwMnAUcDZ5jZ0bGjmlROcPeFBfzefQWwdNRr5wEr3f0IYGX1PDdWsO9+AXym+nNb6O7XTfGYGiW7SQJ4IXCPu29w913AV4FTg8ckRuHuNwE7Rr18KnBp9fhS4LQpHdQkMMZ+FU2Ok8Rc4BddzzdXr5WAAzea2RozOyt6MA1woLvfXz1+ADgwcjCTzLlmdnv1cSS7j1H9yHGSKJmXufsiOh+lzjGzl0cPqCm887v3Un7//gXgMGAhcD/wqdjhTC45ThJbgGd1PT+4ei173H1L9XUrcDWdj1Yl8aCZHQRQfd0aPJ5Jwd0fdPffuvv/AV+msD+3HCeJ1cARZjbfzGYCpwPXBI9pwpjZE81s1u7HwInAuv4/lR3XAG+pHr8FqHe75paze+KreD2F/bll9y94ufujZnYucAMwA1ju7j8JHtZkcCBwtZlB58/lcnf/duyQBsfMrgCOB2ab2WbgAuDjwJVmdiadS//fFDfCwRhjv443s4V0Pj5tBM4OG2ADqJYthOhLjh83hBBTiCYJIURfNEkIIfqiSUII0RdNEkKIvmiSEHthZseb2bXV49f1u8rWzJ5qZn85QMYyM3tP3ddHOSvM7A0JWfO6r9gU6WiSmCZUV88m4e7XuPvH+yhPBZInCZEXmiQyp/o/5V1m9hUzu9PMvm5mT6i+t9HMPmFma4E3mtmJZnazma01s6vM7EmVt7R6j7XAH3a991vN7HPV4wPN7Gozu63afp9OOeqw6h4KF1fee81sdXWx04Vd7/VBM/uZmX0fOKrGfr29ep/bzOxfd+9TxavMbKh6v9dW/gwzu7gru6hCUySaJMrgKOCf3P05wH+z9//dH6ouGvsu8CHgVdXzIeDdZrY/nesNTgEWA88YI+Mfge+5+/OBRcBP6NwP4t7qHgrvNbMTgSPoXLuwEFhsZi83s8V06vMLgZOB42rs07+5+3FV3p3AmV3fm1dlvAb4YrUPZwI73f246v3fbmbza+SIcciuli168gt3X1U9vgx4J/D31fOvVV9fTOcmPauq6vdM4Gbg2cDP3f1uADO7DOh1mforgD8FcPffAjt7XBJ9YrX9uHr+JDqTxizganf/dZVR51qb55rZR+l8pHkSnRr+bq6sLqa628w2VPtwInBs13rFU6rsn9XIEn3QJFEGo7v13c8fqb4a8B13P6NbrK45mCwMuMjdvzQq428GeK8VwGnufpuZvZXO9RK76bW/BvyVu3dPJpjZvAGyRRf6uFEGh5jZS6rHfwx8v4fzQ+ClZnY47Lnq9EjgLmCemR1WeWf0+Fno3G7uHdXPzjCzpwAP0zlL2M0NwNu61jrmmtnTgZuA08zs8dWVrqfU2KdZwP1m9jvAm0d9741mtl815gXA+ir7HZWPmR1ZXU0rJogmiTJYT+cmNXcCB9C5CcpeuPs24K3AFWZ2O9VHDXf/DZ2PF9+qFi7HusfDXwMnmNkdwBrgaHd/iM7Hl3VmdrG73whcDtxceV8HZrn7Wjofe24Drqdzuf94nA/8CFhFZyLr5r+AW6r3+otqH/4Z+CmwtvqV55fQmfKkoKtAM6c6nb7W3Z8bPBRRKDqTEEL0RWcSQoi+6ExCCNEXTRJCiL5okhBC9EWThBCiL5okhBB9+X9lHCG91jPv1gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-s0dN_zw90Jm",
        "outputId": "7724d61e-3b9a-4fad-ea62-94b0d25d4b6e"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                                                                                                                                                                                                                                                                                                                                 precision    recall  f1-score   support\n",
            "\n",
            "                                                                                                                                                                                                                                                                                                     Albert Einstein was born on 14 March 1879.       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                                                                                                                                                     Did Albert Einstein attend the Nobel Prize Award Ceremony?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                                          Did he have any sisters and brothers?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                                                Did he marry and have children?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                              Einstein was rewarded for his many contributions to theoretical physics, and especially for his discovery of the law of the photoelectric effect.       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                                                                                                                                                                       For what did he receive the Nobel Prize?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                           He died 18 April 1955 in Princeton, New Jersey, USA.       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                                                                                                                                                                                  He had one sister named Maja.       0.00      0.00      0.00       0.0\n",
            "He received his main education at the following schools: Catholic elementary school in Munich, Germany (1885-1888)Luitpold Gymnasium in Munich, Germany (1888-1894) Cantonal school in Aarau, Switzerland (1895-1896) Swiss Federal Institute of Technology in Zurich, Switzerland (1896-1900) Ph.D. from Zurich University, Switzerland (1905)       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                                                                                                                                                                                  He was born in Ulm, Germany.        0.00      0.00      0.00       0.0\n",
            "                                                                                                         He was married to Mileva Marić between 1903 and 1919. They had three children, Lieserl (born 1902), Hans Albert (born 1904) and Eduard (born 1910). He married Elsa Löwenthal in 1919 and they lived together until her death in 1936.       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                                                                                                                               His father was Hermann Einstein and his mother was Pauline Einstein (born Koch).       0.00      0.00      0.00       0.0\n",
            "  The Nobel Prize Awarding Institution, the Royal Swedish Academy of Sciences, decided to reserve the Nobel Prize in Physics in 1921, and therefore no Physics Prize was awarded that year. According to the statutes, a reserved prize can be awarded the year after, and Albert Einstein was awarded the 1921 Nobel Prize in Physics in 1922.       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                     The Nobel Prize was announced on 9 November 1922. Being too remote from Sweden, Albert Einstein could not attend the Nobel Prize Award Ceremony in Stockholm on 10 December the same year.       0.00      0.00      0.00       0.0\n",
            "                                                                                                                                                                                                                                                                                                                               When did he die?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                   When was Albert Einstein awarded the Nobel Prize in Physics?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                                                 When was Albert Einstein born?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                                            Where did he receive his education?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                                                             Where was he born?       0.00      0.00      0.00       1.0\n",
            "                                                                                                                                                                                                                                                                                                                          Who were his parents?       0.00      0.00      0.00       1.0\n",
            "\n",
            "                                                                                                                                                                                                                                                                                                                                       accuracy                           0.00      10.0\n",
            "                                                                                                                                                                                                                                                                                                                                      macro avg       0.00      0.00      0.00      10.0\n",
            "                                                                                                                                                                                                                                                                                                                                   weighted avg       0.00      0.00      0.00      10.0\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        }
      ]
    }
  ]
}