Named Entity Recognition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2ebc512c",
   "metadata": {},
   "source": [
    "## Using NLTK Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6fe586e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7ad19649",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('./sample_text.txt', 'r')\n",
    "text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "337de0af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'artificial intelligence (AI), the ability of a digital computer or computer-controlled robot to perform tasks commonly associated with intelligent beings. The term is frequently applied to the project'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[:200]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6713e35f",
   "metadata": {},
   "source": [
    "Word based approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b6f03a2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "words = nltk.word_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2fed7d0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artificial', 'JJ'),\n",
       " ('intelligence', 'NN'),\n",
       " ('(', '('),\n",
       " ('AI', 'NNP'),\n",
       " (')', ')')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Part of speech tagging\n",
    "pos_tags = nltk.pos_tag(words)\n",
    "pos_tags[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fd360243",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NNP: noun, proper, singular\n",
      "    Motown Venneboerger Czestochwa Ranzer Conchita Trumplane Christos\n",
      "    Oceanside Escobar Kreisler Sawyer Cougar Yvette Ervin ODI Darryl CTCA\n",
      "    Shannon A.K.C. Meltex Liverpool ...\n"
     ]
    }
   ],
   "source": [
    "nltk.help.upenn_tagset('NNP')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1d8fa633",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entities</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DataRobot</td>\n",
       "      <td>ORGANIZATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AI</td>\n",
       "      <td>ORGANIZATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sphex</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AI</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Entities        labels\n",
       "0  DataRobot  ORGANIZATION\n",
       "1         AI  ORGANIZATION\n",
       "2      Sphex           GPE\n",
       "3         AI           GPE"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Chunking\n",
    "chunks = nltk.ne_chunk(pos_tags, binary=False)\n",
    "entities = []\n",
    "labels = []\n",
    "for chunk in chunks:\n",
    "    if hasattr(chunk, 'label'):\n",
    "        entities.append(' '.join(c[0] for c in chunk))\n",
    "        labels.append(chunk.label())\n",
    "\n",
    "entities_labels = list(set(zip(entities, labels)))\n",
    "entities_df = pd.DataFrame(entities_labels)\n",
    "entities_df.columns = ['Entities', 'labels']\n",
    "entities_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fd33349",
   "metadata": {},
   "source": [
    "Sentence Based"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e80f03ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entities</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Inductive</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Research</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sphex</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AI</td>\n",
       "      <td>ORGANIZATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DataRobot</td>\n",
       "      <td>ORGANIZATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AI</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Artificial</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Entities        labels\n",
       "0   Inductive           GPE\n",
       "1    Research           GPE\n",
       "2       Sphex           GPE\n",
       "3          AI  ORGANIZATION\n",
       "4   DataRobot  ORGANIZATION\n",
       "5          AI           GPE\n",
       "6  Artificial           GPE"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entities = []\n",
    "labels = []\n",
    "\n",
    "sentences = nltk.sent_tokenize(text)\n",
    "for sent in sentences:\n",
    "    pos_tags = nltk.pos_tag(nltk.word_tokenize(sent))\n",
    "    for chunk in nltk.ne_chunk(pos_tags, binary=False):\n",
    "        if hasattr(chunk, 'label'):\n",
    "            entities.append(' '.join(c[0] for c in chunk))\n",
    "            labels.append(chunk.label())\n",
    "entities_labels = list(set(zip(entities, labels)))\n",
    "entities_df = pd.DataFrame(entities_labels)\n",
    "entities_df.columns = ['Entities', 'labels']\n",
    "entities_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8ecdb468",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_named_entities(text, binary=True):\n",
    "    entities = []\n",
    "    labels = []\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    for sent in sentences:\n",
    "        pos_tags = nltk.pos_tag(nltk.word_tokenize(sent))\n",
    "        for chunk in nltk.ne_chunk(pos_tags, binary=binary):\n",
    "            if hasattr(chunk, 'label'):\n",
    "                entities.append(' '.join(c[0] for c in chunk))\n",
    "                labels.append(chunk.label())\n",
    "    entities_labels = list(set(zip(entities, labels)))\n",
    "    entities_df = pd.DataFrame(entities_labels)\n",
    "    entities_df.columns = ['Entities', 'labels']\n",
    "    return entities_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fa53bba1",
   "metadata": {},
   "outputs": [],
   "source": [
    "text2 = '''Global warming or climate change has today become a major threat to the mankind. The Earth’s temperature is on the rise and there are various reasons for it such as greenhouse gases emanating from carbon dioxide (CO2) emissions, burning of fossil fuels or deforestation. Recent years have been unusually warm, causing worldwide concern. But the fact is that the increase in carbon dioxide actually began in 1800, due to the deforestation of a large chunk of North-eastern American, besides forested parts of the world. The things became worse with emissions in the wake of the industrial revolution, leading to increase in carbon dioxide level by 1900.The sea levels are constantly rising as fresh water marshlands, low-lying cities, and islands have been inundated with seawater\n",
    "There have been changes in rainfall patterns, leading to droughts and fires in some areas, and flooding in other areas.\n",
    "Ice caps are constantly melting posing a threat to polar bears as their feeding season stands reduced.\n",
    "Glaciers are gradually melting.\n",
    "Animal populations are gradually vanishing as there has been a widespread loss of their habitat.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4a3f67cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entities</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>American</td>\n",
       "      <td>NE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Animal</td>\n",
       "      <td>NE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Global</td>\n",
       "      <td>NE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CO2</td>\n",
       "      <td>NE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Earth</td>\n",
       "      <td>NE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Entities labels\n",
       "0  American     NE\n",
       "1    Animal     NE\n",
       "2    Global     NE\n",
       "3       CO2     NE\n",
       "4     Earth     NE"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_named_entities(text2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d90ea6e",
   "metadata": {},
   "source": [
    "## Using Spacy Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b03862b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b0d1717",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "995a9eac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting en-core-web-sm==3.2.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl (13.9 MB)\n",
      "Requirement already satisfied: spacy<3.3.0,>=3.2.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from en-core-web-sm==3.2.0) (3.2.2)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.11.3)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.27.1)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.1)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.7.6)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (8.0.15)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (21.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (4.62.3)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.8.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.6)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.6.1)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.3.0)\n",
      "Requirement already satisfied: setuptools in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (52.0.0.post20210125)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.4.2)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.9)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.22.3)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.6)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.9.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.4.7)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (5.2.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.10.0.2)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.26.7)\n",
      "Requirement already satisfied: colorama in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from tqdm<5.0.0,>=4.38.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.4)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (8.0.3)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\nihal\\anaconda3\\envs\\mlprojects\\lib\\site-packages (from jinja2->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.1.1)\n",
      "Installing collected packages: en-core-web-sm\n",
      "Successfully installed en-core-web-sm-3.2.0\n",
      "[+] Download and installation successful\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "af1ed242",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c0be3756",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entities</th>\n",
       "      <th>Labels</th>\n",
       "      <th>Position_Start</th>\n",
       "      <th>Position_End</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>25</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(the, 1940s)</td>\n",
       "      <td>DATE</td>\n",
       "      <td>430</td>\n",
       "      <td>439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(Sphex)</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>1400</td>\n",
       "      <td>1405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(first)</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>1478</td>\n",
       "      <td>1483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(waspâ€)</td>\n",
       "      <td>GPE</td>\n",
       "      <td>1640</td>\n",
       "      <td>1646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(a, few, inches)</td>\n",
       "      <td>QUANTITY</td>\n",
       "      <td>1704</td>\n",
       "      <td>1716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>GPE</td>\n",
       "      <td>2117</td>\n",
       "      <td>2119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(one)</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>3569</td>\n",
       "      <td>3572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>3609</td>\n",
       "      <td>3611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>(one)</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>3969</td>\n",
       "      <td>3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>(DataRobot)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>4143</td>\n",
       "      <td>4152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>4193</td>\n",
       "      <td>4195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>(DataRobot)</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4460</td>\n",
       "      <td>4469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>4505</td>\n",
       "      <td>4507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>(DataRobot)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>4679</td>\n",
       "      <td>4688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>GPE</td>\n",
       "      <td>4708</td>\n",
       "      <td>4710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>(AI)</td>\n",
       "      <td>ORG</td>\n",
       "      <td>4801</td>\n",
       "      <td>4803</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Entities    Labels  Position_Start  Position_End\n",
       "0               (AI)       ORG              25            27\n",
       "1       (the, 1940s)      DATE             430           439\n",
       "2            (Sphex)    PERSON            1400          1405\n",
       "3            (first)   ORDINAL            1478          1483\n",
       "4           (waspâ€)       GPE            1640          1646\n",
       "5   (a, few, inches)  QUANTITY            1704          1716\n",
       "6               (AI)       GPE            2117          2119\n",
       "7              (one)  CARDINAL            3569          3572\n",
       "8               (AI)       ORG            3609          3611\n",
       "9              (one)  CARDINAL            3969          3972\n",
       "10       (DataRobot)       ORG            4143          4152\n",
       "11              (AI)       ORG            4193          4195\n",
       "12       (DataRobot)    PERSON            4460          4469\n",
       "13              (AI)       ORG            4505          4507\n",
       "14       (DataRobot)       ORG            4679          4688\n",
       "15              (AI)       GPE            4708          4710\n",
       "16              (AI)       ORG            4801          4803"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc = nlp(text)\n",
    "\n",
    "entities = []\n",
    "labels = []\n",
    "position_start = []\n",
    "position_end = []\n",
    "\n",
    "for ent in doc.ents:\n",
    "    entities.append(ent)\n",
    "    labels.append(ent.label_)\n",
    "    position_start.append(ent.start_char)\n",
    "    position_end.append(ent.end_char)\n",
    "\n",
    "df = pd.DataFrame({'Entities': entities, 'Labels':labels, 'Position_Start': position_start, 'Position_End':position_end})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "10208aca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_elements(x):\n",
    "    s = ''\n",
    "    for e in x:\n",
    "        s += str(e)+\" \"\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8b082a85",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_named_entities(text):\n",
    "    nlp = spacy.load('en_core_web_sm')\n",
    "    doc = nlp(text)\n",
    "    entities = []\n",
    "    labels = []\n",
    "    position_start = []\n",
    "    position_end = []\n",
    "\n",
    "    for ent in doc.ents:\n",
    "        entities.append(ent)\n",
    "        labels.append(ent.label_)\n",
    "        position_start.append(ent.start_char)\n",
    "        position_end.append(ent.end_char)\n",
    "\n",
    "    df = pd.DataFrame({'Entities': entities, 'Labels':labels, 'Position_Start': position_start, 'Position_End':position_end})\n",
    "    df['Entities'] = df['Entities'].apply(combine_elements)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "087fa5f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entities</th>\n",
       "      <th>Labels</th>\n",
       "      <th>Position_Start</th>\n",
       "      <th>Position_End</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>today</td>\n",
       "      <td>DATE</td>\n",
       "      <td>37</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Earth</td>\n",
       "      <td>LOC</td>\n",
       "      <td>85</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Recent years</td>\n",
       "      <td>DATE</td>\n",
       "      <td>271</td>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1800</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>407</td>\n",
       "      <td>411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>North - eastern</td>\n",
       "      <td>NORP</td>\n",
       "      <td>458</td>\n",
       "      <td>471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>American</td>\n",
       "      <td>NORP</td>\n",
       "      <td>472</td>\n",
       "      <td>480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1900.The</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>647</td>\n",
       "      <td>655</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Entities    Labels  Position_Start  Position_End\n",
       "0            today       DATE              37            42\n",
       "1            Earth        LOC              85            90\n",
       "2     Recent years       DATE             271           283\n",
       "3             1800   CARDINAL             407           411\n",
       "4  North - eastern       NORP             458           471\n",
       "5         American       NORP             472           480\n",
       "6         1900.The   CARDINAL             647           655"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_named_entities(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a943c8c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}