inltk_language_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting inltk\n",
      "  Downloading https://files.pythonhosted.org/packages/c9/41/2c62e8d080d6d256f3a881b85c22b564aa141d4bc84681056057c018b885/inltk-0.6.1-py3-none-any.whl\n",
      "Requirement already satisfied: pandas in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (0.24.2)\n",
      "Requirement already satisfied: numexpr in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (2.6.5)\n",
      "Requirement already satisfied: Pillow in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (5.2.0)\n",
      "Collecting nvidia-ml-py3 (from inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/6d/64/cce82bddb80c0b0f5c703bbdafa94bfb69a1c5ad7a79cff00b482468f0d3/nvidia-ml-py3-7.352.0.tar.gz\n",
      "Requirement already satisfied: requests in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (2.20.0)\n",
      "Collecting spacy>=2.0.18 (from inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b9/05/e82c888a36f24608664b56abe737f4428410d370791f6112fb3e9b4a4a81/spacy-2.2.2-cp36-cp36m-manylinux1_x86_64.whl (10.3MB)\n",
      "\u001b[K     |████████████████████████████████| 10.3MB 9.3MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: typing in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (3.6.4)\n",
      "Requirement already satisfied: numpy>=1.15 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (1.15.4)\n",
      "Collecting fastprogress>=0.1.19 (from inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/83/db/794db47024a26c75635c35f0ee5431aa8b528e895ad1ed958041290f83f7/fastprogress-0.1.21-py3-none-any.whl\n",
      "Collecting async-timeout>=3.0.1 (from inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/e1/1e/5a4441be21b0726c4464f3f23c8b19628372f606755a9d2e46c187e65ec4/async_timeout-3.0.1-py3-none-any.whl\n",
      "Collecting aiohttp>=3.5.4 (from inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/39/7eb5f98d24904e0f6d3edb505d4aa60e3ef83c0a58d6fe18244a51757247/aiohttp-3.6.2-cp36-cp36m-manylinux1_x86_64.whl (1.2MB)\n",
      "\u001b[K     |████████████████████████████████| 1.2MB 55.0MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: matplotlib in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (2.2.2)\n",
      "Requirement already satisfied: packaging in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (17.1)\n",
      "Requirement already satisfied: beautifulsoup4 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (4.6.0)\n",
      "Requirement already satisfied: scipy in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (1.1.0)\n",
      "Collecting sentencepiece (from inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/3d/efb655a670b98f62ec32d66954e1109f403db4d937c50d779a75b9763a29/sentencepiece-0.1.83-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 55.1MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pyyaml in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (3.12)\n",
      "Requirement already satisfied: bottleneck in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from inltk) (1.2.1)\n",
      "Collecting fastai==1.0.57 (from inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c1/e2/42342ded0385d694e3250e74f43f0dc9a3ff3d5c2241a2ddd98236b5f9de/fastai-1.0.57-py3-none-any.whl (233kB)\n",
      "\u001b[K     |████████████████████████████████| 235kB 63.2MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting dataclasses; python_version < \"3.7\" (from inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/e1/d2/6f02df2616fd4016075f60157c7a0452b38d8f7938ae94343911e0fb0b09/dataclasses-0.7-py3-none-any.whl\n",
      "Requirement already satisfied: pytz>=2011k in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from pandas->inltk) (2018.4)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from pandas->inltk) (2.7.3)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->inltk) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->inltk) (2019.3.9)\n",
      "Requirement already satisfied: idna<2.8,>=2.5 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->inltk) (2.6)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->inltk) (1.23)\n",
      "Collecting srsly<1.1.0,>=0.1.0 (from spacy>=2.0.18->inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/fb/34136c7b2ad04d4472dd9ea86536f5e9fb71fb0eb78edc8dad76a3f9edf2/srsly-0.2.0-cp36-cp36m-manylinux1_x86_64.whl (185kB)\n",
      "\u001b[K     |████████████████████████████████| 194kB 62.9MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting wasabi<1.1.0,>=0.3.0 (from spacy>=2.0.18->inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/3f/35/6dc35bc3b49e160a016e420eb4bdcf1c887db6fd33a463959c06a508c339/wasabi-0.4.0-py3-none-any.whl\n",
      "Collecting preshed<3.1.0,>=3.0.2 (from spacy>=2.0.18->inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/db/6b/e07fad36913879757c90ba03d6fb7f406f7279e11dcefc105ee562de63ea/preshed-3.0.2-cp36-cp36m-manylinux1_x86_64.whl (119kB)\n",
      "\u001b[K     |████████████████████████████████| 122kB 63.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting thinc<7.4.0,>=7.3.0 (from spacy>=2.0.18->inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/07/59/6bb553bc9a5f072d3cd479fc939fea0f6f682892f1f5cff98de5c9b615bb/thinc-7.3.1-cp36-cp36m-manylinux1_x86_64.whl (2.2MB)\n",
      "\u001b[K     |████████████████████████████████| 2.2MB 54.6MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting plac<1.2.0,>=0.9.6 (from spacy>=2.0.18->inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/86/85/40b8f66c2dd8f4fd9f09d59b22720cffecf1331e788b8a0cab5bafb353d1/plac-1.1.3-py2.py3-none-any.whl\n",
      "Collecting cymem<2.1.0,>=2.0.2 (from spacy>=2.0.18->inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/e7/b5/3e1714ebda8fd7c5859f9b216e381adc0a38b962f071568fd00d67e1b1ca/cymem-2.0.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting murmurhash<1.1.0,>=0.28.0 (from spacy>=2.0.18->inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/a6/e6/63f160a4fdf0e875d16b28f972083606d8d54f56cd30cb8929f9a1ee700e/murmurhash-1.0.2-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from spacy>=2.0.18->inltk) (0.23)\n",
      "Collecting blis<0.5.0,>=0.4.0 (from spacy>=2.0.18->inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/41/19/f95c75562d18eb27219df3a3590b911e78d131b68466ad79fdf5847eaac4/blis-0.4.1-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
      "\u001b[K     |████████████████████████████████| 3.7MB 20.2MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: setuptools in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from spacy>=2.0.18->inltk) (39.1.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from aiohttp>=3.5.4->inltk) (18.1.0)\n",
      "Collecting idna-ssl>=1.0; python_version < \"3.7\" (from aiohttp>=3.5.4->inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/46/03/07c4894aae38b0de52b52586b24bf189bb83e4ddabfe2e2c8f2419eec6f4/idna-ssl-1.1.0.tar.gz\n",
      "Collecting multidict<5.0,>=4.5 (from aiohttp>=3.5.4->inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/cc/ceb5b8c76e7a23212b9e0353053cc35a9d86c763d852a76d9b941fe81fbc/multidict-4.5.2-cp36-cp36m-manylinux1_x86_64.whl (309kB)\n",
      "\u001b[K     |████████████████████████████████| 317kB 57.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp>=3.5.4->inltk)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fa/c0/9a73968a9f4e4dac8dffb0ba35f932dd7798fe97901f4942c2d38667862c/yarl-1.3.0-cp36-cp36m-manylinux1_x86_64.whl (251kB)\n",
      "\u001b[K     |████████████████████████████████| 256kB 62.8MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting typing-extensions>=3.6.5; python_version < \"3.7\" (from aiohttp>=3.5.4->inltk)\n",
      "  Downloading https://files.pythonhosted.org/packages/03/92/705fe8aca27678e01bbdd7738173b8e7df0088a2202c80352f664630d638/typing_extensions-3.7.4.1-py3-none-any.whl\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from matplotlib->inltk) (2.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from matplotlib->inltk) (0.10.0)\n",
      "Requirement already satisfied: six>=1.10 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from matplotlib->inltk) (1.11.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from matplotlib->inltk) (1.0.1)\n",
      "Requirement already satisfied: torch>=1.0.0 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from fastai==1.0.57->inltk) (1.1.0)\n",
      "Requirement already satisfied: torchvision in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from fastai==1.0.57->inltk) (0.3.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from thinc<7.4.0,>=7.3.0->spacy>=2.0.18->inltk) (4.32.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->spacy>=2.0.18->inltk) (0.6.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: more-itertools in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from zipp>=0.5->importlib-metadata>=0.20; python_version < \"3.8\"->spacy>=2.0.18->inltk) (4.1.0)\n",
      "Building wheels for collected packages: nvidia-ml-py3, idna-ssl\n",
      "  Building wheel for nvidia-ml-py3 (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/ubuntu/.cache/pip/wheels/e4/1d/06/640c93f5270d67d0247f30be91f232700d19023f9e66d735c7\n",
      "  Building wheel for idna-ssl (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/ubuntu/.cache/pip/wheels/d3/00/b3/32d613e19e08a739751dd6bf998cfed277728f8b2127ad4eb7\n",
      "Successfully built nvidia-ml-py3 idna-ssl\n",
      "Installing collected packages: nvidia-ml-py3, srsly, wasabi, cymem, murmurhash, preshed, plac, blis, thinc, spacy, fastprogress, async-timeout, idna-ssl, multidict, yarl, typing-extensions, aiohttp, sentencepiece, dataclasses, fastai, inltk\n",
      "Successfully installed aiohttp-3.6.2 async-timeout-3.0.1 blis-0.4.1 cymem-2.0.3 dataclasses-0.7 fastai-1.0.57 fastprogress-0.1.21 idna-ssl-1.1.0 inltk-0.6.1 multidict-4.5.2 murmurhash-1.0.2 nvidia-ml-py3-7.352.0 plac-1.1.3 preshed-3.0.2 sentencepiece-0.1.83 spacy-2.2.2 srsly-0.2.0 thinc-7.3.1 typing-extensions-3.7.4.1 wasabi-0.4.0 yarl-1.3.0\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 19.3.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install inltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from inltk.inltk import get_embedding_vectors, get_sentence_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from inltk.inltk import setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup('kn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "kn_sent1 =\"ಚೆಕ್ ಬಂಧಿ;-ಉತ್ತರಕ್ಕೆ ಹಿಂದೂಗಳ ರುದ್ರಭೂವಿ, ಗೇಟ್ ಇರುತ್ತದೆ. ದಕ್ಷಿಣಕ್ಕೆ ನ್ಯೂಮಂಡ್ಲಿ 100 ಅಡಿ ರಸ್ತೆ ಅದರಾಚೆ ರಾಮ ರವರ ಮನೆ ಇರುತ್ತದೆ ಪೂರ್ವಕ್ಕೆ ನ್ಯೂ ಮಂಡ್ಲಿ 100 ಅಡಿ ರಸ್ತೆ ನ್ಯೂ ಮಂಡ್ಲಿ ಸರ್ಕಲ್ ಕಡೆಗೆ ಮುಂದುವರೆದಿದೆ. ಪಶ್ಚಿಮಕ್ಕೆ ನ್ಯೂ ಮಂಡ್ಲಿ 100 ಅಡಿ ರಸ್ತೆ ಇಲಿಯಾಜ್ ನಗರದ ಕಡೆಗೆ ಮುಂದುವರೆದಿದೆ.\"\n",
    "encoding = get_sentence_encoding(kn_sent1, 'kn')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_arr = [kn_sent1, kn_sent1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_arr_emb = list(map(lambda x: get_sentence_encoding(x, 'kn'), sent_arr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_vectors = np.stack(sent_arr_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "Model saved in path: sent_emb/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "# Create some variables.\n",
    "emb = tf.Variable(embeddings_vectors, name='kannada_embeddings')\n",
    "\n",
    "# Add an op to initialize the variable.\n",
    "init_op = tf.global_variables_initializer()\n",
    "\n",
    "# Add ops to save and restore all the variables.\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "# Later, launch the model, initialize the variables and save the\n",
    "# variables to disk.\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    # Save the variables to disk.\n",
    "    save_path = saver.save(sess, \"sent_emb/model.ckpt\")\n",
    "    print(\"Model saved in path: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents = '\\n'.join(sent_arr)\n",
    "\n",
    "with open(os.path.join('sent_emb', 'metadata.tsv'), 'w') as f:\n",
    "    f.write(sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext tensorboard.notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 8000 (pid 25467), started 0:01:00 ago. (Use '!kill 25467' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"600\"\n",
       "            src=\"http://localhost:8000\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fbb7efc7588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir sent_emb --port=8000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hi\n"
     ]
    }
   ],
   "source": [
    "print(\"hi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store -r full_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['TBL_TR_Accident_Driver', 'tbl_CR_Victim_Info', 'TBL_TR_Offence_details', 'tbl_TR_Offence', 'TBL_TR_Person_Details', 'TBL_TR_Accident_Casualties', 'tbl_CR_FIR_Details', 'Masters', 'TR_Accident_Vehicle', 'TBL_TR_Offence_Master', 'tbl_tr_accident_report', 'tbl_CR_Arr_Person_Details', 'TR_Accident_Vehicle_Master'])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_dataset.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unit_ID', 'Crime_No', 'Accident_DateTime', 'Fatal_NonFatal',\n",
       "       'Accident_Classification', 'Severity', 'Collision_Type',\n",
       "       'Accident_Spot', 'Accident_Location', 'Accident_Zone',\n",
       "       'NoOfVehicle_Involved', 'Junction_Control', 'Road_Character',\n",
       "       'Road_Type', 'Road_Classification', 'Road_Width', 'Shoulder_Width',\n",
       "       'Seperation', 'Surface_Type', 'Surface_Condition', 'Road_Condition',\n",
       "       'weather', 'Location_Type', 'Main_Cause', 'Hit_Run', 'XY_Map',\n",
       "       'X_Value', 'AccidentLocation_Map', 'Collision_Diagram', 'Y_Value',\n",
       "       'Route', 'KM', 'HundredM', 'Node_Map', 'Node1', 'Node2',\n",
       "       'Accident_City', 'Accident_Road', 'Reporting_Officer', 'LandMark_First',\n",
       "       'LandMark_Second', 'Distance_LandMark_First',\n",
       "       'Distance_LandMark_Second', 'Junction_Second_Name',\n",
       "       'Distance_Second_Road', 'Research_Code', 'Accident_Description',\n",
       "       'Action_Special_Note', 'UserID', 'Inspection_Officer_Name',\n",
       "       'Inspection_Officer_Rank', 'Inspection_Officer_Code',\n",
       "       'Actual_DateOf_Occurance', 'FromDate_Of_Occurance',\n",
       "       'ToDate_Of_Occurance', 'Lane_Type', 'Road_Markings', 'Spot_Conditions',\n",
       "       'Date_Of_Filling', 'Highway_Patrol_No', 'Aid_Post_No',\n",
       "       'Aid_Post_Distance', 'Trauma_Care_Centre_No',\n",
       "       'Trauma_Care_Centre_Distance', 'Checked', 'Signage', 'Side_Walk',\n",
       "       'Modified_Unit_ID', 'Action_Flag', 'Sync_Id', 'Sync_Id_Old',\n",
       "       'NoOf_Animal_Injured', 'NoOf_Animal_Killed', 'DEDT',\n",
       "       'Accident_Location_id', 'NoOfMinorInjured', 'NoOfNonVehicle',\n",
       "       'RoadNumber', 'Chainage', 'SpeedLimit', 'Road_typeb', 'RoadJunction',\n",
       "       'Accident_spotb', 'PotHoles', 'SteepGradient', 'Footpath',\n",
       "       'FootBridgeSubway', 'ZebraCrossing', 'Collision_Typeb'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_dataset['tbl_tr_accident_report']['tbl_TR_Accident_report'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting geocoder\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4f/6b/13166c909ad2f2d76b929a4227c952630ebaf0d729f6317eb09cbceccbab/geocoder-1.38.1-py2.py3-none-any.whl (98kB)\n",
      "\u001b[K     |████████████████████████████████| 102kB 28.6MB/s ta 0:00:01\n",
      "\u001b[?25hCollecting future (from geocoder)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/0b/38b06fd9b92dc2b68d58b75f900e97884c45bedd2ff83203d933cf5851c9/future-0.18.2.tar.gz (829kB)\n",
      "\u001b[K     |████████████████████████████████| 829kB 55.9MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: requests in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from geocoder) (2.20.0)\n",
      "Requirement already satisfied: six in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from geocoder) (1.11.0)\n",
      "Requirement already satisfied: click in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from geocoder) (6.7)\n",
      "Collecting ratelim (from geocoder)\n",
      "  Downloading https://files.pythonhosted.org/packages/f2/98/7e6d147fd16a10a5f821db6e25f192265d6ecca3d82957a4fdd592cad49c/ratelim-0.1.6-py2.py3-none-any.whl\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->geocoder) (2019.3.9)\n",
      "Requirement already satisfied: idna<2.8,>=2.5 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->geocoder) (2.6)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->geocoder) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from requests->geocoder) (1.23)\n",
      "Requirement already satisfied: decorator in /home/ubuntu/anaconda3/lib/python3.6/site-packages (from ratelim->geocoder) (4.0.11)\n",
      "Building wheels for collected packages: future\n",
      "  Building wheel for future (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/ubuntu/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e\n",
      "Successfully built future\n",
      "Installing collected packages: future, ratelim, geocoder\n",
      "Successfully installed future-0.18.2 geocoder-1.38.1 ratelim-0.1.6\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 19.3.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install geocoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The pastebin URL is:{'error_message': 'The provided API key is invalid.', 'results': [], 'status': 'REQUEST_DENIED'}\n"
     ]
    }
   ],
   "source": [
    "# importing the requests library \n",
    "import requests \n",
    "  \n",
    "?address=1600+Amphitheatre+Parkway,+Mountain+View,+CA&key=YOUR_API_KEY\n",
    "\n",
    "# defining the api-endpoint  \n",
    "API_ENDPOINT = \"https://maps.googleapis.com/maps/api/geocode/json\"\n",
    "\n",
    "address = \"1600 Amphitheatre Parkway, Mountain+View, CA\"\n",
    "\n",
    "PARAMS = {\n",
    "    'key': \"..\",\n",
    "    'address': address\n",
    "}\n",
    "# sending post request and saving response as response object \n",
    "r = requests.get(url = API_ENDPOINT, params = PARAMS) \n",
    "  \n",
    "# extracting response text  \n",
    "pastebin_url = r.json() \n",
    "print(\"The pastebin URL is:%s\"%pastebin_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-84-308ac0b5d413>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;31m# extracting latitude, longitude and formatted address\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;31m# of the first matching location\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0mlatitude\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'results'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'geometry'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'location'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lat'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0mlongitude\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'results'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'geometry'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'location'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lng'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0mformatted_address\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'results'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'formatted_address'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# importing the requests library \n",
    "import requests \n",
    "  \n",
    "# api-endpoint \n",
    "URL = \"http://maps.googleapis.com/maps/api/geocode/json\"\n",
    "  \n",
    "# location given here \n",
    "location = \"delhi technological university\"\n",
    "\n",
    "key = \"\"\n",
    "  \n",
    "# defining a params dict for the parameters to be sent to the API \n",
    "PARAMS = {'address':location,\n",
    "         'key': ''} \n",
    "  \n",
    "    \n",
    "# sending get request and saving the response as response object \n",
    "r = requests.get(url = URL, params = PARAMS) \n",
    "  \n",
    "# extracting data in json format \n",
    "data = r.json() \n",
    "  \n",
    "  \n",
    "# extracting latitude, longitude and formatted address  \n",
    "# of the first matching location \n",
    "latitude = data['results'][0]['geometry']['location']['lat'] \n",
    "longitude = data['results'][0]['geometry']['location']['lng'] \n",
    "formatted_address = data['results'][0]['formatted_address'] \n",
    "  \n",
    "# printing the output \n",
    "print(\"Latitude:%s\\nLongitude:%s\\nFormatted Address:%s\"\n",
    "      %(latitude, longitude,formatted_address))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['FIR_ID', 'Unit_ID', 'Crime_No', 'CCIS_Crime_No', 'FIR_Entry_Type',\n",
       "       'Offence_From_Date', 'Offence_To_Date', 'FIR_Category', 'FIR_Date',\n",
       "       'Delay_Reason', 'Complaint_Mode', 'Info_PS_Date', 'GD_No', 'Address1',\n",
       "       'Address2', 'City', 'State_ID', 'PIN_Code', 'Beat_ID',\n",
       "       'Village_Area_ID', 'Direction_Distance', 'Other_PS_State',\n",
       "       'Other_PS_District', 'Other_PS_Limit_ID', 'UD_Case_No', 'Brief_Facts',\n",
       "       'Action_Taken', 'Transfered_Unit_ID', 'Transfer_Details',\n",
       "       'Dispatch_Court_ID', 'Dispatch_Court_Date', 'Dispatch_Officer_ID',\n",
       "       'Dispatch_Officer_Rank', 'FIR_Type', 'FIR_Status', 'FIR_Stage',\n",
       "       'UserID', 'FIR_Read_Explained', 'Reason_Decline_Investigation',\n",
       "       'Occurence_Place_ID', 'Latitude', 'Longitude', 'Request_Status',\n",
       "       'FIR_Read', 'FIR_Read_Date', 'Copy_To', 'Goonda_Act_case', 'TADA_Case',\n",
       "       'NSA_Case', 'Case_AgainstPP', 'Firing_Case', 'Lathi_Charge_case',\n",
       "       'Tear_Gas', 'Arrest_Date', 'Firing_Type', 'Agitation', 'False_Reason',\n",
       "       'Agitation_Lathy', 'CRPC_Disposal_Date', 'Magistrate_Number',\n",
       "       'Magistrate_date', 'CRPC_Disposal_Remarks', 'Draft', 'With_By_Gov_INV',\n",
       "       'With_By_Gov_Trial', 'Converted_FIR', 'UDRcase_No', 'IOCode', 'IOName',\n",
       "       'GD_Time', 'CCIS_Property', 'CCIS_UD_Ref', 'CCIS_TotPropertyValue',\n",
       "       'missing_disp_upload', 'Trans_Status', 'ForeignerAddress',\n",
       "       'Modified_Unit_ID', 'Action_Flag', 'Sync_Id', 'Sync_Id_Old', 'DEDT',\n",
       "       'Insert_KGSC', 'Update_KGSC', 'GSC_No', 'FIR_Submitted_dt',\n",
       "       'Dispatch_Juvenile_Court_Id'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_dataset['tbl_CR_FIR_Details']['tbl_CR_FIR_Details'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_dataset['tbl_CR_FIR_Details']['tbl_CR_FIR_Details']['Latitude']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "masterData = pd.read_csv('csv_file/accident_report_vehicle_merge.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Unnamed: 0.1', 'Unit_ID', 'Crime_No',\n",
       "       'NoOfVehicle_Involved', 'Accident_Road', 'Accident_Description',\n",
       "       'UserID', 'NoOf_Animal_Injured', 'NoOf_Animal_Killed', 'DEDT',\n",
       "       'NoOfMinorInjured', 'NoOfNonVehicle', 'RoadNumber', 'SpeedLimit',\n",
       "       'Road_typeb', 'SteepGradient', 'Footpath', 'Collision_Typeb',\n",
       "       'Fatal_NonFatal_val', 'Accident_Classification_val', 'Severity_val',\n",
       "       'Collision_Type_val', 'Accident_Spot_val', 'Accident_Location_val',\n",
       "       'Junction_Control_val', 'Road_Character_val', 'Road_Type_val',\n",
       "       'Road_Classification_val', 'Seperation_val', 'Surface_Type_val',\n",
       "       'Surface_Condition_val', 'Road_Condition_val', 'weather_val',\n",
       "       'Location_Type_val', 'Main_Cause_val', 'Hit_Run_val',\n",
       "       'Accident_City_val', 'Accident_Road_val', 'Lane_Type_val',\n",
       "       'Road_Markings_val', 'Spot_Conditions_val', 'Side_Walk_val',\n",
       "       'Accident_Location_id_val', 'RoadJunction_val', 'Accident_spotb_val',\n",
       "       'uc', 'Vehicle_Type', 'Vehicle_Vanoeuvre', 'Vehicle_Model',\n",
       "       'Vehicle_Color', 'Vehicle_Load', 'VehicleSeized', 'Fuel_Type',\n",
       "       'Seating_Capacity', 'Class_Of_Vehicle', 'IsPoliceVehicle',\n",
       "       'IsGov_Vehicle', 'Vehicle_Sub_Type'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "masterData.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            count          mean           std           min  \\\n",
      "Unnamed: 0               152785.0  7.639200e+04  4.410537e+04  0.000000e+00   \n",
      "Unnamed: 0.1             152785.0  1.021218e+05  5.790043e+04  0.000000e+00   \n",
      "Unit_ID                  152785.0  1.193180e+03  6.013267e+02  5.700000e+01   \n",
      "Crime_No                 152785.0  2.016117e+09  1.468408e+06  2.014000e+09   \n",
      "NoOfVehicle_Involved     152785.0  1.553006e+00  5.742412e-01  1.000000e+00   \n",
      "UserID                   142135.0  1.033448e+08  7.954912e+07  5.000000e+01   \n",
      "NoOf_Animal_Injured      152785.0  2.270622e+00  3.855952e+02  0.000000e+00   \n",
      "NoOf_Animal_Killed       152785.0  1.448899e+00  4.180274e+02  0.000000e+00   \n",
      "NoOfMinorInjured            446.0  1.885650e+00  2.606460e+00  0.000000e+00   \n",
      "NoOfNonVehicle              412.0  1.648058e+00  2.072863e+00  0.000000e+00   \n",
      "RoadNumber                  207.0  1.034058e+02  1.164930e+02  1.000000e+00   \n",
      "SpeedLimit                 1527.0  3.656189e+00  2.789526e+00  1.000000e+00   \n",
      "Road_typeb                 1527.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
      "SteepGradient              1527.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
      "Footpath                   1527.0  2.198389e+03  6.275640e+03  0.000000e+00   \n",
      "Collision_Typeb            1527.0  7.259619e+03  2.990362e+03  6.084000e+03   \n",
      "Fatal_NonFatal_val            0.0           NaN           NaN           NaN   \n",
      "Road_Classification_val       0.0           NaN           NaN           NaN   \n",
      "Accident_City_val             0.0           NaN           NaN           NaN   \n",
      "uc                       152785.0  1.193382e+13  6.013267e+12  5.720140e+11   \n",
      "Vehicle_Color            152785.0  2.406200e+03  3.506492e+03  0.000000e+00   \n",
      "Seating_Capacity         152785.0  1.995935e+00  7.294070e+00  0.000000e+00   \n",
      "\n",
      "                                  25%           50%           75%  \\\n",
      "Unnamed: 0               3.819600e+04  7.639200e+04  1.145880e+05   \n",
      "Unnamed: 0.1             5.226600e+04  1.028870e+05  1.523620e+05   \n",
      "Unit_ID                  7.270000e+02  1.212000e+03  1.762000e+03   \n",
      "Crime_No                 2.015000e+09  2.016000e+09  2.017000e+09   \n",
      "NoOfVehicle_Involved     1.000000e+00  2.000000e+00  2.000000e+00   \n",
      "UserID                   4.470016e+07  1.059000e+08  1.512000e+08   \n",
      "NoOf_Animal_Injured      0.000000e+00  1.000000e+00  2.000000e+00   \n",
      "NoOf_Animal_Killed       0.000000e+00  0.000000e+00  0.000000e+00   \n",
      "NoOfMinorInjured         1.000000e+00  1.000000e+00  2.000000e+00   \n",
      "NoOfNonVehicle           1.000000e+00  1.000000e+00  2.000000e+00   \n",
      "RoadNumber               3.250000e+01  6.600000e+01  1.580000e+02   \n",
      "SpeedLimit               2.000000e+00  2.000000e+00  7.000000e+00   \n",
      "Road_typeb               0.000000e+00  0.000000e+00  0.000000e+00   \n",
      "SteepGradient            0.000000e+00  0.000000e+00  0.000000e+00   \n",
      "Footpath                 0.000000e+00  0.000000e+00  0.000000e+00   \n",
      "Collision_Typeb          6.088000e+03  6.093000e+03  6.525000e+03   \n",
      "Fatal_NonFatal_val                NaN           NaN           NaN   \n",
      "Road_Classification_val           NaN           NaN           NaN   \n",
      "Accident_City_val                 NaN           NaN           NaN   \n",
      "uc                       7.272017e+12  1.212202e+13  1.762201e+13   \n",
      "Vehicle_Color            0.000000e+00  0.000000e+00  4.171000e+03   \n",
      "Seating_Capacity         0.000000e+00  0.000000e+00  2.000000e+00   \n",
      "\n",
      "                                  max  \n",
      "Unnamed: 0               1.527840e+05  \n",
      "Unnamed: 0.1             2.012000e+05  \n",
      "Unit_ID                  2.245000e+03  \n",
      "Crime_No                 2.019000e+09  \n",
      "NoOfVehicle_Involved     5.200000e+01  \n",
      "UserID                   1.869000e+09  \n",
      "NoOf_Animal_Injured      1.507200e+05  \n",
      "NoOf_Animal_Killed       1.606200e+05  \n",
      "NoOfMinorInjured         3.100000e+01  \n",
      "NoOfNonVehicle           3.100000e+01  \n",
      "RoadNumber               7.660000e+02  \n",
      "SpeedLimit               8.000000e+00  \n",
      "Road_typeb               0.000000e+00  \n",
      "SteepGradient            0.000000e+00  \n",
      "Footpath                 2.010300e+04  \n",
      "Collision_Typeb          2.010500e+04  \n",
      "Fatal_NonFatal_val                NaN  \n",
      "Road_Classification_val           NaN  \n",
      "Accident_City_val                 NaN  \n",
      "uc                       2.245202e+13  \n",
      "Vehicle_Color            1.108400e+04  \n",
      "Seating_Capacity         1.500000e+02  \n"
     ]
    }
   ],
   "source": [
    "print(masterData.describe().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}