main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/fasttext-crawl-300d-2m/crawl-300d-2M.vec\n",
      "/kaggle/input/news-summary/news_summary.csv\n",
      "/kaggle/input/news-summary/news_summary_more.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.models import model_from_json\n",
    "import numpy as np\n",
    "from nltk import word_tokenize\n",
    "from numpy import array\n",
    "from keras.models import Model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Input\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.preprocessing import sequence\n",
    "from keras.layers import Dense\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import re,codecs\n",
    "from nltk.corpus import stopwords\n",
    "from bs4 import BeautifulSoup \n",
    "import matplotlib.pyplot as plt\n",
    "from keras.preprocessing.text import Tokenizer \n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from tqdm import tqdm\n",
    "from keras.layers import RepeatVector\n",
    "from keras.layers import TimeDistributed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n",
    "\n",
    "                           \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n",
    "\n",
    "                           \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",\n",
    "\n",
    "                           \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n",
    "\n",
    "                           \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
    "\n",
    "                           \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n",
    "\n",
    "                           \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\",\n",
    "\n",
    "                           \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\n",
    "\n",
    "                           \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n",
    "\n",
    "                           \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n",
    "\n",
    "                           \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n",
    "\n",
    "                           \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n",
    "\n",
    "                           \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n",
    "\n",
    "                           \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\",\n",
    "\n",
    "                           \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\",\n",
    "\n",
    "                           \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n",
    "\n",
    "                           \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n",
    "\n",
    "                           \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n",
    "\n",
    "                           \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n",
    "\n",
    "                           \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n",
    "\n",
    "                           \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n",
    "\n",
    "                           \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n",
    "\n",
    "                           \"you're\": \"you are\", \"you've\": \"you have\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "",
    "_uuid": ""
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"../input/news-summary/news_summary.csv\",encoding = \"ISO-8859-1\")\n",
    "#news_summary_more = pd.read_csv(\"../input/news-summary/news_summary_more.csv\",encoding = \"ISO-8859-1\")\n",
    "columns=df.columns\n",
    "new_df=df[['ctext','text']]\n",
    "#print(df.describe())\n",
    "#print(new_df.head())\n",
    "#print(\"Description:\",new_df['ctext'][0])\n",
    "#print(\"Summary:\",new_df['text'][0])\n",
    "#raw=pd.read_csv(\"../input/datafornewssummarisation/summary.csv\",encoding = \"ISO-8859-1\")\n",
    "#raw.to_csv(r'../working/datafornewssummarisation\\summary.csv')\n",
    "new_df.to_csv('summary.csv',encoding = \"ISO-8859-1\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"../working/summary.csv\",encoding = \"ISO-8859-1\")\n",
    "columns=df.columns\n",
    "\n",
    "#print(columns)\n",
    "\n",
    "\n",
    "#An example showing the data and the summary\n",
    "#print(\"Text:\",X[1])\n",
    "#print(\"\\n\")\n",
    "#print(\"Summary:\",y[1])\n",
    "#Text cleaning\n",
    "#Rows having duplicates will be omitted\n",
    "df.drop_duplicates(keep='first',inplace=True)\n",
    "\n",
    "#Rows not having summary or document is deleted\n",
    "df.dropna(axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "    stop_words = set(stopwords.words('english')) \n",
    "    def text_cleaner(text):\n",
    "        newString = text.lower()\n",
    "        newString = BeautifulSoup(newString, \"lxml\").text\n",
    "        newString = re.sub(r'\\([^)]*\\)', '', newString)\n",
    "        newString = re.sub('\"','', newString)\n",
    "        newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])    \n",
    "        newString = re.sub(r\"'s\\b\",\"\",newString)\n",
    "        newString = re.sub(\"[^a-zA-Z]\", \" \", newString) \n",
    "        tokens = [w for w in newString.split() if not w in stop_words]\n",
    "        long_words=[]\n",
    "        for i in tokens:\n",
    "            if len(i)>1:                  #removing short word\n",
    "                long_words.append(i)   \n",
    "        return (\" \".join(long_words)).strip()\n",
    "\n",
    "    cleaned_text = []\n",
    "    for t in df['ctext']:\n",
    "        cleaned_text.append(text_cleaner(t))\n",
    "    #print('done')\n",
    "\n",
    "    def summary_cleaner(text):\n",
    "        newString = re.sub('\"','', text)\n",
    "        newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])    \n",
    "        newString = re.sub(r\"'s\\b\",\"\",newString)\n",
    "        newString = re.sub(\"[^a-zA-Z]\", \" \", newString)\n",
    "        newString = newString.lower()\n",
    "        tokens=newString.split()\n",
    "        newString=''\n",
    "        for i in tokens:\n",
    "            if len(i)>=3:                                 \n",
    "                newString=newString+i+' '  \n",
    "        return newString\n",
    "\n",
    "    #Call the above function\n",
    "    cleaned_summary = []\n",
    "    for t in df['text']:\n",
    "        cleaned_summary.append(summary_cleaner(t))\n",
    "    \n",
    "\n",
    "    df['cleaned_text']=cleaned_text\n",
    "    df['cleaned_summary']=cleaned_summary\n",
    "    df.replace('', np.nan, inplace=True)\n",
    "    df.dropna(axis=0,inplace=True)\n",
    "    print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ctext</th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>cleaned_summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The Daman and Diu administration on Wednesday ...</td>\n",
       "      <td>The Administration of Union Territory Daman an...</td>\n",
       "      <td>daman diu administration wednesday withdrew ci...</td>\n",
       "      <td>the administration union territory daman and d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>From her special numbers to TV?appearances, Bo...</td>\n",
       "      <td>Malaika Arora slammed an Instagram user who tr...</td>\n",
       "      <td>special numbers tv appearances bollywood actor...</td>\n",
       "      <td>malaika arora slammed instagram user who troll...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Indira Gandhi Institute of Medical Science...</td>\n",
       "      <td>The Indira Gandhi Institute of Medical Science...</td>\n",
       "      <td>indira gandhi institute medical sciences patna...</td>\n",
       "      <td>the indira gandhi institute medical sciences i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Lashkar-e-Taiba's Kashmir commander Abu Dujana...</td>\n",
       "      <td>Lashkar-e-Taiba's Kashmir commander Abu Dujana...</td>\n",
       "      <td>lashkar taiba kashmir commander abu dujana kil...</td>\n",
       "      <td>lashkar taiba kashmir commander abu dujana who...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Hotels in Mumbai and other Indian cities are t...</td>\n",
       "      <td>Hotels in Maharashtra will train their staff t...</td>\n",
       "      <td>hotels mumbai indian cities train staff spot s...</td>\n",
       "      <td>hotels maharashtra will train their staff spot...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                              ctext  \\\n",
       "0           0  The Daman and Diu administration on Wednesday ...   \n",
       "1           1  From her special numbers to TV?appearances, Bo...   \n",
       "2           2  The Indira Gandhi Institute of Medical Science...   \n",
       "3           3  Lashkar-e-Taiba's Kashmir commander Abu Dujana...   \n",
       "4           4  Hotels in Mumbai and other Indian cities are t...   \n",
       "\n",
       "                                                text  \\\n",
       "0  The Administration of Union Territory Daman an...   \n",
       "1  Malaika Arora slammed an Instagram user who tr...   \n",
       "2  The Indira Gandhi Institute of Medical Science...   \n",
       "3  Lashkar-e-Taiba's Kashmir commander Abu Dujana...   \n",
       "4  Hotels in Maharashtra will train their staff t...   \n",
       "\n",
       "                                        cleaned_text  \\\n",
       "0  daman diu administration wednesday withdrew ci...   \n",
       "1  special numbers tv appearances bollywood actor...   \n",
       "2  indira gandhi institute medical sciences patna...   \n",
       "3  lashkar taiba kashmir commander abu dujana kil...   \n",
       "4  hotels mumbai indian cities train staff spot s...   \n",
       "\n",
       "                                     cleaned_summary  \n",
       "0  the administration union territory daman and d...  \n",
       "1  malaika arora slammed instagram user who troll...  \n",
       "2  the indira gandhi institute medical sciences i...  \n",
       "3  lashkar taiba kashmir commander abu dujana who...  \n",
       "4  hotels maharashtra will train their staff spot...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dataset_cleaning()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_text =np.array(df['cleaned_text'])\n",
    "cleaned_summary=np.array(df['cleaned_summary'])\n",
    "\n",
    "short_text=[]\n",
    "short_summary=[]\n",
    "\n",
    "for i in range(len(cleaned_text)):\n",
    "    if(len(cleaned_summary[i].split())<=63 and len(cleaned_text[i].split())<=600):\n",
    "        short_text.append(cleaned_text[i])\n",
    "        short_summary.append(cleaned_summary[i])\n",
    "        \n",
    "data=pd.DataFrame({'text':short_text,'summary':short_summary})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>daman diu administration wednesday withdrew ci...</td>\n",
       "      <td>the administration union territory daman and d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>special numbers tv appearances bollywood actor...</td>\n",
       "      <td>malaika arora slammed instagram user who troll...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>indira gandhi institute medical sciences patna...</td>\n",
       "      <td>the indira gandhi institute medical sciences i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lashkar taiba kashmir commander abu dujana kil...</td>\n",
       "      <td>lashkar taiba kashmir commander abu dujana who...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hotels mumbai indian cities train staff spot s...</td>\n",
       "      <td>hotels maharashtra will train their staff spot...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  daman diu administration wednesday withdrew ci...   \n",
       "1  special numbers tv appearances bollywood actor...   \n",
       "2  indira gandhi institute medical sciences patna...   \n",
       "3  lashkar taiba kashmir commander abu dujana kil...   \n",
       "4  hotels mumbai indian cities train staff spot s...   \n",
       "\n",
       "                                             summary  \n",
       "0  the administration union territory daman and d...  \n",
       "1  malaika arora slammed instagram user who troll...  \n",
       "2  the indira gandhi institute medical sciences i...  \n",
       "3  lashkar taiba kashmir commander abu dujana who...  \n",
       "4  hotels maharashtra will train their staff spot...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>daman diu administration wednesday withdrew ci...</td>\n",
       "      <td>sostok the administration union territory dama...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>special numbers tv appearances bollywood actor...</td>\n",
       "      <td>sostok malaika arora slammed instagram user wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>indira gandhi institute medical sciences patna...</td>\n",
       "      <td>sostok the indira gandhi institute medical sci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lashkar taiba kashmir commander abu dujana kil...</td>\n",
       "      <td>sostok lashkar taiba kashmir commander abu duj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hotels mumbai indian cities train staff spot s...</td>\n",
       "      <td>sostok hotels maharashtra will train their sta...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  daman diu administration wednesday withdrew ci...   \n",
       "1  special numbers tv appearances bollywood actor...   \n",
       "2  indira gandhi institute medical sciences patna...   \n",
       "3  lashkar taiba kashmir commander abu dujana kil...   \n",
       "4  hotels mumbai indian cities train staff spot s...   \n",
       "\n",
       "                                             summary  \n",
       "0  sostok the administration union territory dama...  \n",
       "1  sostok malaika arora slammed instagram user wh...  \n",
       "2  sostok the indira gandhi institute medical sci...  \n",
       "3  sostok lashkar taiba kashmir commander abu duj...  \n",
       "4  sostok hotels maharashtra will train their sta...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['summary']=data['summary'].apply(lambda x:'sostok '+x+' eostok')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test=train_test_split(np.array(data['text']),\n",
    "    np.array(data['summary']),test_size=0.1,random_state=0,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3880 3880\n"
     ]
    }
   ],
   "source": [
    "    #This function  will be used to preprocess the data to a form that can be fed into the neural network\n",
    "    #Subparts:\n",
    "    ##1.Find the most_probable_length of the sentences of text_type and summary type \n",
    "    ##(Hint: you may plot the distribution of lengths of sentences)\n",
    "    \"\"\"    \n",
    "    text_count = []\n",
    "    summary_count = []\n",
    "\n",
    "    for t in df_processed['cleaned_text']:\n",
    "        text_count.append(len(t.split()))\n",
    "    for t in df_processed['cleaned_summary']:\n",
    "        summary_count.append(len(t.split()))\n",
    "     \"\"\"\n",
    "      \n",
    "   #graph_df= pd.DataFrame()\n",
    "   #graph_df['text']=text_count\n",
    "    #df2=pd.DataFrame()\n",
    "  # graph_df['summary']=summary_count\n",
    "    #df2['summary']=summary_count\n",
    "  # graph_df.hist(bins = 5,range = [0,600])\n",
    "  # df2.hist(bins = 5,range = [50,63])\n",
    "   #plt.show()\n",
    "    max_sum_len=63\n",
    "    max_text_len=600\n",
    "    \n",
    "\n",
    "    ##2.Tokenizing the data\n",
    "    X_tokenizer = Tokenizer()\n",
    "    X_tokenizer.fit_on_texts(list(X_train))\n",
    "\n",
    "    thresh=4\n",
    "    cnt=0\n",
    "    tot_cnt=0\n",
    "    freq=0\n",
    "    tot_freq=0\n",
    "\n",
    "    for key,value in X_tokenizer.word_counts.items():\n",
    "        tot_cnt=tot_cnt+1\n",
    "        tot_freq=tot_freq+value\n",
    "        if(value<thresh):\n",
    "            cnt=cnt+1\n",
    "            freq=freq+value\n",
    "    \n",
    "    X_tokenizer=Tokenizer(num_words=tot_cnt)\n",
    "    X_tokenizer.fit_on_texts(list(X_train))\n",
    "    X_train_seq = X_tokenizer.texts_to_sequences(X_train)\n",
    "    X_test_seq=X_tokenizer.texts_to_sequences(X_test)\n",
    "    X_train=pad_sequences(X_train_seq,maxlen=max_text_len,padding='post')\n",
    "    X_test= pad_sequences(X_test_seq,maxlen=max_text_len,padding='post')   \n",
    "    X_voc=X_tokenizer.num_words +1\n",
    "    \n",
    "    y_tokenizer = Tokenizer()\n",
    "    y_tokenizer.fit_on_texts(list(y_train))\n",
    "    thresh=6\n",
    "    cnt=0\n",
    "    tot_cnt=0\n",
    "    freq=0\n",
    "    tot_freq=0\n",
    "\n",
    "    for key,value in y_tokenizer.word_counts.items():\n",
    "        tot_cnt=tot_cnt+1\n",
    "        tot_freq=tot_freq+value\n",
    "        if(value<thresh):\n",
    "            cnt=cnt+1\n",
    "            freq=freq+value\n",
    "    y_tokenizer=Tokenizer(num_words=(tot_cnt))\n",
    "    y_tokenizer.fit_on_texts(list(y_train))\n",
    "    y_train_seq = y_tokenizer.texts_to_sequences(y_train)\n",
    "    y_test_seq=y_tokenizer.texts_to_sequences(y_test)\n",
    "    y_train=pad_sequences(y_train_seq,maxlen=max_sum_len,padding='post')\n",
    "    y_test= pad_sequences(y_test_seq,maxlen=max_sum_len,padding='post')  \n",
    "    y_voc=y_tokenizer.num_words +1\n",
    "    \n",
    "    print(y_tokenizer.word_counts['sostok'],len(y_train))\n",
    "    \n",
    "    ind=[]\n",
    "    for i in range(len(y_train)):\n",
    "        cnt=0\n",
    "        for j in y_train[i]:\n",
    "            if j!=0:\n",
    "                cnt=cnt+1\n",
    "        if(cnt==2):\n",
    "            ind.append(i)\n",
    "\n",
    "    y_train=np.delete(y_train,ind, axis=0)\n",
    "    X_train=np.delete(X_train,ind, axis=0)\n",
    "    \n",
    "    ind=[]\n",
    "    for i in range(len(y_test)):\n",
    "        cnt=0\n",
    "        for j in y_test[i]:\n",
    "            if j!=0:\n",
    "                cnt=cnt+1\n",
    "        if(cnt==2):\n",
    "            ind.append(i)\n",
    "\n",
    "    y_test=np.delete(y_test,ind, axis=0)\n",
    "    X_test=np.delete(X_test,ind, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing():\n",
    "##4.Introducing embeddings( We'll be using the Fasttext embeddings)\n",
    "    embeddings_index = {}\n",
    "    f = open('../input/fasttext-crawl-300d-2m/crawl-300d-2M.vec', encoding='utf-8')\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "\n",
    "    word_index = X_tokenizer.word_index\n",
    "    embedding_matrix = np.zeros((len(word_index) + 1, 300))\n",
    "    for word, i in word_index.items():\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    print(\"done\")\n",
    "    \n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "#X_voc,y_voc,x_tr,x_val,y_tr,y_val,X_tokenizer,y_tokenizer=preprocessing(X_train,X_test,y_train,y_test,df_processed)\n",
    "embedding_matrix=preprocessing()\n",
    "#y_tokenizer.word_counts['sostok'],len(y_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42827, (42827, 300))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_sum_len=63\n",
    "max_text_len=600\n",
    "X_voc,embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(embedding_matrix):\n",
    "    from keras import backend as k\n",
    "    k.clear_session()\n",
    "    \n",
    "    encoder_inputs=Input(shape=(max_text_len,))\n",
    "    encoder_emb=Embedding(X_voc,300,weights=[embedding_matrix],trainable=True)(encoder_inputs)\n",
    "    encoder_lstm1 = LSTM(300,return_sequences=True,return_state=True)\n",
    "    encoder_output1, state_h1, state_c1 = encoder_lstm1(encoder_emb)\n",
    "    encoder_lstm2 = LSTM(300,return_sequences=True,return_state=True)\n",
    "    encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1)\n",
    "    encoder_lstm3=LSTM(300, return_state=True, return_sequences=True)\n",
    "    encoder_outputs, state_h, state_c= encoder_lstm3(encoder_output2)\n",
    "\n",
    "\n",
    "    decoder_inputs = Input(shape=(None,))\n",
    "    dec_emb_layer = Embedding(y_voc, 300,trainable=True)\n",
    "    dec_emb = dec_emb_layer(decoder_inputs)\n",
    "    decoder_lstm = LSTM(300, return_sequences=True, return_state=True)\n",
    "    decoder_outputs,decoder_fwd_state, decoder_back_state =decoder_lstm(dec_emb,initial_state=[state_h, state_c])\n",
    "    decoder_dense =TimeDistributed(Dense(y_voc, activation='softmax'))\n",
    "    decoder_outputs =decoder_dense(decoder_outputs)\n",
    "\n",
    "# Define the model \n",
    "    model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "\n",
    "    model.summary()\n",
    "    return model,encoder_inputs,encoder_outputs,state_h,state_c,decoder_inputs,dec_emb_layer,decoder_lstm,decoder_dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 600)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 600, 300)     12848100    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, 600, 300), ( 721200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, 600, 300), ( 721200      lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 300)    5205000     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_3 (LSTM)                   [(None, 600, 300), ( 721200      lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lstm_4 (LSTM)                   [(None, None, 300),  721200      embedding_2[0][0]                \n",
      "                                                                 lstm_3[0][1]                     \n",
      "                                                                 lstm_3[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, None, 17350)  5222350     lstm_4[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 26,160,250\n",
      "Trainable params: 26,160,250\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "m,encoder_inputs,encoder_outputs,state_h,state_c,decoder_inputs,dec_emb_layer,decoder_lstm,decoder_dense=model(embedding_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model is built and summary shown above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_accuracy(model):\n",
    "    #Function to check how the model works on the test data\n",
    "    model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')\n",
    "    from keras.callbacks import EarlyStopping\n",
    "    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=2)\n",
    "    history=model.fit([X_train,y_train[:,:-1]], \n",
    "                      y_train.reshape(y_train.shape[0],y_train.shape[1], 1)\n",
    "                      [:,1:] ,epochs=50,callbacks=[es],\n",
    "                      batch_size=128, validation_data=([X_test,y_test[:,:-1]],\n",
    "                                                       y_test.reshape(y_test.shape[0],y_test.shape[1], 1)[:,1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3880 samples, validate on 432 samples\n",
      "Epoch 1/50\n",
      "3880/3880 [==============================] - 71s 18ms/step - loss: 7.0167 - val_loss: 5.9437\n",
      "Epoch 2/50\n",
      "3880/3880 [==============================] - 69s 18ms/step - loss: 6.1008 - val_loss: 5.5185\n",
      "Epoch 3/50\n",
      "3880/3880 [==============================] - 69s 18ms/step - loss: 5.8917 - val_loss: 5.4534\n",
      "Epoch 4/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 5.8039 - val_loss: 5.4368\n",
      "Epoch 5/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 5.7363 - val_loss: 5.3738\n",
      "Epoch 6/50\n",
      "3880/3880 [==============================] - 68s 18ms/step - loss: 5.6650 - val_loss: 5.3252\n",
      "Epoch 7/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 5.5938 - val_loss: 5.2948\n",
      "Epoch 8/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 5.5114 - val_loss: 5.2435\n",
      "Epoch 9/50\n",
      "3880/3880 [==============================] - 68s 18ms/step - loss: 5.4257 - val_loss: 5.1913\n",
      "Epoch 10/50\n",
      "3880/3880 [==============================] - 69s 18ms/step - loss: 5.3405 - val_loss: 5.1582\n",
      "Epoch 11/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 5.2599 - val_loss: 5.1292\n",
      "Epoch 12/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 5.1811 - val_loss: 5.0971\n",
      "Epoch 13/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 5.1074 - val_loss: 5.0684\n",
      "Epoch 14/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 5.0316 - val_loss: 5.0404\n",
      "Epoch 15/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 4.9562 - val_loss: 5.0329\n",
      "Epoch 16/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.8843 - val_loss: 5.0029\n",
      "Epoch 17/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.8107 - val_loss: 4.9858\n",
      "Epoch 18/50\n",
      "3880/3880 [==============================] - 68s 18ms/step - loss: 4.7377 - val_loss: 4.9714\n",
      "Epoch 19/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.6699 - val_loss: 4.9634\n",
      "Epoch 20/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 4.6031 - val_loss: 4.9602\n",
      "Epoch 21/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.5449 - val_loss: 4.9479\n",
      "Epoch 22/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 4.4724 - val_loss: 4.9525\n",
      "Epoch 23/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.4111 - val_loss: 4.9338\n",
      "Epoch 24/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.3461 - val_loss: 4.9333\n",
      "Epoch 25/50\n",
      "3880/3880 [==============================] - 66s 17ms/step - loss: 4.2847 - val_loss: 4.9291\n",
      "Epoch 26/50\n",
      "3880/3880 [==============================] - 66s 17ms/step - loss: 4.2218 - val_loss: 4.9296\n",
      "Epoch 27/50\n",
      "3880/3880 [==============================] - 68s 17ms/step - loss: 4.1618 - val_loss: 4.9252\n",
      "Epoch 28/50\n",
      "3880/3880 [==============================] - 66s 17ms/step - loss: 4.1023 - val_loss: 4.9235\n",
      "Epoch 29/50\n",
      "3880/3880 [==============================] - 67s 17ms/step - loss: 4.0417 - val_loss: 4.9262\n",
      "Epoch 30/50\n",
      "3880/3880 [==============================] - 66s 17ms/step - loss: 3.9820 - val_loss: 4.9280\n",
      "Epoch 00030: early stopping\n"
     ]
    }
   ],
   "source": [
    "model_accuracy(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Model accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict():\n",
    "    reverse_target_word_index=y_tokenizer.index_word\n",
    "    reverse_source_word_index=X_tokenizer.index_word\n",
    "    target_word_index=y_tokenizer.word_index\n",
    "    latent_dim=300\n",
    "    # Encode the input sequence to get the feature vector\n",
    "    encoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h, state_c])\n",
    "\n",
    "# Decoder setup\n",
    "# Below tensors will hold the states of the previous time step\n",
    "    decoder_state_input_h = Input(shape=(latent_dim,))\n",
    "    decoder_state_input_c = Input(shape=(latent_dim,))\n",
    "    decoder_hidden_state_input = Input(shape=(max_text_len,latent_dim))\n",
    "\n",
    "# Get the embeddings of the decoder sequence\n",
    "    dec_emb2= dec_emb_layer(decoder_inputs) \n",
    "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
    "    decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
    "\n",
    "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
    "    decoder_outputs2 = decoder_dense(decoder_outputs2) \n",
    "\n",
    "# Final decoder model\n",
    "    decoder_model = Model([decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, \n",
    "                                              decoder_state_input_c],[decoder_outputs2] + [state_h2, state_c2])\n",
    "    \n",
    "    def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "        e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
    "    \n",
    "    # Generate empty target sequence of length 1.\n",
    "        target_seq = np.zeros((1,1))\n",
    "    \n",
    "    # Populate the first word of target sequence with the start word.\n",
    "        target_seq[0, 0] = target_word_index['sostok']\n",
    "\n",
    "        stop_condition = False\n",
    "        decoded_sentence = ''\n",
    "        while not stop_condition:\n",
    "      \n",
    "            output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n",
    "\n",
    "        # Sample a token\n",
    "            sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "            sampled_token = reverse_target_word_index[sampled_token_index]\n",
    "        \n",
    "            if(sampled_token!='eostok'):\n",
    "                decoded_sentence += ' '+sampled_token\n",
    "\n",
    "        # Exit condition: either hit max length or find stop word.\n",
    "            if (sampled_token == 'eostok'  or len(decoded_sentence.split()) >= (max_sum_len-1)):\n",
    "                stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "            target_seq = np.zeros((1,1))\n",
    "            target_seq[0, 0] = sampled_token_index\n",
    "\n",
    "        # Update internal states\n",
    "            e_h, e_c = h, c\n",
    "\n",
    "        return decoded_sentence\n",
    "    \n",
    "    def seq2summary(input_seq):\n",
    "        newString=''\n",
    "        for i in input_seq:\n",
    "            if((i!=0 and i!=target_word_index['sostok']) and i!=target_word_index['eostok']):\n",
    "                newString=newString+reverse_target_word_index[i]+' '\n",
    "        return newString\n",
    "\n",
    "    def seq2text(input_seq):\n",
    "        newString=''\n",
    "        for i in input_seq:\n",
    "            if(i!=0):\n",
    "                newString=newString+reverse_source_word_index[i]+' '\n",
    "        return newString\n",
    "    \n",
    "    for i in range(0,4):\n",
    "        print(\"Review:\",seq2text(X_train[i]))\n",
    "        print(\"Original summary:\",seq2summary(y_train[i]))\n",
    "        print(\"Predicted summary:\",decode_sequence(X_train[i].reshape(1,max_text_len)))\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review: new delhi may indian railways install cctv surveillance system stations nirbhaya fund approved cost rs crore strengthen security rail premises tenders floated shortly install high definition cctv cameras stations across country ensure round clock security passengers including women centre created nirbhaya fund rs crore corpus union budget support initiatives government ngos working towards protecting dignity ensuring safety women country project envisages installation cameras platforms waiting areas control rooms stations monitor cctv footages trained rpf personnel constantly station master also given access monitor cctv camera footage part project undertaken public transporter said senior railway ministry official railways stations currently stations already provided cctv cameras helping us great way signboard reading watch preventive effect criminal minds official said besides keeping constant watch cctv surveillance system helps post incident crime investigation railways aims cover stations cctv system phase wise \n",
      "Original summary: the indian railways will install about cctvs stations cost crore using nirbhaya fund present out about stations have already been provided with cctvs besides stations important trains will also equipped with cctvs the centre had created nirbhaya fund with crore after december gang rape incident \n",
      "Predicted summary:  the delhi government has decided allow the first time the country the country the country the country the country the country the country the country the country the state has been taken the state the court said that the government will not have been taken the government the court said\n",
      "\n",
      "\n",
      "Review: says gen bipin rawat manash pratim bhuyan new delhi june transformational move indian army set open combat positions women gender barrier broken countries globally army chief gen bipin rawat said process allow women combat role currently exclusive domain men moving fast initially women recruited positions military police looking women coming jawans going start soon firstly start women military police jawans said giving details move allow women male dominated positions army currently women allowed number select areas including medical legal educational signals engineering wings army combat roles kept limit due operational concerns logistical issues army chief said ready recruit women jawans matter taken government already started process gen rawat told pti exclusive interaction recently said women show grit strength taking challenges combat role shattering glass ceiling countries including germany australia canada us britain denmark finland france norway sweden israel allowed women combat roles roles military police include policing cantonments army establishments prevent breach rules regulations soldiers maintaining movement soldiers well logistics peace war handling prisoners war extending aid civil police whenever required \n",
      "Original summary: army chief general bipin rawat has said women will soon inducted combat roles the army the process for which has started currently women are only allowed medical legal educational signals and engineering wings the army most countries not allow women combat roles with usa and germany being among the few who \n",
      "Predicted summary:  the delhi government has decided allow the first time the country the country the country the country the country the country the country the country the country the state has been taken the state the court said that the government will not have been taken the government the court said\n",
      "\n",
      "\n",
      "Review: mumbai feb fruit juice concentrate maker rasna today said eyeing revenue rs crore next three years snack category looking per cent market share rs crore market growing per cent next three years looking rs crore category next three years rasna chairman managing director piruz khambatta told reporters rasna presence sweet snack category brand vitos also planning launch introduce ethnic snacks western snacks company invested rs crore far vitos plans export snack brand neighbouring countries year looking taking neighbouring countries mostly starting distribution nepal bangladesh sri lanka year added footprint across countries exports contribute per cent companys sales driven largely middle east \n",
      "Original summary: fruit juice concentrate maker rasna eyeing revenue crore the next three years from the snack category chairman piruz khambatta said rasna which has presence the sweet snack category with its brand vitos planning introduce ethnic and western snacks notably rasna enjoys market share the crore powdered drink space \n",
      "Predicted summary:  the delhi government has decided allow the first time the country the country the country the country the country the country the country the country the country the state has been taken the state the court said that the government will not have been taken the government the court said\n",
      "\n",
      "\n",
      "Review: belgian court ruled advertising claims health environmental benefits palm oil free choco spread illegal case brought supermarket chain delhaize ferrero manufactures nutella delhaize ordered repeat claims choco spread better planet health pain fine repetition ferrero argued delhaize campaign unfairly damaged iconic nutella brand placed next choco spread supermarket shelves delhaize earlier court case issue told guardian considering legal actions would appeal verdict issue short lived delhaize advertising campaign claimed belgian outlet best selling choco spread better planet consumer health contained palm oil ferrero argued implied firms used sustainable practices buying palm oil within existing supply chains ethically inferior greenpeace rated ferrero one best global manufacturers deforestation policies said case turned battle sales strategies ferrero definitely one companies best implement sustainable commitments said lambrechts greenpeace belgium spokeswoman case marketing ploys market shares less best initiative palm oil ferrero use palm oil pushed spotlight french environment minister gol ne royal called consumer boycott nutella deforestation concerns royal subsequently backed call offering confectionery titan thousand apologies multiple studies drawn causal links growth palm oil plantations south east asia deforestation court ruled delhaize claims unverifiable ferrero robust palm oil charter says trace almost palm oil sustainable sources company came top wwf sustainability league table palm oil buyers last year member roundtable sustainable palm oil palm oil innovation group court sustained ferrero arguments illegal claim product without palm oil de facto better environment better nutritional qualities one containing palm oil spokesman ferrero said ferrero taken admirable steps reduce impact palm oil within supply chain said glenn hurowitz ceo mighty campaigns deforestation caused palm oil yet impossible delink one company supplies broader continued deforestation palm oil industry whole separate argument ferrero delhaize allowed call spread choco contains chocolate thrown court part delhaize said choco still manufactured sunflower oil cocoa butter coconut oil packaged palm oil free seen impact choco sales contrary said firm communications director roel dekelver despite judgement continue propose present innovative inspirational products customers answer different needs hold us back belgian supermarket merged ahold create bn retail empire last year also alleged health benefits palm oil free produce citing potential health concerns palm oil assessment european food safety authority however assessment also applied vegetable oils products analysis currently reassessed ian barber communications director advertising association said first rule advertising back claim make imagine strategy open delhaize advertise palm oil free leave world make mind good bad thing said sign guardian sustainable business member get stories like direct inbox every week also follow us twitter \n",
      "Original summary: belgian court has ruled that advertising claims about the health and environmental benefits palm oil free choco spread supermarket chain delhaize were illegal the case nutella manufacturer ferrero argued that delhaize campaign unfairly damaged its iconic brand delhaize has now been ordered not repeat claims about its spread being better for the planet health \n",
      "Predicted summary:  the delhi government has decided allow the first time the country the country the country the country the country the country the country the country the country the state has been taken the state the court said that the government will not have been taken the government the court said\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}