main_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv,os,re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(os.path.join(os.getcwd(), ('Data/')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = pd.read_csv('dataset.csv',names=['Author','Text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_df['Author'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df['Text'] = data_df['Text'].map(lambda x: re.sub('\\r|\\n|\\'','',x))\n",
    "data_df['Text'] = data_df['Text'].map(lambda x: re.sub(r'--\\d\\d\\d-\\d\\d\\d-\\d\\d\\d\\d','',x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#data_df.head()\n",
    "#Generate corpus of words for use by the CountVectorizer\n",
    "corpus = []\n",
    "for text in data_df['Text']:\n",
    "    corpus.append(text)\n",
    "#corpus[11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# split the dataset based on attribute 'Author'.\n",
    "from sklearn.model_selection import train_test_split\n",
    "y = data_df.pop(\"Author\")\n",
    "#X = data_df\n",
    "#print(y)\n",
    "X_train, X_test, y_train, y_test = train_test_split(data_df['Text'], y, test_size=0.3, stratify=y, random_state=10) \n",
    "#print((X_train['Text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "50\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Both test and train labels contain instances of all the 50 authors. \n",
    "df_ytrain = pd.DataFrame(y_train)\n",
    "df_ytest = pd.DataFrame(y_test)\n",
    "print(len(df_ytrain['Author'].value_counts()))\n",
    "print(len(df_ytest['Author'].value_counts()))\n",
    "type(df_ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method spmatrix.__str__ of <3500x5000 sparse matrix of type '<class 'numpy.int64'>'\n",
      "\twith 658576 stored elements in Compressed Sparse Row format>>\n",
      "  (0, 637)\t1\n",
      "  (0, 711)\t1\n",
      "  (0, 1520)\t1\n",
      "  (0, 2107)\t1\n",
      "  (0, 3262)\t1\n",
      "  (0, 3051)\t1\n",
      "  (0, 2137)\t1\n",
      "  (0, 552)\t1\n",
      "  (0, 2645)\t1\n",
      "  (0, 773)\t1\n",
      "  (0, 2536)\t1\n",
      "  (0, 4709)\t1\n",
      "  (0, 275)\t1\n",
      "  (0, 4307)\t1\n",
      "  (0, 373)\t1\n",
      "  (0, 2274)\t5\n",
      "  (0, 2640)\t1\n",
      "  (0, 345)\t1\n",
      "  (0, 3796)\t4\n",
      "  (0, 3797)\t3\n",
      "  (0, 4517)\t2\n",
      "  (0, 1294)\t2\n",
      "  (0, 4359)\t1\n",
      "  (0, 2211)\t3\n",
      "  (0, 3185)\t1\n",
      "  :\t:\n",
      "  (3499, 2965)\t1\n",
      "  (3499, 3143)\t1\n",
      "  (3499, 4265)\t4\n",
      "  (3499, 4699)\t4\n",
      "  (3499, 4261)\t1\n",
      "  (3499, 4671)\t3\n",
      "  (3499, 1600)\t1\n",
      "  (3499, 88)\t1\n",
      "  (3499, 1147)\t1\n",
      "  (3499, 3465)\t1\n",
      "  (3499, 3891)\t3\n",
      "  (3499, 2316)\t1\n",
      "  (3499, 2728)\t1\n",
      "  (3499, 4944)\t2\n",
      "  (3499, 2646)\t1\n",
      "  (3499, 1041)\t1\n",
      "  (3499, 4548)\t1\n",
      "  (3499, 1042)\t6\n",
      "  (3499, 4972)\t1\n",
      "  (3499, 1043)\t2\n",
      "  (3499, 199)\t1\n",
      "  (3499, 1713)\t1\n",
      "  (3499, 917)\t1\n",
      "  (3499, 1503)\t1\n",
      "  (3499, 4892)\t1\n"
     ]
    }
   ],
   "source": [
    "from sklearn import preprocessing, naive_bayes, metrics\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "# label encode the target variable \n",
    "encoder = preprocessing.LabelEncoder()\n",
    "y_train = encoder.fit_transform(y_train)\n",
    "y_test = encoder.fit_transform(y_test)\n",
    "\n",
    "# create a count vectorizer object \n",
    "count_vect = CountVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000, max_df=0.5,)\n",
    "#count_vect.fit(corpus)\n",
    "\n",
    "# transform the training and validation data using count vectorizer object\n",
    "Xtrain_count =  count_vect.fit_transform(X_train)\n",
    "Xtest_count =  count_vect.transform(X_test)\n",
    "print(Xtrain_count.__str__)\n",
    "print(Xtrain_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method spmatrix.__str__ of <3500x5000 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 512298 stored elements in Compressed Sparse Row format>>\n",
      "  (0, 1752)\t0.1474346561932368\n",
      "  (0, 4926)\t0.07619913359503527\n",
      "  (0, 741)\t0.18849368746905582\n",
      "  (0, 1113)\t0.09401888358022331\n",
      "  (0, 1409)\t0.156472484860133\n",
      "  (0, 4127)\t0.1934728669070515\n",
      "  (0, 4426)\t0.18127505571405708\n",
      "  (0, 740)\t0.1431506713577788\n",
      "  (0, 1735)\t0.15036230357848548\n",
      "  (0, 2063)\t0.08486588826455495\n",
      "  (0, 1733)\t0.1570309089335085\n",
      "  (0, 1678)\t0.08093796477474303\n",
      "  (0, 900)\t0.17388571637216244\n",
      "  (0, 1802)\t0.16484260939277687\n",
      "  (0, 2553)\t0.1435377232905835\n",
      "  (0, 4068)\t0.3420454946459797\n",
      "  (0, 1815)\t0.41627426981463295\n",
      "  (0, 4691)\t0.10768717475366485\n",
      "  (0, 4703)\t0.13879987720537834\n",
      "  (0, 2276)\t0.17007667534192272\n",
      "  (0, 3686)\t0.1950295301870916\n",
      "  (0, 2646)\t0.14295867056868772\n",
      "  (0, 2091)\t0.14220059024409096\n",
      "  (0, 2764)\t0.097827924610463\n",
      "  (0, 335)\t0.1950295301870916\n",
      "  :\t:\n",
      "  (3499, 3428)\t0.07650243213300482\n",
      "  (3499, 2577)\t0.08453022916658148\n",
      "  (3499, 2889)\t0.14979491286478125\n",
      "  (3499, 964)\t0.1077196115867431\n",
      "  (3499, 694)\t0.0975765765685133\n",
      "  (3499, 3707)\t0.08496421380730272\n",
      "  (3499, 3631)\t0.09316089633412745\n",
      "  (3499, 1283)\t0.0975765765685133\n",
      "  (3499, 591)\t0.09929490119317813\n",
      "  (3499, 2863)\t0.09087465439345052\n",
      "  (3499, 2670)\t0.08812585175861534\n",
      "  (3499, 2145)\t0.1063981541988263\n",
      "  (3499, 3257)\t0.08972949466856586\n",
      "  (3499, 1196)\t0.08453022916658148\n",
      "  (3499, 3098)\t0.10184243489808607\n",
      "  (3499, 3754)\t0.09870559868410313\n",
      "  (3499, 2714)\t0.08761861350481852\n",
      "  (3499, 3091)\t0.11013637412490955\n",
      "  (3499, 3129)\t0.10682902759785227\n",
      "  (3499, 1926)\t0.08829782569742407\n",
      "  (3499, 4399)\t0.11119108294737398\n",
      "  (3499, 3070)\t0.10515849033241212\n",
      "  (3499, 278)\t0.10150570695736355\n",
      "  (3499, 4410)\t0.1077196115867431\n",
      "  (3499, 424)\t0.10437270958611365\n"
     ]
    }
   ],
   "source": [
    "# word level tf-idf\n",
    "tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000,sublinear_tf=True, max_df=0.5,  )\n",
    "#tfidf_vect.fit(corpus)\n",
    "Xtrain_tfidf = tfidf_vect.fit_transform(X_train)\n",
    "Xtest_tfidf = tfidf_vect.transform(X_test)\n",
    "\n",
    "#print(Xtrain_tfidf.__str__)\n",
    "#print(Xtrain_tfidf)\n",
    "\n",
    "# ngram level tf-idf \n",
    "tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3), max_features=5000,sublinear_tf=True, max_df=0.5,)\n",
    "#tfidf_vect_ngram.fit(corpus)\n",
    "Xtrain_tfidf_ngram = tfidf_vect_ngram.fit_transform(X_train)\n",
    "Xtest_tfidf_ngram = tfidf_vect_ngram.transform(X_test)\n",
    "\n",
    "print(Xtrain_tfidf_ngram.__str__)\n",
    "print(Xtrain_tfidf_ngram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(classifier, feature_vector_train, label, feature_vector_test, is_neural_net=False):\n",
    "    # fit the training dataset on the classifier\n",
    "    classifier.fit(feature_vector_train, label)\n",
    "    \n",
    "    # predict the labels on validation dataset\n",
    "    predictions = classifier.predict(feature_vector_test)\n",
    "    \n",
    "    if is_neural_net:\n",
    "        predictions = predictions.argmax(axis=-1)\n",
    "    \n",
    "    return metrics.accuracy_score(predictions, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3500, 5000)\n",
      "NB, Count Vectors:  0.8126666666666666\n"
     ]
    }
   ],
   "source": [
    "print(Xtrain_tfidf.shape)\n",
    "# Naive Bayes on Count Vectors\n",
    "accuracy = train_model(naive_bayes.MultinomialNB(), Xtrain_count, y_train, Xtest_count)\n",
    "print (\"NB, Count Vectors: \", accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NB, WordLevel TF-IDF:  0.7833333333333333\n",
      "NB, N-Gram Vectors:  0.754\n"
     ]
    }
   ],
   "source": [
    "# Naive Bayes on Word Level TF IDF Vectors\n",
    "accuracy = train_model(naive_bayes.MultinomialNB(), Xtrain_tfidf, y_train, Xtest_tfidf)\n",
    "print (\"NB, WordLevel TF-IDF: \", accuracy)\n",
    "\n",
    "# Naive Bayes on Ngram Level TF IDF Vectors\n",
    "accuracy = train_model(naive_bayes.MultinomialNB(), Xtrain_tfidf_ngram, y_train, Xtest_tfidf_ngram)\n",
    "print (\"NB, N-Gram Vectors: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}