Content based recommendation system.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import time\n",
    "from datasketch import MinHash, MinHashLSHForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    text = re.sub(r'[^\\w\\s]','',text)\n",
    "    tokens = text.lower()\n",
    "    tokens = tokens.split()\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "permutations = 128\n",
    "\n",
    "#Number of Recommendations to return\n",
    "num_recommendations = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shingles (tokens) are: ['the', 'devil', 'went', 'down', 'to', 'georgia']\n"
     ]
    }
   ],
   "source": [
    "text = 'The devil went down to Georgia'\n",
    "print('The shingles (tokens) are:', preprocess(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_forest(data, perms):\n",
    "    start_time = time.time()\n",
    "    \n",
    "    minhash = []\n",
    "    \n",
    "    for text in data['text']:\n",
    "        tokens = preprocess(text)\n",
    "        m = MinHash(num_perm=perms)\n",
    "        for s in tokens:\n",
    "            m.update(s.encode('utf8'))\n",
    "        minhash.append(m)\n",
    "        \n",
    "    forest = MinHashLSHForest(num_perm=perms)\n",
    "    \n",
    "    for i,m in enumerate(minhash):\n",
    "        forest.add(i,m)\n",
    "        \n",
    "    forest.index()\n",
    "    \n",
    "    print('It took %s seconds to build forest.' %(time.time()-start_time))\n",
    "    \n",
    "    return forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(text, database, perms, num_results, forest):\n",
    "    start_time = time.time()\n",
    "    \n",
    "    tokens = preprocess(text)\n",
    "    m = MinHash(num_perm=perms)\n",
    "    for s in tokens:\n",
    "        m.update(s.encode('utf8'))\n",
    "        \n",
    "    idx_array = np.array(forest.query(m, num_results))\n",
    "    if len(idx_array) == 0:\n",
    "        return None # if your query is empty, return none\n",
    "    \n",
    "    result = database.iloc[idx_array]['title']\n",
    "    \n",
    "    print('It took %s seconds to query forest.' %(time.time()-start_time))\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It took 40.99619197845459 seconds to build forest.\n"
     ]
    }
   ],
   "source": [
    "mylist = []\n",
    "\n",
    "for chunk in  pd.read_csv('C:\\Python37\\Recommender System\\papers.csv',chunksize=2000):\n",
    "    mylist.append(chunk)\n",
    "\n",
    "db = pd.concat(mylist, axis= 0)\n",
    "del mylist\n",
    "db['text'] = db['title'] + ' ' + db['abstract']\n",
    "forest = get_forest(db, permutations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It took 0.03798627853393555 seconds to query forest.\n",
      "\n",
      " Top Recommendation(s) is(are) \n",
      " 995     Neural Network Weight Matrix Synthesis Using O...\n",
      "5       Using a neural net to instantiate a deformable...\n",
      "5191    A Self-Organizing Integrated Segmentation and ...\n",
      "2069    Analytic Solutions to the Formation of Feature...\n",
      "2457    Inferring Neural Firing Rates from Spike Train...\n",
      "Name: title, dtype: object\n"
     ]
    }
   ],
   "source": [
    "num_recommendations = 5\n",
    "title = 'Using a neural net to instantiate a deformable model'\n",
    "result = predict(title, db, permutations, num_recommendations, forest)\n",
    "print('\\n Top Recommendation(s) is(are) \\n', result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}