MetricsOnValidationSet_reply.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "unable to import 'smart_open.gcs', disabling that module\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "from smart_open import open\n",
    "import boto3\n",
    "from io import StringIO # python3; python2: BytesIO \n",
    "from boto3.s3.transfer import TransferConfig\n",
    "import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Divide train into 90 % training and 10 % testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"text_ tokens\", \"tweet_id\" ,\"language\", \"tweet_timestamp\", \"engaged_with_user_id\",\\\n",
    "               \"engaging_user_id\", \"engagee_follows_engager\", \"reply_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [0, 2, 7, 8, 9, 14, 19, 20], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100)))\n",
    "val_set['reply_bool'] = val_set.reply_timestamp.fillna(0)\n",
    "val_set[val_set.reply_bool != 0.0] = 1.0\n",
    "gt = list(val_set.reply_bool)\n",
    "pr = [0.5] * len(gt)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prauc)\n",
    "print(rce)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Factorization Machine Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"tweet_timestamp\", \"reply_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [8, 20], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100))).reset_index()\n",
    "val_set['reply_bool'] = val_set.reply_timestamp.fillna(0)\n",
    "val_set.loc[val_set.reply_bool != 0.0, 'reply_bool'] = 1.0\n",
    "gt = list(val_set.reply_bool)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/data/val_set_result.csv',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'prauc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-fc3b1d158529>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprauc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'prauc' is not defined"
     ]
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.809286688284587"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"tweet_timestamp\", \"reply_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [8, 20], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100))).reset_index()\n",
    "val_set['reply_bool'] = val_set.reply_timestamp.fillna(0)\n",
    "val_set.loc[val_set.reply_bool != 0.0, 'reply_bool'] = 1.0\n",
    "gt = list(val_set.reply_bool)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.csv',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### ./libFM -task c -train /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/train_set.csv.libfm -test /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set.csv.libfm -dim 1,1,16 -method sgd -iter 8 -learn_rate 0.00001 -init_stdev 0.1 -regular '0.028,0.01,0.01' -out /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07596418651654453"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.5819812126435835"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"tweet_timestamp\", \"reply_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [8, 20], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100))).reset_index()\n",
    "val_set['reply_bool'] = val_set.reply_timestamp.fillna(0)\n",
    "val_set.loc[val_set.reply_bool != 0.0, 'reply_bool'] = 1.0\n",
    "gt = list(val_set.reply_bool)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.cs',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.cs',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ./libFM -task c -train /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/train_set.csv.libfm -test /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set.csv.libfm -dim 1,1,16 -method sgd -iter 8 -learn_rate 0.01 -init_stdev 0.1 -regular '0.028,0.01,0.01' -out /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1255242588436767"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.22150346471027"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_als.csv',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23713872027556365"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-12784.705715202648"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"tweet_timestamp\", \"reply_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [8, 20], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100))).reset_index()\n",
    "val_set['reply_bool'] = val_set.reply_timestamp.fillna(0)\n",
    "val_set.loc[val_set.reply_bool != 0.0, 'reply_bool'] = 1.0\n",
    "gt = list(val_set.reply_bool)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.csv',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ./libFM -task c -train /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/train_set.csv.libfm -test /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set.csv.libfm -dim 1,1,16 -method sgd -iter 10 -learn_rate 0.01 -init_stdev 0.1 -regular '0.028,0.01,0.01' -out /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13159767869843486"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.65676682277217"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ffm/reply/gt_val_set.csv',\\\n",
    "            names = ['ground_truth'])\n",
    "gt = list(gt_df.ground_truth)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ffm/reply/val_set.output',\\\n",
    "                      names = ['probability_reply'])\n",
    "pr = list(pr_df.probability_reply)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13344144016918683"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13.762694675393494"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On Leaderboard:\n",
    "##### ./libFM -task c -train /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/train_df.csv -test /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/test_df.csv -dim 1,1,16 -method sgd -iter 10 -learn_rate 0.01 -init_stdev 0.1 -regular '0.028,0.01,0.01' -out /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/test_df_result_sgd.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Next thing to try by increasing number of iterations with SGD:\n",
    "##### ./libFM -task c -train /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/train_set.csv.libfm -test /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set.csv.libfm -dim 1,1,16 -method sgd -iter 20 -learn_rate 0.01 -init_stdev 0.1 -regular '0.028,0.01,0.01' -out /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_sgd.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  With ALS with right regularization and parameters\n",
    "#####  ./libFM -task c -train /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/train_set.csv.libfm -test /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set.csv.libfm -dim 1,1,16 -method als -iter 12  -init_stdev 0.1 -regular '0.028,0.01,0.01' -out /home/ubuntu/recsys_challenge_2020/classification_type_models/f1/reply/val_set_result_als.csv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}