MetricsOnValidationSet_retweet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "unable to import 'smart_open.gcs', disabling that module\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "from smart_open import open\n",
    "import boto3\n",
    "from io import StringIO # python3; python2: BytesIO \n",
    "from boto3.s3.transfer import TransferConfig\n",
    "import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Divide train into 90 % training and 10 % testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"text_ tokens\", \"tweet_id\" ,\"language\", \"tweet_timestamp\", \"engaged_with_user_id\",\\\n",
    "               \"engaging_user_id\", \"engagee_follows_engager\", \"retweet_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [0, 2, 7, 8, 9, 14, 19, 21], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100)))\n",
    "val_set['retweet_bool'] = val_set.retweet_timestamp.fillna(0)\n",
    "val_set[val_set.retweet_bool != 0.0] = 1.0\n",
    "gt = list(val_set.retweet_bool)\n",
    "pr = [0.5] * len(gt)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prauc)\n",
    "print(rce)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics on FM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"tweet_timestamp\", \"retweet_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [8, 21], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100))).reset_index()\n",
    "val_set['retweet_bool'] = val_set.retweet_timestamp.fillna(0)\n",
    "val_set.loc[val_set.retweet_bool != 0.0, 'retweet_bool'] = 1.0\n",
    "gt = list(val_set.retweet_bool)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/data/retweet/val_set_result.csv',\\\n",
    "                      names = ['probability_retweet'])\n",
    "pr = list(pr_df.probability_retweet)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42136331079050626"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18.46998582987537"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"tweet_timestamp\", \"retweet_timestamp\"]\n",
    "train = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [8, 21], sep=\"\\x01\"\n",
    "                   )\n",
    "train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10\n",
    "train_set = train_sorted.head(int(len(train_sorted)*(n_head/100)))\n",
    "val_set = train_sorted.tail(int(len(train_sorted)*(n_tail/100))).reset_index()\n",
    "val_set['retweet_bool'] = val_set.retweet_timestamp.fillna(0)\n",
    "val_set.loc[val_set.retweet_bool != 0.0, 'retweet_bool'] = 1.0\n",
    "gt = list(val_set.retweet_bool)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ffm/retweet/val_set.output',\\\n",
    "                      names = ['probability_retweet'])\n",
    "pr = list(pr_df.probability_retweet)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4696163566300273"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23.62294508164885"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ffm/retweet/val_set_gt.csv',\\\n",
    "            names = ['ground_truth'])\n",
    "gt = list(gt_df.ground_truth)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ffm/retweet/val_set.output',\\\n",
    "                      names = ['probability_retweet'])\n",
    "pr = list(pr_df.probability_retweet)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47210513831601136"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23.841255320153753"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ft_embedding_based_features/ffm/retweet/val_set_gt.csv',\\\n",
    "            names = ['ground_truth'])\n",
    "gt = list(gt_df.ground_truth)\n",
    "pr_df = pd.read_csv('/home/ubuntu/recsys_challenge_2020/classification_type_models/ft_embedding_based_features/ffm/retweet/val_set.output',\\\n",
    "                      names = ['probability_retweet'])\n",
    "pr = list(pr_df.probability_retweet)\n",
    "prauc = metrics.compute_prauc(pr,gt)\n",
    "rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ./ffm-train -p ../../ft_embedding_based_features/ffm/retweet/val_set.ffm -l 0.0002 --auto-stop -s 96 ../../ft_embedding_based_features/ffm/retweet/train_set.ffm\n",
    "##### ./ffm-predict ../../ft_embedding_based_features/ffm/retweet/val_set.ffm train_set.ffm.model ../../ft_embedding_based_features/ffm/retweet/val_set.output\n",
    "##### iter   tr_logloss   va_logloss      tr_time\n",
    "#####   1      0.29747      0.25389       1129.7\n",
    "#####   2      0.24647      0.24008       2212.0\n",
    "#####   3      0.21755      0.23859       3215.6\n",
    "#####   4      0.19949      0.24003\n",
    "##### Auto-stop. Use model at 3th iteration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49112814598621485"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prauc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25.663740331606256"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rce"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}