SaveUserTokens.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "from smart_open import open\n",
    "import boto3\n",
    "from io import StringIO # python3; python2: BytesIO \n",
    "from boto3.s3.transfer import TransferConfig\n",
    "import metrics\n",
    "import torch\n",
    "from transformers import *\n",
    "import numpy as np\n",
    "import ast\n",
    "import time\n",
    "import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"text_ tokens\", \"engaging_user_id\", \"reply_timestamp\"]\n",
    "train_set = pd.read_csv('s3://recsys-challenge-2020/training.tsv', encoding=\"utf-8\",  sep=\"\\x01\",\n",
    "                     usecols= [0, 14, 20], names = column_of_interest)\n",
    "column_of_interest = [\"engaging_user_id\"]\n",
    "val_set = pd.read_csv('s3://recsys-challenge-2020/val.tsv', encoding=\"utf-8\", sep=\"\\x01\",\n",
    "                     usecols= [14], names=column_of_interest)\n",
    "unique_val_set_users = val_set[['engaging_user_id']].drop_duplicates()\n",
    "train_set_positive = train_set[train_set.reply_timestamp.notnull()]\n",
    "warm_train_set_positive = pd.merge(train_set_positive, unique_val_set_users, how = 'inner', on = 'engaging_user_id')\n",
    "user_tokens = warm_train_set_positive.groupby(['engaging_user_id'])['text_ tokens'].agg(list).reset_index()\n",
    "user_tokens.to_csv('s3://recsys-challenge-2020/user_tokens_reply.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"text_ tokens\", \"engaging_user_id\", \"retweet_timestamp\"]\n",
    "train_set = pd.read_csv('s3://recsys-challenge-2020/training.tsv', encoding=\"utf-8\",  sep=\"\\x01\",\n",
    "                     usecols= [0, 14, 21], names = column_of_interest)\n",
    "column_of_interest = [\"engaging_user_id\"]\n",
    "val_set = pd.read_csv('s3://recsys-challenge-2020/val.tsv', encoding=\"utf-8\", sep=\"\\x01\",\n",
    "                     usecols= [14], names=column_of_interest)\n",
    "unique_val_set_users = val_set[['engaging_user_id']].drop_duplicates()\n",
    "train_set_positive = train_set[train_set.retweet_timestamp.notnull()]\n",
    "warm_train_set_positive = pd.merge(train_set_positive, unique_val_set_users, how = 'inner', on = 'engaging_user_id')\n",
    "user_tokens = warm_train_set_positive.groupby(['engaging_user_id'])['text_ tokens'].agg(list).reset_index()\n",
    "user_tokens.to_csv('s3://recsys-challenge-2020/user_tokens_retweet.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retweet with comment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"text_ tokens\", \"engaging_user_id\", \"retweet_with_comment_timestamp\"]\n",
    "train_set = pd.read_csv('s3://recsys-challenge-2020/training.tsv', encoding=\"utf-8\",  sep=\"\\x01\",\n",
    "                     usecols= [0, 14, 22], names = column_of_interest)\n",
    "column_of_interest = [\"engaging_user_id\"]\n",
    "val_set = pd.read_csv('s3://recsys-challenge-2020/val.tsv', encoding=\"utf-8\", sep=\"\\x01\",\n",
    "                     usecols= [14], names=column_of_interest)\n",
    "unique_val_set_users = val_set[['engaging_user_id']].drop_duplicates()\n",
    "train_set_positive = train_set[train_set.retweet_with_comment_timestamp.notnull()]\n",
    "warm_train_set_positive = pd.merge(train_set_positive, unique_val_set_users, how = 'inner', on = 'engaging_user_id')\n",
    "user_tokens = warm_train_set_positive.groupby(['engaging_user_id'])['text_ tokens'].agg(list).reset_index()\n",
    "user_tokens.to_csv('s3://recsys-challenge-2020/user_tokens_retweet_with_comment.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"text_ tokens\", \"engaging_user_id\", \"like_timestamp\"]\n",
    "train_set = pd.read_csv('s3://recsys-challenge-2020/training.tsv', encoding=\"utf-8\",  sep=\"\\x01\",\n",
    "                     usecols= [0, 14, 23], names = column_of_interest)\n",
    "column_of_interest = [\"engaging_user_id\"]\n",
    "val_set = pd.read_csv('s3://recsys-challenge-2020/val.tsv', encoding=\"utf-8\", sep=\"\\x01\",\n",
    "                     usecols= [14], names=column_of_interest)\n",
    "unique_val_set_users = val_set[['engaging_user_id']].drop_duplicates()\n",
    "train_set_positive = train_set[train_set.like_timestamp.notnull()]\n",
    "warm_train_set_positive = pd.merge(train_set_positive, unique_val_set_users, how = 'inner', on = 'engaging_user_id')\n",
    "user_tokens = warm_train_set_positive.groupby(['engaging_user_id'])['text_ tokens'].agg(list).reset_index()\n",
    "user_tokens.to_csv('s3://recsys-challenge-2020/user_tokens_like.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokens exceed 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"engaging_user_id\"]\n",
    "val_set = pd.read_csv('s3://recsys-challenge-2020/val.tsv', encoding=\"utf-8\", sep=\"\\x01\",\n",
    "                     usecols= [14], names=column_of_interest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "model = BertModel.from_pretrained('/dev/bert/')\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "engaging_user_id    12E9952B6B5F58BA0A83B8116CCF19D1\n",
       "Name: 1119671, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_set.loc[1119671]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_tokens = pd.read_csv('s3://recsys-challenge-2020/user_tokens_reply.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}