LanguageDistribution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "from smart_open import open\n",
    "import boto3\n",
    "from io import StringIO # python3; python2: BytesIO \n",
    "from boto3.s3.transfer import TransferConfig\n",
    "import metrics\n",
    "import torch\n",
    "from transformers import *\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = [\"Language\", \"tweet_timestamp\"]\n",
    "train_language = pd.read_csv(open(\"s3://recsys-challenge-2020/training.tsv\"), encoding=\"utf-8\",\n",
    "                    names = all_features, usecols= [7, 8], sep=\"\\x01\"\n",
    "                   )\n",
    "# train_sorted = train.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "# n_head = 90\n",
    "# n_tail = 10\n",
    "\n",
    "# gt = list(val_set.reply_bool)\n",
    "# pr = [0.5] * len(gt)\n",
    "# prauc = metrics.compute_prauc(pr,gt)\n",
    "# rce = metrics.compute_rce(pr,gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sorted = train_language.sort_values(by=['tweet_timestamp']).reset_index()\n",
    "n_head = 90\n",
    "n_tail = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_language = train_sorted.head(int(len(train_sorted)*(n_head/100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_set_language = train_sorted.tail(int(len(train_sorted)*(n_tail/100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_language.to_csv('s3://recsys-challenge-2020/train_set_language.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_set_language.to_csv('s3://recsys-challenge-2020/val_set_language.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "D3164C7FBCF2565DDF915B1B3AEFB1DC    60722973\n",
       "22C448FF81263D4BAF2A176145EE9EAD    23652265\n",
       "06D61DCBBE938971E1EA0C38BD9B5446    13273364\n",
       "ECED8A16BE2A5E8871FD55F4842F16B1    10280788\n",
       "B9175601E87101A984A50F8A62A1C374     7884887\n",
       "                                      ...   \n",
       "D18801336202297E6484F634CAC6592E         614\n",
       "4690215948DBF6872B8ED1C2BC87B17E         590\n",
       "B2235C8B73239FDC5780DD132419833A         363\n",
       "2E18F6F53E3CF073911AF0A93BBE5373         138\n",
       "515E873C86EE1577E75FA2387B7FA59E          14\n",
       "Name: Language, Length: 66, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_language.Language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_language.Language.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Language</th>\n",
       "      <th>tweet_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>22C448FF81263D4BAF2A176145EE9EAD</td>\n",
       "      <td>1581258144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>B9175601E87101A984A50F8A62A1C374</td>\n",
       "      <td>1581018910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>22C448FF81263D4BAF2A176145EE9EAD</td>\n",
       "      <td>1581244866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>22C448FF81263D4BAF2A176145EE9EAD</td>\n",
       "      <td>1581250124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>22C448FF81263D4BAF2A176145EE9EAD</td>\n",
       "      <td>1580958805</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Language  tweet_timestamp\n",
       "0  22C448FF81263D4BAF2A176145EE9EAD       1581258144\n",
       "1  B9175601E87101A984A50F8A62A1C374       1581018910\n",
       "2  22C448FF81263D4BAF2A176145EE9EAD       1581244866\n",
       "3  22C448FF81263D4BAF2A176145EE9EAD       1581250124\n",
       "4  22C448FF81263D4BAF2A176145EE9EAD       1580958805"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_language.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Language</th>\n",
       "      <th>tweet_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>19856362</td>\n",
       "      <td>06D61DCBBE938971E1EA0C38BD9B5446</td>\n",
       "      <td>1580947200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>70014084</td>\n",
       "      <td>D3164C7FBCF2565DDF915B1B3AEFB1DC</td>\n",
       "      <td>1580947200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>45640141</td>\n",
       "      <td>D3164C7FBCF2565DDF915B1B3AEFB1DC</td>\n",
       "      <td>1580947200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>103063682</td>\n",
       "      <td>ECED8A16BE2A5E8871FD55F4842F16B1</td>\n",
       "      <td>1580947200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>54351326</td>\n",
       "      <td>D3164C7FBCF2565DDF915B1B3AEFB1DC</td>\n",
       "      <td>1580947200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       index                          Language  tweet_timestamp\n",
       "0   19856362  06D61DCBBE938971E1EA0C38BD9B5446       1580947200\n",
       "1   70014084  D3164C7FBCF2565DDF915B1B3AEFB1DC       1580947200\n",
       "2   45640141  D3164C7FBCF2565DDF915B1B3AEFB1DC       1580947200\n",
       "3  103063682  ECED8A16BE2A5E8871FD55F4842F16B1       1580947200\n",
       "4   54351326  D3164C7FBCF2565DDF915B1B3AEFB1DC       1580947200"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set_language.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14807523"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_set_language)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "133267714"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_set_language)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}