TweetText.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "from smart_open import open\n",
    "import boto3\n",
    "from io import StringIO # python3; python2: BytesIO \n",
    "from boto3.s3.transfer import TransferConfig\n",
    "import metrics\n",
    "import torch\n",
    "from transformers import *\n",
    "import numpy as np\n",
    "import ast\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"text_ tokens\"]\n",
    "train_set = pd.read_csv('s3://recsys-challenge-2020/train_set_reply.csv', encoding=\"utf-8\",\n",
    "                     usecols= [1])\n",
    "val_set = pd.read_csv('s3://recsys-challenge-2020/val_set_reply.csv', encoding=\"utf-8\",\n",
    "                     usecols= [1])\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_ tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>101\\t108\\t31278\\t90939\\t70325\\t196\\t199\\t71436\\t10107\\t29190\\t10107\\t106\\t16680\\t68314\\t10153\\t17015\\t15934\\t10104\\t108\\t10233\\t12396\\t14945\\t10107\\t10858\\t11405\\t13600\\t13597\\t169\\t57343\\t64482\\t119\\t119\\t119\\t100\\t11741\\t16381\\t10109\\t68830\\t10110\\t20886\\t108\\t10233\\t11127\\t21768\\t100\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t12132\\t10884\\t10157\\t11490\\t12022\\t10113\\t10731\\t10729\\t11565\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t162\\t11211\\t11703\\t12022\\t11211\\t10240\\t44466\\t100886\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>101\\t95026\\t90141\\t54922\\t10287\\t13567\\t10169\\t10105\\t10751\\t25899\\t22094\\t10111\\t30372\\t14504\\t48865\\t10107\\t10106\\t10105\\t10377\\t10451\\t44132\\t11858\\t106\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t183\\t15417\\t11281\\t13034\\t12396\\t10174\\t10858\\t10410\\t11166\\t10133\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>101\\t10117\\t12518\\t45415\\t10472\\t10114\\t19072\\t11357\\t95171\\t81690\\t11850\\t24448\\t34387\\t10161\\t15862\\t117\\t169\\t18299\\t16879\\t13617\\t10393\\t10106\\t19037\\t10855\\t13337\\t109369\\t10336\\t10105\\t11284\\t118\\t12103\\t11356\\t119\\t119\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t32992\\t13034\\t10410\\t11537\\t77593\\t10240\\t10174\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>101\\t36084\\t11372\\t26095\\t117\\t12900\\t39122\\t16114\\t10132\\t70231\\t100\\t10247\\t10104\\t22284\\t14444\\t10114\\t10812\\t10192\\t17796\\t10149\\t137\\t10974\\t10350\\t168\\t48495\\t173\\t16213\\t188\\t16218\\t16159\\t11744\\t51122\\t69857\\t86095\\t10165\\t12655\\t13137\\t119\\t56824\\t104767\\t131\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t13802\\t10115\\t10477\\t12396\\t10129\\t11274\\t10759\\t11127\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t16986\\t15417\\t104107\\t74178\\t11305\\t10162\\t10759\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>101\\t34764\\t26905\\t117\\t10479\\t92486\\t10559\\t99565\\t10939\\t71027\\t10107\\t117\\t10114\\t10347\\t16150\\t10135\\t108\\t13681\\t11490\\t13998\\t119\\t119\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t185\\t11403\\t13695\\t43125\\t10162\\t11369\\t10147\\t11259\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t47114\\t11447\\t13966\\t10457\\t10858\\t23377\\t10858\\t10477\\t102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         text_ tokens\n",
       "0  101\\t108\\t31278\\t90939\\t70325\\t196\\t199\\t71436\\t10107\\t29190\\t10107\\t106\\t16680\\t68314\\t10153\\t17015\\t15934\\t10104\\t108\\t10233\\t12396\\t14945\\t10107\\t10858\\t11405\\t13600\\t13597\\t169\\t57343\\t64482\\t119\\t119\\t119\\t100\\t11741\\t16381\\t10109\\t68830\\t10110\\t20886\\t108\\t10233\\t11127\\t21768\\t100\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t12132\\t10884\\t10157\\t11490\\t12022\\t10113\\t10731\\t10729\\t11565\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t162\\t11211\\t11703\\t12022\\t11211\\t10240\\t44466\\t100886\\t102\n",
       "1  101\\t95026\\t90141\\t54922\\t10287\\t13567\\t10169\\t10105\\t10751\\t25899\\t22094\\t10111\\t30372\\t14504\\t48865\\t10107\\t10106\\t10105\\t10377\\t10451\\t44132\\t11858\\t106\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t183\\t15417\\t11281\\t13034\\t12396\\t10174\\t10858\\t10410\\t11166\\t10133\\t102                                                                                                                                                                                                                                  \n",
       "2  101\\t10117\\t12518\\t45415\\t10472\\t10114\\t19072\\t11357\\t95171\\t81690\\t11850\\t24448\\t34387\\t10161\\t15862\\t117\\t169\\t18299\\t16879\\t13617\\t10393\\t10106\\t19037\\t10855\\t13337\\t109369\\t10336\\t10105\\t11284\\t118\\t12103\\t11356\\t119\\t119\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t32992\\t13034\\t10410\\t11537\\t77593\\t10240\\t10174\\t102                                                                                                                                                                               \n",
       "3  101\\t36084\\t11372\\t26095\\t117\\t12900\\t39122\\t16114\\t10132\\t70231\\t100\\t10247\\t10104\\t22284\\t14444\\t10114\\t10812\\t10192\\t17796\\t10149\\t137\\t10974\\t10350\\t168\\t48495\\t173\\t16213\\t188\\t16218\\t16159\\t11744\\t51122\\t69857\\t86095\\t10165\\t12655\\t13137\\t119\\t56824\\t104767\\t131\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t13802\\t10115\\t10477\\t12396\\t10129\\t11274\\t10759\\t11127\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t16986\\t15417\\t104107\\t74178\\t11305\\t10162\\t10759\\t102                               \n",
       "4  101\\t34764\\t26905\\t117\\t10479\\t92486\\t10559\\t99565\\t10939\\t71027\\t10107\\t117\\t10114\\t10347\\t16150\\t10135\\t108\\t13681\\t11490\\t13998\\t119\\t119\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t185\\t11403\\t13695\\t43125\\t10162\\t11369\\t10147\\t11259\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t47114\\t11447\\t13966\\t10457\\t10858\\t23377\\t10858\\t10477\\t102                                                                                                                                                           "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_ tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>101\\t17713\\t9405\\t119152\\t119002\\t118823\\t9531\\t10530\\t13767\\t9665\\t118649\\t33323\\t23466\\t9405\\t18623\\t27023\\t58303\\t48345\\t119\\t119\\t119\\t9665\\t118649\\t33323\\t23466\\t11018\\t9405\\t119152\\t9531\\t11489\\t9580\\t118762\\t9519\\t119285\\t130\\t14040\\t17655\\t9580\\t31531\\t125\\t14040\\t18382\\t68943\\t9576\\t16605\\t58303\\t48345\\t119\\t119\\t119\\t137\\t49866\\t27572\\t10112\\t11259\\t12674\\t11403\\t108\\t80426\\t11403\\t108\\t9560\\t22333\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t171\\t10731\\t48205\\t11259\\t12396\\t11211\\t11281\\t10174\\t11090\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>101\\t64002\\t9247\\t23665\\t9251\\t17342\\t23990\\t9764\\t36553\\t100543\\t9358\\t56789\\t9247\\t18108\\t12310\\t9485\\t38709\\t12178\\t28911\\t1729\\t1729\\t1729\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t191\\t19282\\t10305\\t83148\\t13966\\t10305\\t13966\\t11166\\t10161\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>101\\t56898\\t137\\t44592\\t11090\\t13503\\t10157\\t131\\t17144\\t112\\t187\\t12820\\t43662\\t10271\\t10950\\t119\\t119\\t119\\t14156\\t10114\\t13028\\t10479\\t10124\\t10105\\t14361\\t10139\\t10165\\t16297\\t96509\\t13328\\t11762\\t10551\\t136\\t119\\t108\\t10882\\t11759\\t11273\\t10280\\t10305\\t108\\t31091\\t34567\\t10237\\t10731\\t10237\\t100\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>101\\t137\\t15595\\t21840\\t14590\\t75268\\t57493\\t10759\\t872\\t13874\\t32629\\t28462\\t102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>101\\t56898\\t137\\t13069\\t93870\\t10284\\t11274\\t12396\\t168\\t60801\\t11211\\t131\\t9331\\t30842\\t10739\\t9580\\t118762\\t9491\\t66540\\t17594\\t100\\t119\\t9076\\t68833\\t8843\\t19105\\t18108\\t9425\\t12424\\t9390\\t119086\\t9609\\t11018\\t8863\\t100\\t119\\t9150\\t12965\\t8996\\t26737\\t101202\\t9137\\t11018\\t9144\\t9551\\t23811\\t16439\\t100\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t147\\t11373\\t11259\\t10759\\t12674\\t10157\\t11733\\t10116\\t11259\\t10575\\t102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                text_ tokens\n",
       "0  101\\t17713\\t9405\\t119152\\t119002\\t118823\\t9531\\t10530\\t13767\\t9665\\t118649\\t33323\\t23466\\t9405\\t18623\\t27023\\t58303\\t48345\\t119\\t119\\t119\\t9665\\t118649\\t33323\\t23466\\t11018\\t9405\\t119152\\t9531\\t11489\\t9580\\t118762\\t9519\\t119285\\t130\\t14040\\t17655\\t9580\\t31531\\t125\\t14040\\t18382\\t68943\\t9576\\t16605\\t58303\\t48345\\t119\\t119\\t119\\t137\\t49866\\t27572\\t10112\\t11259\\t12674\\t11403\\t108\\t80426\\t11403\\t108\\t9560\\t22333\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t171\\t10731\\t48205\\t11259\\t12396\\t11211\\t11281\\t10174\\t11090\\t102\n",
       "1  101\\t64002\\t9247\\t23665\\t9251\\t17342\\t23990\\t9764\\t36553\\t100543\\t9358\\t56789\\t9247\\t18108\\t12310\\t9485\\t38709\\t12178\\t28911\\t1729\\t1729\\t1729\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t191\\t19282\\t10305\\t83148\\t13966\\t10305\\t13966\\t11166\\t10161\\t102                                                                                                                                                                                                                                                                             \n",
       "2  101\\t56898\\t137\\t44592\\t11090\\t13503\\t10157\\t131\\t17144\\t112\\t187\\t12820\\t43662\\t10271\\t10950\\t119\\t119\\t119\\t14156\\t10114\\t13028\\t10479\\t10124\\t10105\\t14361\\t10139\\t10165\\t16297\\t96509\\t13328\\t11762\\t10551\\t136\\t119\\t108\\t10882\\t11759\\t11273\\t10280\\t10305\\t108\\t31091\\t34567\\t10237\\t10731\\t10237\\t100\\t102                                                                                                                                                                                                                       \n",
       "3  101\\t137\\t15595\\t21840\\t14590\\t75268\\t57493\\t10759\\t872\\t13874\\t32629\\t28462\\t102                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
       "4  101\\t56898\\t137\\t13069\\t93870\\t10284\\t11274\\t12396\\t168\\t60801\\t11211\\t131\\t9331\\t30842\\t10739\\t9580\\t118762\\t9491\\t66540\\t17594\\t100\\t119\\t9076\\t68833\\t8843\\t19105\\t18108\\t9425\\t12424\\t9390\\t119086\\t9609\\t11018\\t8863\\t100\\t119\\t9150\\t12965\\t8996\\t26737\\t101202\\t9137\\t11018\\t9144\\t9551\\t23811\\t16439\\t100\\t14120\\t131\\t120\\t120\\t188\\t119\\t11170\\t120\\t147\\t11373\\t11259\\t10759\\t12674\\t10157\\t11733\\t10116\\t11259\\t10575\\t102                                                                                                   "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_text(row):\n",
    "    tweet_tokens = tokenizer.decode(list(map(int, row.split('\\t'))))\n",
    "    return tweet_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set['user_text'] = train_set['text_ tokens'].apply(lambda x: calculate_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_set['user_text'] = val_set['text_ tokens'].apply(lambda x: calculate_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set[['user_text']].to_csv('s3://recsys-challenge-2020/train_set_text.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_set.to_csv('s3://recsys-challenge-2020/val_set_text.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[CLS] # ENVIVO | ¡ Buenas noches! Comienza una nueva edición de # 24HorasCentral junto a Iván Núñez... [UNK] Sigue la señal en vivo # 24Play [UNK] https : / / t. co / mi3yDUaS2F https : / / t. co / Y6qU6gwdCL [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[CLS] Celebrate Lunar New Year with the new Tigeress and Swift Outfits in the Item Shop now! https : / / t. co / oJIWHkCp5o [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[CLS] The media tend not to feature abortion survivors like Claire Culwell, a woman whose story has in recent years captivated the pro - life world.. https : / / t. co / COWpNipsgk [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[CLS] porto alegre, tem novidade na agenda [UNK] 14 de março eu toco no teatro do @ sesc _ rs e já tô muito ansiosa pra rever todo mundo. ingressos aqui : https : / / t. co / ihrnvHrB1P https : / / t. co / HyJf3917d1 [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[CLS] Karnataka boy, who guided ambulance during floods, to be awarded on # RepublicDay.. https : / / t. co / qOtzvndLmE https : / / t. co / HRGZbClnCv [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133267709</td>\n",
       "      <td>[CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133267710</td>\n",
       "      <td>[CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133267711</td>\n",
       "      <td>[CLS] RT @ colenunez7 : once i get comfortable with you i [UNK] m goofy af [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133267712</td>\n",
       "      <td>[CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133267713</td>\n",
       "      <td>[CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133267714 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                user_text\n",
       "0          [CLS] # ENVIVO | ¡ Buenas noches! Comienza una nueva edición de # 24HorasCentral junto a Iván Núñez... [UNK] Sigue la señal en vivo # 24Play [UNK] https : / / t. co / mi3yDUaS2F https : / / t. co / Y6qU6gwdCL [SEP]        \n",
       "1          [CLS] Celebrate Lunar New Year with the new Tigeress and Swift Outfits in the Item Shop now! https : / / t. co / oJIWHkCp5o [SEP]                                                                                             \n",
       "2          [CLS] The media tend not to feature abortion survivors like Claire Culwell, a woman whose story has in recent years captivated the pro - life world.. https : / / t. co / COWpNipsgk [SEP]                                    \n",
       "3          [CLS] porto alegre, tem novidade na agenda [UNK] 14 de março eu toco no teatro do @ sesc _ rs e já tô muito ansiosa pra rever todo mundo. ingressos aqui : https : / / t. co / ihrnvHrB1P https : / / t. co / HyJf3917d1 [SEP]\n",
       "4          [CLS] Karnataka boy, who guided ambulance during floods, to be awarded on # RepublicDay.. https : / / t. co / qOtzvndLmE https : / / t. co / HRGZbClnCv [SEP]                                                                 \n",
       "...                                                                                                                                                                  ...                                                                 \n",
       "133267709  [CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]                                                                                  \n",
       "133267710  [CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]                                                                                  \n",
       "133267711  [CLS] RT @ colenunez7 : once i get comfortable with you i [UNK] m goofy af [SEP]                                                                                                                                              \n",
       "133267712  [CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]                                                                                  \n",
       "133267713  [CLS] SM 사옥빌딩 앞에 있는 전광판차 사진들입니다... 전광판차는 사옥 앞에서 오늘 아침 9시부터 오후 4시까지 있을 예정입니다... @ weareoneEXO # EXO # 엑소 https : / / t. co / cS70EH6IkT [SEP]                                                                                  \n",
       "\n",
       "[133267714 rows x 1 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set[['user_text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_of_interest = [\"language\"]\n",
    "train_set_language = pd.read_csv('s3://recsys-challenge-2020/train_set_language.csv', encoding=\"utf-8\",\n",
    "                     usecols= [1])\n",
    "val_set_language = pd.read_csv('s3://recsys-challenge-2020/val_set_language.csv', encoding=\"utf-8\",\n",
    "                     usecols= [1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>06D61DCBBE938971E1EA0C38BD9B5446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>D3164C7FBCF2565DDF915B1B3AEFB1DC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>D3164C7FBCF2565DDF915B1B3AEFB1DC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>ECED8A16BE2A5E8871FD55F4842F16B1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>D3164C7FBCF2565DDF915B1B3AEFB1DC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Language\n",
       "0  06D61DCBBE938971E1EA0C38BD9B5446\n",
       "1  D3164C7FBCF2565DDF915B1B3AEFB1DC\n",
       "2  D3164C7FBCF2565DDF915B1B3AEFB1DC\n",
       "3  ECED8A16BE2A5E8871FD55F4842F16B1\n",
       "4  D3164C7FBCF2565DDF915B1B3AEFB1DC"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set_language.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "133267714"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_set_language)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}