ver002a.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bu kernel'da Güneşin çözümü detaylı olarak incelenecek ve bazı fonksiyonlar taslak oluşturmak için, türkçeleştirilecek."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "yapılan işlemler onucu skor yükselmedi. Lokasyon, ve keyword gibi değişkenler concatlanıp bakılacak.\n",
    "Skoru yükselten asıl kaynağın model kurulması ve çalıştırılması olduğu anlaşıldı...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "df_train = pd.read_csv('train.csv')\n",
    "df_test = pd.read_csv('test.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-19093e8e3af9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mglove_embeddings\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'glove.840B.300d.pkl'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mallow_pickle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\ProgramFiles\\lib\\site-packages\\numpy\\lib\\npyio.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(file, mmap_mode, allow_pickle, fix_imports, encoding)\u001b[0m\n\u001b[0;32m    458\u001b[0m                                  \"when allow_pickle=False\")\n\u001b[0;32m    459\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 460\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mpickle_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    461\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    462\u001b[0m                 raise IOError(\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "glove_embeddings = np.load('glove.840B.300d.pkl', allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GloVe Embeddings cover 52.06% of vocabulary and 82.68% of text in Training Set\n",
      "GloVe Embeddings cover 57.21% of vocabulary and 81.85% of text in Test Set\n"
     ]
    }
   ],
   "source": [
    "################################################################\n",
    "# build_vocab fonksiyonu, toplam datada ki kelimeleri saydırıyor...\n",
    "def build_vocab(X):\n",
    "    \n",
    "    cümleler = X.apply(lambda s: s.split()).values      \n",
    "    kelimeler = {}\n",
    "    \n",
    "    for i in cümleler:\n",
    "        for j in i:\n",
    "            try:\n",
    "                kelimeler[j] += 1\n",
    "            except KeyError:\n",
    "                kelimeler[j] = 1                \n",
    "    return kelimeler\n",
    "##############################################################\n",
    "\n",
    "def check_embeddings_coverage(X, embeddings):\n",
    "    \n",
    "    kelimeler = build_vocab(X)    \n",
    "    \n",
    "    covered = {}  #Datasetimizin ve sözlükte olan kesişen kelimelerin listesi\n",
    "    oov = {}      # Datasetimizde var sözlükte olmayan Kelimeler\n",
    "    n_covered = 0\n",
    "    n_oov = 0\n",
    "    \n",
    "    for j in kelimeler:\n",
    "        try:\n",
    "            covered[j] = embeddings[j]\n",
    "            n_covered += kelimeler[j]\n",
    "        except:\n",
    "            oov[j] = kelimeler[j]\n",
    "            n_oov += kelimeler[j]\n",
    "            \n",
    "    vocab_coverage = len(covered) / len(kelimeler)\n",
    "    text_coverage = (n_covered / (n_covered + n_oov))\n",
    "    \n",
    "    sorted_oov = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n",
    "    \n",
    "    #sorted_oov      -> oov (Out Of Vocab) lerin sıralanmış hali\n",
    "    #vocab_coverage  -> Her bir kelime unuqie kabul edildiğinde (kesişen kelimeler / Data Set Kelime sayısı) \n",
    "    #text_covarage  ->  Her bir Kelime unuqie kabul edilmediği zaman (kesişen kelime / dataset Kelime sayısı)   \n",
    "    \n",
    "    return sorted_oov, vocab_coverage, text_coverage\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "train_glove_oov, train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(df_train['text'], glove_embeddings)\n",
    "test_glove_oov, test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(df_test['text'], glove_embeddings)\n",
    "print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))\n",
    "print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('\\x89Û_', 50),\n",
       " ('MH370:', 39),\n",
       " ('\\x89ÛÒ', 38),\n",
       " ('Latest:', 31),\n",
       " (\"Legionnaires'\", 31),\n",
       " ('#prebreak', 30),\n",
       " (\"he's\", 28),\n",
       " ('re\\x89Û_', 27),\n",
       " ('w/', 26),\n",
       " (\"there's\", 26),\n",
       " (\"'The\", 26),\n",
       " ('\\x89ÛÓ', 26),\n",
       " ('Legionnaires:', 25),\n",
       " ('a...', 24),\n",
       " ('PM:', 24),\n",
       " (\"Confirmed'\", 23),\n",
       " (\"'Conclusively\", 23),\n",
       " ('Typhoon-Devastated', 23),\n",
       " ('#???', 23),\n",
       " ('Saipan:', 22),\n",
       " ('.:', 21),\n",
       " ('were...', 20),\n",
       " ('(Detonation)', 20),\n",
       " ('MH370', 20),\n",
       " ('VIDEO:', 20),\n",
       " ('#Hiroshima', 20),\n",
       " ('#??', 20),\n",
       " ('disea...', 19),\n",
       " ('Photo:', 19),\n",
       " ('bestnaijamade', 18),\n",
       " (\"'We're\", 18),\n",
       " (\"Reddit's\", 18),\n",
       " ('Soudelor', 18),\n",
       " ('news!', 17),\n",
       " ('2015]', 17),\n",
       " ('[Black', 17),\n",
       " ('Funtenna:', 17),\n",
       " (\"That's\", 17),\n",
       " (\"won't\", 17),\n",
       " (\"they're\", 17),\n",
       " (\"'I\", 17),\n",
       " (\"What's\", 17),\n",
       " ('Crash:', 16),\n",
       " (\"water':\", 16),\n",
       " (\"neighbour's\", 16),\n",
       " ('Unconfirmed!', 16),\n",
       " ('#GBBO', 16),\n",
       " ('fires.', 16),\n",
       " (\"she's\", 16),\n",
       " (\"China's\", 15),\n",
       " ('quiz:', 15),\n",
       " (\"ain't\", 15),\n",
       " (\"Can't\", 15),\n",
       " ('MP:', 14),\n",
       " (\"wasn't\", 14),\n",
       " (\"crematoria'\", 13),\n",
       " ('UTC2015-08-05', 13),\n",
       " ('don\\x89Ûªt', 13),\n",
       " (\"aren't\", 13),\n",
       " (\"isn't\", 13),\n",
       " ('News:', 13),\n",
       " ('\\x89ÛÏWhen', 12),\n",
       " ('horror:', 12),\n",
       " ('Sensor-Senso', 12),\n",
       " ('Report:', 12),\n",
       " (\"We're\", 12),\n",
       " (\"hasn't\", 12),\n",
       " ('t...', 12),\n",
       " ('#ISIS', 11),\n",
       " ('Rubble?', 11),\n",
       " ('Past:', 11),\n",
       " ('Derailment:', 11),\n",
       " ('self-image?', 11),\n",
       " (\"haven't\", 11),\n",
       " ('&gt;&gt;', 11),\n",
       " (\"There's\", 11),\n",
       " ('you!', 10),\n",
       " ('sh...', 10),\n",
       " ('Toddler:', 10),\n",
       " (\"'food\", 10),\n",
       " ('Time2015-08-06', 10),\n",
       " ('California:', 10),\n",
       " ('#Sismo', 10),\n",
       " ('b...', 10),\n",
       " (\"'It's\", 10),\n",
       " (\"pay'\", 10),\n",
       " ('derailment:', 10),\n",
       " (\"Let's\", 10),\n",
       " (\"He's\", 10),\n",
       " ('&gt;', 10),\n",
       " ('9:', 10),\n",
       " ('the...', 10),\n",
       " ('(Photos)', 9),\n",
       " ('http://t.co/qew4c5M1xd', 9),\n",
       " ('**Trfc', 9),\n",
       " ('injuries.', 9),\n",
       " ('here:', 9),\n",
       " ('UPDATE:', 9),\n",
       " ('attack:', 9),\n",
       " ('#Directioners', 9),\n",
       " ('now!', 9),\n",
       " (\"You're\", 9),\n",
       " ('@Change', 9),\n",
       " ('@POTUS', 9),\n",
       " ('Mediterran...', 8),\n",
       " ('(HWO)', 8),\n",
       " ('Update:', 8),\n",
       " ('#Nursing', 8),\n",
       " ('#abstorm', 8),\n",
       " (\"let's\", 8),\n",
       " ('crash:', 8),\n",
       " ('invoices.', 8),\n",
       " ('@emmerdale', 8),\n",
       " (\"I'M\", 8),\n",
       " (\"shouldn't\", 8),\n",
       " ('O784', 8),\n",
       " (\"today's\", 8),\n",
       " ('rea\\x89Û_', 8),\n",
       " ('#Beyhive', 8),\n",
       " ('RT:', 8),\n",
       " ('Horse...', 8),\n",
       " ('Forest:', 8),\n",
       " (\"we're\", 8),\n",
       " (':(', 8),\n",
       " (\"what's\", 8),\n",
       " ('Island:', 8),\n",
       " ('#DnB', 8),\n",
       " ('..', 8),\n",
       " ('Cramer:', 7),\n",
       " ('Chile:', 7),\n",
       " ('#Lies', 7),\n",
       " ('#Quran', 7),\n",
       " ('Truth...', 7),\n",
       " ('you\\x89Ûªve', 7),\n",
       " ('China\\x89Ûªs', 7),\n",
       " ('#animalrescue', 7),\n",
       " ('link:', 7),\n",
       " ('Canaanites?', 7),\n",
       " (\"Disney's\", 7),\n",
       " ('Linkury', 7),\n",
       " ('53inch', 7),\n",
       " ('#WX', 7),\n",
       " (\"Parley's\", 7),\n",
       " (\"'Save\", 7),\n",
       " ('wreck.', 7),\n",
       " ('Hobbit:', 7),\n",
       " ('RÌ©union', 7),\n",
       " ('mhtw4fnet', 7),\n",
       " (\"'avoiding\", 7),\n",
       " ('casualties.', 7),\n",
       " ('it!', 7),\n",
       " ('out!', 7),\n",
       " ('&lt;3', 7),\n",
       " ('#IranDeal', 7),\n",
       " ('BREAKING:', 7),\n",
       " ('and...', 7),\n",
       " ('them!', 7),\n",
       " ('#Ices\\x89Û_', 7),\n",
       " ('#TrapMusic', 7),\n",
       " ('#Dubstep', 7),\n",
       " ('@djicemoon', 7),\n",
       " ('[AFTERSHOCK]', 7),\n",
       " ('[IR]', 7),\n",
       " (\"Iger's\", 6),\n",
       " ('Wildfire):', 6),\n",
       " (\"33'\", 6),\n",
       " ('be\\x89Û_', 6),\n",
       " ('http://t.co/KSAwlYuX02', 6),\n",
       " ('#Bestnaijamade:', 6),\n",
       " ('@Towel', 6),\n",
       " ('@Stretcher', 6),\n",
       " ('Meat-Loving', 6),\n",
       " ('Panic!', 6),\n",
       " ('#Antioch', 6),\n",
       " (\"charity.'\", 6),\n",
       " ('him)', 6),\n",
       " ('(peace', 6),\n",
       " ('NWS:', 6),\n",
       " ('fatalities.', 6),\n",
       " ('by\\x89Û_', 6),\n",
       " ('exploded.', 6),\n",
       " ('Here:', 6),\n",
       " (\"wouldn't\", 6),\n",
       " ('#ProphetMuhammad', 6),\n",
       " ('Madinah.', 6),\n",
       " ('#drought', 6),\n",
       " ('I\\x89Ûªm', 6),\n",
       " ('Stand-user?', 6),\n",
       " ('w/heavenly', 6),\n",
       " ('GM!', 6),\n",
       " ('It\\x89Ûªs', 6),\n",
       " ('@MikeParrActor', 6),\n",
       " ('s...', 6),\n",
       " ('#Rohingya', 6),\n",
       " ('#course', 6),\n",
       " ('Inj**', 6),\n",
       " (\"'It\", 6),\n",
       " (\"'I'm\", 6),\n",
       " ('catastrophe.', 6),\n",
       " ('#\\x89Û_', 6),\n",
       " ('Video:', 6),\n",
       " ('buildings?', 6),\n",
       " ('2015:', 6),\n",
       " ('read\\x89Û_', 6),\n",
       " ('#Womens', 6),\n",
       " ('#Handbag', 6),\n",
       " ('me?', 6),\n",
       " ('?Please', 6),\n",
       " ('RT?', 6),\n",
       " ('100%', 6),\n",
       " ('post:', 6),\n",
       " ('#MTVHottest', 6),\n",
       " ('http://t.co/eNCmhz6y34', 6),\n",
       " ('http://t.co/q2eBlOKeVE', 6),\n",
       " (\"Isn't\", 6),\n",
       " (\"'If\", 6),\n",
       " (\"Here's\", 6),\n",
       " (\"you've\", 6),\n",
       " ('Police:', 6),\n",
       " ('#KerrickTrial', 5),\n",
       " ('trauma.', 5),\n",
       " ('starring...', 5),\n",
       " ('@invalid', 5),\n",
       " ('Med:', 5),\n",
       " ('Officials:', 5),\n",
       " ('obliteration.', 5),\n",
       " ('http://t.co/vVPLFQv58P', 5),\n",
       " ('#PantherAttack', 5),\n",
       " ('@YoungHeroesID', 5),\n",
       " ('97/Georgia', 5),\n",
       " ('injury:I-495', 5),\n",
       " ('America\\x89Ûªs', 5),\n",
       " ('#islam!', 5),\n",
       " ('#Afterlife', 5),\n",
       " ('#hailstorm', 5),\n",
       " ('#socialnews', 5),\n",
       " ('#Allah', 5),\n",
       " ('(Reuters)', 5),\n",
       " ('of\\x89Û_', 5),\n",
       " ('emergency.', 5),\n",
       " ('http://t.co/cybKsXHF7d', 5),\n",
       " ('horror;', 5),\n",
       " ('people;', 5),\n",
       " ('refugees;', 5),\n",
       " ('#Genocide;', 5),\n",
       " ('#StrategicPatience', 5),\n",
       " ('.POTUS', 5),\n",
       " ('#Sittwe', 5),\n",
       " ('#Myanmar', 5),\n",
       " ('M.O.P.)', 5),\n",
       " ('#1', 5),\n",
       " ('an...', 5),\n",
       " (\"Fire'\", 5),\n",
       " ('desolate.', 5),\n",
       " (\"accidents'\", 5),\n",
       " ('Here\\x89Ûªs', 5),\n",
       " ('th...', 5),\n",
       " ('#Emmerdale', 5),\n",
       " ('curfew.', 5),\n",
       " ('(Vine', 5),\n",
       " ('Rear-', 5),\n",
       " ('Collision-No', 5),\n",
       " ('Broadway:', 5),\n",
       " ('Correction:', 5),\n",
       " ('now:', 5),\n",
       " ('#preparedness', 5),\n",
       " ('#wildfire', 5),\n",
       " (\"Australia's\", 5),\n",
       " (\"Japan's\", 5),\n",
       " (\"'A\", 5),\n",
       " ('soon!', 5),\n",
       " ('@USAgov', 5),\n",
       " ('@usatoday', 5),\n",
       " ('(Official', 5),\n",
       " ('i...', 5),\n",
       " ('grief\\x89Ûª', 5),\n",
       " ('\\x89Û÷politics', 5),\n",
       " ('Times:', 5),\n",
       " (\"America's\", 5),\n",
       " (\"who's\", 5),\n",
       " (\"we've\", 5),\n",
       " ('#wildfires', 5),\n",
       " ('Training:', 4),\n",
       " ('#IDFire', 4),\n",
       " ('Hawaii:', 4),\n",
       " (\"attack'\", 4),\n",
       " (\"'suicide\", 4),\n",
       " ('@Rexyy', 4),\n",
       " ('@Grazed', 4),\n",
       " ('sinking.', 4),\n",
       " ('Selfies:', 4),\n",
       " ('#JapÌ_n', 4),\n",
       " ('DETECTADO', 4),\n",
       " ('*screams', 4),\n",
       " (\"i'm\", 4),\n",
       " ('#360WiseNews', 4),\n",
       " ('Rubble?:', 4),\n",
       " ('carryi...', 4),\n",
       " ('whao..', 4),\n",
       " ('...//..//', 4),\n",
       " ('I...', 4),\n",
       " ('specif...', 4),\n",
       " ('Content:', 4),\n",
       " ('offensive\\x89Ûª', 4),\n",
       " ('\\x89Û÷extremely', 4),\n",
       " ('101:', 4),\n",
       " ('story:', 4),\n",
       " ('#refugees', 4),\n",
       " ('#Nuclear', 4),\n",
       " ('lulgzimbestpicts', 4),\n",
       " ('#Mayhem', 4),\n",
       " ('#RAmag.', 4),\n",
       " ('#7', 4),\n",
       " ('@DiamondKesawn', 4),\n",
       " ('@RaynbowAffair', 4),\n",
       " ('http://t.co/vIwXY1XDYK', 4),\n",
       " ('http://t.co/LvlH3W3aWO', 4),\n",
       " ('murderer.', 4),\n",
       " ('murder.', 4),\n",
       " (\"Floyd's\", 4),\n",
       " ('poll:', 4),\n",
       " ('let\\x89Ûªs', 4),\n",
       " ('Hellfire!', 4),\n",
       " ('#Hellfire', 4),\n",
       " ('IN]', 4),\n",
       " ('hail:', 4),\n",
       " ('[wind:', 4),\n",
       " ('floods.', 4),\n",
       " ('...THE', 4),\n",
       " (\"1'\", 4),\n",
       " ('Saturday!', 4),\n",
       " (\"'We\", 4),\n",
       " ('them?', 4),\n",
       " ('Guide:', 4),\n",
       " ('#OTRAMETLIFE', 4),\n",
       " ('#WorldNews', 4),\n",
       " ('(AP)', 4),\n",
       " ('flames.', 4),\n",
       " ('#hiring!', 4),\n",
       " ('plan?', 4),\n",
       " ('alarm:', 4),\n",
       " ('M1.94', 4),\n",
       " ('#USGS', 4),\n",
       " ('#EarthQuake', 4),\n",
       " ('Time2015-08-05', 4),\n",
       " ('EQ:', 4),\n",
       " ('Clev:', 4),\n",
       " (\"couldn't\", 4),\n",
       " ('drought.', 4),\n",
       " ('https://t.co/rqWuoy1fm4', 4),\n",
       " ('#SummerFate', 4),\n",
       " ('I\\x89Ûªve', 4),\n",
       " (\"'Detonate'\", 4),\n",
       " ('(feat.', 4),\n",
       " ('Flood:', 4),\n",
       " (\"y'all\", 4),\n",
       " ('i\\x89Û_', 4),\n",
       " ('you?', 4),\n",
       " ('#tubestrike', 4),\n",
       " ('#Terrorist', 4),\n",
       " ('#Zionist', 4),\n",
       " ('AbbsWinston:', 4),\n",
       " ('MH370;', 4),\n",
       " ('#emmerdale', 4),\n",
       " ('life:', 4),\n",
       " ('can\\x89Ûªt', 4),\n",
       " ('https://t.co/moLL5vd8yD', 4),\n",
       " ('OPPRESSIONS', 4),\n",
       " ('#SLANGLUCCI', 4),\n",
       " ('#FETTILOOTCH', 4),\n",
       " ('danger.', 4),\n",
       " ('today:', 4),\n",
       " ('crush:', 4),\n",
       " (\"would've\", 4),\n",
       " ('åÊ', 4),\n",
       " ('Now!', 4),\n",
       " ('collapsed.', 4),\n",
       " ('cliff.', 4),\n",
       " ('#Dependency', 4),\n",
       " ('#Chemical', 4),\n",
       " ('åÈ', 4),\n",
       " (\"they'll\", 4),\n",
       " ('Vote:', 4),\n",
       " ('Catastrophe.', 4),\n",
       " ('library:', 4),\n",
       " ('\\x89ÛÏThe', 4),\n",
       " ('in...', 4),\n",
       " ('[Video]', 4),\n",
       " ('Mitt.\\x89Û\\x9d', 4),\n",
       " ('\\x89ÛÏWe', 4),\n",
       " ('Bush:', 4),\n",
       " ('@Michael5SOS', 4),\n",
       " (\"you'd\", 4),\n",
       " ('PHOTOS:', 4),\n",
       " ('collapse.', 4),\n",
       " (\"'the\", 4),\n",
       " ('beforeitsnews', 4),\n",
       " ('http://t.co/zujwUiomb3', 4),\n",
       " (\"they'd\", 4),\n",
       " ('it?', 4),\n",
       " ('@LoneWolffur', 4),\n",
       " ('Video)', 4),\n",
       " ('&lt;', 4),\n",
       " ('to...', 4),\n",
       " ('@local_arsonist', 4),\n",
       " ('(via', 4),\n",
       " ('http://t.co/WvJ39a3BGM', 4),\n",
       " ('http://t.co/thoYhrHkfJ', 4),\n",
       " ('http://t.co/o91f3cYy0R', 4),\n",
       " ('http://t.co/nnMQlz91o9', 4),\n",
       " ('&gt;&gt;&gt;', 4),\n",
       " (\"XP'\", 4),\n",
       " (\"@'aRmageddon\", 4),\n",
       " ('(Temporary:300)', 4),\n",
       " ('#PBBan', 4),\n",
       " ('MORE!)', 4),\n",
       " ('(BOB', 4),\n",
       " ('3:', 4),\n",
       " ('on...', 4),\n",
       " ('#VoteJKT48ID', 4),\n",
       " ('#KCA', 4),\n",
       " ('o...', 4),\n",
       " (';ACCIDENT', 4),\n",
       " ('accident:', 4),\n",
       " (\"they've\", 4),\n",
       " ('#RockyFire', 4),\n",
       " ('wreck?', 3),\n",
       " ('Shots:', 3),\n",
       " ('(Parker', 3),\n",
       " (\"California's\", 3),\n",
       " ('Throwingknifes!', 3),\n",
       " ('@dannyonpc', 3),\n",
       " (\"Jupiter's\", 3),\n",
       " (\"Ferguson's\", 3),\n",
       " ('@NASAHurricane', 3),\n",
       " ('Taiwan;', 3),\n",
       " (\"Soudelor's\", 3),\n",
       " ('Map:', 3),\n",
       " ('#thankU', 3),\n",
       " ('GodsLove', 3),\n",
       " ('traumatised.', 3),\n",
       " ('#usNWSgov', 3),\n",
       " ('failure:', 3),\n",
       " ('@witter', 3),\n",
       " ('#bookboost', 3),\n",
       " ('#ibooklove', 3),\n",
       " ('#AoMS', 3),\n",
       " ('::', 3),\n",
       " ('unhappiness?', 3),\n",
       " ('150-Foot', 3),\n",
       " ('[Report', 3),\n",
       " ('#???????', 3),\n",
       " ('#Gabon', 3),\n",
       " ('#BlowMandyUp', 3),\n",
       " ('https://t.co/dehMym5lpk', 3),\n",
       " ('@iTunes', 3),\n",
       " ('#RIOT', 3),\n",
       " ('@realmandyrain', 3),\n",
       " ('Listen/Buy', 3),\n",
       " ('@NickCannon', 3),\n",
       " ('prefer:', 3),\n",
       " ('quiz!', 3),\n",
       " ('@Zak_Bagans', 3),\n",
       " ('link...', 3),\n",
       " ('OffensiveåÊContent', 3),\n",
       " ('#onlinecommunities', 3),\n",
       " ('CNN:', 3),\n",
       " (\"chief's\", 3),\n",
       " ('Oops:', 3),\n",
       " ('Nigeria:', 3),\n",
       " ('terrorism.', 3),\n",
       " ('panic.', 3),\n",
       " ('Disco:', 3),\n",
       " ('Disease:', 3),\n",
       " ('S\\x89Û_', 3),\n",
       " ('Grows:', 3),\n",
       " ('projected:', 3),\n",
       " ('@worldnetdaily', 3),\n",
       " (\"DON'T\", 3),\n",
       " ('obliterated.', 3),\n",
       " ('BB4SP', 3),\n",
       " ('Women!', 3),\n",
       " ('/r/worldnews', 3),\n",
       " ('ministers:', 3),\n",
       " ('\\x89ÛÏa', 3),\n",
       " ('http://t.co/BbDpnj8XSx', 3),\n",
       " (\"They're\", 3),\n",
       " ('#mudslide', 3),\n",
       " ('Dorret', 3),\n",
       " (\"mudslide'\", 3),\n",
       " (\"'mudslide'\", 3),\n",
       " ('@business', 3),\n",
       " ('--&gt;', 3),\n",
       " ('#FaroeIslands', 3),\n",
       " (';-;', 3),\n",
       " ('bang.', 3),\n",
       " ('#landslide', 3),\n",
       " ('3)', 3),\n",
       " ('me??', 3),\n",
       " ('report:', 3),\n",
       " ('CLEARED:incident', 3),\n",
       " ('p...', 3),\n",
       " ('injured:', 3),\n",
       " ('Photos:', 3),\n",
       " ('#guillermo', 3),\n",
       " ('WED.AUG.5TH', 3),\n",
       " ('#TDP', 3),\n",
       " ('#TRS', 3),\n",
       " ('#NRI', 3),\n",
       " ('#hostages', 3),\n",
       " ('hostages.', 3),\n",
       " ('(PHOTO)', 3),\n",
       " ('@IAN_Hellfire', 3),\n",
       " ('#Reflect', 3),\n",
       " ('Humaza.', 3),\n",
       " ('#wealth', 3),\n",
       " ('tongue!', 3),\n",
       " ('Health!', 3),\n",
       " ('pres:', 3),\n",
       " ('#humanconsumption', 3),\n",
       " ('#ban', 3),\n",
       " ('#Magginoodle', 3),\n",
       " ('#NestleIndia', 3),\n",
       " ('#offers2go', 3),\n",
       " ('#foodscare', 3),\n",
       " ('http://t.co/0wrATkA2jL', 3),\n",
       " ('HARM/KID', 3),\n",
       " ('picked!', 3),\n",
       " ('winds.', 3),\n",
       " ('#arwx', 3),\n",
       " ('MPH.', 3),\n",
       " ('PRAY!', 3),\n",
       " ('#QUOTE', 3),\n",
       " ('#FARRAKHAN', 3),\n",
       " ('flames...', 3),\n",
       " ('*NEW*', 3),\n",
       " ('@MNPDNashville', 3),\n",
       " (\"it'll\", 3),\n",
       " ('Falcon)', 3),\n",
       " ('(Captain', 3),\n",
       " ('S@X', 3),\n",
       " ('d...', 3),\n",
       " ('#9', 3),\n",
       " (\"'Food\", 3),\n",
       " ('Denmark:', 3),\n",
       " ('[0', 3),\n",
       " ('update:', 3),\n",
       " ('explode...', 3),\n",
       " ('@GOP', 3),\n",
       " (':/', 3),\n",
       " ('#Wildfire', 3),\n",
       " ('too!', 3),\n",
       " ('tr...', 3),\n",
       " ('#Casper', 3),\n",
       " ('evacuated.', 3),\n",
       " ('evacuate.', 3),\n",
       " ('live:', 3),\n",
       " ('rescue!', 3),\n",
       " ('@TfLBusAlerts', 3),\n",
       " ('Plan.', 3),\n",
       " ('floods:', 3),\n",
       " ('at\\x89Û_', 3),\n",
       " ('http://t.co/V3aZWOAmzK', 3),\n",
       " ('UTC!', 3),\n",
       " ('http://t.co/9NwAJLi9cr', 3),\n",
       " ('Grace:', 3),\n",
       " ('#GamerGate', 3),\n",
       " ('Articles:', 3),\n",
       " ('drought:', 3),\n",
       " ('Fire.', 3),\n",
       " ('him!', 3),\n",
       " ('@Refugees', 3),\n",
       " ('wom...', 3),\n",
       " ('IDPs:', 3),\n",
       " ('#genocide', 3),\n",
       " ('#Prison', 3),\n",
       " ('@smh', 3),\n",
       " ('AVERTED:', 3),\n",
       " ('the\\x89Û_', 3),\n",
       " ('Jam...', 3),\n",
       " ('sitting\\x89Û_:', 3),\n",
       " ('\\x89ÛÏRichmond', 3),\n",
       " ('tab\\x89Û_', 3),\n",
       " ('Connector-Connecto', 3),\n",
       " ('@spinningbot', 3),\n",
       " ('@TinyJecht', 3),\n",
       " ('demonstratio...', 3),\n",
       " ('Multiplayer:', 3),\n",
       " ('Kaduna:', 3),\n",
       " ('favorite:', 3),\n",
       " ('http://t.co/cEdCUgEuWs', 3),\n",
       " ('train...', 3),\n",
       " (\"passengers'\", 3),\n",
       " ('Lives:', 3),\n",
       " ('#ModiMinistry', 3),\n",
       " ('home:', 3),\n",
       " ('#wmata', 3),\n",
       " ('derailed.', 3),\n",
       " ('@unsuckdcmetro', 3),\n",
       " ('#Metro', 3),\n",
       " ('#WMATA', 3),\n",
       " ('@DavidVonderhaar', 3),\n",
       " ('saddlebrooke...', 3),\n",
       " ('just\\x89Û_:', 3),\n",
       " ('day:', 3),\n",
       " ('å¨', 3),\n",
       " ('@gerenciatodos', 3),\n",
       " ('@accionempresa', 3),\n",
       " ('for...', 3),\n",
       " (\"tonight's\", 3),\n",
       " ('#MH370', 3),\n",
       " ('#MH370.', 3),\n",
       " ('#????', 3),\n",
       " ('40%', 3),\n",
       " ('#CecilTheLion', 3),\n",
       " (\"'Death\", 3),\n",
       " ('this?', 3),\n",
       " ('#ArtistsUnited', 3),\n",
       " ('@ClaytonBryant', 3),\n",
       " ('#rockin', 3),\n",
       " ('(GTA', 3),\n",
       " ('TMP:', 3),\n",
       " ('NAVBL.', 3),\n",
       " ('TAXIWAYS', 3),\n",
       " ('APCH.', 3),\n",
       " ('WND:', 3),\n",
       " ('#WorstSummerJob', 3),\n",
       " (\"companies'\", 3),\n",
       " (\"'You\", 3),\n",
       " ('8/5/2015', 3),\n",
       " ('Enrt**', 3),\n",
       " ('Collision-1141', 3),\n",
       " ('local\\x89Û_', 3),\n",
       " ('#HarryBeCareful', 3),\n",
       " ('County:', 3),\n",
       " ('collide.', 3),\n",
       " ('Duty:', 3),\n",
       " (\"everyone's\", 3),\n",
       " ('animal!', 3),\n",
       " (\"Dorret's\", 3),\n",
       " (\"'My\", 3),\n",
       " ('that?', 3),\n",
       " (\"weren't\", 3),\n",
       " ('millions.', 3),\n",
       " ('@Calum5SOS', 3),\n",
       " ('Crash.', 3),\n",
       " (\"'When\", 3),\n",
       " (\"'you\", 3),\n",
       " ('2...', 3),\n",
       " ('now...', 3),\n",
       " ('Day:', 3),\n",
       " (\"'protest'\", 3),\n",
       " ('Australia\\x89Ûªs', 3),\n",
       " ('@po_st', 3),\n",
       " ('(and', 3),\n",
       " ('@KurtSchlichter', 3),\n",
       " (\"i'll\", 3),\n",
       " ('civilians.', 3),\n",
       " ('??.', 3),\n",
       " ('Meek.', 3),\n",
       " ('#BB17', 3),\n",
       " ('again!', 3),\n",
       " (\"Ain't\", 3),\n",
       " ('@Blizzard_draco', 3),\n",
       " ('of...', 3),\n",
       " ('blight:', 3),\n",
       " (\"we'd\", 3),\n",
       " (\"'What\", 3),\n",
       " (\"someone's\", 3),\n",
       " ('IDis', 3),\n",
       " ('lgl/org', 3),\n",
       " ('w/Bioterrorism&amp;use', 3),\n",
       " ('Hostage&amp;2', 3),\n",
       " ('90BLKs&amp;8WHTs', 3),\n",
       " ('#bioterrorism', 3),\n",
       " ('#GOPDebate', 3),\n",
       " ('#RickPerry', 3),\n",
       " ('sanctions.', 3),\n",
       " ('#Bioterror', 3),\n",
       " ('#frontpage:', 3),\n",
       " ('your...', 3),\n",
       " (\"we'll\", 3),\n",
       " ('Control?\\x89Û\\x9d', 3),\n",
       " ('\\x89ÛÏHatchet', 3),\n",
       " ('Attack:', 3),\n",
       " ('Fire:', 3),\n",
       " ('A...', 3),\n",
       " ('students...', 3),\n",
       " ('#NewsInTweets', 3),\n",
       " ('night!', 3),\n",
       " ('#UltimaLucha', 3),\n",
       " ('Armageddon.', 3),\n",
       " ('#ARMAGEDDON', 3),\n",
       " ('@TrubGME', 3),\n",
       " ('Minecraft:', 3),\n",
       " ('next?', 3),\n",
       " (\"She's\", 3),\n",
       " ('https://t.co/e8dl1lNCVu', 3),\n",
       " ('#gilbert23', 3),\n",
       " ('@ViralSpell', 3),\n",
       " ('die!', 3),\n",
       " ('#til_now', 3),\n",
       " ('#Accident', 3),\n",
       " ('(29-07-2015)', 3),\n",
       " ('tonight?', 3),\n",
       " ('do?', 3),\n",
       " (\"'There\", 3),\n",
       " ('accident?', 3),\n",
       " ('more!', 3),\n",
       " ('day!', 3),\n",
       " ('#flooding', 3),\n",
       " (\"confirmed'\", 2),\n",
       " (\"'conclusively\", 2),\n",
       " ('wounds.', 2),\n",
       " ('http://t.co/sAF9MoSkSN', 2),\n",
       " ('#Kashmir', 2),\n",
       " ('#JonathanFerrell', 2),\n",
       " ('chest/torso', 2),\n",
       " ('bicep.', 2),\n",
       " ('wounds;', 2),\n",
       " ('ushed', 2),\n",
       " ('@wocowae', 2),\n",
       " ('shots:', 2),\n",
       " ('veterans.', 2),\n",
       " ('exc...', 2),\n",
       " ('Gunfire:', 2),\n",
       " ('v...', 2),\n",
       " ('(TWIA)', 2),\n",
       " ('hike:', 2),\n",
       " ('#Trump', 2),\n",
       " ('$2.99', 2),\n",
       " (\"single'\", 2),\n",
       " ('pa...', 2),\n",
       " ('Email:', 2),\n",
       " ('208-267-6773', 2),\n",
       " ('Phone:', 2),\n",
       " ('troubling.', 2),\n",
       " ('@TheTawniest', 2),\n",
       " ('@aria_ahrary', 2),\n",
       " ('Whirlwind!', 2),\n",
       " ('whirlwind.', 2),\n",
       " ('weapons!', 2),\n",
       " ('#cheyenne', 2),\n",
       " ('#rapidcity', 2),\n",
       " ('#billings', 2),\n",
       " (\"'blue'\", 2),\n",
       " ('God?', 2),\n",
       " ('http://t.co/zDtoyd8EbJ', 2),\n",
       " ('UTC]?5km', 2),\n",
       " ('[01:04', 2),\n",
       " ('volcanoåÊinåÊRussia.', 2),\n",
       " ('sixpenceee:', 2),\n",
       " ('#Volcano', 2),\n",
       " ('Raung', 2),\n",
       " (\"'@NASASolarSystem:\", 2),\n",
       " ('http://t.co/XsSgEdSbH4', 2),\n",
       " ('Radio:', 2),\n",
       " ('http://t.co/I27Oa0HISp', 2),\n",
       " ('http://t.co/3SICroAaNz', 2),\n",
       " ('#stormchase', 2),\n",
       " ('violent.', 2),\n",
       " ('King:', 2),\n",
       " (\"Diageo's\", 2),\n",
       " (\"day'\", 2),\n",
       " (\"'looking\", 2),\n",
       " ('upheaval:', 2),\n",
       " ('#TheGame', 2),\n",
       " ('Ferguson\\x89Ûªs', 2),\n",
       " (\"'bullseye'\", 2),\n",
       " ('OutBid', 2),\n",
       " ('@tsunami_esh', 2),\n",
       " ('Pc.', 2),\n",
       " ('tutorial.:', 2),\n",
       " ('#traumatised', 2),\n",
       " ('TRAUMATISED', 2),\n",
       " ('lavenderpoetrycafe:', 2),\n",
       " ('(?EudryLantiqua?)', 2),\n",
       " ('wa...', 2),\n",
       " ('tragedy;', 2),\n",
       " ('tragedy:', 2),\n",
       " ('for\\x89Û_', 2),\n",
       " ('thunderstorm.', 2),\n",
       " ('#OK.', 2),\n",
       " ('#Thunderstorm', 2),\n",
       " ('@LivingSafely:', 2),\n",
       " ('Oklahoma-OK', 2),\n",
       " ('10:38PM', 2),\n",
       " ('[OK]', 2),\n",
       " ('IMPACT...', 2),\n",
       " ('10:30pm.', 2),\n",
       " ('BUDDYS!!!!', 2),\n",
       " ('@Spencers', 2),\n",
       " ('@OriginalFunko', 2),\n",
       " (\"Haven't\", 2),\n",
       " ('http://t.co/J5mKcbKcov', 2),\n",
       " ('#Palestinians', 2),\n",
       " ('terrorism!', 2),\n",
       " ('humans!', 2),\n",
       " ('terrorism:', 2),\n",
       " (\"Story'\", 2),\n",
       " (\"'Epic\", 2),\n",
       " (\"80's\", 2),\n",
       " ('http://t.co/YdEtWgRibk', 2),\n",
       " ('#Autoinsurance', 2),\n",
       " (\"Survive'\", 2),\n",
       " (':S', 2),\n",
       " ('Suruc', 2),\n",
       " ('http://t.co/iIkSsJGBDn', 2),\n",
       " ('#premonitions', 2),\n",
       " (\"named]...'\", 2),\n",
       " ('[location', 2),\n",
       " (\"'Suicide\", 2),\n",
       " ('SuruÌ¤', 2),\n",
       " ('police:', 2),\n",
       " ('Mosque:', 2),\n",
       " (\"mosque'\", 2),\n",
       " ('#mercados', 2),\n",
       " ('-/', 2),\n",
       " ('//./../..', 2),\n",
       " ('NTSB:', 2),\n",
       " ('@rightwaystan', 2),\n",
       " ('http://t.co/vz1irH0Nmm', 2),\n",
       " ('Rightways:', 2),\n",
       " ('Deck\\x89Û\\x9d', 2),\n",
       " ('\\x89ÛÏStretcher', 2),\n",
       " ('@PLlolz', 2),\n",
       " (\"sky's\", 2),\n",
       " ('storm????', 2),\n",
       " (\"snowstorm'\", 2),\n",
       " ('#treeporn', 2),\n",
       " ('#Hampshire', 2),\n",
       " ('#SouthDowns', 2),\n",
       " ('battlefield.', 2),\n",
       " ('on?', 2),\n",
       " ('Sirens.', 2),\n",
       " ('@FEVWarrior', 2),\n",
       " ('married??????...', 2),\n",
       " ('(2009)', 2),\n",
       " ('Maria..', 2),\n",
       " (\"'@WEYREY_gidi:\", 2),\n",
       " ('Coursing*', 2),\n",
       " ('#YoNews', 2),\n",
       " ('Brooklyn:', 2),\n",
       " ('Sinkhole!:', 2),\n",
       " ('http://t.co/btdjGWeKqx', 2),\n",
       " ('*screams*', 2),\n",
       " ('@camilacabello97', 2),\n",
       " (\"'shell\", 2),\n",
       " ('Tryout:', 2),\n",
       " ('ha...', 2),\n",
       " ('hunt...', 2),\n",
       " ('rubble.', 2),\n",
       " ('alternatives?', 2),\n",
       " ('@BLutz10', 2),\n",
       " ('`bbcnews', 2),\n",
       " ('#UndergroundRailraod', 2),\n",
       " ('#YAHIstorical', 2),\n",
       " ('#Rioting', 2),\n",
       " ('Noonan@CindyNoonan-Heartbreak', 2),\n",
       " ('rioting.', 2),\n",
       " (\"'Without\", 2),\n",
       " ('Arrived:', 2),\n",
       " ('migrants\\x89Û_', 2),\n",
       " ('Home:', 2),\n",
       " (\"water'\", 2),\n",
       " ('me!', 2),\n",
       " ('#c4news', 2),\n",
       " ('https://t.co/Is2iDC3UBJ', 2),\n",
       " ('Trucks)', 2),\n",
       " ('(14', 2),\n",
       " ('Collapse)', 2),\n",
       " ('(Structural', 2),\n",
       " ('UD:', 2),\n",
       " ('https://t.co/wUDlkq7ncx', 2),\n",
       " ('Earth:', 2),\n",
       " ('http://t.co/po19h8YCND', 2),\n",
       " ('recap/', 2),\n",
       " (\"World's\", 2),\n",
       " (\"'Couple\", 2),\n",
       " ('@ViralSpell:', 2),\n",
       " ('/(', 2),\n",
       " ('./.....hmm', 2),\n",
       " ('wowo--===', 2),\n",
       " ('#ZippedNews', 2),\n",
       " ('@Robot_Rainstorm', 2),\n",
       " ('three:', 2),\n",
       " ('Trust]', 2),\n",
       " ('[Daily', 2),\n",
       " ('State:', 2),\n",
       " ('#radiation', 2),\n",
       " ('case:', 2),\n",
       " (\"Content'\", 2),\n",
       " (\"'Reddit\", 2),\n",
       " ('#freespeech', 2),\n",
       " ('#amageddon', 2),\n",
       " ('h...', 2),\n",
       " ('India:', 2),\n",
       " ('@dailykos', 2),\n",
       " ('police-run', 2),\n",
       " ('theater:', 2),\n",
       " ('panicking.', 2),\n",
       " (\"TV'\", 2),\n",
       " (\"'People\", 2),\n",
       " ('@montetjwitter11', 2),\n",
       " ('VIDEO]', 2),\n",
       " ('[OFFICIAL', 2),\n",
       " ('pandemonium...', 2),\n",
       " ('Pandemonium.', 2),\n",
       " ('http://t.co/7hAnPcr5rK', 2),\n",
       " ('Outbreak:', 2),\n",
       " ('Legio...', 2),\n",
       " ('Bronx;', 2),\n",
       " ('http://t.co/jhpdSSVhvE', 2),\n",
       " ('sp...', 2),\n",
       " ('West:', 2),\n",
       " (\"'California:\", 2),\n",
       " ('Pakistan.', 2),\n",
       " ('Moments)', 2),\n",
       " ('Obliteration?', 2),\n",
       " ('#GlobalWarming!!', 2),\n",
       " ('#MicheleBachman', 2),\n",
       " ('#JonVoyage', 2),\n",
       " ('):', 2),\n",
       " ('5:', 2),\n",
       " ('#breakingnews', 2),\n",
       " ('reactor.', 2),\n",
       " ('individuals...', 2),\n",
       " ('#NYTimes', 2),\n",
       " ('#guardian', 2),\n",
       " ('?#FUKUSHIMA?#TEPCO?', 2),\n",
       " ('#Chernobyl', 2),\n",
       " ('happens?', 2),\n",
       " ('be?', 2),\n",
       " ('Angry?', 2),\n",
       " ('RaÌ¼l', 2),\n",
       " ('#BakeOffFriends', 2),\n",
       " ('MUDSLIDE', 2),\n",
       " ('off!', 2),\n",
       " ('@TeamHendrick', 2),\n",
       " ('Betrayed:', 2),\n",
       " ('LINK:', 2),\n",
       " ('#ushanka', 2),\n",
       " ('?????FOLLOW', 2),\n",
       " ('More!', 2),\n",
       " ('Results:', 2),\n",
       " ('#NoSurrender', 2),\n",
       " ('(18+)', 2),\n",
       " ('Text.', 2),\n",
       " ('#4PlayThursdays.', 2),\n",
       " ('@alexbelloli', 2),\n",
       " ('MAYHEM!', 2),\n",
       " ('@itsjustinstuart', 2),\n",
       " ('mayhem...', 2),\n",
       " (\"Hanneman's\", 2),\n",
       " (\"guy's\", 2),\n",
       " ('5%', 2),\n",
       " (\"murderer's\", 2),\n",
       " ('terrorist.', 2),\n",
       " ('#gunsense', 2),\n",
       " ('murderer!', 2),\n",
       " ('MURDERER.', 2),\n",
       " ('Hiroshima:', 2),\n",
       " ('#DebateQuestionsWeWantToHear', 2),\n",
       " ('year?', 2),\n",
       " ('@RoyalCarribean', 2),\n",
       " ('http://t.co/c1H7JECFrV', 2),\n",
       " ('murder?', 2),\n",
       " ('@noah_anyname', 2),\n",
       " ('@samanthaturne19', 2),\n",
       " ('-a', 2),\n",
       " ('http://t.co/iDmhSwewQw', 2),\n",
       " ('/r/pics', 2),\n",
       " ('rained.', 2),\n",
       " ('6+', 2),\n",
       " ('2.1A', 2),\n",
       " (\"mineral's\", 2),\n",
       " ('Forecast.', 2),\n",
       " ('a\\x89Û_', 2),\n",
       " ('made)', 2),\n",
       " ('(home', 2),\n",
       " ('democracy.', 2),\n",
       " ('Italy:', 2),\n",
       " ('landslide.', 2),\n",
       " ('&amp;...', 2),\n",
       " ('#sinkhole', 2),\n",
       " ('#Cowboys:', 2),\n",
       " (\"Wednesday's\", 2),\n",
       " ('won...', 2),\n",
       " ('THIS!', 2),\n",
       " ('wrong?', 2),\n",
       " ('(with', 2),\n",
       " ('video:', 2),\n",
       " ('#renew911health', 2),\n",
       " ('responders!', 2),\n",
       " ('injured.', 2),\n",
       " ('post;', 2),\n",
       " ('moment:', 2),\n",
       " ('http://t.co/mfckpVzfV8', 2),\n",
       " ('#MeteoEarth?', 2),\n",
       " ('@Hurricane_Dolce', 2),\n",
       " ...]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_glove_oov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def clean(tweet): \n",
    "    # Special characters\n",
    "    '''\n",
    "    tweet = re.sub(r\"\\x89Û_\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89ÛÒ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89ÛÓ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89ÛÏWhen\", \"When\", tweet)\n",
    "    tweet = re.sub(r\"\\x89ÛÏ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"China\\x89Ûªs\", \"China's\", tweet)\n",
    "    tweet = re.sub(r\"let\\x89Ûªs\", \"let's\", tweet)\n",
    "    tweet = re.sub(r\"\\x89Û÷\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89Ûª\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89Û\\x9d\", \"\", tweet)\n",
    "    tweet = re.sub(r\"å_\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89Û¢\", \"\", tweet)\n",
    "    tweet = re.sub(r\"\\x89Û¢åÊ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"fromåÊwounds\", \"from wounds\", tweet)\n",
    "    tweet = re.sub(r\"åÊ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"åÈ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"JapÌ_n\", \"Japan\", tweet)    \n",
    "    tweet = re.sub(r\"Ì©\", \"e\", tweet)\n",
    "    tweet = re.sub(r\"å¨\", \"\", tweet)\n",
    "    tweet = re.sub(r\"SuruÌ¤\", \"Suruc\", tweet)\n",
    "    tweet = re.sub(r\"åÇ\", \"\", tweet)\n",
    "    tweet = re.sub(r\"å£3million\", \"3 million\", tweet)\n",
    "    tweet = re.sub(r\"åÀ\", \"\", tweet)\n",
    "    \n",
    "    # Contractions\n",
    "    tweet = re.sub(r\"he's\", \"he is\", tweet)\n",
    "    tweet = re.sub(r\"there's\", \"there is\", tweet)\n",
    "    tweet = re.sub(r\"We're\", \"We are\", tweet)\n",
    "    tweet = re.sub(r\"That's\", \"That is\", tweet)\n",
    "    tweet = re.sub(r\"won't\", \"will not\", tweet)\n",
    "    tweet = re.sub(r\"they're\", \"they are\", tweet)\n",
    "    tweet = re.sub(r\"Can't\", \"Cannot\", tweet)\n",
    "    tweet = re.sub(r\"wasn't\", \"was not\", tweet)\n",
    "    tweet = re.sub(r\"don\\x89Ûªt\", \"do not\", tweet)\n",
    "    tweet = re.sub(r\"aren't\", \"are not\", tweet)\n",
    "    tweet = re.sub(r\"isn't\", \"is not\", tweet)\n",
    "    tweet = re.sub(r\"What's\", \"What is\", tweet)\n",
    "    tweet = re.sub(r\"haven't\", \"have not\", tweet)\n",
    "    tweet = re.sub(r\"hasn't\", \"has not\", tweet)\n",
    "    tweet = re.sub(r\"There's\", \"There is\", tweet)\n",
    "    tweet = re.sub(r\"He's\", \"He is\", tweet)\n",
    "    tweet = re.sub(r\"It's\", \"It is\", tweet)\n",
    "    tweet = re.sub(r\"You're\", \"You are\", tweet)\n",
    "    tweet = re.sub(r\"I'M\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"shouldn't\", \"should not\", tweet)\n",
    "    tweet = re.sub(r\"wouldn't\", \"would not\", tweet)\n",
    "    tweet = re.sub(r\"i'm\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"I\\x89Ûªm\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"I'm\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"Isn't\", \"is not\", tweet)\n",
    "    tweet = re.sub(r\"Here's\", \"Here is\", tweet)\n",
    "    tweet = re.sub(r\"you've\", \"you have\", tweet)\n",
    "    tweet = re.sub(r\"you\\x89Ûªve\", \"you have\", tweet)\n",
    "    tweet = re.sub(r\"we're\", \"we are\", tweet)\n",
    "    tweet = re.sub(r\"what's\", \"what is\", tweet)\n",
    "    tweet = re.sub(r\"couldn't\", \"could not\", tweet)\n",
    "    tweet = re.sub(r\"we've\", \"we have\", tweet)\n",
    "    tweet = re.sub(r\"it\\x89Ûªs\", \"it is\", tweet)\n",
    "    tweet = re.sub(r\"doesn\\x89Ûªt\", \"does not\", tweet)\n",
    "    tweet = re.sub(r\"It\\x89Ûªs\", \"It is\", tweet)\n",
    "    tweet = re.sub(r\"Here\\x89Ûªs\", \"Here is\", tweet)\n",
    "    tweet = re.sub(r\"who's\", \"who is\", tweet)\n",
    "    tweet = re.sub(r\"I\\x89Ûªve\", \"I have\", tweet)\n",
    "    tweet = re.sub(r\"y'all\", \"you all\", tweet)\n",
    "    tweet = re.sub(r\"can\\x89Ûªt\", \"cannot\", tweet)\n",
    "    tweet = re.sub(r\"would've\", \"would have\", tweet)\n",
    "    tweet = re.sub(r\"it'll\", \"it will\", tweet)\n",
    "    tweet = re.sub(r\"we'll\", \"we will\", tweet)\n",
    "    tweet = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", tweet)\n",
    "    tweet = re.sub(r\"We've\", \"We have\", tweet)\n",
    "    tweet = re.sub(r\"he'll\", \"he will\", tweet)\n",
    "    tweet = re.sub(r\"Y'all\", \"You all\", tweet)\n",
    "    tweet = re.sub(r\"Weren't\", \"Were not\", tweet)\n",
    "    tweet = re.sub(r\"Didn't\", \"Did not\", tweet)\n",
    "    tweet = re.sub(r\"they'll\", \"they will\", tweet)\n",
    "    tweet = re.sub(r\"they'd\", \"they would\", tweet)\n",
    "    tweet = re.sub(r\"DON'T\", \"DO NOT\", tweet)\n",
    "    tweet = re.sub(r\"That\\x89Ûªs\", \"That is\", tweet)\n",
    "    tweet = re.sub(r\"they've\", \"they have\", tweet)\n",
    "    tweet = re.sub(r\"i'd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"should've\", \"should have\", tweet)\n",
    "    tweet = re.sub(r\"You\\x89Ûªre\", \"You are\", tweet)\n",
    "    tweet = re.sub(r\"where's\", \"where is\", tweet)\n",
    "    tweet = re.sub(r\"Don\\x89Ûªt\", \"Do not\", tweet)\n",
    "    tweet = re.sub(r\"we'd\", \"we would\", tweet)\n",
    "    tweet = re.sub(r\"i'll\", \"I will\", tweet)\n",
    "    tweet = re.sub(r\"weren't\", \"were not\", tweet)\n",
    "    tweet = re.sub(r\"They're\", \"They are\", tweet)\n",
    "    tweet = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", tweet)\n",
    "    tweet = re.sub(r\"you\\x89Ûªll\", \"you will\", tweet)\n",
    "    tweet = re.sub(r\"I\\x89Ûªd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"let's\", \"let us\", tweet)\n",
    "    tweet = re.sub(r\"it's\", \"it is\", tweet)\n",
    "    tweet = re.sub(r\"can't\", \"cannot\", tweet)\n",
    "    tweet = re.sub(r\"don't\", \"do not\", tweet)\n",
    "    tweet = re.sub(r\"you're\", \"you are\", tweet)\n",
    "    tweet = re.sub(r\"i've\", \"I have\", tweet)\n",
    "    tweet = re.sub(r\"that's\", \"that is\", tweet)\n",
    "    tweet = re.sub(r\"i'll\", \"I will\", tweet)\n",
    "    tweet = re.sub(r\"doesn't\", \"does not\", tweet)\n",
    "    tweet = re.sub(r\"i'd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"didn't\", \"did not\", tweet)\n",
    "    tweet = re.sub(r\"ain't\", \"am not\", tweet)\n",
    "    tweet = re.sub(r\"you'll\", \"you will\", tweet)\n",
    "    tweet = re.sub(r\"I've\", \"I have\", tweet)\n",
    "    tweet = re.sub(r\"Don't\", \"do not\", tweet)\n",
    "    tweet = re.sub(r\"I'll\", \"I will\", tweet)\n",
    "    tweet = re.sub(r\"I'd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"Let's\", \"Let us\", tweet)\n",
    "    tweet = re.sub(r\"you'd\", \"You would\", tweet)\n",
    "    tweet = re.sub(r\"It's\", \"It is\", tweet)\n",
    "    tweet = re.sub(r\"Ain't\", \"am not\", tweet)\n",
    "    tweet = re.sub(r\"Haven't\", \"Have not\", tweet)\n",
    "    tweet = re.sub(r\"Could've\", \"Could have\", tweet)\n",
    "    tweet = re.sub(r\"youve\", \"you have\", tweet)  \n",
    "    tweet = re.sub(r\"donå«t\", \"do not\", tweet)   \n",
    "            \n",
    "    # Character entity references\n",
    "    tweet = re.sub(r\"&gt;\", \">\", tweet)\n",
    "    tweet = re.sub(r\"&lt;\", \"<\", tweet)\n",
    "    tweet = re.sub(r\"&amp;\", \"&\", tweet)\n",
    "    \n",
    "    # Typos, slang and informal abbreviations\n",
    "    tweet = re.sub(r\"w/e\", \"whatever\", tweet)\n",
    "    tweet = re.sub(r\"w/\", \"with\", tweet)\n",
    "    tweet = re.sub(r\"USAgov\", \"USA government\", tweet)\n",
    "    tweet = re.sub(r\"recentlu\", \"recently\", tweet)\n",
    "    tweet = re.sub(r\"Ph0tos\", \"Photos\", tweet)\n",
    "    tweet = re.sub(r\"amirite\", \"am I right\", tweet)\n",
    "    tweet = re.sub(r\"exp0sed\", \"exposed\", tweet)\n",
    "    tweet = re.sub(r\"<3\", \"love\", tweet)\n",
    "    tweet = re.sub(r\"amageddon\", \"armageddon\", tweet)\n",
    "    tweet = re.sub(r\"Trfc\", \"Traffic\", tweet)\n",
    "    tweet = re.sub(r\"8/5/2015\", \"2015-08-05\", tweet)\n",
    "    tweet = re.sub(r\"WindStorm\", \"Wind Storm\", tweet)\n",
    "    tweet = re.sub(r\"8/6/2015\", \"2015-08-06\", tweet)\n",
    "    tweet = re.sub(r\"10:38PM\", \"10:38 PM\", tweet)\n",
    "    tweet = re.sub(r\"10:30pm\", \"10:30 PM\", tweet)\n",
    "    tweet = re.sub(r\"16yr\", \"16 year\", tweet)\n",
    "    tweet = re.sub(r\"lmao\", \"laughing my ass off\", tweet)   \n",
    "    tweet = re.sub(r\"TRAUMATISED\", \"traumatized\", tweet)\n",
    "    \n",
    "    # Hashtags and usernames\n",
    "    tweet = re.sub(r\"IranDeal\", \"Iran Deal\", tweet)\n",
    "    tweet = re.sub(r\"ArianaGrande\", \"Ariana Grande\", tweet)\n",
    "    tweet = re.sub(r\"camilacabello97\", \"camila cabello\", tweet) \n",
    "    tweet = re.sub(r\"RondaRousey\", \"Ronda Rousey\", tweet)     \n",
    "    tweet = re.sub(r\"MTVHottest\", \"MTV Hottest\", tweet)\n",
    "    tweet = re.sub(r\"TrapMusic\", \"Trap Music\", tweet)\n",
    "    tweet = re.sub(r\"ProphetMuhammad\", \"Prophet Muhammad\", tweet)\n",
    "    tweet = re.sub(r\"PantherAttack\", \"Panther Attack\", tweet)\n",
    "    tweet = re.sub(r\"StrategicPatience\", \"Strategic Patience\", tweet)\n",
    "    tweet = re.sub(r\"socialnews\", \"social news\", tweet)\n",
    "    tweet = re.sub(r\"NASAHurricane\", \"NASA Hurricane\", tweet)\n",
    "    tweet = re.sub(r\"onlinecommunities\", \"online communities\", tweet)\n",
    "    tweet = re.sub(r\"humanconsumption\", \"human consumption\", tweet)\n",
    "    tweet = re.sub(r\"Typhoon-Devastated\", \"Typhoon Devastated\", tweet)\n",
    "    tweet = re.sub(r\"Meat-Loving\", \"Meat Loving\", tweet)\n",
    "    tweet = re.sub(r\"facialabuse\", \"facial abuse\", tweet)\n",
    "    tweet = re.sub(r\"LakeCounty\", \"Lake County\", tweet)\n",
    "    tweet = re.sub(r\"BeingAuthor\", \"Being Author\", tweet)\n",
    "    tweet = re.sub(r\"withheavenly\", \"with heavenly\", tweet)\n",
    "    tweet = re.sub(r\"thankU\", \"thank you\", tweet)\n",
    "    tweet = re.sub(r\"iTunesMusic\", \"iTunes Music\", tweet)\n",
    "    tweet = re.sub(r\"OffensiveContent\", \"Offensive Content\", tweet)\n",
    "    tweet = re.sub(r\"WorstSummerJob\", \"Worst Summer Job\", tweet)\n",
    "    tweet = re.sub(r\"HarryBeCareful\", \"Harry Be Careful\", tweet)\n",
    "    tweet = re.sub(r\"NASASolarSystem\", \"NASA Solar System\", tweet)\n",
    "    tweet = re.sub(r\"animalrescue\", \"animal rescue\", tweet)\n",
    "    tweet = re.sub(r\"KurtSchlichter\", \"Kurt Schlichter\", tweet)\n",
    "    tweet = re.sub(r\"aRmageddon\", \"armageddon\", tweet)\n",
    "    tweet = re.sub(r\"Throwingknifes\", \"Throwing knives\", tweet)\n",
    "    tweet = re.sub(r\"GodsLove\", \"God's Love\", tweet)\n",
    "    tweet = re.sub(r\"bookboost\", \"book boost\", tweet)\n",
    "    tweet = re.sub(r\"ibooklove\", \"I book love\", tweet)\n",
    "    tweet = re.sub(r\"NestleIndia\", \"Nestle India\", tweet)\n",
    "    tweet = re.sub(r\"realDonaldTrump\", \"Donald Trump\", tweet)\n",
    "    tweet = re.sub(r\"DavidVonderhaar\", \"David Vonderhaar\", tweet)\n",
    "    tweet = re.sub(r\"CecilTheLion\", \"Cecil The Lion\", tweet)\n",
    "    tweet = re.sub(r\"weathernetwork\", \"weather network\", tweet)\n",
    "    tweet = re.sub(r\"withBioterrorism&use\", \"with Bioterrorism & use\", tweet)\n",
    "    tweet = re.sub(r\"Hostage&2\", \"Hostage & 2\", tweet)\n",
    "    tweet = re.sub(r\"GOPDebate\", \"GOP Debate\", tweet)\n",
    "    tweet = re.sub(r\"RickPerry\", \"Rick Perry\", tweet)\n",
    "    tweet = re.sub(r\"frontpage\", \"front page\", tweet)\n",
    "    tweet = re.sub(r\"NewsInTweets\", \"News In Tweets\", tweet)\n",
    "    tweet = re.sub(r\"ViralSpell\", \"Viral Spell\", tweet)\n",
    "    tweet = re.sub(r\"til_now\", \"until now\", tweet)\n",
    "    tweet = re.sub(r\"volcanoinRussia\", \"volcano in Russia\", tweet)\n",
    "    tweet = re.sub(r\"ZippedNews\", \"Zipped News\", tweet)\n",
    "    tweet = re.sub(r\"MicheleBachman\", \"Michele Bachman\", tweet)\n",
    "    tweet = re.sub(r\"53inch\", \"53 inch\", tweet)\n",
    "    tweet = re.sub(r\"KerrickTrial\", \"Kerrick Trial\", tweet)\n",
    "    tweet = re.sub(r\"abstorm\", \"Alberta Storm\", tweet)\n",
    "    tweet = re.sub(r\"Beyhive\", \"Beyonce hive\", tweet)\n",
    "    tweet = re.sub(r\"IDFire\", \"Idaho Fire\", tweet)\n",
    "    tweet = re.sub(r\"DETECTADO\", \"Detected\", tweet)\n",
    "    tweet = re.sub(r\"RockyFire\", \"Rocky Fire\", tweet)\n",
    "    tweet = re.sub(r\"Listen/Buy\", \"Listen / Buy\", tweet)\n",
    "    tweet = re.sub(r\"NickCannon\", \"Nick Cannon\", tweet)\n",
    "    tweet = re.sub(r\"FaroeIslands\", \"Faroe Islands\", tweet)\n",
    "    tweet = re.sub(r\"yycstorm\", \"Calgary Storm\", tweet)\n",
    "    tweet = re.sub(r\"IDPs:\", \"Internally Displaced People :\", tweet)\n",
    "    tweet = re.sub(r\"ArtistsUnited\", \"Artists United\", tweet)\n",
    "    tweet = re.sub(r\"ClaytonBryant\", \"Clayton Bryant\", tweet)\n",
    "    tweet = re.sub(r\"jimmyfallon\", \"jimmy fallon\", tweet)\n",
    "    tweet = re.sub(r\"justinbieber\", \"justin bieber\", tweet)  \n",
    "    tweet = re.sub(r\"UTC2015\", \"UTC 2015\", tweet)\n",
    "    tweet = re.sub(r\"Time2015\", \"Time 2015\", tweet)\n",
    "    tweet = re.sub(r\"djicemoon\", \"dj icemoon\", tweet)\n",
    "    tweet = re.sub(r\"LivingSafely\", \"Living Safely\", tweet)\n",
    "    tweet = re.sub(r\"FIFA16\", \"Fifa 2016\", tweet)\n",
    "    tweet = re.sub(r\"thisiswhywecanthavenicethings\", \"this is why we cannot have nice things\", tweet)\n",
    "    tweet = re.sub(r\"bbcnews\", \"bbc news\", tweet)\n",
    "    tweet = re.sub(r\"UndergroundRailraod\", \"Underground Railraod\", tweet)\n",
    "    tweet = re.sub(r\"c4news\", \"c4 news\", tweet)\n",
    "    tweet = re.sub(r\"OBLITERATION\", \"obliteration\", tweet)\n",
    "    tweet = re.sub(r\"MUDSLIDE\", \"mudslide\", tweet)\n",
    "    tweet = re.sub(r\"NoSurrender\", \"No Surrender\", tweet)\n",
    "    tweet = re.sub(r\"NotExplained\", \"Not Explained\", tweet)\n",
    "    tweet = re.sub(r\"greatbritishbakeoff\", \"great british bake off\", tweet)\n",
    "    tweet = re.sub(r\"LondonFire\", \"London Fire\", tweet)\n",
    "    tweet = re.sub(r\"KOTAWeather\", \"KOTA Weather\", tweet)\n",
    "    tweet = re.sub(r\"LuchaUnderground\", \"Lucha Underground\", tweet)\n",
    "    tweet = re.sub(r\"KOIN6News\", \"KOIN 6 News\", tweet)\n",
    "    tweet = re.sub(r\"LiveOnK2\", \"Live On K2\", tweet)\n",
    "    tweet = re.sub(r\"9NewsGoldCoast\", \"9 News Gold Coast\", tweet)\n",
    "    tweet = re.sub(r\"nikeplus\", \"nike plus\", tweet)\n",
    "    tweet = re.sub(r\"david_cameron\", \"David Cameron\", tweet)\n",
    "    tweet = re.sub(r\"peterjukes\", \"Peter Jukes\", tweet)\n",
    "    tweet = re.sub(r\"JamesMelville\", \"James Melville\", tweet)\n",
    "    tweet = re.sub(r\"megynkelly\", \"Megyn Kelly\", tweet)\n",
    "    tweet = re.sub(r\"cnewslive\", \"C News Live\", tweet)\n",
    "    tweet = re.sub(r\"JamaicaObserver\", \"Jamaica Observer\", tweet)\n",
    "    tweet = re.sub(r\"TweetLikeItsSeptember11th2001\", \"Tweet like it is september 11th 2001\", tweet)\n",
    "    tweet = re.sub(r\"cbplawyers\", \"cbp lawyers\", tweet)\n",
    "    tweet = re.sub(r\"fewmoretweets\", \"few more tweets\", tweet)\n",
    "    tweet = re.sub(r\"BlackLivesMatter\", \"Black Lives Matter\", tweet)\n",
    "    tweet = re.sub(r\"cjoyner\", \"Chris Joyner\", tweet)\n",
    "    tweet = re.sub(r\"ENGvAUS\", \"England vs Australia\", tweet)\n",
    "    tweet = re.sub(r\"ScottWalker\", \"Scott Walker\", tweet)\n",
    "    tweet = re.sub(r\"MikeParrActor\", \"Michael Parr\", tweet)\n",
    "    tweet = re.sub(r\"4PlayThursdays\", \"Foreplay Thursdays\", tweet)\n",
    "    tweet = re.sub(r\"TGF2015\", \"Tontitown Grape Festival\", tweet)\n",
    "    tweet = re.sub(r\"realmandyrain\", \"Mandy Rain\", tweet)\n",
    "    tweet = re.sub(r\"GraysonDolan\", \"Grayson Dolan\", tweet)\n",
    "    tweet = re.sub(r\"ApolloBrown\", \"Apollo Brown\", tweet)\n",
    "    tweet = re.sub(r\"saddlebrooke\", \"Saddlebrooke\", tweet)\n",
    "    tweet = re.sub(r\"TontitownGrape\", \"Tontitown Grape\", tweet)\n",
    "    tweet = re.sub(r\"AbbsWinston\", \"Abbs Winston\", tweet)\n",
    "    tweet = re.sub(r\"ShaunKing\", \"Shaun King\", tweet)\n",
    "    tweet = re.sub(r\"MeekMill\", \"Meek Mill\", tweet)\n",
    "    tweet = re.sub(r\"TornadoGiveaway\", \"Tornado Giveaway\", tweet)\n",
    "    tweet = re.sub(r\"GRupdates\", \"GR updates\", tweet)\n",
    "    tweet = re.sub(r\"SouthDowns\", \"South Downs\", tweet)\n",
    "    tweet = re.sub(r\"braininjury\", \"brain injury\", tweet)\n",
    "    tweet = re.sub(r\"auspol\", \"Australian politics\", tweet)\n",
    "    tweet = re.sub(r\"PlannedParenthood\", \"Planned Parenthood\", tweet)\n",
    "    tweet = re.sub(r\"calgaryweather\", \"Calgary Weather\", tweet)\n",
    "    tweet = re.sub(r\"weallheartonedirection\", \"we all heart one direction\", tweet)\n",
    "    tweet = re.sub(r\"edsheeran\", \"Ed Sheeran\", tweet)\n",
    "    tweet = re.sub(r\"TrueHeroes\", \"True Heroes\", tweet)\n",
    "    tweet = re.sub(r\"S3XLEAK\", \"sex leak\", tweet)\n",
    "    tweet = re.sub(r\"ComplexMag\", \"Complex Magazine\", tweet)\n",
    "    tweet = re.sub(r\"TheAdvocateMag\", \"The Advocate Magazine\", tweet)\n",
    "    tweet = re.sub(r\"CityofCalgary\", \"City of Calgary\", tweet)\n",
    "    tweet = re.sub(r\"EbolaOutbreak\", \"Ebola Outbreak\", tweet)\n",
    "    tweet = re.sub(r\"SummerFate\", \"Summer Fate\", tweet)\n",
    "    tweet = re.sub(r\"RAmag\", \"Royal Academy Magazine\", tweet)\n",
    "    tweet = re.sub(r\"offers2go\", \"offers to go\", tweet)\n",
    "    tweet = re.sub(r\"foodscare\", \"food scare\", tweet)\n",
    "    tweet = re.sub(r\"MNPDNashville\", \"Metropolitan Nashville Police Department\", tweet)\n",
    "    tweet = re.sub(r\"TfLBusAlerts\", \"TfL Bus Alerts\", tweet)\n",
    "    tweet = re.sub(r\"GamerGate\", \"Gamer Gate\", tweet)\n",
    "    tweet = re.sub(r\"IHHen\", \"Humanitarian Relief\", tweet)\n",
    "    tweet = re.sub(r\"spinningbot\", \"spinning bot\", tweet)\n",
    "    tweet = re.sub(r\"ModiMinistry\", \"Modi Ministry\", tweet)\n",
    "    tweet = re.sub(r\"TAXIWAYS\", \"taxi ways\", tweet)\n",
    "    tweet = re.sub(r\"Calum5SOS\", \"Calum Hood\", tweet)\n",
    "    tweet = re.sub(r\"po_st\", \"po.st\", tweet)\n",
    "    tweet = re.sub(r\"scoopit\", \"scoop.it\", tweet)\n",
    "    tweet = re.sub(r\"UltimaLucha\", \"Ultima Lucha\", tweet)\n",
    "    tweet = re.sub(r\"JonathanFerrell\", \"Jonathan Ferrell\", tweet)\n",
    "    tweet = re.sub(r\"aria_ahrary\", \"Aria Ahrary\", tweet)\n",
    "    tweet = re.sub(r\"rapidcity\", \"Rapid City\", tweet)\n",
    "    tweet = re.sub(r\"OutBid\", \"outbid\", tweet)\n",
    "    tweet = re.sub(r\"lavenderpoetrycafe\", \"lavender poetry cafe\", tweet)\n",
    "    tweet = re.sub(r\"EudryLantiqua\", \"Eudry Lantiqua\", tweet)\n",
    "    tweet = re.sub(r\"15PM\", \"15 PM\", tweet)\n",
    "    tweet = re.sub(r\"OriginalFunko\", \"Funko\", tweet)\n",
    "    tweet = re.sub(r\"rightwaystan\", \"Richard Tan\", tweet)\n",
    "    tweet = re.sub(r\"CindyNoonan\", \"Cindy Noonan\", tweet)\n",
    "    tweet = re.sub(r\"RT_America\", \"RT America\", tweet)\n",
    "    tweet = re.sub(r\"narendramodi\", \"Narendra Modi\", tweet)\n",
    "    tweet = re.sub(r\"BakeOffFriends\", \"Bake Off Friends\", tweet)\n",
    "    tweet = re.sub(r\"TeamHendrick\", \"Hendrick Motorsports\", tweet)\n",
    "    tweet = re.sub(r\"alexbelloli\", \"Alex Belloli\", tweet)\n",
    "    tweet = re.sub(r\"itsjustinstuart\", \"Justin Stuart\", tweet)\n",
    "    tweet = re.sub(r\"gunsense\", \"gun sense\", tweet)\n",
    "    tweet = re.sub(r\"DebateQuestionsWeWantToHear\", \"debate questions we want to hear\", tweet)\n",
    "    tweet = re.sub(r\"RoyalCarribean\", \"Royal Carribean\", tweet)\n",
    "    tweet = re.sub(r\"samanthaturne19\", \"Samantha Turner\", tweet)\n",
    "    tweet = re.sub(r\"JonVoyage\", \"Jon Stewart\", tweet)\n",
    "    tweet = re.sub(r\"renew911health\", \"renew 911 health\", tweet)\n",
    "    tweet = re.sub(r\"SuryaRay\", \"Surya Ray\", tweet)\n",
    "    tweet = re.sub(r\"pattonoswalt\", \"Patton Oswalt\", tweet)\n",
    "    tweet = re.sub(r\"minhazmerchant\", \"Minhaz Merchant\", tweet)\n",
    "    tweet = re.sub(r\"TLVFaces\", \"Israel Diaspora Coalition\", tweet)\n",
    "    tweet = re.sub(r\"pmarca\", \"Marc Andreessen\", tweet)\n",
    "    tweet = re.sub(r\"pdx911\", \"Portland Police\", tweet)\n",
    "    tweet = re.sub(r\"jamaicaplain\", \"Jamaica Plain\", tweet)\n",
    "    tweet = re.sub(r\"Japton\", \"Arkansas\", tweet)\n",
    "    tweet = re.sub(r\"RouteComplex\", \"Route Complex\", tweet)\n",
    "    tweet = re.sub(r\"INSubcontinent\", \"Indian Subcontinent\", tweet)\n",
    "    tweet = re.sub(r\"NJTurnpike\", \"New Jersey Turnpike\", tweet)\n",
    "    tweet = re.sub(r\"Politifiact\", \"PolitiFact\", tweet)\n",
    "    tweet = re.sub(r\"Hiroshima70\", \"Hiroshima\", tweet)\n",
    "    tweet = re.sub(r\"GMMBC\", \"Greater Mt Moriah Baptist Church\", tweet)\n",
    "    tweet = re.sub(r\"versethe\", \"verse the\", tweet)\n",
    "    tweet = re.sub(r\"TubeStrike\", \"Tube Strike\", tweet)\n",
    "    tweet = re.sub(r\"MissionHills\", \"Mission Hills\", tweet)\n",
    "    tweet = re.sub(r\"ProtectDenaliWolves\", \"Protect Denali Wolves\", tweet)\n",
    "    tweet = re.sub(r\"NANKANA\", \"Nankana\", tweet)\n",
    "    tweet = re.sub(r\"SAHIB\", \"Sahib\", tweet)\n",
    "    tweet = re.sub(r\"PAKPATTAN\", \"Pakpattan\", tweet)\n",
    "    tweet = re.sub(r\"Newz_Sacramento\", \"News Sacramento\", tweet)\n",
    "    tweet = re.sub(r\"gofundme\", \"go fund me\", tweet)\n",
    "    tweet = re.sub(r\"pmharper\", \"Stephen Harper\", tweet)\n",
    "    tweet = re.sub(r\"IvanBerroa\", \"Ivan Berroa\", tweet)\n",
    "    tweet = re.sub(r\"LosDelSonido\", \"Los Del Sonido\", tweet)\n",
    "    tweet = re.sub(r\"bancodeseries\", \"banco de series\", tweet)\n",
    "    tweet = re.sub(r\"timkaine\", \"Tim Kaine\", tweet)\n",
    "    tweet = re.sub(r\"IdentityTheft\", \"Identity Theft\", tweet)\n",
    "    tweet = re.sub(r\"AllLivesMatter\", \"All Lives Matter\", tweet)\n",
    "    tweet = re.sub(r\"mishacollins\", \"Misha Collins\", tweet)\n",
    "    tweet = re.sub(r\"BillNeelyNBC\", \"Bill Neely\", tweet)\n",
    "    tweet = re.sub(r\"BeClearOnCancer\", \"be clear on cancer\", tweet)\n",
    "    tweet = re.sub(r\"Kowing\", \"Knowing\", tweet)\n",
    "    tweet = re.sub(r\"ScreamQueens\", \"Scream Queens\", tweet)\n",
    "    tweet = re.sub(r\"AskCharley\", \"Ask Charley\", tweet)\n",
    "    tweet = re.sub(r\"BlizzHeroes\", \"Heroes of the Storm\", tweet)\n",
    "    tweet = re.sub(r\"BradleyBrad47\", \"Bradley Brad\", tweet)\n",
    "    tweet = re.sub(r\"HannaPH\", \"Typhoon Hanna\", tweet)\n",
    "    tweet = re.sub(r\"meinlcymbals\", \"MEINL Cymbals\", tweet)\n",
    "    tweet = re.sub(r\"Ptbo\", \"Peterborough\", tweet)\n",
    "    tweet = re.sub(r\"cnnbrk\", \"CNN Breaking News\", tweet)\n",
    "    tweet = re.sub(r\"IndianNews\", \"Indian News\", tweet)\n",
    "    tweet = re.sub(r\"savebees\", \"save bees\", tweet)\n",
    "    tweet = re.sub(r\"GreenHarvard\", \"Green Harvard\", tweet)\n",
    "    tweet = re.sub(r\"StandwithPP\", \"Stand with planned parenthood\", tweet)\n",
    "    tweet = re.sub(r\"hermancranston\", \"Herman Cranston\", tweet)\n",
    "    tweet = re.sub(r\"WMUR9\", \"WMUR-TV\", tweet)\n",
    "    tweet = re.sub(r\"RockBottomRadFM\", \"Rock Bottom Radio\", tweet)\n",
    "    tweet = re.sub(r\"ameenshaikh3\", \"Ameen Shaikh\", tweet)\n",
    "    tweet = re.sub(r\"ProSyn\", \"Project Syndicate\", tweet)\n",
    "    tweet = re.sub(r\"Daesh\", \"ISIS\", tweet)\n",
    "    tweet = re.sub(r\"s2g\", \"swear to god\", tweet)\n",
    "    tweet = re.sub(r\"listenlive\", \"listen live\", tweet)\n",
    "    tweet = re.sub(r\"CDCgov\", \"Centers for Disease Control and Prevention\", tweet)\n",
    "    tweet = re.sub(r\"FoxNew\", \"Fox News\", tweet)\n",
    "    tweet = re.sub(r\"CBSBigBrother\", \"Big Brother\", tweet)\n",
    "    tweet = re.sub(r\"JulieDiCaro\", \"Julie DiCaro\", tweet)\n",
    "    tweet = re.sub(r\"theadvocatemag\", \"The Advocate Magazine\", tweet)\n",
    "    tweet = re.sub(r\"RohnertParkDPS\", \"Rohnert Park Police Department\", tweet)\n",
    "    tweet = re.sub(r\"THISIZBWRIGHT\", \"Bonnie Wright\", tweet)\n",
    "    tweet = re.sub(r\"Popularmmos\", \"Popular MMOs\", tweet)\n",
    "    tweet = re.sub(r\"WildHorses\", \"Wild Horses\", tweet)\n",
    "    tweet = re.sub(r\"FantasticFour\", \"Fantastic Four\", tweet)\n",
    "    tweet = re.sub(r\"HORNDALE\", \"Horndale\", tweet)\n",
    "    tweet = re.sub(r\"PINER\", \"Piner\", tweet)\n",
    "    tweet = re.sub(r\"BathAndNorthEastSomerset\", \"Bath and North East Somerset\", tweet)\n",
    "    tweet = re.sub(r\"thatswhatfriendsarefor\", \"that is what friends are for\", tweet)\n",
    "    tweet = re.sub(r\"residualincome\", \"residual income\", tweet)\n",
    "    tweet = re.sub(r\"YahooNewsDigest\", \"Yahoo News Digest\", tweet)\n",
    "    tweet = re.sub(r\"MalaysiaAirlines\", \"Malaysia Airlines\", tweet)\n",
    "    tweet = re.sub(r\"AmazonDeals\", \"Amazon Deals\", tweet)\n",
    "    tweet = re.sub(r\"MissCharleyWebb\", \"Charley Webb\", tweet)\n",
    "    tweet = re.sub(r\"shoalstraffic\", \"shoals traffic\", tweet)\n",
    "    tweet = re.sub(r\"GeorgeFoster72\", \"George Foster\", tweet)\n",
    "    tweet = re.sub(r\"pop2015\", \"pop 2015\", tweet)\n",
    "    tweet = re.sub(r\"_PokemonCards_\", \"Pokemon Cards\", tweet)\n",
    "    tweet = re.sub(r\"DianneG\", \"Dianne Gallagher\", tweet)\n",
    "    tweet = re.sub(r\"KashmirConflict\", \"Kashmir Conflict\", tweet)\n",
    "    tweet = re.sub(r\"BritishBakeOff\", \"British Bake Off\", tweet)\n",
    "    tweet = re.sub(r\"FreeKashmir\", \"Free Kashmir\", tweet)\n",
    "    tweet = re.sub(r\"mattmosley\", \"Matt Mosley\", tweet)\n",
    "    tweet = re.sub(r\"BishopFred\", \"Bishop Fred\", tweet)\n",
    "    tweet = re.sub(r\"EndConflict\", \"End Conflict\", tweet)\n",
    "    tweet = re.sub(r\"EndOccupation\", \"End Occupation\", tweet)\n",
    "    tweet = re.sub(r\"UNHEALED\", \"unhealed\", tweet)\n",
    "    tweet = re.sub(r\"CharlesDagnall\", \"Charles Dagnall\", tweet)\n",
    "    tweet = re.sub(r\"Latestnews\", \"Latest news\", tweet)\n",
    "    tweet = re.sub(r\"KindleCountdown\", \"Kindle Countdown\", tweet)\n",
    "    tweet = re.sub(r\"NoMoreHandouts\", \"No More Handouts\", tweet)\n",
    "    tweet = re.sub(r\"datingtips\", \"dating tips\", tweet)\n",
    "    tweet = re.sub(r\"charlesadler\", \"Charles Adler\", tweet)\n",
    "    tweet = re.sub(r\"twia\", \"Texas Windstorm Insurance Association\", tweet)\n",
    "    tweet = re.sub(r\"txlege\", \"Texas Legislature\", tweet)\n",
    "    tweet = re.sub(r\"WindstormInsurer\", \"Windstorm Insurer\", tweet)\n",
    "    tweet = re.sub(r\"Newss\", \"News\", tweet)\n",
    "    tweet = re.sub(r\"hempoil\", \"hemp oil\", tweet)\n",
    "    tweet = re.sub(r\"CommoditiesAre\", \"Commodities are\", tweet)\n",
    "    tweet = re.sub(r\"tubestrike\", \"tube strike\", tweet)\n",
    "    tweet = re.sub(r\"JoeNBC\", \"Joe Scarborough\", tweet)\n",
    "    tweet = re.sub(r\"LiteraryCakes\", \"Literary Cakes\", tweet)\n",
    "    tweet = re.sub(r\"TI5\", \"The International 5\", tweet)\n",
    "    tweet = re.sub(r\"thehill\", \"the hill\", tweet)\n",
    "    tweet = re.sub(r\"3others\", \"3 others\", tweet)\n",
    "    tweet = re.sub(r\"stighefootball\", \"Sam Tighe\", tweet)\n",
    "    tweet = re.sub(r\"whatstheimportantvideo\", \"what is the important video\", tweet)\n",
    "    tweet = re.sub(r\"ClaudioMeloni\", \"Claudio Meloni\", tweet)\n",
    "    tweet = re.sub(r\"DukeSkywalker\", \"Duke Skywalker\", tweet)\n",
    "    tweet = re.sub(r\"carsonmwr\", \"Fort Carson\", tweet)\n",
    "    tweet = re.sub(r\"offdishduty\", \"off dish duty\", tweet)\n",
    "    tweet = re.sub(r\"andword\", \"and word\", tweet)\n",
    "    tweet = re.sub(r\"rhodeisland\", \"Rhode Island\", tweet)\n",
    "    tweet = re.sub(r\"easternoregon\", \"Eastern Oregon\", tweet)\n",
    "    tweet = re.sub(r\"WAwildfire\", \"Washington Wildfire\", tweet)\n",
    "    tweet = re.sub(r\"fingerrockfire\", \"Finger Rock Fire\", tweet)\n",
    "    tweet = re.sub(r\"57am\", \"57 am\", tweet)\n",
    "    tweet = re.sub(r\"fingerrockfire\", \"Finger Rock Fire\", tweet)\n",
    "    tweet = re.sub(r\"JacobHoggard\", \"Jacob Hoggard\", tweet)\n",
    "    tweet = re.sub(r\"newnewnew\", \"new new new\", tweet)\n",
    "    tweet = re.sub(r\"under50\", \"under 50\", tweet)\n",
    "    tweet = re.sub(r\"getitbeforeitsgone\", \"get it before it is gone\", tweet)\n",
    "    tweet = re.sub(r\"freshoutofthebox\", \"fresh out of the box\", tweet)\n",
    "    tweet = re.sub(r\"amwriting\", \"am writing\", tweet)\n",
    "    tweet = re.sub(r\"Bokoharm\", \"Boko Haram\", tweet)\n",
    "    tweet = re.sub(r\"Nowlike\", \"Now like\", tweet)\n",
    "    tweet = re.sub(r\"seasonfrom\", \"season from\", tweet)\n",
    "    tweet = re.sub(r\"epicente\", \"epicenter\", tweet)\n",
    "    tweet = re.sub(r\"epicenterr\", \"epicenter\", tweet)\n",
    "    tweet = re.sub(r\"sicklife\", \"sick life\", tweet)\n",
    "    tweet = re.sub(r\"yycweather\", \"Calgary Weather\", tweet)\n",
    "    tweet = re.sub(r\"calgarysun\", \"Calgary Sun\", tweet)\n",
    "    tweet = re.sub(r\"approachng\", \"approaching\", tweet)\n",
    "    tweet = re.sub(r\"evng\", \"evening\", tweet)\n",
    "    tweet = re.sub(r\"Sumthng\", \"something\", tweet)\n",
    "    tweet = re.sub(r\"EllenPompeo\", \"Ellen Pompeo\", tweet)\n",
    "    tweet = re.sub(r\"shondarhimes\", \"Shonda Rhimes\", tweet)\n",
    "    tweet = re.sub(r\"ABCNetwork\", \"ABC Network\", tweet)\n",
    "    tweet = re.sub(r\"SushmaSwaraj\", \"Sushma Swaraj\", tweet)\n",
    "    tweet = re.sub(r\"pray4japan\", \"Pray for Japan\", tweet)\n",
    "    tweet = re.sub(r\"hope4japan\", \"Hope for Japan\", tweet)\n",
    "    tweet = re.sub(r\"Illusionimagess\", \"Illusion images\", tweet)\n",
    "    tweet = re.sub(r\"SummerUnderTheStars\", \"Summer Under The Stars\", tweet)\n",
    "    tweet = re.sub(r\"ShallWeDance\", \"Shall We Dance\", tweet)\n",
    "    tweet = re.sub(r\"TCMParty\", \"TCM Party\", tweet)\n",
    "    tweet = re.sub(r\"marijuananews\", \"marijuana news\", tweet)\n",
    "    tweet = re.sub(r\"onbeingwithKristaTippett\", \"on being with Krista Tippett\", tweet)\n",
    "    tweet = re.sub(r\"Beingtweets\", \"Being tweets\", tweet)\n",
    "    tweet = re.sub(r\"newauthors\", \"new authors\", tweet)\n",
    "    tweet = re.sub(r\"remedyyyy\", \"remedy\", tweet)\n",
    "    tweet = re.sub(r\"44PM\", \"44 PM\", tweet)\n",
    "    tweet = re.sub(r\"HeadlinesApp\", \"Headlines App\", tweet)\n",
    "    tweet = re.sub(r\"40PM\", \"40 PM\", tweet)\n",
    "    tweet = re.sub(r\"myswc\", \"Severe Weather Center\", tweet)\n",
    "    tweet = re.sub(r\"ithats\", \"that is\", tweet)\n",
    "    tweet = re.sub(r\"icouldsitinthismomentforever\", \"I could sit in this moment forever\", tweet)\n",
    "    tweet = re.sub(r\"FatLoss\", \"Fat Loss\", tweet)\n",
    "    tweet = re.sub(r\"02PM\", \"02 PM\", tweet)\n",
    "    tweet = re.sub(r\"MetroFmTalk\", \"Metro Fm Talk\", tweet)\n",
    "    tweet = re.sub(r\"Bstrd\", \"bastard\", tweet)\n",
    "    tweet = re.sub(r\"bldy\", \"bloody\", tweet)\n",
    "    tweet = re.sub(r\"MetrofmTalk\", \"Metro Fm Talk\", tweet)\n",
    "    tweet = re.sub(r\"terrorismturn\", \"terrorism turn\", tweet)\n",
    "    tweet = re.sub(r\"BBCNewsAsia\", \"BBC News Asia\", tweet)\n",
    "    tweet = re.sub(r\"BehindTheScenes\", \"Behind The Scenes\", tweet)\n",
    "    tweet = re.sub(r\"GeorgeTakei\", \"George Takei\", tweet)\n",
    "    tweet = re.sub(r\"WomensWeeklyMag\", \"Womens Weekly Magazine\", tweet)\n",
    "    tweet = re.sub(r\"SurvivorsGuidetoEarth\", \"Survivors Guide to Earth\", tweet)\n",
    "    tweet = re.sub(r\"incubusband\", \"incubus band\", tweet)\n",
    "    tweet = re.sub(r\"Babypicturethis\", \"Baby picture this\", tweet)\n",
    "    tweet = re.sub(r\"BombEffects\", \"Bomb Effects\", tweet)\n",
    "    tweet = re.sub(r\"win10\", \"Windows 10\", tweet)\n",
    "    tweet = re.sub(r\"idkidk\", \"I do not know I do not know\", tweet)\n",
    "    tweet = re.sub(r\"TheWalkingDead\", \"The Walking Dead\", tweet)\n",
    "    tweet = re.sub(r\"amyschumer\", \"Amy Schumer\", tweet)\n",
    "    tweet = re.sub(r\"crewlist\", \"crew list\", tweet)\n",
    "    tweet = re.sub(r\"Erdogans\", \"Erdogan\", tweet)\n",
    "    tweet = re.sub(r\"BBCLive\", \"BBC Live\", tweet)\n",
    "    tweet = re.sub(r\"TonyAbbottMHR\", \"Tony Abbott\", tweet)\n",
    "    tweet = re.sub(r\"paulmyerscough\", \"Paul Myerscough\", tweet)\n",
    "    tweet = re.sub(r\"georgegallagher\", \"George Gallagher\", tweet)\n",
    "    tweet = re.sub(r\"JimmieJohnson\", \"Jimmie Johnson\", tweet)\n",
    "    tweet = re.sub(r\"pctool\", \"pc tool\", tweet)\n",
    "    tweet = re.sub(r\"DoingHashtagsRight\", \"Doing Hashtags Right\", tweet)\n",
    "    tweet = re.sub(r\"ThrowbackThursday\", \"Throwback Thursday\", tweet)\n",
    "    tweet = re.sub(r\"SnowBackSunday\", \"Snowback Sunday\", tweet)\n",
    "    tweet = re.sub(r\"LakeEffect\", \"Lake Effect\", tweet)\n",
    "    tweet = re.sub(r\"RTphotographyUK\", \"Richard Thomas Photography UK\", tweet)\n",
    "    tweet = re.sub(r\"BigBang_CBS\", \"Big Bang CBS\", tweet)\n",
    "    tweet = re.sub(r\"writerslife\", \"writers life\", tweet)\n",
    "    tweet = re.sub(r\"NaturalBirth\", \"Natural Birth\", tweet)\n",
    "    tweet = re.sub(r\"UnusualWords\", \"Unusual Words\", tweet)\n",
    "    tweet = re.sub(r\"wizkhalifa\", \"Wiz Khalifa\", tweet)\n",
    "    tweet = re.sub(r\"acreativedc\", \"a creative DC\", tweet)\n",
    "    tweet = re.sub(r\"vscodc\", \"vsco DC\", tweet)\n",
    "    tweet = re.sub(r\"VSCOcam\", \"vsco camera\", tweet)\n",
    "    tweet = re.sub(r\"TheBEACHDC\", \"The beach DC\", tweet)\n",
    "    tweet = re.sub(r\"buildingmuseum\", \"building museum\", tweet)\n",
    "    tweet = re.sub(r\"WorldOil\", \"World Oil\", tweet)\n",
    "    tweet = re.sub(r\"redwedding\", \"red wedding\", tweet)\n",
    "    tweet = re.sub(r\"AmazingRaceCanada\", \"Amazing Race Canada\", tweet)\n",
    "    tweet = re.sub(r\"WakeUpAmerica\", \"Wake Up America\", tweet)\n",
    "    tweet = re.sub(r\"\\\\Allahuakbar\\\\\", \"Allahu Akbar\", tweet)\n",
    "    tweet = re.sub(r\"bleased\", \"blessed\", tweet)\n",
    "    tweet = re.sub(r\"nigeriantribune\", \"Nigerian Tribune\", tweet)\n",
    "    tweet = re.sub(r\"HIDEO_KOJIMA_EN\", \"Hideo Kojima\", tweet)\n",
    "    tweet = re.sub(r\"FusionFestival\", \"Fusion Festival\", tweet)\n",
    "    tweet = re.sub(r\"50Mixed\", \"50 Mixed\", tweet)\n",
    "    tweet = re.sub(r\"NoAgenda\", \"No Agenda\", tweet)\n",
    "    tweet = re.sub(r\"WhiteGenocide\", \"White Genocide\", tweet)\n",
    "    tweet = re.sub(r\"dirtylying\", \"dirty lying\", tweet)\n",
    "    tweet = re.sub(r\"SyrianRefugees\", \"Syrian Refugees\", tweet)\n",
    "    tweet = re.sub(r\"changetheworld\", \"change the world\", tweet)\n",
    "    tweet = re.sub(r\"Ebolacase\", \"Ebola case\", tweet)\n",
    "    tweet = re.sub(r\"mcgtech\", \"mcg technologies\", tweet)\n",
    "    tweet = re.sub(r\"withweapons\", \"with weapons\", tweet)\n",
    "    tweet = re.sub(r\"advancedwarfare\", \"advanced warfare\", tweet)\n",
    "    tweet = re.sub(r\"letsFootball\", \"let us Football\", tweet)\n",
    "    tweet = re.sub(r\"LateNiteMix\", \"late night mix\", tweet)\n",
    "    tweet = re.sub(r\"PhilCollinsFeed\", \"Phil Collins\", tweet)\n",
    "    tweet = re.sub(r\"RudyHavenstein\", \"Rudy Havenstein\", tweet)\n",
    "    tweet = re.sub(r\"22PM\", \"22 PM\", tweet)\n",
    "    tweet = re.sub(r\"54am\", \"54 AM\", tweet)\n",
    "    tweet = re.sub(r\"38am\", \"38 AM\", tweet)\n",
    "    tweet = re.sub(r\"OldFolkExplainStuff\", \"Old Folk Explain Stuff\", tweet)\n",
    "    tweet = re.sub(r\"BlacklivesMatter\", \"Black Lives Matter\", tweet)\n",
    "    tweet = re.sub(r\"InsaneLimits\", \"Insane Limits\", tweet)\n",
    "    tweet = re.sub(r\"youcantsitwithus\", \"you cannot sit with us\", tweet)\n",
    "    tweet = re.sub(r\"2k15\", \"2015\", tweet)\n",
    "    tweet = re.sub(r\"TheIran\", \"Iran\", tweet)\n",
    "    tweet = re.sub(r\"JimmyFallon\", \"Jimmy Fallon\", tweet)\n",
    "    tweet = re.sub(r\"AlbertBrooks\", \"Albert Brooks\", tweet)\n",
    "    tweet = re.sub(r\"defense_news\", \"defense news\", tweet)\n",
    "    tweet = re.sub(r\"nuclearrcSA\", \"Nuclear Risk Control Self Assessment\", tweet)\n",
    "    tweet = re.sub(r\"Auspol\", \"Australia Politics\", tweet)\n",
    "    tweet = re.sub(r\"NuclearPower\", \"Nuclear Power\", tweet)\n",
    "    tweet = re.sub(r\"WhiteTerrorism\", \"White Terrorism\", tweet)\n",
    "    tweet = re.sub(r\"truthfrequencyradio\", \"Truth Frequency Radio\", tweet)\n",
    "    tweet = re.sub(r\"ErasureIsNotEquality\", \"Erasure is not equality\", tweet)\n",
    "    tweet = re.sub(r\"ProBonoNews\", \"Pro Bono News\", tweet)\n",
    "    tweet = re.sub(r\"JakartaPost\", \"Jakarta Post\", tweet)\n",
    "    tweet = re.sub(r\"toopainful\", \"too painful\", tweet)\n",
    "    tweet = re.sub(r\"melindahaunton\", \"Melinda Haunton\", tweet)\n",
    "    tweet = re.sub(r\"NoNukes\", \"No Nukes\", tweet)\n",
    "    tweet = re.sub(r\"curryspcworld\", \"Currys PC World\", tweet)\n",
    "    tweet = re.sub(r\"ineedcake\", \"I need cake\", tweet)\n",
    "    tweet = re.sub(r\"blackforestgateau\", \"black forest gateau\", tweet)\n",
    "    tweet = re.sub(r\"BBCOne\", \"BBC One\", tweet)\n",
    "    tweet = re.sub(r\"AlexxPage\", \"Alex Page\", tweet)\n",
    "    tweet = re.sub(r\"jonathanserrie\", \"Jonathan Serrie\", tweet)\n",
    "    tweet = re.sub(r\"SocialJerkBlog\", \"Social Jerk Blog\", tweet)\n",
    "    tweet = re.sub(r\"ChelseaVPeretti\", \"Chelsea Peretti\", tweet)\n",
    "    tweet = re.sub(r\"irongiant\", \"iron giant\", tweet)\n",
    "    tweet = re.sub(r\"RonFunches\", \"Ron Funches\", tweet)\n",
    "    tweet = re.sub(r\"TimCook\", \"Tim Cook\", tweet)\n",
    "    tweet = re.sub(r\"sebastianstanisaliveandwell\", \"Sebastian Stan is alive and well\", tweet)\n",
    "    tweet = re.sub(r\"Madsummer\", \"Mad summer\", tweet)\n",
    "    tweet = re.sub(r\"NowYouKnow\", \"Now you know\", tweet)\n",
    "    tweet = re.sub(r\"concertphotography\", \"concert photography\", tweet)\n",
    "    tweet = re.sub(r\"TomLandry\", \"Tom Landry\", tweet)\n",
    "    tweet = re.sub(r\"showgirldayoff\", \"show girl day off\", tweet)\n",
    "    tweet = re.sub(r\"Yougslavia\", \"Yugoslavia\", tweet)\n",
    "    tweet = re.sub(r\"QuantumDataInformatics\", \"Quantum Data Informatics\", tweet)\n",
    "    tweet = re.sub(r\"FromTheDesk\", \"From The Desk\", tweet)\n",
    "    tweet = re.sub(r\"TheaterTrial\", \"Theater Trial\", tweet)\n",
    "    tweet = re.sub(r\"CatoInstitute\", \"Cato Institute\", tweet)\n",
    "    tweet = re.sub(r\"EmekaGift\", \"Emeka Gift\", tweet)\n",
    "    tweet = re.sub(r\"LetsBe_Rational\", \"Let us be rational\", tweet)\n",
    "    tweet = re.sub(r\"Cynicalreality\", \"Cynical reality\", tweet)\n",
    "    tweet = re.sub(r\"FredOlsenCruise\", \"Fred Olsen Cruise\", tweet)\n",
    "    tweet = re.sub(r\"NotSorry\", \"not sorry\", tweet)\n",
    "    tweet = re.sub(r\"UseYourWords\", \"use your words\", tweet)\n",
    "    tweet = re.sub(r\"WordoftheDay\", \"word of the day\", tweet)\n",
    "    tweet = re.sub(r\"Dictionarycom\", \"Dictionary.com\", tweet)\n",
    "    tweet = re.sub(r\"TheBrooklynLife\", \"The Brooklyn Life\", tweet)\n",
    "    tweet = re.sub(r\"jokethey\", \"joke they\", tweet)\n",
    "    tweet = re.sub(r\"nflweek1picks\", \"NFL week 1 picks\", tweet)\n",
    "    tweet = re.sub(r\"uiseful\", \"useful\", tweet)\n",
    "    tweet = re.sub(r\"JusticeDotOrg\", \"The American Association for Justice\", tweet)\n",
    "    tweet = re.sub(r\"autoaccidents\", \"auto accidents\", tweet)\n",
    "    tweet = re.sub(r\"SteveGursten\", \"Steve Gursten\", tweet)\n",
    "    tweet = re.sub(r\"MichiganAutoLaw\", \"Michigan Auto Law\", tweet)\n",
    "    tweet = re.sub(r\"birdgang\", \"bird gang\", tweet)\n",
    "    tweet = re.sub(r\"nflnetwork\", \"NFL Network\", tweet)\n",
    "    tweet = re.sub(r\"NYDNSports\", \"NY Daily News Sports\", tweet)\n",
    "    tweet = re.sub(r\"RVacchianoNYDN\", \"Ralph Vacchiano NY Daily News\", tweet)\n",
    "    tweet = re.sub(r\"EdmontonEsks\", \"Edmonton Eskimos\", tweet)\n",
    "    tweet = re.sub(r\"david_brelsford\", \"David Brelsford\", tweet)\n",
    "    tweet = re.sub(r\"TOI_India\", \"The Times of India\", tweet)\n",
    "    tweet = re.sub(r\"hegot\", \"he got\", tweet)\n",
    "    tweet = re.sub(r\"SkinsOn9\", \"Skins on 9\", tweet)\n",
    "    tweet = re.sub(r\"sothathappened\", \"so that happened\", tweet)\n",
    "    tweet = re.sub(r\"LCOutOfDoors\", \"LC Out Of Doors\", tweet)\n",
    "    tweet = re.sub(r\"NationFirst\", \"Nation First\", tweet)\n",
    "    tweet = re.sub(r\"IndiaToday\", \"India Today\", tweet)\n",
    "    tweet = re.sub(r\"HLPS\", \"helps\", tweet)\n",
    "    tweet = re.sub(r\"HOSTAGESTHROSW\", \"hostages throw\", tweet)\n",
    "    tweet = re.sub(r\"SNCTIONS\", \"sanctions\", tweet)\n",
    "    tweet = re.sub(r\"BidTime\", \"Bid Time\", tweet)\n",
    "    tweet = re.sub(r\"crunchysensible\", \"crunchy sensible\", tweet)\n",
    "    tweet = re.sub(r\"RandomActsOfRomance\", \"Random acts of romance\", tweet)\n",
    "    tweet = re.sub(r\"MomentsAtHill\", \"Moments at hill\", tweet)\n",
    "    tweet = re.sub(r\"eatshit\", \"eat shit\", tweet)\n",
    "    tweet = re.sub(r\"liveleakfun\", \"live leak fun\", tweet)\n",
    "    tweet = re.sub(r\"SahelNews\", \"Sahel News\", tweet)\n",
    "    tweet = re.sub(r\"abc7newsbayarea\", \"ABC 7 News Bay Area\", tweet)\n",
    "    tweet = re.sub(r\"facilitiesmanagement\", \"facilities management\", tweet)\n",
    "    tweet = re.sub(r\"facilitydude\", \"facility dude\", tweet)\n",
    "    tweet = re.sub(r\"CampLogistics\", \"Camp logistics\", tweet)\n",
    "    tweet = re.sub(r\"alaskapublic\", \"Alaska public\", tweet)\n",
    "    tweet = re.sub(r\"MarketResearch\", \"Market Research\", tweet)\n",
    "    tweet = re.sub(r\"AccuracyEsports\", \"Accuracy Esports\", tweet)\n",
    "    tweet = re.sub(r\"TheBodyShopAust\", \"The Body Shop Australia\", tweet)\n",
    "    tweet = re.sub(r\"yychail\", \"Calgary hail\", tweet)\n",
    "    tweet = re.sub(r\"yyctraffic\", \"Calgary traffic\", tweet)\n",
    "    tweet = re.sub(r\"eliotschool\", \"eliot school\", tweet)\n",
    "    tweet = re.sub(r\"TheBrokenCity\", \"The Broken City\", tweet)\n",
    "    tweet = re.sub(r\"OldsFireDept\", \"Olds Fire Department\", tweet)\n",
    "    tweet = re.sub(r\"RiverComplex\", \"River Complex\", tweet)\n",
    "    tweet = re.sub(r\"fieldworksmells\", \"field work smells\", tweet)\n",
    "    tweet = re.sub(r\"IranElection\", \"Iran Election\", tweet)\n",
    "    tweet = re.sub(r\"glowng\", \"glowing\", tweet)\n",
    "    tweet = re.sub(r\"kindlng\", \"kindling\", tweet)\n",
    "    tweet = re.sub(r\"riggd\", \"rigged\", tweet)\n",
    "    tweet = re.sub(r\"slownewsday\", \"slow news day\", tweet)\n",
    "    tweet = re.sub(r\"MyanmarFlood\", \"Myanmar Flood\", tweet)\n",
    "    tweet = re.sub(r\"abc7chicago\", \"ABC 7 Chicago\", tweet)\n",
    "    tweet = re.sub(r\"copolitics\", \"Colorado Politics\", tweet)\n",
    "    tweet = re.sub(r\"AdilGhumro\", \"Adil Ghumro\", tweet)\n",
    "    tweet = re.sub(r\"netbots\", \"net bots\", tweet)\n",
    "    tweet = re.sub(r\"byebyeroad\", \"bye bye road\", tweet)\n",
    "    tweet = re.sub(r\"massiveflooding\", \"massive flooding\", tweet)\n",
    "    tweet = re.sub(r\"EndofUS\", \"End of United States\", tweet)\n",
    "    tweet = re.sub(r\"35PM\", \"35 PM\", tweet)\n",
    "    tweet = re.sub(r\"greektheatrela\", \"Greek Theatre Los Angeles\", tweet)\n",
    "    tweet = re.sub(r\"76mins\", \"76 minutes\", tweet)\n",
    "    tweet = re.sub(r\"publicsafetyfirst\", \"public safety first\", tweet)\n",
    "    tweet = re.sub(r\"livesmatter\", \"lives matter\", tweet)\n",
    "    tweet = re.sub(r\"myhometown\", \"my hometown\", tweet)\n",
    "    tweet = re.sub(r\"tankerfire\", \"tanker fire\", tweet)\n",
    "    tweet = re.sub(r\"MEMORIALDAY\", \"memorial day\", tweet)\n",
    "    tweet = re.sub(r\"MEMORIAL_DAY\", \"memorial day\", tweet)\n",
    "    tweet = re.sub(r\"instaxbooty\", \"instagram booty\", tweet)\n",
    "    tweet = re.sub(r\"Jerusalem_Post\", \"Jerusalem Post\", tweet)\n",
    "    tweet = re.sub(r\"WayneRooney_INA\", \"Wayne Rooney\", tweet)\n",
    "    tweet = re.sub(r\"VirtualReality\", \"Virtual Reality\", tweet)\n",
    "    tweet = re.sub(r\"OculusRift\", \"Oculus Rift\", tweet)\n",
    "    tweet = re.sub(r\"OwenJones84\", \"Owen Jones\", tweet)\n",
    "    tweet = re.sub(r\"jeremycorbyn\", \"Jeremy Corbyn\", tweet)\n",
    "    tweet = re.sub(r\"paulrogers002\", \"Paul Rogers\", tweet)\n",
    "    tweet = re.sub(r\"mortalkombatx\", \"Mortal Kombat X\", tweet)\n",
    "    tweet = re.sub(r\"mortalkombat\", \"Mortal Kombat\", tweet)\n",
    "    tweet = re.sub(r\"FilipeCoelho92\", \"Filipe Coelho\", tweet)\n",
    "    tweet = re.sub(r\"OnlyQuakeNews\", \"Only Quake News\", tweet)\n",
    "    tweet = re.sub(r\"kostumes\", \"costumes\", tweet)\n",
    "    tweet = re.sub(r\"YEEESSSS\", \"yes\", tweet)\n",
    "    tweet = re.sub(r\"ToshikazuKatayama\", \"Toshikazu Katayama\", tweet)\n",
    "    tweet = re.sub(r\"IntlDevelopment\", \"Intl Development\", tweet)\n",
    "    tweet = re.sub(r\"ExtremeWeather\", \"Extreme Weather\", tweet)\n",
    "    tweet = re.sub(r\"WereNotGruberVoters\", \"We are not gruber voters\", tweet)\n",
    "    tweet = re.sub(r\"NewsThousands\", \"News Thousands\", tweet)\n",
    "    tweet = re.sub(r\"EdmundAdamus\", \"Edmund Adamus\", tweet)\n",
    "    tweet = re.sub(r\"EyewitnessWV\", \"Eye witness WV\", tweet)\n",
    "    tweet = re.sub(r\"PhiladelphiaMuseu\", \"Philadelphia Museum\", tweet)\n",
    "    tweet = re.sub(r\"DublinComicCon\", \"Dublin Comic Con\", tweet)\n",
    "    tweet = re.sub(r\"NicholasBrendon\", \"Nicholas Brendon\", tweet)\n",
    "    tweet = re.sub(r\"Alltheway80s\", \"All the way 80s\", tweet)\n",
    "    tweet = re.sub(r\"FromTheField\", \"From the field\", tweet)\n",
    "    tweet = re.sub(r\"NorthIowa\", \"North Iowa\", tweet)\n",
    "    tweet = re.sub(r\"WillowFire\", \"Willow Fire\", tweet)\n",
    "    tweet = re.sub(r\"MadRiverComplex\", \"Mad River Complex\", tweet)\n",
    "    tweet = re.sub(r\"feelingmanly\", \"feeling manly\", tweet)\n",
    "    tweet = re.sub(r\"stillnotoverit\", \"still not over it\", tweet)\n",
    "    tweet = re.sub(r\"FortitudeValley\", \"Fortitude Valley\", tweet)\n",
    "    tweet = re.sub(r\"CoastpowerlineTramTr\", \"Coast powerline\", tweet)\n",
    "    tweet = re.sub(r\"ServicesGold\", \"Services Gold\", tweet)\n",
    "    tweet = re.sub(r\"NewsbrokenEmergency\", \"News broken emergency\", tweet)\n",
    "    tweet = re.sub(r\"Evaucation\", \"evacuation\", tweet)\n",
    "    tweet = re.sub(r\"leaveevacuateexitbe\", \"leave evacuate exit be\", tweet)\n",
    "    tweet = re.sub(r\"P_EOPLE\", \"PEOPLE\", tweet)\n",
    "    tweet = re.sub(r\"Tubestrike\", \"tube strike\", tweet)\n",
    "    tweet = re.sub(r\"CLASS_SICK\", \"CLASS SICK\", tweet)\n",
    "    tweet = re.sub(r\"localplumber\", \"local plumber\", tweet)\n",
    "    tweet = re.sub(r\"awesomejobsiri\", \"awesome job siri\", tweet)\n",
    "    tweet = re.sub(r\"PayForItHow\", \"Pay for it how\", tweet)\n",
    "    tweet = re.sub(r\"ThisIsAfrica\", \"This is Africa\", tweet)\n",
    "    tweet = re.sub(r\"crimeairnetwork\", \"crime air network\", tweet)\n",
    "    tweet = re.sub(r\"KimAcheson\", \"Kim Acheson\", tweet)\n",
    "    tweet = re.sub(r\"cityofcalgary\", \"City of Calgary\", tweet)\n",
    "    tweet = re.sub(r\"prosyndicate\", \"pro syndicate\", tweet)\n",
    "    tweet = re.sub(r\"660NEWS\", \"660 NEWS\", tweet)\n",
    "    tweet = re.sub(r\"BusInsMagazine\", \"Business Insurance Magazine\", tweet)\n",
    "    tweet = re.sub(r\"wfocus\", \"focus\", tweet)\n",
    "    tweet = re.sub(r\"ShastaDam\", \"Shasta Dam\", tweet)\n",
    "    tweet = re.sub(r\"go2MarkFranco\", \"Mark Franco\", tweet)\n",
    "    tweet = re.sub(r\"StephGHinojosa\", \"Steph Hinojosa\", tweet)\n",
    "    tweet = re.sub(r\"Nashgrier\", \"Nash Grier\", tweet)\n",
    "    tweet = re.sub(r\"NashNewVideo\", \"Nash new video\", tweet)\n",
    "    tweet = re.sub(r\"IWouldntGetElectedBecause\", \"I would not get elected because\", tweet)\n",
    "    tweet = re.sub(r\"SHGames\", \"Sledgehammer Games\", tweet)\n",
    "    tweet = re.sub(r\"bedhair\", \"bed hair\", tweet)\n",
    "    tweet = re.sub(r\"JoelHeyman\", \"Joel Heyman\", tweet)\n",
    "    tweet = re.sub(r\"viaYouTube\", \"via YouTube\", tweet)\n",
    "    '''       \n",
    "           \n",
    "    # Urls\n",
    "    tweet = re.sub(r\"https?:\\/\\/t.co\\/[A-Za-z0-9]+\", \"\", tweet)\n",
    "        \n",
    "    # Words with punctuations and special characters\n",
    "    punctuations = '@#!?+&*[]-%.:/();$=><|{}^' + \"'`\"\n",
    "    for p in punctuations:\n",
    "        tweet = tweet.replace(p, f' {p} ')\n",
    "        \n",
    "    # ... and ..\n",
    "    tweet = tweet.replace('...', ' ... ')\n",
    "    if '...' not in tweet:\n",
    "        tweet = tweet.replace('..', ' ... ')      \n",
    "        \n",
    "    # Acronyms\n",
    "    tweet = re.sub(r\"MH370\", \"Malaysia Airlines Flight 370\", tweet)\n",
    "    tweet = re.sub(r\"mÌ¼sica\", \"music\", tweet)\n",
    "    tweet = re.sub(r\"okwx\", \"Oklahoma City Weather\", tweet)\n",
    "    tweet = re.sub(r\"arwx\", \"Arkansas Weather\", tweet)    \n",
    "    tweet = re.sub(r\"gawx\", \"Georgia Weather\", tweet)  \n",
    "    tweet = re.sub(r\"scwx\", \"South Carolina Weather\", tweet)  \n",
    "    tweet = re.sub(r\"cawx\", \"California Weather\", tweet)\n",
    "    tweet = re.sub(r\"tnwx\", \"Tennessee Weather\", tweet)\n",
    "    tweet = re.sub(r\"azwx\", \"Arizona Weather\", tweet)  \n",
    "    tweet = re.sub(r\"alwx\", \"Alabama Weather\", tweet)\n",
    "    tweet = re.sub(r\"wordpressdotcom\", \"wordpress\", tweet)    \n",
    "    tweet = re.sub(r\"usNWSgov\", \"United States National Weather Service\", tweet)\n",
    "    tweet = re.sub(r\"Suruc\", \"Sanliurfa\", tweet)   \n",
    "    \n",
    "    # Grouping same words without embeddings\n",
    "    tweet = re.sub(r\"Bestnaijamade\", \"bestnaijamade\", tweet)\n",
    "    tweet = re.sub(r\"SOUDELOR\", \"Soudelor\", tweet)\n",
    "    \n",
    "    return tweet\n",
    "\n",
    "\n",
    "df_train['text_cleaned'] = df_train['text'].apply(lambda s : clean(s))\n",
    "df_test['text_cleaned'] = df_test['text'].apply(lambda s : clean(s))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GloVe Embeddings cover 82.89% of vocabulary and 97.14% of text in Training Set\n",
      "GloVe Embeddings cover 88.09% of vocabulary and 97.32% of text in Test Set\n"
     ]
    }
   ],
   "source": [
    "train_glove_oov, train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(df_train['text_cleaned'], glove_embeddings)\n",
    "test_glove_oov, test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(df_test['text_cleaned'], glove_embeddings)\n",
    "print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))\n",
    "print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit',\n",
       " 'Hellfire! We don\\x89Ûªt even want to think about it or mention it so let\\x89Ûªs not do anything that leads to it #islam!',\n",
       " \"The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.'\",\n",
       " 'In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!',\n",
       " 'To fight bioterrorism sir.',\n",
       " 'Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her\\n \\n#FARRAKHAN #QUOTE',\n",
       " '#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption',\n",
       " '#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect',\n",
       " 'He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam',\n",
       " 'RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG',\n",
       " 'Hellfire is surrounded by desires so be careful and don\\x89Ûªt let your desires control you! #Afterlife',\n",
       " 'CLEARED:incident with injury:I-495  inner loop Exit 31 - MD 97/Georgia Ave Silver Spring',\n",
       " \"Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...\",\n",
       " 'wowo--=== 12000 Nigerian refugees repatriated from Cameroon',\n",
       " '.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4',\n",
       " 'Caution: breathing may be hazardous to your health.',\n",
       " 'I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????',\n",
       " 'that horrible sinking feeling when you\\x89Ûªve been at home on your phone for a while and you realise its been on 3G this whole time']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mislabeled = df_train.groupby(['text']).nunique().sort_values(by='target', ascending=False)\n",
    "df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']\n",
    "df_mislabeled.index.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['target_relabeled'] = df_train['target'].copy() \n",
    "\n",
    "df_train.loc[df_train['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you! #Afterlife', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == '.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == 'CLEARED:incident with injury:I-495  inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == '#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her\\n \\n#FARRAKHAN #QUOTE', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == \"Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"wowo--=== 12000 Nigerian refugees repatriated from Cameroon\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it #islam!\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.'\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"Caution: breathing may be hazardous to your health.\", 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == \"I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time\", 'target_relabeled'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7613 entries, 0 to 7612\n",
      "Data columns (total 6 columns):\n",
      "id              7613 non-null int64\n",
      "keyword         7552 non-null object\n",
      "location        5080 non-null object\n",
      "text            7613 non-null object\n",
      "target          7613 non-null int64\n",
      "text_cleaned    7613 non-null object\n",
      "dtypes: int64(2), object(4)\n",
      "memory usage: 357.0+ KB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test1 = df_test[[\"text_cleaned\"]] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "#nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "sw = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word_count\n",
    "df_train['word_count'] = df_train['text'].apply(lambda x: len(str(x).split()))\n",
    "df_test['word_count'] = df_test['text'].apply(lambda x: len(str(x).split()))\n",
    "\n",
    "# unique_word_count\n",
    "df_train['unique_word_count'] = df_train['text'].apply(lambda x: len(set(str(x).split())))\n",
    "df_test['unique_word_count'] = df_test['text'].apply(lambda x: len(set(str(x).split())))\n",
    "\n",
    "# stop_word_count\n",
    "df_train['stop_word_count'] = df_train['text'].apply(lambda x: len([w for w in str(x).lower().split() if w in sw]))\n",
    "df_test['stop_word_count'] = df_test['text'].apply(lambda x: len([w for w in str(x).lower().split() if w in sw]))\n",
    "\n",
    "# mean_word_length\n",
    "df_train['mean_word_length'] = df_train['text'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n",
    "df_test['mean_word_length'] = df_test['text'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n",
    "\n",
    "# char_count\n",
    "df_train['char_count'] = df_train['text'].apply(lambda x: len(str(x)))\n",
    "df_test['char_count'] = df_test['text'].apply(lambda x: len(str(x)))\n",
    "\n",
    "# punctuation_count\n",
    "df_train['punctuation_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n",
    "df_test['punctuation_count'] = df_test['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>text_cleaned</th>\n",
       "      <th>word_count</th>\n",
       "      <th>unique_word_count</th>\n",
       "      <th>stop_word_count</th>\n",
       "      <th>mean_word_length</th>\n",
       "      <th>char_count</th>\n",
       "      <th>punctuation_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this  # earthquake...</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>6</td>\n",
       "      <td>4.384615</td>\n",
       "      <td>69</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "      <td>Forest fire near La Ronge Sask .  Canada</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>4.571429</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "      <td>All residents asked to  ' shelter in place '  ...</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>5.090909</td>\n",
       "      <td>133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "      <td>13,000 people receive  # wildfires evacuation ...</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>7.125000</td>\n",
       "      <td>65</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Just got sent this photo from Ruby  # Alaska a...</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>7</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>88</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  target                                       text_cleaned  word_count  unique_word_count  stop_word_count  mean_word_length  char_count  punctuation_count\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...       1  Our Deeds are the Reason of this  # earthquake...          13                 13                6          4.384615          69                  1\n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada       1           Forest fire near La Ronge Sask .  Canada           7                  7                0          4.571429          38                  1\n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...       1  All residents asked to  ' shelter in place '  ...          22                 20               11          5.090909         133                  3\n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...       1  13,000 people receive  # wildfires evacuation ...           8                  8                1          7.125000          65                  2\n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...       1  Just got sent this photo from Ruby  # Alaska a...          16                 15                7          4.500000          88                  2"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train1 = df_train[[\"word_count\", \"unique_word_count\", \"mean_word_length\", \"char_count\", \"punctuation_count\"]] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tf_idf_word_vectorizer = TfidfVectorizer()\n",
    "tf_idf_word_vectorizer.fit_transform(df_train[\"text_cleaned\"])\n",
    "\n",
    "test1_tf_idf_word = tf_idf_word_vectorizer.transform(test1)\n",
    "\n",
    "train_tf_idf_word = tf_idf_word_vectorizer.transform(df_train[\"text_cleaned\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(df_train1)\n",
    "\n",
    "df_train1 = scaler.transform(df_train1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.4        0.42857143 0.1249531  0.41333333 0.01639344]\n",
      " [0.2        0.21428571 0.1358885  0.20666667 0.01639344]\n",
      " [0.7        0.67857143 0.16629712 0.84       0.04918033]\n",
      " ...\n",
      " [0.23333333 0.25       0.29268293 0.38666667 0.18032787]\n",
      " [0.6        0.64285714 0.23491656 0.86666667 0.08196721]\n",
      " [0.4        0.42857143 0.23752345 0.58       0.1147541 ]]\n"
     ]
    }
   ],
   "source": [
    "print(df_train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7613, 17034)\n",
      "(7613, 17039)\n"
     ]
    }
   ],
   "source": [
    "dense = train_tf_idf_word.todense()\n",
    "print(dense.shape) \n",
    "\n",
    "newCol = [[1],[0],[1]]\n",
    "\n",
    "\n",
    "\n",
    "allData = np.append(dense, df_train1, 1)\n",
    "\n",
    "print(allData.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MODEL KURMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_x, test_x, train_y, test_y = train_test_split(allData,\n",
    "                                                                   df_train[\"target\"], \n",
    "                                                                    random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tf_idf_word_vectorizer = TfidfVectorizer()\n",
    "tf_idf_word_vectorizer.fit_transform(train_x)\n",
    "\n",
    "test1_tf_idf_word = tf_idf_word_vectorizer.transform(test1)\n",
    "\n",
    "x_train_tf_idf_word = tf_idf_word_vectorizer.transform(train_x)\n",
    "x_test_tf_idf_word = tf_idf_word_vectorizer.transform(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 1., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 1.],\n",
       "       [0., 0., 0., 0., 1., 0.],\n",
       "       [0., 1., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 1., 0., 0., 0.]])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_tf_idf_word.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5709, 14244)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "np.shape(x_train_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 5709 entries, 5242 to 5157\n",
      "Data columns (total 1 columns):\n",
      "target    5709 non-null int64\n",
      "dtypes: int64(1)\n",
      "memory usage: 89.2 KB\n"
     ]
    }
   ],
   "source": [
    "train_y.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word-Level TF-IDF Doğruluk Oranı: 0.7615513915679251\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import model_selection\n",
    "nb = MultinomialNB()\n",
    "nb_model = nb.fit(x_train_tf_idf_word,train_y)\n",
    "accuracy = model_selection.cross_val_score(nb_model, \n",
    "                                           x_test_tf_idf_word, \n",
    "                                           test_y, \n",
    "                                           cv = 10).mean()\n",
    "\n",
    "print(\"Word-Level TF-IDF Doğruluk Oranı:\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word-Level TF-IDF Doğruluk Oranı: 0.731110498759989\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import model_selection\n",
    "nb = MultinomialNB()\n",
    "nb_model = nb.fit(train_x,train_y)\n",
    "accuracy = model_selection.cross_val_score(nb_model, \n",
    "                                           test_x, \n",
    "                                           test_y, \n",
    "                                           cv = 10).mean()\n",
    "\n",
    "print(\"Word-Level TF-IDF Doğruluk Oranı:\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}