ver004c-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "eksik sonuc verdi."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "yapılan işlemler onucu skor yükselmedi. Lokasyon, ve keyword gibi değişkenler concatlanıp bakılacak.\n",
    "Skoru yükselten asıl kaynağın model kurulması ve çalıştırılması olduğu anlaşıldı...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "df_train = pd.read_csv('train.csv')\n",
    "df_test = pd.read_csv('test.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def clean(tweet): \n",
    "   \n",
    "    # Urls\n",
    "    tweet = re.sub(r\"https?:\\/\\/t.co\\/[A-Za-z0-9]+\", \"\", tweet)\n",
    "        \n",
    "    # Words with punctuations and special characters\n",
    "    punctuations = '@#!?+&*[]-%.:/();$=><|{}^' + \"'`\"\n",
    "    for p in punctuations:\n",
    "        tweet = tweet.replace(p, f' {p} ')\n",
    "        \n",
    "    # ... and ..\n",
    "    tweet = tweet.replace('...', ' ... ')\n",
    "    if '...' not in tweet:\n",
    "        tweet = tweet.replace('..', ' ... ')      \n",
    "        \n",
    "    # Acronyms\n",
    "    tweet = re.sub(r\"MH370\", \"Malaysia Airlines Flight 370\", tweet)\n",
    "    tweet = re.sub(r\"mÌ¼sica\", \"music\", tweet)\n",
    "    tweet = re.sub(r\"okwx\", \"Oklahoma City Weather\", tweet)\n",
    "    tweet = re.sub(r\"arwx\", \"Arkansas Weather\", tweet)    \n",
    "    tweet = re.sub(r\"gawx\", \"Georgia Weather\", tweet)  \n",
    "    tweet = re.sub(r\"scwx\", \"South Carolina Weather\", tweet)  \n",
    "    tweet = re.sub(r\"cawx\", \"California Weather\", tweet)\n",
    "    tweet = re.sub(r\"tnwx\", \"Tennessee Weather\", tweet)\n",
    "    tweet = re.sub(r\"azwx\", \"Arizona Weather\", tweet)  \n",
    "    tweet = re.sub(r\"alwx\", \"Alabama Weather\", tweet)\n",
    "    tweet = re.sub(r\"wordpressdotcom\", \"wordpress\", tweet)    \n",
    "    tweet = re.sub(r\"usNWSgov\", \"United States National Weather Service\", tweet)\n",
    "    tweet = re.sub(r\"Suruc\", \"Sanliurfa\", tweet)   \n",
    "    \n",
    "    # Grouping same words without embeddings\n",
    "    tweet = re.sub(r\"Bestnaijamade\", \"bestnaijamade\", tweet)\n",
    "    tweet = re.sub(r\"SOUDELOR\", \"Soudelor\", tweet)\n",
    "    \n",
    "    return tweet\n",
    "\n",
    "\n",
    "df_train['text_cleaned'] = df_train['text'].apply(lambda s : clean(s))\n",
    "df_test['text_cleaned'] = df_test['text'].apply(lambda s : clean(s))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit',\n",
       " 'Hellfire! We don\\x89Ûªt even want to think about it or mention it so let\\x89Ûªs not do anything that leads to it #islam!',\n",
       " \"The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.'\",\n",
       " 'In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!',\n",
       " 'To fight bioterrorism sir.',\n",
       " 'Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her\\n \\n#FARRAKHAN #QUOTE',\n",
       " '#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption',\n",
       " '#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect',\n",
       " 'He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam',\n",
       " 'RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG',\n",
       " 'Hellfire is surrounded by desires so be careful and don\\x89Ûªt let your desires control you! #Afterlife',\n",
       " 'CLEARED:incident with injury:I-495  inner loop Exit 31 - MD 97/Georgia Ave Silver Spring',\n",
       " \"Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...\",\n",
       " 'wowo--=== 12000 Nigerian refugees repatriated from Cameroon',\n",
       " '.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4',\n",
       " 'Caution: breathing may be hazardous to your health.',\n",
       " 'I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????',\n",
       " 'that horrible sinking feeling when you\\x89Ûªve been at home on your phone for a while and you realise its been on 3G this whole time']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mislabeled = df_train.groupby(['text']).nunique().sort_values(by='target', ascending=False)\n",
    "df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']\n",
    "df_mislabeled.index.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['target_relabeled'] = df_train['target'].copy() \n",
    "\n",
    "df_train.loc[df_train['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you! #Afterlife', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == '.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == 'CLEARED:incident with injury:I-495  inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == '#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!', 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == 'Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her\\n \\n#FARRAKHAN #QUOTE', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == \"Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"wowo--=== 12000 Nigerian refugees repatriated from Cameroon\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it #islam!\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.'\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"Caution: breathing may be hazardous to your health.\", 'target_relabeled'] = 1\n",
    "df_train.loc[df_train['text'] == \"I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect\", 'target_relabeled'] = 0\n",
    "df_train.loc[df_train['text'] == \"that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time\", 'target_relabeled'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_x, test_x, train_y, test_y = train_test_split( df_train['text_cleaned']\n",
    "                                                     , df_train[\"target_relabeled\"],\n",
    "                                                       random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5709, 14244)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(x_train_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<5709x14244 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 74394 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_idf_word_vectorizer = TfidfVectorizer()\n",
    "tf_idf_word_vectorizer.fit_transform(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_tf_idf_word = tf_idf_word_vectorizer.transform(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_tf_idf_word = tf_idf_word_vectorizer.transform(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test1 = df_test[\"text_cleaned\"]\n",
    "test1_ft = tf_idf_word_vectorizer.transform(test1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 70)                997150    \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 70)                0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 70)                4970      \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 70)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 70)                4970      \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 71        \n",
      "=================================================================\n",
      "Total params: 1,007,161\n",
      "Trainable params: 1,007,161\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import to_categorical\n",
    "from keras import models\n",
    "from keras import layers\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from keras.models import Sequential\n",
    "\n",
    "# Input - Layer\n",
    "model = Sequential()\n",
    "model.add(layers.Dense(70, activation = \"relu\", input_shape=(14244, )))\n",
    "# Hidden - Layers\n",
    "model.add(layers.Dropout(0.3, noise_shape=None, seed=None))\n",
    "model.add(layers.Dense(70, activation = \"relu\"))\n",
    "model.add(layers.Dropout(0.2, noise_shape=None, seed=None))\n",
    "model.add(layers.Dense(70, activation = \"relu\"))\n",
    "\n",
    "model.add(layers.Dense(1, activation = \"sigmoid\"))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    " optimizer = \"adam\",\n",
    " loss = \"binary_crossentropy\",\n",
    " metrics = [\"accuracy\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5709 samples, validate on 1904 samples\n",
      "Epoch 1/3\n",
      "5709/5709 [==============================] - 5s 944us/step - loss: 0.6863 - accuracy: 0.5712 - val_loss: 0.6713 - val_accuracy: 0.5751\n",
      "Epoch 2/3\n",
      "5709/5709 [==============================] - 1s 206us/step - loss: 0.6494 - accuracy: 0.5709 - val_loss: 0.6213 - val_accuracy: 0.5751\n",
      "Epoch 3/3\n",
      "5709/5709 [==============================] - 1s 204us/step - loss: 0.5641 - accuracy: 0.6334 - val_loss: 0.5509 - val_accuracy: 0.7668\n"
     ]
    }
   ],
   "source": [
    "results = model.fit(\n",
    " x_train_tf_idf_word, train_y,\n",
    " epochs= 3,\n",
    " batch_size = 500,\n",
    " validation_data = (x_test_tf_idf_word, test_y)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(test1_ft)\n",
    "y_pred = (y_pred > 0.5)\n",
    "y_pred1 = pd.DataFrame(data = y_pred, index = range(3263), columns=['target'])\n",
    "y_pred1[\"target\"] = y_pred1[\"target\"].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv('sample_submission.csv') \n",
    "submission= submission[[\"id\"]]\n",
    "final_submission=pd.concat([submission,y_pred1],axis=1)\n",
    "final_submission.to_csv('11062020_7.csv',index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target\n",
       "0   0       1\n",
       "1   2       0\n",
       "2   3       1\n",
       "3   9       1\n",
       "4  11       1"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3263, 2)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(final_submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5709, 7123)"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(x_train_tf_idf_word)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}