-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathver006c.py
313 lines (233 loc) · 13 KB
/
ver006c.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
import numpy as np
import pandas as pd
import operator
from wordcloud import STOPWORDS
import string
pd.set_option('display.max_rows', 5000000)
pd.set_option('display.max_columns', 5000000)
pd.set_option('display.width', 10000000)
pd.set_option('display.max_colwidth', 4000)
df_train = pd.read_csv('train.csv')
df_test = pd.read_csv('test.csv')
import re
def clean(tweet):
# Special characters
# Urls
tweet = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", tweet)
# Words with punctuations and special characters
punctuations = '@#!?+&*[]-%.:/();$=><|{}^' + "'`"
for p in punctuations:
tweet = tweet.replace(p, f' {p} ')
# ... and ..
tweet = tweet.replace('...', ' ... ')
if '...' not in tweet:
tweet = tweet.replace('..', ' ... ')
# Acronyms
tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
tweet = re.sub(r"m̼sica", "music", tweet)
tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
tweet = re.sub(r"gawx", "Georgia Weather", tweet)
tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
tweet = re.sub(r"cawx", "California Weather", tweet)
tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
tweet = re.sub(r"azwx", "Arizona Weather", tweet)
tweet = re.sub(r"alwx", "Alabama Weather", tweet)
tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
# Grouping same words without embeddings
tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
return tweet
df_train['text_cleaned'] = df_train['text'].apply(lambda s : clean(s))
df_test['text_cleaned'] = df_test['text'].apply(lambda s : clean(s))
df_mislabeled = df_train.groupby(['text']).nunique().sort_values(by='target', ascending=False)
df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
df_mislabeled.index.tolist()
df_train['target_relabeled'] = df_train['target'].copy()
df_train.loc[df_train['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you! #Afterlife', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == '.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4', 'target_relabeled'] = 1
df_train.loc[df_train['text'] == 'CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1
df_train.loc[df_train['text'] == '#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == 'In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == 'Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her\n \n#FARRAKHAN #QUOTE', 'target_relabeled'] = 1
df_train.loc[df_train['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1
df_train.loc[df_train['text'] == "Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "wowo--=== 12000 Nigerian refugees repatriated from Cameroon", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it #islam!", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.'", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "Caution: breathing may be hazardous to your health.", 'target_relabeled'] = 1
df_train.loc[df_train['text'] == "I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time", 'target_relabeled'] = 0
# word_count
df_train['word_count'] = df_train['text'].apply(lambda x: len(str(x).split()))
df_test['word_count'] = df_test['text'].apply(lambda x: len(str(x).split()))
# unique_word_count
df_train['unique_word_count'] = df_train['text'].apply(lambda x: len(set(str(x).split())))
df_test['unique_word_count'] = df_test['text'].apply(lambda x: len(set(str(x).split())))
# stop_word_count
df_train['stop_word_count'] = df_train['text'].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))
df_test['stop_word_count'] = df_test['text'].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))
# url_count
df_train['url_count'] = df_train['text'].apply(lambda x: len([w for w in str(x).lower().split() if 'http' in w or 'https' in w]))
df_test['url_count'] = df_test['text'].apply(lambda x: len([w for w in str(x).lower().split() if 'http' in w or 'https' in w]))
# mean_word_length
df_train['mean_word_length'] = df_train['text'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
df_test['mean_word_length'] = df_test['text'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
# char_count
df_train['char_count'] = df_train['text'].apply(lambda x: len(str(x)))
df_test['char_count'] = df_test['text'].apply(lambda x: len(str(x)))
# punctuation_count
df_train['punctuation_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))
df_test['punctuation_count'] = df_test['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))
# hashtag_count
df_train['hashtag_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c == '#']))
df_test['hashtag_count'] = df_test['text'].apply(lambda x: len([c for c in str(x) if c == '#']))
# mention_count
df_train['mention_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c == '@']))
df_test['mention_count'] = df_test['text'].apply(lambda x: len([c for c in str(x) if c == '@']))
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
train_x, test_x, train_y, test_y = train_test_split( df_train[["text",'text_cleaned', "word_count", "unique_word_count","stop_word_count","mean_word_length","char_count","punctuation_count"]]
, df_train["target_relabeled"], test_size=0.2,
random_state = 1)
train_x1 = train_x["text_cleaned"]
train_x2 = train_x["word_count"]
train_x3 = train_x["unique_word_count"]
train_x4 = train_x["stop_word_count"]
train_x5 = train_x["mean_word_length"]
train_x6 = train_x["char_count"]
train_x7 = train_x["punctuation_count"]
test_x1 = test_x["text_cleaned"]
test_x2 = test_x["word_count"]
test_x3 = test_x["unique_word_count"]
test_x4 = test_x["stop_word_count"]
test_x5 = test_x["mean_word_length"]
test_x6 = test_x["char_count"]
test_x7 = test_x["punctuation_count"]
tf_idf_word_vectorizer = TfidfVectorizer()
tf_idf_word_vectorizer.fit_transform(train_x1)
x_train_tf_idf_word1 = tf_idf_word_vectorizer.transform(train_x1)
from scipy.sparse import coo_matrix, hstack
x_train_tf_idf_word2 = coo_matrix(train_x2)
x_train_tf_idf_word2 = x_train_tf_idf_word2.reshape(6090,1)
x_train_tf_idf_word3 = coo_matrix(train_x3)
x_train_tf_idf_word3 = x_train_tf_idf_word3.reshape(6090,1)
x_train_tf_idf_word4 = coo_matrix(train_x4)
x_train_tf_idf_word4 = x_train_tf_idf_word4.reshape(6090,1)
x_train_tf_idf_word5 = coo_matrix(train_x5)
x_train_tf_idf_word5 = x_train_tf_idf_word5.reshape(6090,1)
x_train_tf_idf_word6 = coo_matrix(train_x6)
x_train_tf_idf_word6 = x_train_tf_idf_word6.reshape(6090,1)
x_train_tf_idf_word7 = coo_matrix(train_x7)
x_train_tf_idf_word7 = x_train_tf_idf_word7.reshape(6090,1)
x_train_tf_idf_word = hstack([x_train_tf_idf_word1,x_train_tf_idf_word2,x_train_tf_idf_word3, x_train_tf_idf_word4, x_train_tf_idf_word5, x_train_tf_idf_word6, x_train_tf_idf_word7])
x_train_tf_idf_word = x_train_tf_idf_word.toarray()
x_test_tf_idf_word1 = tf_idf_word_vectorizer.transform(test_x1)
x_test_tf_idf_word2 = coo_matrix(test_x2)
x_test_tf_idf_word2 = x_test_tf_idf_word2.reshape(1523,1)
x_test_tf_idf_word3 = coo_matrix(test_x3)
x_test_tf_idf_word3 = x_test_tf_idf_word3.reshape(1523,1)
x_test_tf_idf_word4 = coo_matrix(test_x4)
x_test_tf_idf_word4 = x_test_tf_idf_word4.reshape(1523,1)
x_test_tf_idf_word5 = coo_matrix(test_x5)
x_test_tf_idf_word5 = x_test_tf_idf_word5.reshape(1523,1)
x_test_tf_idf_word6 = coo_matrix(test_x6)
x_test_tf_idf_word6 = x_test_tf_idf_word6.reshape(1523,1)
x_test_tf_idf_word7 = coo_matrix(test_x7)
x_test_tf_idf_word7 = x_test_tf_idf_word7.reshape(1523,1)
x_test_tf_idf_word = hstack([x_test_tf_idf_word1, x_test_tf_idf_word2,x_test_tf_idf_word3,x_test_tf_idf_word4,x_test_tf_idf_word5,x_test_tf_idf_word6,x_test_tf_idf_word7])
x_test_tf_idf_word = x_test_tf_idf_word.toarray()
test1 = df_test["text_cleaned"]
test1 = tf_idf_word_vectorizer.transform(test1)
test2 = df_test["word_count"]
test2 = coo_matrix(test2)
test2 = test2.reshape(3263,1)
test3 = df_test["unique_word_count"]
test3 = coo_matrix(test3)
test3 = test3.reshape(3263,1)
test4 = df_test["stop_word_count"]
test4 = coo_matrix(test4)
test4 = test4.reshape(3263,1)
test5 = df_test["mean_word_length"]
test5 = coo_matrix(test5)
test5 = test5.reshape(3263,1)
test6 = df_test["char_count"]
test6 = coo_matrix(test6)
test6 = test6.reshape(3263,1)
test7 = df_test["punctuation_count"]
test7 = coo_matrix(test7)
test7 = test7.reshape(3263,1)
test = hstack([test1, test2,test3,test4,test5,test6,test7])
test = test.toarray()
from keras.utils import to_categorical
from keras import models
from keras import layers
import numpy as np
from keras.models import Sequential
# Input - Layer
#BU YENİ MODEL
from keras.layers import LeakyReLU
from keras.layers import Dense
from keras.layers import Conv1D, MaxPooling1D
from keras.layers import Flatten
from keras.layers import Embedding
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Dropout, Activation
from keras.layers import LSTM
#model = Sequential()
#model.add(Dense(128, kernel_initializer ='glorot_uniform',input_dim=XX.shape[1]))
#model.add(LeakyReLU(alpha=0.01))
#model.add(Dropout(0.20))
#model.add(Dense(128, kernel_initializer ='glorot_uniform'))
#model.add(LeakyReLU(alpha=0.01))
#model.add(Dropout(0.20))
#model.add(Dense(output_dim = 1, kernel_initializer ='glorot_uniform', activation = 'sigmoid'))
#model.compile(loss='binary_crossentropy',
# optimizer='adamax',
# metrics=['acc',f1_m,precision_m, recall_m])
#
#es = keras.callbacks.EarlyStopping(monitor='val_f1_m', min_delta=0, patience=3, verbose=0, mode='auto', baseline=None, restore_best_weights=False)
#
#model.summary()
#
#model.fit(XX, Y, batch_size = 12, nb_epoch = 100,callbacks=[es],validation_split=0.2)
#BU KISIM ORJİNAL MODEL
model = Sequential()
model.add(layers.Dense(50, activation = "relu", input_shape=(14847, )))
# Hidden - Layers
model.add(layers.Dense(1, activation = "sigmoid"))
model.summary()
model.compile(
optimizer = "adam",
loss = "binary_crossentropy",
metrics = ["accuracy"]
)
results = model.fit(
x_train_tf_idf_word, train_y,
epochs= 25,
batch_size = 600,
validation_data = (x_test_tf_idf_word, test_y)
)
y_pred_test = model.predict(x_test_tf_idf_word)
y_pred_test = model.predict(x_test_tf_idf_word)
y_pred_test = (y_pred_test > 0.5)
y_pred_test = pd.DataFrame(data = y_pred_test, index = range(1523), columns=['target'])
y_pred_test["target"] = y_pred_test["target"].astype(int)
y_pred = model.predict(test)
y_pred = (y_pred > 0.5)
y_pred1 = pd.DataFrame(data = y_pred, index = range(3263), columns=['target'])
y_pred1["target"] = y_pred1["target"].astype(int)
from sklearn.metrics import classification_report,confusion_matrix
cm = confusion_matrix(test_y, y_pred_test)
print(cm)
test_x1 = test_x1.reset_index()
test_y = test_y.reset_index()
veri_inceleme_test = pd.concat([test_x1["text_cleaned"], y_pred_test, test_y], axis=1)
veri_inceleme_test.drop(["index"],axis=1,inplace=True) ###Drop Sütun Etmek