eda_and_text_preprocessing_on_jigsaw.py

# -*- coding: utf-8 -*-
"""EDA and Text  Preprocessing on jigsaw.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1eNFEdzARZiN-ssHOxh7_B5c45wcgnXc9

# Load Dataset and Parameter Initialization
"""

from google.colab import drive
import numpy as np 
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
# import wordcloud
from wordcloud import WordCloud
# import NLTK mainly for stopwords
import nltk
from nltk.tokenize.treebank import TreebankWordTokenizer
from nltk.tokenize import  word_tokenize
nltk.download('punkt')
nltk.download('stopwords')
from nltk.corpus import stopwords
from PIL import Image
!pip install emoji
import emoji
from collections import Counter
from itertools import chain
import pickle
import operator
import string
import tensorflow as tf

"""Mount Google Drive"""

!python --version

!pip freeze

drive.mount('/content/drive')

"""We will use only train dataset for EDA and pretrained word vectors from GloVe Common Crawl (840B tokens, 2.2M vocab, cased, 300d vectors) ."""

TRAIN_DATASET_PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/train.csv'
TEST_PUBLIC_DATASET_PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/test_public_expanded.csv'
TEST_PRIVATE_DATASET_PATH = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/test_private_expanded.csv'
GLOVE_EMBEDDING_PATH = '/content/drive/My Drive/Glove/glove.840B.300d.pkl'
CRAWL_EMBEDDING_PATH = "/content/drive/My Drive/Crawl/crawl-300d-2M.pkl"

"""Read the datasets """

train_df = pd.read_csv(TRAIN_DATASET_PATH)
test_public_df =  pd.read_csv(TEST_PUBLIC_DATASET_PATH)
test_private_df = pd.read_csv(TEST_PRIVATE_DATASET_PATH)

len(test_public_df)

len(test_private_df)

"""Create lists with column names """

TARGET_COLUMN = 'target'
GENDER_IDENTITIES = ['male', 'female', 'transgender', 'other_gender']
SEXUAL_ORIENTATION_IDENTITIES = ['heterosexual', 'homosexual_gay_or_lesbian', 'bisexual',
'other_sexual_orientation']
RELIGION_IDENTINTIES = ['christian', 'jewish', 'muslim', 'hindu', 'buddhist',
'atheist', 'other_religion']
RACE_IDENTINTIES = ['black', 'white', 'latino', 'asian',
'other_race_or_ethnicity']
DISABILITY_IDENTINTIES = ['physical_disability','intellectual_or_learning_disability',
                          'psychiatric_or_mental_illness', 'other_disability']
IDENTITY_COLUMNS = GENDER_IDENTITIES + SEXUAL_ORIENTATION_IDENTITIES + RELIGION_IDENTINTIES + RACE_IDENTINTIES + DISABILITY_IDENTINTIES
IDENTITY_COLUMNS  = [
    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
    'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
  ] 
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
REACTION_COLUMNS = ['funny', 'wow',	'sad',	'likes',	'disagree']

"""# Exploratory Data Analysis

## Understanding the Data scheme

Print the first 5 samples
"""

train_df = train_df[['id']+["comment_text"]+AUX_COLUMNS+IDENTITY_COLUMNS]
train_df.head()

num_of_samples = len(train_df)
print('Train size: {:d}'.format(num_of_samples))

# check for amount of missing(null) values in every column αnd print the precentage of them. 
null_columns=train_df.columns[train_df.isnull().any()]
print(train_df[null_columns].isnull().sum() / num_of_samples * 100)

"""Check the frequency of lengths of training sentences"""

# tokenize
tokenizer = tf.keras.preprocessing.text.Tokenizer(lower=False)
tokenizer.fit_on_texts(list(train_df["comment_text"].astype(str)))
x_train = tokenizer.texts_to_sequences(list(train_df["comment_text"].astype(str)))

# count lengths 
training_sentence_lengths = [len(tokens) for tokens in x_train]
print("Max sentence length is %s" % max(training_sentence_lengths))
targets_n_lens = pd.DataFrame(data=train_df[TARGET_COLUMN],columns=[TARGET_COLUMN])
targets_n_lens['lens'] = training_sentence_lengths

#plot hist 
plt.hist(training_sentence_lengths, density=True, bins=10)  # arguments are passed to np.histogram
plt.title("Histogram of lenghts of comments")
plt.ylabel('Frequency')
plt.xlabel('Lengths of training comments')
plt.show()

plt.hist(targets_n_lens[targets_n_lens[TARGET_COLUMN] >= .5]['lens'].values, density=True, bins=10)  # arguments are passed to np.histogram
plt.title("Histogram of lenghts of toxic comments")
plt.ylabel('Frequency')
plt.xlabel('Lengths of training toxic comments');
plt.show()

plt.hist(targets_n_lens[targets_n_lens[TARGET_COLUMN] < .5]['lens'].values, density=True, bins=10)  # arguments are passed to np.histogram
plt.title("Histogram of lenghts of non-toxic comments")
plt.ylabel('Frequency')
plt.xlabel('Lengths of training non-toxic comments');
plt.show()

"""## Understanding the Toxic Comments with Identity"""

num_of_non_toxic_samples = int(train_df[train_df.target < 0.5][TARGET_COLUMN].count())
num_of_toxic_samples = num_of_samples - num_of_non_toxic_samples

print('Nummer of samples: ',num_of_samples)
print('Nummer of non-toxic samples: {:d} ,percentage: {:.2f}%'.format(num_of_non_toxic_samples , (num_of_non_toxic_samples/num_of_samples)*100)) 
print('Nummer of toxic samples: {:d} ,percentage: {:.2f}%'.format(num_of_toxic_samples , (num_of_toxic_samples/num_of_samples)*100))

"""Let's drop the samples without identity and calculate the previous stats """

train_df_with_identity = train_df.loc[:, AUX_COLUMNS + IDENTITY_COLUMNS ].dropna()
num_of_samples_with_identity =  len(train_df_with_identity)
num_of_toxic_samples_with_identity = int(train_df_with_identity[train_df_with_identity.target >= 0.5][TARGET_COLUMN].count())
num_of_non_toxic_samples_with_identity = num_of_samples_with_identity - num_of_toxic_samples_with_identity

print('Number of samples with identity: {:d}'.format(num_of_samples_with_identity)) 
print('Nummer of non-toxic samples with identity: {:d} ,percentage: {:.2f}%'.format(num_of_non_toxic_samples_with_identity , (num_of_non_toxic_samples_with_identity/num_of_samples_with_identity)*100)) 
print('Nummer of toxic samples with identity: {:d} ,percentage: {:.2f}%'.format(num_of_toxic_samples_with_identity , (num_of_toxic_samples_with_identity/num_of_samples_with_identity)*100))

"""Let's plot the results """

counts_of_samples = pd.DataFrame([['All Samples',num_of_toxic_samples,num_of_non_toxic_samples],
                                  ['Samples with Identity',num_of_toxic_samples_with_identity,num_of_non_toxic_samples_with_identity]],
                                 columns = ['','Toxic','Non-toxic'])
counts_of_samples.set_index('',inplace=True)
plot1 = counts_of_samples.plot(kind='barh', stacked=True, figsize=(10,8), fontsize=10)
plot1.ticklabel_format(style='plain', axis='x')

"""Count the samples in every subgroup"""

num_of_samples_in_subgroups = train_df_with_identity[(train_df_with_identity[IDENTITY_COLUMNS] > 0)][IDENTITY_COLUMNS].count()
num_of_samples_in_subgroups = num_of_samples_in_subgroups.to_frame(name='count')

"""Count the toxic samples in every subgroup"""

# first get all samples that have an identity and are toxic 
toxic_samples_with_identity = train_df_with_identity[train_df_with_identity[TARGET_COLUMN] >= 0.5 ]
num_of__toxic_samples_in_subgroups = toxic_samples_with_identity[toxic_samples_with_identity[IDENTITY_COLUMNS] > 0][IDENTITY_COLUMNS].count()
num_of__toxic_samples_in_subgroups = num_of__toxic_samples_in_subgroups.to_frame(name='toxic_count')

subgroup_stats =  (num_of__toxic_samples_in_subgroups.toxic_count /  num_of_samples_in_subgroups['count'] ) * 100
subgroup_stats = subgroup_stats.to_frame(name='toxic percentage')
subgroup_stats['toxic'] = num_of__toxic_samples_in_subgroups.toxic_count
subgroup_stats['non_toxic'] = num_of_samples_in_subgroups['count'] - num_of__toxic_samples_in_subgroups.toxic_count 
subgroup_stats['total'] = num_of_samples_in_subgroups['count']

subgroup_stats.sort_values(by= 'toxic' , ascending=False , inplace=True)

subgroup_stats[['toxic','non_toxic']].plot(kind='bar', stacked=True, figsize=(20,8), fontsize=15).legend(prop={'size': 15})

# multiply each identity with the target
weighted_toxic_percentage = train_df_with_identity[IDENTITY_COLUMNS].apply(lambda x: np.asarray(x) * np.asarray(train_df_with_identity[TARGET_COLUMN])).sum()
# devide the number of samples in each subgroup 
weighted_toxic_percentage = (weighted_toxic_percentage / num_of_samples_in_subgroups['count']) * 100
weighted_toxic_percentage.sort_values(ascending=False, inplace=True)

plt.figure(figsize=(15,8))
sns.set(font_scale=1)
ax = sns.barplot(x = weighted_toxic_percentage.values , y = weighted_toxic_percentage.index)
plt.ylabel('Subgroups')
plt.xlabel('Weighted Toxicity(%)')
plt.show()

"""## Correlation and Heatmap of Identities"""

rows = [{c:train_df_with_identity[f].corr(train_df_with_identity[c]) for c in [TARGET_COLUMN] + AUX_COLUMNS} for f in IDENTITY_COLUMNS]
poptoxicity_correlations = pd.DataFrame(rows, index=IDENTITY_COLUMNS)
plt.figure(figsize=(12, 8))
sns.set(font_scale=1)
ax = sns.heatmap(poptoxicity_correlations, vmin=-0.1, vmax=0.1, center=0.0)

# Compute the correlation matrix
corr = train_df_with_identity[IDENTITY_COLUMNS].corr()
# Generate a mask for the upper triangle
mask = np.zeros_like(corr, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
# Set up the matplotlib figure
sns.set(font_scale = 1)
f, ax = plt.subplots(figsize=(11, 9))
# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)
# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})

"""The heatmap plot of the correlation between the identities is very insightful. I will summarize my observations below. As always, if you see something interesting please mention it to me in the comment section.


*   It is interesting to see that strong correlations form triangular area at the edge of diagonal.

*   This basically means that there is a strong correlation between the groups of the identity (gender, religion, races, disabilities). This means, the comments where male identity is the target, female identity is also very likely to be the target.

*   In another words, in toxic and non-toxic comments, people tend to make it about one group vs another quiet frequently.

## Word Clouds
"""

# we will write a simple function to generate the wordcloud per identity group
def generate_word_cloud(identity, toxic_comments, non_toxic_comments):
    # convert stop words to sets as required by the wordcloud library
    stop_words = set(stopwords.words("english"))
    # create toxic wordcloud
    toxic_picture = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/images/toxic_sign.png'
    toxic_mask=np.array(Image.open(toxic_picture))
    toxic_mask=toxic_mask[:,:]
    wordcloud_toxic = WordCloud(max_words=1000, background_color="black", mask=toxic_mask ,stopwords=stop_words).generate(toxic_comments)
    # create non-toxic wordcloud
    peace_picture = '/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/images/peace_sign.png'
    peace_mask = np.array(Image.open(peace_picture))
    wordcloud_non_toxic = WordCloud(max_words=1000, background_color="black", mask=peace_mask, stopwords=stop_words).generate(non_toxic_comments)
    # draw the two wordclouds side by side using subplot
    fig = plt.figure(figsize=[20,5])
    fig.add_subplot(1, 2, 1).set_title("Toxic Wordcloud", fontsize=10)
    plt.imshow(wordcloud_toxic, interpolation="bilinear")
    plt.axis("off")
    fig.add_subplot(1, 2, 2).set_title("Non Toxic Wordcloud", fontsize=10)
    plt.imshow(wordcloud_non_toxic, interpolation="bilinear")
    plt.axis("off")
    plt.subplots_adjust(top=0.85)
    plt.suptitle('Word Cloud - {} Identity'.format(identity), size = 16)
    plt.show()

"""### White and Black """

identity = 'white'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

identity = 'black'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""### Homosexual """

identity = 'homosexual_gay_or_lesbian'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""### Muslim and Jewish """

identity = 'muslim'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

identity = 'jewish'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""### Female and Male"""

identity = 'female'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

identity = 'male'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""### Psychiatric or mental illness"""

identity = 'psychiatric_or_mental_illness'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""### Transgender"""

identity = 'transgender'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""### Atheist"""

identity = 'atheist'
# get the comments for the given identity
identity_comments = train_df[train_df[identity] > 0 ][[TARGET_COLUMN,'comment_text']]
# lets convert the comments as one long string (as needed by wordcloud)
toxic_comments = ' '.join(identity_comments[identity_comments.target >= 0.5]['comment_text'])
non_toxic_comments = ' '.join(identity_comments[identity_comments.target < 0.5]['comment_text'])
# draw the wordcloud using the function we created earlier
generate_word_cloud(identity, toxic_comments, non_toxic_comments)

"""The wordcloud above is really interesting. Looking at it, I have made the following observations:

* Although the sentiment within the sentences (probably) varies in toxic and non-toxic comments, looking at it from top word frequencies, the differences are not that big.
* Between comments about White and Black identity, there is a huge overlap!
* Comments towards homosexual have more unique set of words (as imagined) from the other identity groups. However, between toxic and non-toxic comment there isn't a big variation in terms of the high frequenty words.
* For comments about Jewish identity, the word Muslim appears frequently. After reviewing a lot of the samples of such comments I noticed that a large number of comments about Jewish identity is toxic towards Muslim identity.
* Ironically, Trump is a very frequent common topic of discussion in toxic and non-toxic comments. However, frequency of Trump appearing is more in toxic comments. "Trump" or "Trump Supporters" could be a identitity in itself =)

Do you see other interesting patterns in the visualization above? Did I make a mistake? Can I do something better? Write them down in the comment section if possible :)

## Emojis
"""

# we will use this simple function to process a string and return all the emojis as a list
def extract_emojis(str):
    return [c for c in str if c in emoji.UNICODE_EMOJI]

# create a new column to state if  a row / comment has emojis 
train_df['emoji_count'] = train_df['comment_text'].apply(lambda x:1 if len(extract_emojis(x)) > 0 else 0)

emoji_mean_per_identity = []

for identity in IDENTITY_COLUMNS:
    toxic_emoji_mean = train_df[(train_df[identity]> 0) & (train_df[TARGET_COLUMN] >= .5)]['emoji_count'].mean()
    non_toxic_emoji_mean = train_df[(train_df[identity]> 0) & (train_df[TARGET_COLUMN] < .5)]['emoji_count'].mean()
    emoji_mean_per_identity.append([identity, toxic_emoji_mean, non_toxic_emoji_mean])

emoji_mean_per_identity_df = pd.DataFrame(emoji_mean_per_identity, columns = ['identity','toxic','non toxic']).set_index('identity')

# now we can plot our dataframe and see what we have
emoji_mean_per_identity_df.plot.bar(figsize=(15,5))
plt.ylabel('mean emojis per comment')
plt.title('Emojis usage in comments for different identities - Normalized')

"""This clears the picture up much better. First of all, the overall use of emoji is pretty low compared to what I see nowaydays. Furthermore, there are definetly a few comments with a rediculous number of emojis which are responsible for skewing our data in the last plot. Finally, as you can imagine; the use of emoji varies and doesn't really differentiate toxic comments and non-toxic comments.

# Preprocessing

## Word Embeddings
"""

def load_embeddings(path):
    with open(path,'rb') as f:
        embedding_index = pickle.load(f)
    return embedding_index

def build_matrix(word_index, path):
    embedding_index = load_embeddings(path)
    embedding_matrix = np.zeros((len(word_index) + 1, 300))
    unknown_words = []
    
    for word, i in word_index.items():
        try:
            embedding_matrix[i] = embedding_index[word]
        except KeyError:
            unknown_words.append(word)
    return embedding_matrix, unknown_words

def build_vocab(sentences, verbose =  True):
    """
    build_vocab builds a ordered dictionary of words and their frequency in your text corpus.
    :param sentences: list of list of words
    :return: dictionary of words and their count
    """
    vocab = {}
    for sentence in sentences:
        for word in sentence:
            try:
                vocab[word] += 1
            except KeyError:
                vocab[word] = 1
    return vocab

def check_coverage(vocab,embeddings_index):
  """
  goes through a given vocabulary and tries to find word vectors in your embedding matrix
  """
  known_words = {}
  unknown_words = {}
  num_known_words = 0
  num_unknown_words = 0
  for word in vocab.keys():
    try:
      known_words[word] = embeddings_index[word]
      num_known_words += vocab[word]
    except:
      unknown_words[word] = vocab[word]
      num_unknown_words += vocab[word]
      pass
    
  print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))
  print('Found embeddings for  {:.2%} of all text'.format(num_known_words / (num_known_words + num_unknown_words)))
  unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]

  return unknown_words

import time 
tic = time.time()
glove_embeddings = load_embeddings(GLOVE_EMBEDDING_PATH)
print(f'loaded {len(glove_embeddings)} word vectors in {time.time()-tic}s')

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""Seems like ' and other punctuation directly on or in a word is an issue. We could simply delete punctuation to fix that words, but there are better methods. Lets explore the embeddings, in particular symbols a bit. For that we first need to define "what is a symbol" in contrast to a regular letter. I nowadays use the following list for "regular" letters. And symbols are all characters not in that list.

## Delete symbols that we have no embeddings and split contractions
"""

latin_similar = "’'‘ÆÐƎƏƐƔIJŊŒẞÞǷȜæðǝəɛɣijŋœĸſßþƿȝĄƁÇĐƊĘĦĮƘŁØƠŞȘŢȚŦŲƯY̨Ƴąɓçđɗęħįƙłøơşșţțŧųưy̨ƴÁÀÂÄǍĂĀÃÅǺĄÆǼǢƁĆĊĈČÇĎḌĐƊÐÉÈĖÊËĚĔĒĘẸƎƏƐĠĜǦĞĢƔáàâäǎăāãåǻąæǽǣɓćċĉčçďḍđɗðéèėêëěĕēęẹǝəɛġĝǧğģɣĤḤĦIÍÌİÎÏǏĬĪĨĮỊIJĴĶƘĹĻŁĽĿʼNŃN̈ŇÑŅŊÓÒÔÖǑŎŌÕŐỌØǾƠŒĥḥħıíìiîïǐĭīĩįịijĵķƙĸĺļłľŀʼnńn̈ňñņŋóòôöǒŏōõőọøǿơœŔŘŖŚŜŠŞȘṢẞŤŢṬŦÞÚÙÛÜǓŬŪŨŰŮŲỤƯẂẀŴẄǷÝỲŶŸȲỸƳŹŻŽẒŕřŗſśŝšşșṣßťţṭŧþúùûüǔŭūũűůųụưẃẁŵẅƿýỳŷÿȳỹƴźżžẓ"
white_list = string.ascii_letters + string.digits + latin_similar + ' ' + "'"
print(white_list)

"""Print all symbols that we have an embedding vector for."""

glove_chars = ''.join([c for c in glove_embeddings if len(c) == 1])
glove_symbols = ''.join([c for c in glove_chars if not c in white_list])
glove_symbols

"""Print symbols in our comments """

jigsaw_chars = build_vocab(list(train_df["comment_text"]))
jigsaw_symbols = ''.join([c for c in jigsaw_chars if not c in white_list])
jigsaw_symbols

"""Delete all symbols we have no embeddings for"""

symbols_to_delete = ''.join([c for c in jigsaw_symbols if not c in glove_symbols])
symbols_to_isolate = ''.join([c for c in jigsaw_symbols if c in glove_symbols]) # we are keeping them

isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}

def handle_punctuation(x):
    x = x.translate(remove_dict)
    x = x.translate(isolate_dict)
    return x

train_df['comment_text'] = train_df['comment_text'].apply(lambda x:handle_punctuation(x))
test_public_df['comment_text'] = test_public_df['comment_text'].apply(lambda x:handle_punctuation(x))
test_private_df['comment_text'] = test_private_df['comment_text'].apply(lambda x:handle_punctuation(x))

"""Check Coverage"""

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""Now lets split standard contraction that will fix the issue with the ' punctuation"""

tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
  x = tokenizer.tokenize(x)
  x = ' '.join(x)
  return x

train_df['comment_text'] = train_df['comment_text'].apply(lambda x:handle_contractions(x))
test_public_df['comment_text'] = test_public_df['comment_text'].apply(lambda x:handle_contractions(x))
test_private_df['comment_text'] = test_private_df['comment_text'].apply(lambda x:handle_contractions(x))

"""Check Coverage"""

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""## Check if lowercase/uppercase a word without embedding , find embedding  """

def check_case(comment,embeddings_index):
  comment = comment.split()
  
  comment = [word if word in embeddings_index else word.lower() if word.lower() in embeddings_index else word.title() if word.title() in embeddings_index else word
             for word in comment ]

  comment = ' '.join(comment)
  return comment

train_df['comment_text'] = train_df['comment_text'].apply(lambda x:check_case(x,glove_embeddings))
test_public_df['comment_text'] = test_public_df['comment_text'].apply(lambda x:check_case(x,glove_embeddings))
test_private_df['comment_text'] = test_private_df['comment_text'].apply(lambda x:check_case(x,glove_embeddings))

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""## More cleaning of the contractions """

contraction_mapping = {
    "daesh" : "isis", "Qur'an" : "quran",
    "Trump's" : 'trump is',"'cause": 'because',',cause': 'because',';cause': 'because',"ain't": 'am not','ain,t': 'am not',
    'ain;t': 'am not','ain´t': 'am not','ain’t': 'am not',"aren't": 'are not',
    'aren,t': 'are not','aren;t': 'are not','aren´t': 'are not','aren’t': 'are not',"can't": 'cannot',"can't've": 'cannot have','can,t': 'cannot','can,t,ve': 'cannot have',
    'can;t': 'cannot','can;t;ve': 'cannot have',
    'can´t': 'cannot','can´t´ve': 'cannot have','can’t': 'cannot','can’t’ve': 'cannot have',
    "could've": 'could have','could,ve': 'could have','could;ve': 'could have',"couldn't": 'could not',"couldn't've": 'could not have','couldn,t': 'could not','couldn,t,ve': 'could not have','couldn;t': 'could not',
    'couldn;t;ve': 'could not have','couldn´t': 'could not',
    'couldn´t´ve': 'could not have','couldn’t': 'could not','couldn’t’ve': 'could not have','could´ve': 'could have',
    'could’ve': 'could have',"didn't": 'did not','didn,t': 'did not','didn;t': 'did not','didn´t': 'did not',
    'didn’t': 'did not',"doesn't": 'does not','doesn,t': 'does not','doesn;t': 'does not','doesn´t': 'does not',
    'doesn’t': 'does not',"don't": 'do not','don,t': 'do not','don;t': 'do not','don´t': 'do not','don’t': 'do not',
    "hadn't": 'had not',"hadn't've": 'had not have','hadn,t': 'had not','hadn,t,ve': 'had not have','hadn;t': 'had not',
    'hadn;t;ve': 'had not have','hadn´t': 'had not','hadn´t´ve': 'had not have','hadn’t': 'had not','hadn’t’ve': 'had not have',"hasn't": 'has not','hasn,t': 'has not','hasn;t': 'has not','hasn´t': 'has not','hasn’t': 'has not',
    "haven't": 'have not','haven,t': 'have not','haven;t': 'have not','haven´t': 'have not','haven’t': 'have not',"he'd": 'he would',
    "he'd've": 'he would have',"he'll": 'he will',
    "he's": 'he is','he,d': 'he would','he,d,ve': 'he would have','he,ll': 'he will','he,s': 'he is','he;d': 'he would',
    'he;d;ve': 'he would have','he;ll': 'he will','he;s': 'he is','he´d': 'he would','he´d´ve': 'he would have','he´ll': 'he will',
    'he´s': 'he is','he’d': 'he would','he’d’ve': 'he would have','he’ll': 'he will','he’s': 'he is',"how'd": 'how did',"how'll": 'how will',
    "how's": 'how is','how,d': 'how did','how,ll': 'how will','how,s': 'how is','how;d': 'how did','how;ll': 'how will',
    'how;s': 'how is','how´d': 'how did','how´ll': 'how will','how´s': 'how is','how’d': 'how did','how’ll': 'how will',
    'how’s': 'how is',"i'd": 'i would',"i'll": 'i will',"i'm": 'i am',"i've": 'i have','i,d': 'i would','i,ll': 'i will',
    'i,m': 'i am','i,ve': 'i have','i;d': 'i would','i;ll': 'i will','i;m': 'i am','i;ve': 'i have',"isn't": 'is not',
    'isn,t': 'is not','isn;t': 'is not','isn´t': 'is not','isn’t': 'is not',"it'd": 'it would',"it'll": 'it will',"It's":'it is',
    "it's": 'it is','it,d': 'it would','it,ll': 'it will','it,s': 'it is','it;d': 'it would','it;ll': 'it will','it;s': 'it is','it´d': 'it would','it´ll': 'it will','it´s': 'it is',
    'it’d': 'it would','it’ll': 'it will','it’s': 'it is',
    'i´d': 'i would','i´ll': 'i will','i´m': 'i am','i´ve': 'i have','i’d': 'i would','i’ll': 'i will','i’m': 'i am',
    'i’ve': 'i have',"let's": 'let us','let,s': 'let us','let;s': 'let us','let´s': 'let us',
    'let’s': 'let us',"ma'am": 'madam','ma,am': 'madam','ma;am': 'madam',"mayn't": 'may not','mayn,t': 'may not','mayn;t': 'may not',
    'mayn´t': 'may not','mayn’t': 'may not','ma´am': 'madam','ma’am': 'madam',"might've": 'might have','might,ve': 'might have','might;ve': 'might have',"mightn't": 'might not','mightn,t': 'might not','mightn;t': 'might not','mightn´t': 'might not',
    'mightn’t': 'might not','might´ve': 'might have','might’ve': 'might have',"must've": 'must have','must,ve': 'must have','must;ve': 'must have',
    "mustn't": 'must not','mustn,t': 'must not','mustn;t': 'must not','mustn´t': 'must not','mustn’t': 'must not','must´ve': 'must have',
    'must’ve': 'must have',"needn't": 'need not','needn,t': 'need not','needn;t': 'need not','needn´t': 'need not','needn’t': 'need not',"oughtn't": 'ought not','oughtn,t': 'ought not','oughtn;t': 'ought not',
    'oughtn´t': 'ought not','oughtn’t': 'ought not',"sha'n't": 'shall not','sha,n,t': 'shall not','sha;n;t': 'shall not',"shan't": 'shall not',
    'shan,t': 'shall not','shan;t': 'shall not','shan´t': 'shall not','shan’t': 'shall not','sha´n´t': 'shall not','sha’n’t': 'shall not',
    "she'd": 'she would',"she'll": 'she will',"she's": 'she is','she,d': 'she would','she,ll': 'she will',
    'she,s': 'she is','she;d': 'she would','she;ll': 'she will','she;s': 'she is','she´d': 'she would','she´ll': 'she will',
    'she´s': 'she is','she’d': 'she would','she’ll': 'she will','she’s': 'she is',"should've": 'should have','should,ve': 'should have','should;ve': 'should have',
    "shouldn't": 'should not','shouldn,t': 'should not','shouldn;t': 'should not','shouldn´t': 'should not','shouldn’t': 'should not','should´ve': 'should have',
    'should’ve': 'should have',"that'd": 'that would',"that's": 'that is','that,d': 'that would','that,s': 'that is','that;d': 'that would',
    'that;s': 'that is','that´d': 'that would','that´s': 'that is','that’d': 'that would','that’s': 'that is',"there'd": 'there had',
    "there's": 'there is','there,d': 'there had','there,s': 'there is','there;d': 'there had','there;s': 'there is',
    'there´d': 'there had','there´s': 'there is','there’d': 'there had','there’s': 'there is',
    "they'd": 'they would',"they'll": 'they will',"they're": 'they are',"they've": 'they have',
    'they,d': 'they would','they,ll': 'they will','they,re': 'they are','they,ve': 'they have','they;d': 'they would','they;ll': 'they will','they;re': 'they are',
    'they;ve': 'they have','they´d': 'they would','they´ll': 'they will','they´re': 'they are','they´ve': 'they have','they’d': 'they would','they’ll': 'they will',
    'they’re': 'they are','they’ve': 'they have',"wasn't": 'was not','wasn,t': 'was not','wasn;t': 'was not','wasn´t': 'was not',
    'wasn’t': 'was not',"we'd": 'we would',"we'll": 'we will',"we're": 'we are',"we've": 'we have','we,d': 'we would','we,ll': 'we will',
    'we,re': 'we are','we,ve': 'we have','we;d': 'we would','we;ll': 'we will','we;re': 'we are','we;ve': 'we have',
    "weren't": 'were not','weren,t': 'were not','weren;t': 'were not','weren´t': 'were not','weren’t': 'were not','we´d': 'we would','we´ll': 'we will',
    'we´re': 'we are','we´ve': 'we have','we’d': 'we would','we’ll': 'we will','we’re': 'we are','we’ve': 'we have',"what'll": 'what will',"what're": 'what are',"what's": 'what is',
    "what've": 'what have','what,ll': 'what will','what,re': 'what are','what,s': 'what is','what,ve': 'what have','what;ll': 'what will','what;re': 'what are',
    'what;s': 'what is','what;ve': 'what have','what´ll': 'what will',
    'what´re': 'what are','what´s': 'what is','what´ve': 'what have','what’ll': 'what will','what’re': 'what are','what’s': 'what is',
    'what’ve': 'what have',"where'd": 'where did',"where's": 'where is','where,d': 'where did','where,s': 'where is','where;d': 'where did',
    'where;s': 'where is','where´d': 'where did','where´s': 'where is','where’d': 'where did','where’s': 'where is',
    "who'll": 'who will',"who's": 'who is','who,ll': 'who will','who,s': 'who is','who;ll': 'who will','who;s': 'who is',
    'who´ll': 'who will','who´s': 'who is','who’ll': 'who will','who’s': 'who is',"won't": 'will not','won,t': 'will not','won;t': 'will not',
    'won´t': 'will not','won’t': 'will not',"wouldn't": 'would not','wouldn,t': 'would not','wouldn;t': 'would not','wouldn´t': 'would not',
    'wouldn’t': 'would not',"you'd": 'you would',"you'll": 'you will',"you're": 'you are','you,d': 'you would','you,ll': 'you will',
    'you,re': 'you are','you;d': 'you would','you;ll': 'you will',
    'you;re': 'you are','you´d': 'you would','you´ll': 'you will','you´re': 'you are','you’d': 'you would','you’ll': 'you will','you’re': 'you are',
    '´cause': 'because','’cause': 'because',"you've": "you have","could'nt": 'could not',
    "havn't": 'have not',"here’s": "here is",'i""m': 'i am',"i'am": 'i am',"i'l": "i will","i'v": 'i have',"wan't": 'want',"was'nt": "was not","who'd": "who would",
    "who're": "who are","who've": "who have","why'd": "why would","would've": "would have","y'all": "you all","y'know": "you know","you.i": "you i",
    "your'e": "you are","arn't": "are not","agains't": "against","c'mon": "common","doens't": "does not",'don""t': "do not","dosen't": "does not",
    "dosn't": "does not","shoudn't": "should not","that'll": "that will","there'll": "there will","there're": "there are",
    "this'll": "this all","u're": "you are", "ya'll": "you all","you'r": "you are","you’ve": "you have","d'int": "did not","did'nt": "did not","din't": "did not","dont't": "do not","gov't": "government",
    "i'ma": "i am","is'nt": "is not","‘I":'I',
    'ᴀɴᴅ':'and','ᴛʜᴇ':'the','ʜᴏᴍᴇ':'home','ᴜᴘ':'up','ʙʏ':'by','ᴀᴛ':'at','…and':'and','civilbeat':'civil beat',\
    'TrumpCare':'Trump care','Trumpcare':'Trump care', 'OBAMAcare':'Obama care','ᴄʜᴇᴄᴋ':'check','ғᴏʀ':'for','ᴛʜɪs':'this','ᴄᴏᴍᴘᴜᴛᴇʀ':'computer',\
    'ᴍᴏɴᴛʜ':'month','ᴡᴏʀᴋɪɴɢ':'working','ᴊᴏʙ':'job','ғʀᴏᴍ':'from','Sᴛᴀʀᴛ':'start','gubmit':'submit','CO₂':'carbon dioxide','ғɪʀsᴛ':'first',\
    'ᴇɴᴅ':'end','ᴄᴀɴ':'can','ʜᴀᴠᴇ':'have','ᴛᴏ':'to','ʟɪɴᴋ':'link','ᴏғ':'of','ʜᴏᴜʀʟʏ':'hourly','ᴡᴇᴇᴋ':'week','ᴇɴᴅ':'end','ᴇxᴛʀᴀ':'extra',\
    'Gʀᴇᴀᴛ':'great','sᴛᴜᴅᴇɴᴛs':'student','sᴛᴀʏ':'stay','ᴍᴏᴍs':'mother','ᴏʀ':'or','ᴀɴʏᴏɴᴇ':'anyone','ɴᴇᴇᴅɪɴɢ':'needing','ᴀɴ':'an','ɪɴᴄᴏᴍᴇ':'income',\
    'ʀᴇʟɪᴀʙʟᴇ':'reliable','ғɪʀsᴛ':'first','ʏᴏᴜʀ':'your','sɪɢɴɪɴɢ':'signing','ʙᴏᴛᴛᴏᴍ':'bottom','ғᴏʟʟᴏᴡɪɴɢ':'following','Mᴀᴋᴇ':'make',\
    'ᴄᴏɴɴᴇᴄᴛɪᴏɴ':'connection','ɪɴᴛᴇʀɴᴇᴛ':'internet','financialpost':'financial post', 'ʜaᴠᴇ':' have ', 'ᴄaɴ':' can ', 'Maᴋᴇ':' make ', 'ʀᴇʟɪaʙʟᴇ':' reliable ', 'ɴᴇᴇᴅ':' need ',
    'ᴏɴʟʏ':' only ', 'ᴇxᴛʀa':' extra ', 'aɴ':' an ', 'aɴʏᴏɴᴇ':' anyone ', 'sᴛaʏ':' stay ', 'Sᴛaʀᴛ':' start', 'SHOPO':'shop',
    }

def clean_contr(x, dic):
    x = x.split()
    x = [dic[word] if word in dic else dic[word.lower()] if word.lower() in dic else word for word in x ]
    x = ' '.join(x)
    return x

train_df['comment_text'] = train_df['comment_text'].apply(lambda x:clean_contr(x,contraction_mapping))
test_public_df['comment_text'] = test_public_df['comment_text'].apply(lambda x:clean_contr(x,contraction_mapping))
test_private_df['comment_text'] = test_private_df['comment_text'].apply(lambda x:clean_contr(x,contraction_mapping))

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""## Correct mispelled words

Correct spelling for a range of known mispelled words taking by https://www.kaggle.com/nz0722/simple-eda-text-preprocessing-jigsaw
"""

mispell_dict = {'SB91':'senate bill','tRump':'trump','utmterm':'utm term','FakeNews':'fake news','Gʀᴇat':'great','ʙᴏᴛtoᴍ':'bottom','washingtontimes':'washington times','garycrum':'gary crum','htmlutmterm':'html utm term','RangerMC':'car','TFWs':'tuition fee waiver','SJWs':'social justice warrior','Koncerned':'concerned','Vinis':'vinys','Yᴏᴜ':'you','Trumpsters':'trump','Trumpian':'trump','bigly':'big league','Trumpism':'trump','Yoyou':'you','Auwe':'wonder','Drumpf':'trump','utmterm':'utm term','Brexit':'british exit','utilitas':'utilities','ᴀ':'a', '😉':'wink','😂':'joy','😀':'stuck out tongue', 'theguardian':'the guardian','deplorables':'deplorable', 'theglobeandmail':'the globe and mail', 'justiciaries': 'justiciary','creditdation': 'Accreditation','doctrne':'doctrine','fentayal': 'fentanyl','designation-': 'designation','CONartist' : 'con-artist','Mutilitated' : 'Mutilated','Obumblers': 'bumblers','negotiatiations': 'negotiations','dood-': 'dood','irakis' : 'iraki','cooerate': 'cooperate','COx':'cox','racistcomments':'racist comments','envirnmetalists': 'environmentalists',}

def correct_spelling(x, dic):
    x = x.split()
    x = [dic[word] if word in dic else dic[word.lower()] if word.lower() in dic else word for word in x ]
    x = ' '.join(x)
    return x

train_df['comment_text'] = train_df['comment_text'].apply(lambda x:correct_spelling(x,mispell_dict))
test_public_df['comment_text'] = test_public_df['comment_text'].apply(lambda x:correct_spelling(x,mispell_dict))
test_private_df['comment_text'] = test_private_df['comment_text'].apply(lambda x:correct_spelling(x,mispell_dict))

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""## Need to fix the punctuation ' in the start """

def del_punctuation_from_start(x,punc):
  x = [word[1:] if word.startswith(punc) else word for word in x.split()]
  x = ' '.join(x)
  return x

train_df['comment_text'] = train_df['comment_text'].apply(lambda x:del_punctuation_from_start(x,"'"))
test_public_df['comment_text'] = test_public_df['comment_text'].apply(lambda x:del_punctuation_from_start(x,"'"))
test_private_df['comment_text'] = test_private_df['comment_text'].apply(lambda x:del_punctuation_from_start(x,"'"))

vocab = build_vocab(list(train_df['comment_text'].apply(lambda x:x.split())))
unknown_words = check_coverage(vocab, glove_embeddings)
unknown_words[:10]

"""# Save cleared data sets"""

train_df.to_csv(r'/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/train_cleared.csv', index = False)
test_public_df.to_csv(r'/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/test_public_cleared.csv', index = False)
test_private_df.to_csv(r'/content/drive/My Drive/Jigsaw Unintended Bias in Toxicity Classification/data/test_private_cleared.csv', index = False)