# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="xBF9RPBhw2we" colab_type="text"
# ### Dataset Reading

# + id="pN4tMIn1w2wg" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 546} executionInfo={"status": "ok", "timestamp": 1600416147678, "user_tz": -330, "elapsed": 1764, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="68547974-f81e-4d73-ee32-3b99f34ed037"
import pandas as pd 
data = pd.read_excel('drive/My Drive/HASOC Competition Data/hasoc_2020_de_train_new.xlsx')
pd.set_option('display.max_colwidth',150)
data.head(10)

# + id="O9ABoWjOw2wl" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416149723, "user_tz": -330, "elapsed": 3792, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="74642ab3-6a6a-4818-9591-f0c09e1ad3ef"
data.shape

# + id="JSKI3CX6w2wp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} executionInfo={"status": "ok", "timestamp": 1600416149724, "user_tz": -330, "elapsed": 3778, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="ccf532b6-2204-4b65-9948-8ac0bb00de6d"
print(data.dtypes)

# + [markdown] id="XNsif5VGw2ws" colab_type="text"
# ### Making of "label" Variable

# + id="gwE60IAxw2ws" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} executionInfo={"status": "ok", "timestamp": 1600416149725, "user_tz": -330, "elapsed": 3764, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="b54efb15-b56f-4d5f-8d1f-e05d07d39f8e"
label = data['task1']
label.head()

# + [markdown] id="ShrD5Y7ew2wv" colab_type="text"
# ### Checking Dataset Balancing

# + id="kFui_Mz3w2wv" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 347} executionInfo={"status": "ok", "timestamp": 1600416149726, "user_tz": -330, "elapsed": 3748, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="cfb2b170-4b33-4155-c4d2-e04074a78181"
print(label.value_counts())
import matplotlib.pyplot as plt
label.value_counts().plot(kind='bar', color='blue')

# + [markdown] id="MRSdKLNiw2wx" colab_type="text"
# ### Convering label into "0" or "1"

# + id="0ESnvF3Vw2wy" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416149728, "user_tz": -330, "elapsed": 3733, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="40b4eab9-a652-42cc-d06c-1a30e08b62e5"
import numpy as np
classes_list = ["HOF","NOT"]
label_index = data['task1'].apply(classes_list.index)
final_label = np.asarray(label_index)
print(final_label[:10])

# + id="NSuVpENKGBWU" colab_type="code" colab={}
from keras.utils.np_utils import to_categorical
label_twoDimension = to_categorical(final_label, num_classes=2)

# + id="GtlQzqdpGMBM" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 191} executionInfo={"status": "ok", "timestamp": 1600416149730, "user_tz": -330, "elapsed": 3713, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="6234a9f9-f693-4ed1-8ab6-31607766eee6"
print(label_twoDimension[:10])

# + [markdown] id="s2JSVKo3w2w0" colab_type="text"
# ### Making of "text" Variable

# + id="-VK4ScnGw2w1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416149731, "user_tz": -330, "elapsed": 3697, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="a2239842-521d-437e-8ee3-6e04b93957ba"
text = data['text']
text.head(10)

# + [markdown] id="tbKm17HIw2w3" colab_type="text"
# ### Dataset Pre-processing

# + id="BkcaKofp22ry" colab_type="code" colab={}
import re 

def text_clean(text):
    ''' Pre process and convert texts to a list of words '''
    
    text=text.lower()

    # Clean the text
    text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
    text = re.sub('https?://\S+|www\.\S+', "", text)
    text = re.sub(r"rt", " ", text)

    return text


# + id="F7u-edjI3DMD" colab_type="code" colab={}
clean_text = text.apply(lambda x:text_clean(x))

# + id="MXqRTrZA3DiA" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416149736, "user_tz": -330, "elapsed": 3645, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="0110ead6-2acd-47f4-fb22-bd7b2a49384d"
clean_text.head(10)

# + [markdown] id="AGYA06eJw2xJ" colab_type="text"
# ### Removing stopwords

# + id="JBLDOKifw2xK" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 52} executionInfo={"status": "ok", "timestamp": 1600416149737, "user_tz": -330, "elapsed": 3628, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="fd680d09-1cba-49a8-acd4-db6ed43db770"
import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
def stop_words_removal(text1):
    text1=[w for w in  text1.split(" ")  if w not in stopwords.words('german')]
    return " ".join(text1)



# + id="dwSLSw3Nw2xN" colab_type="code" colab={}
clean_text_ns=clean_text.apply(lambda x: stop_words_removal(x))

# + id="OFjJCsd_w2xQ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416156179, "user_tz": -330, "elapsed": 10046, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="a707cec2-df51-4846-d65e-47a77d606fc4"
print(clean_text_ns.head(10))

# + [markdown] id="Vxq3KDt4w2xS" colab_type="text"
# ### Lemmatization

# + [markdown] id="96IyUsaow2xa" colab_type="text"
# ### Stemming

# + id="2TuWAy4bw2xb" colab_type="code" colab={}
# Stemming
from nltk.stem import SnowballStemmer 
stemmer = SnowballStemmer("german")

def word_stemmer(text):
    stem_text = "".join([stemmer.stem(i) for i in text])
    return stem_text


# + id="ivl__lJWw2xe" colab_type="code" colab={}
clean_text_stem = clean_text_ns.apply(lambda x : word_stemmer(x))

# + id="qoMbymPmw2xf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 141} executionInfo={"status": "ok", "timestamp": 1600416157513, "user_tz": -330, "elapsed": 11272, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="e4a7b80f-1f2a-42ac-a413-5b9c720c5370"
print(clean_text_stem.head())

# + [markdown] id="LyORidvKw2xl" colab_type="text"
# ### Tokenization using "keras"

# + id="feW2fI8Dw2xl" colab_type="code" colab={}
import keras
import tensorflow
from keras.preprocessing.text import Tokenizer
tok_all = Tokenizer(filters='!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~', lower=True, char_level = False)
tok_all.fit_on_texts(clean_text_stem)

# + [markdown] id="pVf7lAKJw2xo" colab_type="text"
# ### Making Vocab for words

# + id="LtBxjGZKw2xo" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416157517, "user_tz": -330, "elapsed": 11240, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="beed2e64-0df9-47a1-96dc-8f32d4002aa5"
vocabulary_all = len(tok_all.word_counts)
print(vocabulary_all)

# + id="PKAhcecYw2xr" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 55} executionInfo={"status": "ok", "timestamp": 1600416157517, "user_tz": -330, "elapsed": 11226, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="630bdcfb-a3ac-4e9d-9565-5934480ed371"
l = tok_all.word_index
print(l)

# + [markdown] id="wLKyeIYHw2xu" colab_type="text"
# ### encoding or sequencing

# + id="5tTNFeyrw2xu" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 52} executionInfo={"status": "ok", "timestamp": 1600416157518, "user_tz": -330, "elapsed": 11213, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="5b50fd86-9ccc-41d6-8449-8dd52f0139e3"
encoded_clean_text_stem = tok_all.texts_to_sequences(clean_text_stem)
print(clean_text_stem[0])
print(encoded_clean_text_stem[0])

# + [markdown] id="ao425zSrw2xw" colab_type="text"
# ### Pre-padding

# + id="mJB28ImAw2xw" colab_type="code" colab={}
from keras.preprocessing import sequence
max_length = 100
padded_clean_text_stem = sequence.pad_sequences(encoded_clean_text_stem, maxlen=max_length, padding='pre')

# + [markdown] id="2iiYSp1DS6u0" colab_type="text"
# # Reading Test data and Pre-processing

# + id="Iv9nGusFTDXd" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 581} executionInfo={"status": "ok", "timestamp": 1600416157519, "user_tz": -330, "elapsed": 11196, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="e11e7665-7426-47e2-a419-71d6fccaa19f"
data_t = pd.read_csv('drive/My Drive/HASOC Competition Data/german_test_1509.csv')
pd.set_option('display.max_colwidth',150)
data_t.head(10)

# + id="Pi7LVxNbTDf6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416157519, "user_tz": -330, "elapsed": 11183, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="7ae040b5-97ff-4c57-ad2a-1f967ea07149"
data_t.shape

# + id="tTXDrYq7TDn2" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} executionInfo={"status": "ok", "timestamp": 1600416157520, "user_tz": -330, "elapsed": 11170, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="47442ab5-91ee-4713-df2c-f230899f2a7d"
print(data_t.dtypes)

# + id="-3tAM1pRTDwH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} executionInfo={"status": "ok", "timestamp": 1600416158162, "user_tz": -330, "elapsed": 11794, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="46f59bba-2d8c-412c-c5d6-57985e714025"
label_t = data_t['task1']
label_t.head()

# + id="5iwTbXm1TD7z" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 347} executionInfo={"status": "ok", "timestamp": 1600416158163, "user_tz": -330, "elapsed": 11777, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="1114a56f-3cfd-41b8-8f52-1c7df4ae047e"
print(label_t.value_counts())
import matplotlib.pyplot as plt
label_t.value_counts().plot(kind='bar', color='red')

# + id="dqlgFxcOTD_7" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416158165, "user_tz": -330, "elapsed": 11761, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="c0096dda-1ac8-4489-94ab-108c77bcedc3"
import numpy as np
classes_list_t = ["HOF","NOT"]
label_t_index = data_t['task1'].apply(classes_list_t.index)
final_label_t = np.asarray(label_t_index)
print(final_label_t[:10])

# + id="akgKVRsWTD5P" colab_type="code" colab={}
from keras.utils.np_utils import to_categorical
label_twoDimension_t = to_categorical(final_label_t, num_classes=2)

# + id="CE4LCQg6TD3b" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 191} executionInfo={"status": "ok", "timestamp": 1600416158167, "user_tz": -330, "elapsed": 11742, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="041312a3-997b-4d90-9ea9-5c73c53a75a7"
print(label_twoDimension_t[:10])

# + id="8skXvlKrTDzo" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416158168, "user_tz": -330, "elapsed": 11726, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="11650a92-0235-4606-e996-2866b403398f"
text_t = data_t['text']
text_t.head(10)

# + id="C4hTPizJTDt3" colab_type="code" colab={}
import re 

def text_clean(text):
    ''' Pre process and convert texts to a list of words '''
    
    text=text.lower()

    # Clean the text
    text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
    text = re.sub('https?://\S+|www\.\S+', "", text)
    text = re.sub(r"rt", " ", text)

    return text


# + id="jtuvaaFATDr8" colab_type="code" colab={}
clean_text_t = text_t.apply(lambda x:text_clean(x))

# + id="rvpiq8GgTDlR" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416158169, "user_tz": -330, "elapsed": 11695, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="5beaa6c4-13f6-40f5-950d-294df380411a"
clean_text_t.head(10)

# + id="3pb-M3Z1TDjY" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 52} executionInfo={"status": "ok", "timestamp": 1600416158170, "user_tz": -330, "elapsed": 11681, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="5a84a869-2259-4f0b-bc21-a5ac175af330"
import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
def stop_words_removal(text1):
    text1=[w for w in  text1.split(" ")  if w not in stopwords.words('german')]
    return " ".join(text1)



# + id="fTC-kVrzTDde" colab_type="code" colab={}
clean_text_t_ns=clean_text_t.apply(lambda x: stop_words_removal(x))

# + id="cZ_LUoSxTDa2" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416158877, "user_tz": -330, "elapsed": 12367, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="92d78aa2-c3d9-4cfd-8eab-dd4f254670aa"
print(clean_text_t_ns.head(10))

# + id="0QoR4ZA_TDVo" colab_type="code" colab={}
# Stemming
from nltk.stem import SnowballStemmer 
stemmer = SnowballStemmer("german")

def word_stemmer(text):
    stem_text = "".join([stemmer.stem(i) for i in text])
    return stem_text


# + id="8KN0S_XyTDRz" colab_type="code" colab={}
clean_text_t_stem = clean_text_t_ns.apply(lambda x : word_stemmer(x))

# + id="0qc9VWx5TDO4" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 228} executionInfo={"status": "ok", "timestamp": 1600416158879, "user_tz": -330, "elapsed": 12338, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="85288a2a-da1b-44fe-e785-f7a7ede68292"
print(clean_text_t_stem.head(10))

# + id="ktwBAVboTDMS" colab_type="code" colab={}
import keras
import tensorflow
from keras.preprocessing.text import Tokenizer
tok_test = Tokenizer(filters='!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~', lower=True, char_level = False)
tok_test.fit_on_texts(clean_text_t_stem)

# + id="Tu74MfqmUIei" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416159642, "user_tz": -330, "elapsed": 13080, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="89de5461-4019-41b9-e60e-19e1a0c7d061"
vocabulary_all_test = len(tok_test.word_counts)
print(vocabulary_all_test)

# + id="-2OTf7ARUIbJ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 55} executionInfo={"status": "ok", "timestamp": 1600416159642, "user_tz": -330, "elapsed": 13063, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="0f3ff920-1729-4c39-ae46-a8f569639869"
test_list = tok_test.word_index
print(test_list)

# + id="P3zkZa1-UIY0" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 52} executionInfo={"status": "ok", "timestamp": 1600416159643, "user_tz": -330, "elapsed": 13048, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="3e469e46-9e86-44b1-83a1-ce3537b86afd"
encoded_clean_text_t_stem = tok_all.texts_to_sequences(clean_text_t_stem)
print(clean_text_t_stem[0])
print(encoded_clean_text_t_stem[0])

# + id="vueZ6NY1UOfj" colab_type="code" colab={}
from keras.preprocessing import sequence
max_length = 100
padded_clean_text_t_stem = sequence.pad_sequences(encoded_clean_text_t_stem, maxlen=max_length, padding='pre')

# + [markdown] id="NOt-cuGYY61S" colab_type="text"
# # FastText Embeddings

# + id="UGCoESHFZALJ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600416326657, "user_tz": -330, "elapsed": 179955, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="1ec35476-bca6-4eaa-c67f-9ad906da0082"
# FastText Embedding link - https://fasttext.cc/docs/en/crawl-vectors.html

import os
embeddings_index = {}
f = open('drive/My Drive/HASOC Competition Data/Copy of cc.de.300.vec',encoding='utf-8',errors='ignore')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))

# + id="zTpcIkp4ZAT_" colab_type="code" colab={}
embedding_matrix = np.zeros((vocabulary_all+1, 300))
for word, i in tok_all.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
         embedding_matrix[i] = embedding_vector

# + [markdown] id="oM5OmlqZgrLy" colab_type="text"
# # **CNN 1 Model**

# + id="r2VGeKXv0vOz" colab_type="code" colab={}
from keras.preprocessing import sequence
from keras.preprocessing import text
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding, LSTM
from keras.layers import Conv1D, Flatten
from keras.preprocessing import text
from keras.models import Sequential,Model
from keras.layers import Dense ,Activation,MaxPool1D,Conv1D,Flatten,Dropout,Activation,Dropout,Input,Lambda,concatenate
from keras.utils import np_utils
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from nltk.stem.porter import PorterStemmer
import nltk
import csv
import pandas as pd

from keras.preprocessing import text as keras_text, sequence as keras_seq

# + id="kisP_dcfzxnO" colab_type="code" colab={}
Embedding_Layer = Embedding(vocabulary_all+1, 300, weights=[embedding_matrix], input_length=max_length)

# + id="qr8uLf-q0lPJ" colab_type="code" colab={}
CNN1_network=Sequential([Embedding_Layer,
                        Conv1D(128,3,activation="relu",padding='same'),
                       Dropout(0.2),
                       MaxPool1D(2),
                       Flatten(),
                       Dense(64,activation="relu"),
                       Dropout(0.3),
                       Dense(2,activation="sigmoid")
                       ])

# + id="iqV6VLZ83HH6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 433} executionInfo={"status": "ok", "timestamp": 1600416906283, "user_tz": -330, "elapsed": 3133, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="e4966939-b9af-4889-cf40-d471d8d46150"
CNN1_network.summary()

# + id="80QTgAc6BMJ1" colab_type="code" colab={}
from keras.optimizers import Adam
CNN1_network.compile(loss = "binary_crossentropy", optimizer=Adam(lr=0.00003), metrics=["accuracy"])


# + id="h9c9ECvp7P9f" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 976} executionInfo={"status": "ok", "timestamp": 1600416906286, "user_tz": -330, "elapsed": 3113, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="b2be7e0a-aca8-4f16-ba97-a270c288214a"
from keras.utils.vis_utils import plot_model
plot_model(CNN1_network, to_file='CNN2_model.png', show_shapes=True, show_layer_names=True)

# + id="LR0JsV_kAcRY" colab_type="code" colab={}
from keras.callbacks import EarlyStopping, ReduceLROnPlateau,ModelCheckpoint
earlystopper = EarlyStopping(patience=8, verbose=1)

reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.9,
                              patience=2, min_lr=0.00001, verbose=1)

# + [markdown] id="fMnqgj6rhDVR" colab_type="text"
# ### **Model Fitting or Training**

# + id="5nbnfnRZAv1M" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} executionInfo={"status": "ok", "timestamp": 1600417419606, "user_tz": -330, "elapsed": 516364, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="312da534-2b14-4891-9a6c-54808a2feaea"
hist = CNN1_network.fit(padded_clean_text_stem,label_twoDimension,epochs=200,batch_size=32,callbacks=[earlystopper, reduce_lr])

# + [markdown] id="T5W_uxCThTLl" colab_type="text"
# # log loss

# + id="X9DBoQg8Cf1G" colab_type="code" colab={}
CNN1_network_predictions = CNN1_network.predict(padded_clean_text_t_stem)

# + id="bJQznoSlJ5bT" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} executionInfo={"status": "ok", "timestamp": 1600417419615, "user_tz": -330, "elapsed": 516350, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="090f60f4-f8ac-4c1b-8e25-9987993fded2"
from sklearn.metrics import log_loss
log_loss_test= log_loss(label_twoDimension_t,CNN1_network_predictions)
log_loss_test

# + [markdown] id="MryQdO5YhdEz" colab_type="text"
# # Classification Report

# + id="3UhoxZljKBVs" colab_type="code" colab={}
predictions = np.zeros_like(CNN1_network_predictions)
predictions[np.arange(len(CNN1_network_predictions)), CNN1_network_predictions.argmax(1)] = 1

# + id="pNAHulQqKP80" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 433} executionInfo={"status": "ok", "timestamp": 1600417419619, "user_tz": -330, "elapsed": 516334, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="9bcf8dfc-5051-41da-8263-1f54c7d92ae6"
predictionInteger=(np.argmax(predictions, axis=1))
predictionInteger

# + id="aU_GMP6-HxrH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 260} executionInfo={"status": "ok", "timestamp": 1600417419620, "user_tz": -330, "elapsed": 516318, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="1167216a-e125-4a81-e607-1c811e78ac99"
pred_label = np.array(predictionInteger)
df = pd.DataFrame(data=pred_label , columns=["task1"])
print(df)

# + id="YRERiboiHx24" colab_type="code" colab={}
df.to_csv("submission_DE_A.csv", index=False)

# + id="HE-j9PERKXBE" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 191} executionInfo={"status": "ok", "timestamp": 1600417419623, "user_tz": -330, "elapsed": 516296, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="371aaecc-9775-4334-c45e-a6bf6eb4618e"
from sklearn.metrics import classification_report
print(classification_report(label_twoDimension_t,predictions))

# + [markdown] id="WqNX-4ljhwsu" colab_type="text"
# # Epoch v/s Loss Plot

# + id="Dk322X4pKjEQ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 295} executionInfo={"status": "ok", "timestamp": 1600417420623, "user_tz": -330, "elapsed": 517281, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="2de3c1b6-d6c6-4f22-c1d4-a12faa86c8a3"
from matplotlib import pyplot as plt
plt.plot(hist.history["loss"],color = 'red', label = 'train_loss')
#plt.plot(hist.history["val_loss"],color = 'blue', label = 'val_loss')
plt.title('Loss Visualisation')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.savefig('CNN1_HASOC_Ger_lossPlot.pdf',dpi=1000)

from google.colab import files
files.download('CNN1_HASOC_Ger_lossPlot.pdf') 

# + [markdown] id="A5eYuEVbh0Qi" colab_type="text"
# # Epoch v/s Accuracy Plot

# + id="BSDEzNM1LKmp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 295} executionInfo={"status": "ok", "timestamp": 1600417420625, "user_tz": -330, "elapsed": 517270, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="f6c87009-31cb-4023-dc4f-0c4b31cc3de8"
plt.plot(hist.history["accuracy"],color = 'red', label = 'train_accuracy')
#plt.plot(hist.history["val_accuracy"],color = 'blue', label = 'val_accuracy')
plt.title('Accuracy Visualisation')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig('CNN1_HASOC_Ger_accuracyPlot.pdf',dpi=1000)
files.download('CNN1_HASOC_Ger_accuracyPlot.pdf') 

# + [markdown] id="5v-PNBwUh6fK" colab_type="text"
# # Area under Curve-ROC

# + id="rIga22ZbL5Lg" colab_type="code" colab={}
pred_train = CNN1_network.predict(padded_clean_text_stem)
pred_test = CNN1_network.predict(padded_clean_text_t_stem)

# + id="rWKVJtN1Mz_d" colab_type="code" colab={}
import numpy as np
import matplotlib.pyplot as plt
from itertools import cycle

from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
from scipy import interp

def plot_AUC_ROC(y_true, y_pred):
    n_classes = 2 #change this value according to class value

    # Compute ROC curve and ROC area for each class
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_pred[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])

    # Compute micro-average ROC curve and ROC area
    fpr["micro"], tpr["micro"], _ = roc_curve(y_true.ravel(), y_pred.ravel())
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
############################################################################################
    lw = 2
    # Compute macro-average ROC curve and ROC area

    # First aggregate all false positive rates
    all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

    # Then interpolate all ROC curves at this points
    mean_tpr = np.zeros_like(all_fpr)
    for i in range(n_classes):
        mean_tpr += interp(all_fpr, fpr[i], tpr[i])

    # Finally average it and compute AUC
    mean_tpr /= n_classes

    fpr["macro"] = all_fpr
    tpr["macro"] = mean_tpr
    roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

    # Plot all ROC curves
    plt.figure()
    
    plt.plot(fpr["micro"], tpr["micro"],
            label='micro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["micro"]),
            color='deeppink', linestyle=':', linewidth=4)

    plt.plot(fpr["macro"], tpr["macro"],
            label='macro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["macro"]),
            color='navy', linestyle=':', linewidth=4)
    
    colors = cycle(['aqua', 'darkorange'])
    #classes_list1 = ["DE","NE","DK"]
    classes_list1 = ["Non-duplicate","Duplicate"]
    for i, color,c in zip(range(n_classes), colors,classes_list1):
        plt.plot(fpr[i], tpr[i], color=color, lw=lw,
                label='{0} (AUC = {1:0.2f})'
                ''.format(c, roc_auc[i]))

    plt.plot([0, 1], [0, 1], 'k--', lw=lw)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic curve')
    plt.legend(loc="lower right")
    #plt.show()
    plt.savefig('CNN1_HASOC_Ger_Area_RocPlot.pdf',dpi=1000)
    files.download('CNN1_HASOC_Ger_Area_RocPlot.pdf')

  

# + id="i3tsqxDENNB6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 333} executionInfo={"status": "ok", "timestamp": 1600417421779, "user_tz": -330, "elapsed": 518395, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggzx2UtLRgfAA9F990pxIS41WDo9WctnzL8mXg3LQ=s64", "userId": "17758832831689054457"}} outputId="8495aeee-95fb-4112-c484-8c6a063d71f7"
plot_AUC_ROC(label_twoDimension_t,pred_test)

# + id="6boPbARON83n" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered