# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="Vkr-bw_b1JQX"
# # Hate speech classification by k-fold cross validation on movies dataset
#
#
#
# The class labels depict the following:
#
# 0: Normal speech, 
# 1: Offensive speech
# 2: Hate speech
# -

# #### To work with this, the following folder paths needs to be created in the directory of this notebook:
# classification_reports/   : This will contain all the classification reports generated by the model
#
# movies/       : contains all_movies.csv file
#
# movies/for_training/:    contains 6 movies used for cross validation training and testing
#

# + colab={"base_uri": "https://localhost:8080/"} id="3cqyGp3sQg0h" outputId="aee25fd1-fdcf-4611-a59b-29d2c111f329"
# ! pip install transformers==2.6.0

# + id="3YwUM7V6OewJ"
import pandas as pd
from matplotlib import pyplot as plt
import numpy as np
import re
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import os
import glob

# + colab={"base_uri": "https://localhost:8080/", "height": 213, "referenced_widgets": ["b61e58ec33e34a57bb354bbe4c4d1e13", "699ecfecf9c64a59b4fb1f30215f843a", "de3bab8399f54373af8c5730eca6c2ec", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "6cc29a0852e74d5ba923bf30b79983ea", "<KEY>", "5a4a1a1fea8347369f180c549e26a308", "<KEY>", "376985f2bc034027919a4817338caa32", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "4c4d8f38728a4c9eb377be33105da4ec", "<KEY>", "<KEY>", "d0565a23c1cc4955ac4e3b5a81e1fd7b", "ed0ece8debcd45b288320e1a8797be60", "c4b32aec61c047e5b7c903e1991d940c", "93d1f8af4a894615b404ed17091919d5", "<KEY>", "c86279a8f86e4f75ad048c4d6677fdab", "<KEY>", "0dccce07c4024c4496d2b68b0d9077e5", "<KEY>", "6e2363d5466f46459873c2a5b36eea1c", "<KEY>", "<KEY>", "<KEY>"]} id="SI4UAcYcOpxc" outputId="70b42db8-15d3-40f0-e727-87c65e07b07f"
from transformers import BertTokenizer, TFBertForSequenceClassification
from transformers import InputExample, InputFeatures

tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")


# + [markdown] id="2J1kcn3M5nEi"
#
# ---
# ### Cross validation
#

# + [markdown] id="ccJl8pYO5yI-"
# #### 6-fold cross validation on movies 
# -

# Methods to convert the data into the data required by the model for training and testing

# + id="xRSnyXKl5xol"
def convert_data_to_examples_cv(train, DATA_COLUMN, LABEL_COLUMN):
    train_InputExamples = train.apply(
        lambda x: InputExample(guid=None,  # Globally unique ID for bookkeeping, unused in this case
                               text_a=x[DATA_COLUMN],
                               text_b=None,
                               label=x[LABEL_COLUMN]), axis=1)

    return train_InputExamples


def convert_examples_to_tf_dataset_cv(examples, tokenizer, max_length=128):
    features = []  # -> will hold InputFeatures to be converted later

    for e in examples:
        # Documentation is really strong for this method, so please take a look at it
        input_dict = tokenizer.encode_plus(
            e.text_a,
            add_special_tokens=True,
            max_length=max_length,  # truncates if len(s) > max_length
            return_token_type_ids=True,
            return_attention_mask=True,
            pad_to_max_length=True,  # pads to the right by default # CHECK THIS for pad_to_max_length
            truncation=True
        )

        input_ids, token_type_ids, attention_mask = (input_dict["input_ids"],
                                                     input_dict["token_type_ids"], input_dict['attention_mask'])

        features.append(
            InputFeatures(
                input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, label=e.label
            )
        )

    def gen():
        for f in features:
            yield (
                {
                    "input_ids": f.input_ids,
                    "attention_mask": f.attention_mask,
                    "token_type_ids": f.token_type_ids,
                },
                f.label,
            )

    return tf.data.Dataset.from_generator(
        gen,
        ({"input_ids": tf.int32, "attention_mask": tf.int32, "token_type_ids": tf.int32}, tf.int64),
        (
            {
                "input_ids": tf.TensorShape([None]),
                "attention_mask": tf.TensorShape([None]),
                "token_type_ids": tf.TensorShape([None]),
            },
            tf.TensorShape([]),
        ),
    )


# + id="8fqcU2hAHFc_"
def train_bert(df_train, df_test):
    # initialize model with 3 labels, for hate, offensive and normal class classification
    model = TFBertForSequenceClassification.from_pretrained("bert-base-uncased",
                                                            trainable=True,
                                                            num_labels=3)
    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")

    train = df_train[['text', 'majority_answer']]
    train.columns = ['DATA_COLUMN', 'LABEL_COLUMN']

    test = df_test[['text', 'majority_answer']]
    test.columns = ['DATA_COLUMN', 'LABEL_COLUMN']

    DATA_COLUMN = 'DATA_COLUMN'
    LABEL_COLUMN = 'LABEL_COLUMN'

    train_InputExamples = convert_data_to_examples_cv(train, DATA_COLUMN, LABEL_COLUMN)
    test_InputExamples = convert_data_to_examples_cv(test, DATA_COLUMN, LABEL_COLUMN)

    train_data = convert_examples_to_tf_dataset_cv(list(train_InputExamples), tokenizer)
    train_data = train_data.batch(32)

    valid_data = convert_examples_to_tf_dataset_cv(list(test_InputExamples), tokenizer)
    valid_data = valid_data.batch(32)

    # compile and fit
    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=3e-6, epsilon=1e-08, clipnorm=1.0),
                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                  metrics=[tf.keras.metrics.SparseCategoricalAccuracy('accuracy')])
    print('train data type',type(train_data))
    model.fit(train_data, epochs=6, validation_data=valid_data)

    test_data = convert_examples_to_tf_dataset_cv(list(test_InputExamples), tokenizer)
    test_data = test_data.batch(32)

    print('predicting')
    preds = model.predict(test_data)

    # classification
    return classification_report(pd.DataFrame(test['LABEL_COLUMN']), np.argmax(preds[0], axis=1), output_dict=True)


# + id="DWsFTqlHlgv3"
def load_movies_to_df(path):
    df_movies = []

    for filename in glob.glob(path + '*.csv'):
        df_movies.append(pd.read_csv(filename))

    return df_movies


# + id="Xz1rCrpvB2RA"
df_movies = load_movies_to_df('movies/for_training/')
classification_reports = []
df_main = pd.DataFrame()

# + colab={"base_uri": "https://localhost:8080/"} id="DpchIXr0lciQ" outputId="1b21ce5b-279e-415e-8be6-240b63a196f5"
# perform cross folding
for i in range(len(df_movies)):
    df_train = pd.concat(df_movies[0:i] + df_movies[i + 1:])
    df_test = df_movies[i]

    train_movies = df_train['movie_name'].unique()
    test_movie = df_test['movie_name'].unique()
    print(','.join(train_movies))
    print(test_movie[0])

    report = train_bert(df_train, df_test)
    classification_reports.append(report)
    
    print('Train movies: ', str(','.join(train_movies)))
    print('Test movie: ', str(test_movie[0]))
    print('Classification report: \n', classification_reports[i])
    print('------------------------------------------------')

    df_cr = pd.DataFrame(classification_reports[i]).transpose()
    df_cr['movie_train'] =  str(','.join(train_movies))
    df_cr['movie_test'] = str(test_movie[0])
    df_cr.to_csv('classification_reports/'+'bert_cv_testmovie_'+str(test_movie[0])+'.csv')
    df_main = df_main.append(df_cr)

# + id="LyfPoHfVB63k"
df_main.to_csv('classification_reports/bert_crossvalid_movies.csv')

# + colab={"base_uri": "https://localhost:8080/"} id="e9c-yeiBHfGG" outputId="468178dc-04e5-4d4d-a08e-109f144ac458"
 print(df_main)

# + colab={"base_uri": "https://localhost:8080/"} id="iD3D9odZB7de" outputId="89dc5eb0-87c9-4e5a-b57e-38cedf0c52e9"
len(classification_reports[0])

# + colab={"base_uri": "https://localhost:8080/", "height": 261} id="68L-CNl6GGg8" outputId="3dbacce7-0569-49ca-861b-c943455b37a5"
df_main.head()


# + id="NRGObHolGHHF"
def get_precision_recall_f1(category, result_df):
    precision = result_df[result_df.label==category].precision.mean()
    recall = result_df[result_df.label==category].recall.mean()
    f1 = result_df[result_df.label==category]['f1-score'].mean()
    
    return {'label': category, 'precision': precision, 'recall': recall, 'f1': f1}


# + id="vj_ZEFYdRdF4"
df_cv= pd.read_csv('classification_reports/bert_crossvalid_movies.csv')

# + colab={"base_uri": "https://localhost:8080/"} id="LUdxiuhqHcJx" outputId="3d84fcc2-d3aa-4d00-c15d-62f52c2fbae6"
len(classification_reports[0])

# + colab={"base_uri": "https://localhost:8080/", "height": 261} id="tuiVb3fUHcJy" outputId="b68aca66-e1e1-47ec-f417-8b2dde6b33de"
df_main.head()


# + id="dRL1UIIsHcJz"
def get_precision_recall_f1(category, result_df):
    precision = result_df[result_df.label==category].precision.mean()
    recall = result_df[result_df.label==category].recall.mean()
    f1 = result_df[result_df.label==category]['f1-score'].mean()
    
    return {'label': category, 'precision': precision, 'recall': recall, 'f1': f1}


# + id="PVRT0b1thTtB"
df_cv= pd.read_csv('classification_reports/bert_crossvalid_movies.csv')

# + colab={"base_uri": "https://localhost:8080/", "height": 261} id="SnXkEFJlmfSU" outputId="8505fc77-294d-47f6-e5e6-5b50b4068373"
df_cv = df_cv.rename(columns={'Unnamed: 0': 'label', 'b': 'Y'})
df_cv.head()

# + id="V81usTykmsnl"
normal_dict = get_precision_recall_f1('0', df_cv)
offensive_dict = get_precision_recall_f1('1',df_cv)
hate_dict = get_precision_recall_f1('2',df_cv)
# -

# #### Aggregated classification results for all 6 folds

# + colab={"base_uri": "https://localhost:8080/", "height": 143} id="9xnoFFrOmgu1" outputId="76b36a6e-56ff-49d6-afeb-2793df164ad3"
df_result = pd.DataFrame([normal_dict, offensive_dict, hate_dict])
df_result

# + colab={"base_uri": "https://localhost:8080/"} id="cLHfd4BFnloL" outputId="b0df1c6c-2066-48d4-f897-8fbcdf26e439"
for cr in classification_reports:
  print(cr)

# + id="bZEsT4VuKY1a"


StarcoderdataJupyterScriptsDedupFiltered