# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # import packages

# +
import numpy as np
import pandas as pd

# Visualization
import matplotlib.pyplot as plt
import seaborn as sns

# Preprocessing
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
# Text representation
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

# ML Algo
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC

# Evaluvation
from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score

from time import time
# -

import string
import re
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk import WordNetLemmatizer

from scripts.utils import plotConfusionMatrixHeatmap

# # Load Trianing file

data_path = '../../data/train.csv'

df = pd.read_csv(data_path)

df.shape

df.columns

df.head()

df.drop(columns=['Unnamed: 0', 'index'], inplace=True)

df.columns

df.head()

df['label'].value_counts()

# Class distribution
df['label'].value_counts()/df.shape[0] * 100


# # Pre Processing

def clean_text(doc):
    """
      1. Converting all text into lower case
      2. Removing classified words like xxx
      3. Remove stop words
      4. remove punctuation
      5. remove digits
      6. Wordnet lemmatizer
      """
    # Set stop word as english
    stop_word = set(stopwords.words('english'))
    
    # Tokenize the sentence and make all character lower case
    doc = [x.lower() for x in word_tokenize(doc)]
    
    # Remove classified texts
    doc = [x for x in doc if x.lower() != 'xxxx' and x.lower() != 'xx' and x.lower() != 'xx/xx/xxxx']
    
    # Remove stop words
    doc = [x for x in doc if x not in stop_word]
    
    # Remove Punctuation
    doc = [x for x in doc if x not in string.punctuation]
    
    # Remove Digits
    doc = [x for x in doc if not x.isdigit()]
    
    # Set NLTK Wordnet lemmatizer and lemmatize the sentence
    lemmatizer = WordNetLemmatizer()
    doc = " ".join([lemmatizer.lemmatize(word) for word in doc])
    
    return doc


df['text_processed'] = df.apply(lambda row : clean_text(row['full_text']), axis = 1)

df.head()

# # Encoding and Modeling

from sklearn.preprocessing import LabelEncoder
label_encoder = LabelEncoder()

df['label_id'] = label_encoder.fit_transform(df['label'])

# Put the label category into dict for future use
label_map = df.set_index('label_id').to_dict()['label']
label_map

df.head()

X = df.text_processed
y = df.label_id

print(X.shape, y.shape)

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)

print(X_train.shape, y_train.shape)
print(X_test.shape, y_test.shape)

# # Multinomial Naive bayes

# Parameter values to test
param_grid = {
    'TfIdf__max_features' : [5000, 10000, 20000, 25000],
    'TfIdf__ngram_range' : [(1,1),(1,2),(2,2)],
    'TfIdf__use_idf' : [True],
    'MultinomialNB__alpha' : [0.01, 0.02, 0.05, 0.10]
    }

# Creating pipeline for Naive Bayes Model
pipeline_mnb = Pipeline(steps = [('TfIdf', TfidfVectorizer()),
                              ('MultinomialNB', MultinomialNB())])

grid_search_mnb = GridSearchCV(pipeline_mnb, param_grid, cv=5, verbose=1, n_jobs=6)

grid_search_mnb.fit(X_train, y_train)

print(grid_search_mnb.best_params_)

print(grid_search_mnb.best_estimator_)

grid_search_mnb.score(X_test, y_test)

y_predicted = grid_search_mnb.predict(X_test)

classification_report_mnb = classification_report(y_test, y_predicted)
print(classification_report_mnb)

key_to_label_name = [x[1] for x in sorted(label_map.items())]

# +
conf_matrix_df = pd.DataFrame(data=confusion_matrix(y_test, y_predicted),
                              index=key_to_label_name, columns=key_to_label_name)

plotConfusionMatrixHeatmap(conf_matrix_df, model_name='Multinomial Naive bayes', figsize=(12, 10))

# +
best_model = grid_search_mnb
best_model.version = 1.0
best_model.pandas_version = pd.__version__
best_model.numpy_version = np.__version__
best_model.sklearn_version = sklearn_version
best_model.build_datetime = datetime.now()
    
modelpath = '../../data/models'
if not os.path.exists(modelpath):
    os.mkdir(modelpath)
mnbmodel_path = os.path.join(modelpath, 'Multinomial_naive_bayes_with_7_class.pkl')
if not os.path.exists(mnbmodel_path):
    with open(mnbmodel_path, 'wb') as f:
        pickle.dump(best_model, f)
# -

# # Logistic Regression

param_grid = {
    'TfIdf__max_features' : [5000, 10000, 20000, 25000],
    'TfIdf__ngram_range' : [(1,1),(1,2),(2,2)],
    'TfIdf__use_idf' : [True]
    }

# Creating pipeline for Logistice Regression model
pipeline_lr = Pipeline(steps = [('TfIdf', TfidfVectorizer()),
                              ('LogisticRegression', LogisticRegression(class_weight="balanced"))])

grid_search_lr = GridSearchCV(pipeline_lr, param_grid_lr, cv=5, verbose=1, n_jobs=-1)

grid_search_lr.fit(X_train, y_train)

print(grid_search_lr.best_params_)

print(grid_search_lr.best_estimator_)

grid_search_lr.score(X_test, y_test)

y_predicted = grid_search_lr.predict(X_test)

classification_report_lr = classification_report(y_test, y_predicted)
print(classification_report_lr)

# +
conf_matrix_df = pd.DataFrame(data=confusion_matrix(y_test, y_predicted),
                              index=key_to_label_name, columns=key_to_label_name)

plotConfusionMatrixHeatmap(conf_matrix_df, model_name='Multinomial', figsize=(12, 10))
# -

# # Random forest clasifier

from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer

vectorizer = TfidfVectorizer(min_df=3, stop_words="english", sublinear_tf=True, norm='l2', ngram_range=(1, 2))

pipeline_rf = Pipeline(steps = [('countvectorizer', vectorizer),
                              ('clf', RandomForestClassifier())])

model = pipeline_rf.fit(X_train, y_train)

y_pred = model.predict(X_test)

y_pred_prob = model.predict_proba(X_test)

lr_probs = y_pred_prob[:,1]

accuracy_score(y_test, y_pred)

conf_matrix_df = pd.DataFrame(data=confusion_matrix(y_test, y_pred),index=key_to_label_name,
                              columns=key_to_label_name)

classification_rep = classification_report(y_test, y_pred,target_names=key_to_label_name)
print(classification_rep)

plotConfusionMatrixHeatmap(conf_matrix_df, model_name='Random forest', figsize=(12, 10))

# # Doc 2 Vec with logistic regression

from gensim.models.doc2vec import Doc2Vec, TaggedDocument

#prepare training data in doc2vec format:
train_doc2vec = [TaggedDocument((d), tags=[str(i)]) for i, d in enumerate(X_train)]

#Train a doc2vec model to learn
model = Doc2Vec(vector_size=50, alpha=0.025, min_count=5, dm =1, epochs=100)
model.build_vocab(train_doc2vec)
model.train(train_doc2vec, total_examples=model.corpus_count, epochs=model.epochs)
model.save("../../data/models/d2v.model")
print("Model Saved")

#Infer the feature representation for training and test data using the trained model
model= Doc2Vec.load("../../data/models/d2v.model")

#infer in multiple steps to get a stable representation. 
train_vectors =  [model.infer_vector(list_of_tokens, steps=50) for list_of_tokens in X_train]
test_vectors = [model.infer_vector(list_of_tokens, steps=50) for list_of_tokens in X_test]

clf = LogisticRegression(class_weight="balanced")
clf.fit(train_vectors, y_train)





StarcoderdataJupyterScriptsDedupFiltered