# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
#import libraries
import re
import time
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib inline

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split

from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.externals import joblib 
# -

#load data
train_data = pd.read_csv("undersampled/train.tsv", delimiter='\t', lineterminator='\n', header=None)
dev_data = pd.read_csv("undersampled/dev.tsv", delimiter='\t', lineterminator='\n', header=None)
test_data = pd.read_csv("undersampled/test.tsv", delimiter='\t', lineterminator='\n', header=None)

(train_data.shape[0], dev_data.shape[0], test_data.shape[0])

train_dev_data = pd.concat([train_data, dev_data], axis=0)
train_dev_data.shape[0]

train_dev_data.head()

#assign x and y
train_x = train_dev_data[0]
test_x = test_data[0]
train_y = train_dev_data[1]
test_y = test_data[1]

print(f"training data: {np.round(train_y.value_counts()[1]/train_data.shape[0],4)*100}% positive class")
print(f"test data: {np.round(test_y.value_counts()[1]/test_data.shape[0],4)*100}% positive class")


# +
def preprocessor(s):
    s = s.lower()
    s = re.sub(r'\d+', 'DG', s)
    s = re.sub(r'@\w+', "@USER", s)
    return s

vect = CountVectorizer(preprocessor=preprocessor)
nb = MultinomialNB(fit_prior=False)
pipe = Pipeline(steps=[("vectorizer", vect), ("naivebayes", nb)])
param_grid = {"vectorizer__ngram_range": [(1,1),(1,2),(1,3)],
              "vectorizer__max_df": [0.8,0.9,1.0],
              "naivebayes__alpha": [0.01, 0.1, 1.0, 10.0]}

search = GridSearchCV(pipe, param_grid, cv=3, verbose=1)
search.fit(train_x, train_y)
# -

search.best_params_

search_results = pd.DataFrame(search.cv_results_)[["mean_fit_time","mean_test_score","mean_train_score",
                                                   "param_naivebayes__alpha","param_vectorizer__max_df", 
                                                   "param_vectorizer__ngram_range"]]
search_results.sort_values("mean_test_score", ascending=False)

train_pred = search.predict(train_x)
print(f"accuracy: {np.round(accuracy_score(train_pred, train_y),3)}")
print(f"f1-score: {np.round(f1_score(train_pred, train_y),3)}")

confusion_matrix(train_pred, train_y)

test_pred = search.predict(test_x)
print(f"accuracy: {np.round(accuracy_score(test_pred, test_y),3)}")
print(f"f1-score: {np.round(f1_score(test_pred, test_y),3)}")

confusion_matrix(test_pred, test_y)



StarcoderdataJupyterScriptsDedupFiltered