# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import pandas as pd
import os
from os.path import isfile, join
import time
import re
import numpy as np
from tqdm.notebook import tqdm
import pickle
import pandas as pd
import itertools
from collections import Counter
import numpy as np
import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords
from sklearn.linear_model import LogisticRegression
from gensim.models import word2vec
import os
import string
import spacy
import nltk
nltk.download('averaged_perceptron_tagger')
from nltk import pos_tag
from scipy.sparse import csr_matrix
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
sentiment_analyzer = SentimentIntensityAnalyzer()

# + jupyter={"outputs_hidden": true} tags=[]
# !pip3 install nltk
# !pip3 install vaderSentiment
# !pip3 install gensim

# + tags=[]
from sklearn.linear_model import LogisticRegression
# -

from sklearn.metrics import roc_auc_score, auc, f1_score, accuracy_score, roc_curve

# ### Load Data

os.getcwd()

os.listdir()

filePath = ["./aclImdb/train/", "./aclImdb/test/"]
'/Users/ryola/Documents/Master/Winter/MGTA\ 415/Project/aclImdb/test'
tag = ["pos/", "neg/"]
for i in filePath:
    for j in tag:
        start = time.time()
        path = i+j
        onlyfiles = [f for f in os.listdir(path) if isfile(join(path, f))]
        end = time.time()
        print(f"Detect {len(onlyfiles)} files at the current path:'{path}', time: {round(end-start, 4)}s.")
# onlyfiles

os.getcwd()

# %%time
testPosPath = '/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/test/pos/'
testNegPath = '/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/test/neg/'
testfiles = [f for f in os.listdir(testPosPath) if isfile(join(testPosPath, f))]
testfiles += [f for f in os.listdir(testNegPath) if isfile(join(testNegPath, f))]

# %%time
Xtest = []
ytest = [1] * 12500 + [0] * 12500
# filedir = "./aclImdb/test/neg/"
start = time.time()
for i in range(len(testfiles)):
    file = testfiles[i]
    if i < 12500:
        filedir = "/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/test/pos/"
    else:
        filedir = "/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/test/neg/"
    path = filedir + file
    rating = re.findall("[0-9]+_([0-9]+).txt", file)[0]
    with open(path, "r") as f:
        contents = f.read()
        Xtest.append([contents, rating])
end = time.time()
# print("Time :{}s.".format(round(end- start, 4)))

test = pd.DataFrame(Xtest, columns = ["text", 'rating'])
test['label'] = ytest
print(test.shape)
test.head()

# %%time
trainPosPath = '/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/train/pos/'
trainNegPath = '/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/train/neg/'
trainfiles = [f for f in os.listdir(trainPosPath) if isfile(join(trainPosPath, f))]
trainfiles += [f for f in os.listdir(trainNegPath) if isfile(join(trainNegPath, f))]

# %%time
Xtrain = []
ytrain = [1] * 12500 + [0] * 12500
# filedir = "./aclImdb/test/neg/"
start = time.time()
for i in range(len(trainfiles)):
    file = trainfiles[i]
    if i < 12500:
        filedir = "/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/train/pos/"
    else:
        filedir = "/home/jovyan/Documents/Master/Winter/MGTA 415/Project/aclImdb/train/neg/"
    path = filedir + file
    rating = re.findall("[0-9]+_([0-9]+).txt", file)[0]
    with open(path, "r") as f:
        contents = f.read()
        Xtrain.append([contents, rating])
end = time.time()
print("Time :{}s.".format(round(end- start, 4)))

train = pd.DataFrame(Xtrain, columns = ["text", 'rating'])
train['label'] = ytrain
print(train.shape)
train.head()

# ### Basic Info About Train Set

Xtrain[0]

doc_lens = []
for doc in Xtrain:
    l = len(doc[0].split())
    doc_lens.append(l)
lens = np.array(doc_lens)
print(lens.shape, lens.mean())

print(lens.max(), lens.min())

# avg doc lengths for pos & neg samples
lens[:12500].mean().round(), lens[12500:].mean().round()

import seaborn as sns
import matplotlib.pyplot as plt
plt.figure(figsize = (16,4))
plt.title("Raw Corpus Length Histogram")
_ = plt.hist(lens, bins='auto')
plt.savefig('data/raw_corpus_length_hist')
plt.show()


# ### Baseline using VADER

def get_polarity_label(sentence):
    score = sentiment_analyzer.polarity_scores(sentence)
    neg, neu, pos = score['neg'], score['neu'], score['pos']
    if pos >= neg: #and pos > neu:
        return 1 #'positive'
    else:
    #if neg > pos and neg > neu:
        return 0 #'negative'
    # return 'neutral'
# get_polarity_label('I feel good')


test['vader'] = test.text.apply(lambda x: get_polarity_label(x))
test.vader.mean()

accuracy_score(test.label, test.vader), f1_score(test.label, test.vader)

test[['label', 'vader']].to_csv("../data/res/vader.csv", index = False)


# ### Preprocess

def preprocess_df(df, col = "text", target_col = 'clean_text'):
    # get English stopwords
    stop_words = set(stopwords.words('english'))
    stop_words.add('would')
    # prepare translation table to translate punctuation to space
    translator = str.maketrans(string.punctuation, ' ' * len(string.punctuation))
    preprocessed_sentences = []
    for i, row in df.iterrows():
        sent = row[col]
        sent_nopuncts = sent.translate(translator)
        words_list = sent_nopuncts.strip().split()
        filtered_words = [word for word in words_list if word not in stop_words and len(word) != 1] # also skip space from above translation
        preprocessed_sentences.append(" ".join(filtered_words))
    df[target_col] = preprocessed_sentences
    return df


train = preprocess_df(train)
test = preprocess_df(test)

import en_core_web_sm
import re
nlp_processor = en_core_web_sm.load()
nltk.download('stopwords') # <--- this is new
from nltk.corpus import stopwords
stop = set(stopwords.words('english'))
def pre_processing_by_spacy(doc, lemma = True, need_sent = False):
    doc_class = nlp_processor(doc)
    tokens = []
    # step 1: get sentences
    for sent in doc_class.sents:
        # step 2: get tokens
        # step 3 (optional): lemma
        if lemma:
            words = [token.lemma_ for token in sent]
        else:
            words = [token.text for token in sent]
        if need_sent:
            tokens.append(words)
        else:
            tokens += words
            
    raw_corpus = re.sub(r'[^\w\s]', ' ', ' '.join(tokens)).split()
    corpus = [w.lower() for w in raw_corpus if w.lower() not in stop]
    
    return corpus


# tokenization -> preprocess by spacy 
tagged_data = [pre_processing_by_spacy(_d) for i, _d in tqdm(enumerate(train["clean_text"]))]

# tokenization -> preprocess by spacy 
tagged_test_data = [pre_processing_by_spacy(_d) for i, _d in tqdm(enumerate(test["clean_text"]))]

# save tagged train data
with open("data/tagged_train_data.txt", "w") as f:
    for s in tqdm(tagged_data):
        f.write(str(s) +"\n")
with open("data/tagged_test_data.txt", "w") as f:
    for s in tqdm(tagged_test_data):
        f.write(str(s) +"\n")

# + [markdown] tags=[]
# ### Rule-based
# -

ypred1 = []
for l in tqdm(tagged_test_data):
    dct = sentiment_analyzer.polarity_scores(' '.join(l))
    if dct['pos'] >= dct['neg']:
        ypred1.append(1)
    else:
        ypred1.append(0)

print(accuracy_score(ytest, ypred1))

ypred1 = []
for l in tqdm(train.clean_text):
    dct = sentiment_analyzer.polarity_scores(l)
    if dct['pos'] >= dct['neg']:
        ypred1.append(1)
    else:
        ypred1.append(0)

print(accuracy_score(ytest, ypred1))

# ### Binary Encoding For Adj + Adv

# + tags=[]
tagged_sents = []
tags = ['JJ', 'JJR', 'JJS', 'RB', 'RBR', 'RBs']
for i in tqdm(range(len(tagged_data))):
    tagged_sents+= [w for w, tag in pos_tag(tagged_data[i]) if tag in tags]
# -

vocab = list(set(tagged_sents))
print("Size of adj/adv vocab = {}".format(len(vocab)))

Xtrain = []
for tokens in tqdm(tagged_data):
    s = list(set(tokens))
    Xtrain.append([s.count(w) for w in vocab])

Xtest = []
for tokens in tqdm(tagged_test_data):
    s = list(set(tokens))
    Xtest.append([s.count(w) for w in vocab])

# save tagged train data
with open("data/Xtrain_sent_bin.txt", "w") as f:
    for s in tqdm(Xtrain):
        f.write(str(s) +"\n")
with open("data/Xtest_sent_bin.txt", "w") as f:
    for s in tqdm(Xtest):
        f.write(str(s) +"\n")

trans_dict = {"[": "", ',':"", "]":""}
translator = str.maketrans(trans_dict)
ytrain = [1] * 12500 + [0] * 12500
ytest = ytrain
Xtrain = []
Xtest = []
with open("data/Xtrain_sent_bin.txt", "r") as f:
    for line in tqdm(f):
        Xtrain.append(line.translate(translator).split())

with open("data/Xtest_sent_bin.txt", "r") as f:
    for line in tqdm(f):
        Xtest.append(line.translate(translator).split())

# + tags=[]
# %%time
# Xtrain_ = csr_matrix(Xtrain)
# Xtest_ = csr_matrix(Xtest)
print("====START FITING REGRESSION====")
clf = LogisticRegression(C = 20)
clf.fit(Xtrain, ytrain)


# -

def evaluate(ytest, ypred, name):
    accu = accuracy_score(ytest, ypred)
    f1 = f1_score(ytest, ypred)
    AUC = roc_auc_score(ytest, ypred)
    fpr, tpr = roc_curve(ytest, ypred)
    
    lw = 2
    plt.figure(figsize = (6,6))
    plt.plot(fpr, tpr, lw = lw, label="ROC curve of model {}, AUC = ({1:0.2f})".format(name, AUC))
    plt.plot([0, 1], [0, 1], "k--", lw=lw)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate")
    plt.title("ROC Curve")
    plt.legend(loc="lower right")
    plt.show()
    
    df = pd.DataFrame()
    df.assign(
        model = name,
        accuracy = [accu],
        f1_score = [f1],
        AUC = [AUC]
    )
    print(df)
    return df


ypred2 = clf.predict(Xtest)
evaluate(ytest, ypred2)

























sentiment_analyzer = SentimentIntensityAnalyzer()
scores = sentiment_analyzer.lexicon

a = scores.get("1")
[a]

tagged_adj_sent = []
for i in tqdm(range(len(tagged_adj))):
    l = []
    for w in tagged_adj[i]:
        if scores.get(w) is not None:
            l.append(w)
    tagged_adj_sent.append(l)

tagged_adj = []
for i in tqdm(range(len(tagged_adj_sent))):
    tagged_adj.append([scores[w] for w in tagged_adj_sent[i]])

np.array(tagged_adj)



ypred1 = []
for l in tqdm(tagged_test_data):
    dct = sentiment_analyzer.polarity_scores(' '.join(l))
    if dct['pos'] >= dct['neg'] - 0.1:
        ypred1.append(1)
    else:
        ypred1.append(0)

print(accuracy_score(ytest, ypred1))



StarcoderdataJupyterScriptsDedupFiltered