# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import sys
sys.path.append('../../Utils/')

import tensorflow as tf 
import numpy as np 
import pandas as pd 
import nltk
import math
import random
from sklearn.utils import shuffle
from load_imdb_data import load_imdb_data
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score 
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np 
import matplotlib.pyplot as plt 
np.random.seed(0)

imdb_data = load_imdb_data()

imdb_data = shuffle(imdb_data)
imdb_data.reset_index(drop=True, inplace=True)


def performanceTest(X, y, method='tf-idf'):
    X_train, X_test, y_train, y_test = train_test_split(X,
                                                        y, 
                                                        test_size=0.2,  
                                                        random_state=42)
    
    if method == "tf-idf":
        vectorizer = TfidfVectorizer(min_df=1)
        X_train = vectorizer.fit_transform(X_train)
        X_test = vectorizer.transform(X_test)
        clf = LogisticRegression()
        clf.fit(X_train, y_train)
        train_pred = clf.predict(X_train)
        test_pred = clf.predict(X_test)
        print("Train F1-score : ", f1_score(y_train, train_pred))
        print("Test F1-score : ", f1_score(y_test, test_pred))
    else:
        clf = LogisticRegression()
        clf.fit(X_train, y_train)
        train_pred = clf.predict(X_train)
        test_pred = clf.predict(X_test)
        print("Train F1-score : ", f1_score(y_train, train_pred))
        print("Test F1-score : ", f1_score(y_test, test_pred))


# cd skip-thoughts

# ls -lh ../

import skipthoughts
model = skipthoughts.load_model()
encoder = skipthoughts.Encoder(model)

encoder.encode(X=['<NAME>'])

skip_thoughts = encoder.encode(X = list(imdb_data['review']))

performanceTest(skip_thoughts, imdb_data['sentiment'], method=None)

StarcoderdataJupyterScriptsDedupFiltered