# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import numpy as np
import scipy.sparse as sp
import matplotlib.pyplot as plt
import json

from sklearn.metrics import f1_score

import sys
import os

path = os.path.abspath(os.getcwd()) + '/../data_load'
sys.path.insert(0, path)

from movie_data import MovieData

# %matplotlib inline
# -

moviedata = MovieData(min_genre_frequency=0.05)
print(moviedata.genre_labels)

# +
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

vectorizer = TfidfVectorizer(
    max_df=0.7,
    analyzer='word',
    ngram_range=(1, 1),
    max_features=10000,
    stop_words='english'
)

X = vectorizer.fit_transform(moviedata.plots)
# -

#vectorizer.get_feature_names()
X.todense().shape

# +
from sklearn.decomposition import PCA

pca = PCA(n_components=2000)
XX = pca.fit_transform(X.todense())
# -

plt.imshow(XX, aspect='auto')

# +
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X.todense(),
    moviedata.one_hot_genres,
    test_size=0.4, 
    random_state=42)
# -

XX.shape

# ## Least Squares

# +
genre_coeffs = []
scores = []

for i, label in enumerate(moviedata.genre_labels):
    coeffs = np.linalg.lstsq(X_train, y_train[:, i])[0]
    genre_coeffs.append(coeffs)
    y_pred = np.dot(X_test, coeffs)
    y_pred = np.array(y_pred > 0.25, dtype=int)
    score = f1_score(y_test[:, i], y_pred)
    scores.append(score)
    print("Genre: {}, Score: {}".format(labels[i], score))
    
    
#np.linalg.lstsq(X_train, y_train[:, 0])[0]
# -

np.mean(scores)

# ## RandomForrest Model

# +
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score

clf = RandomForestClassifier(n_estimators=25, max_features='auto', max_depth=None)
clf.fit(X_train, y_train)
y_guess_probs = clf.predict_proba(X_test)
y_guess = clf.predict(X_test)

scores = f1_score(y_test, y_guess, average=None)

for item in list(zip(moviedata.genre_labels, scores)):
    print("{} {:.2f}".format(item[0], item[1]))


print("Average {:.3f}".format(np.mean(scores)))

# -

np.mean(scores)

# ## Logrithmic

# +
from sklearn.linear_model import LogisticRegression 
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.utils import shuffle
from sklearn import preprocessing
from sklearn import utils

X_train, X_test, y_train, y_test = train_test_split(
    XX,
    moviedata.one_hot_genres,
    test_size=0.1,
    random_state=43
)

# +
scores = []

for i, label in enumerate(moviedata.genre_labels):
    clf = LogisticRegression()
    clf.fit(X_train, y_train[:, i])
    y_pred_probs = clf.predict_proba(X_test)[:, 1]
    plt.hist(y_pred_probs)
    y_pred = np.array(y_pred_probs > np.mean(y_pred_probs), dtype=int)
    score = f1_score(y_test[:, i], y_pred)
    scores.append(score)
    print("{} {:.2f}".format(label, score))
    
print('Mean f1 score: {:.3f}'.format(np.mean(scores)))
# -



StarcoderdataJupyterScriptsDedupFiltered