import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score, recall_score

df = pd.read_table('data/preprocessed.tsv', usecols=['title', 'description', 'selected'])
df.fillna(value="", inplace=True)

y = df['selected'].astype(int).values

"""
Explanation: Machine Learning with Text
Problem: Use the title and description of a talk to predict whether it might be selected.
Starting with the imports...
End of explanation
"""


corpus = df['title']

"""
Explanation: The Training & Prediction pipeline

Let's use the 'title' column as the corpus
End of explanation
"""


vect = TfidfVectorizer(sublinear_tf=True, stop_words='english')
X = vect.fit_transform(corpus)
pd.DataFrame(X.toarray(), columns=vect.get_feature_names()).head()

"""
Explanation: Text Vectorization & The TD Matrix
End of explanation
"""


svd = TruncatedSVD(n_components=250)
X = svd.fit_transform(X)
pd.DataFrame(X).head()

"""
Explanation: Dimensionality Reduction
End of explanation
"""


gnb = GaussianNB()
gnb.fit(X, y)

"""
Explanation: Training the Classifier
End of explanation
"""


predictions = gnb.predict(X)
print((predictions == y).sum() / 290)

"""
Explanation: Testing the classifier
End of explanation
"""


# Retrieve the corpus from the dataset

# Obtain the TD Matrix

# Reduce the dimensionality of the TD matrix to 250

# Train the classifier

# Test the classifier

"""
Explanation: Exercise 1: Use the 'description' column of the dataset as a corpus for the predictions
End of explanation
"""


