# Load data
import pandas as pd
with open('./data_files/8lWZYw-u-yNbGBkC4B--ip77K1oVwwyZTHKLeD7rm7k.csv') as data_file:
    df = pd.read_csv(data_file)
df.head()

"""
Explanation: Hypothesis
Non-linear models will be more accurate than a logistic regression model
End of explanation
"""


# Remove messages without a Subject
print df.shape
df = df.dropna(subset=['Subject'])
print df.shape

# Perform bag of words feature extraction
# TODO: Why are there only 3000 words in the vocabulary?
from sklearn.feature_extraction.text import CountVectorizer
count_vect = CountVectorizer(stop_words='english', lowercase=True)
train_counts = count_vect.fit_transform(df['Subject'])
print 'Dimensions of vocabulary feature matrix are:'
print train_counts.shape

# Add TF/IDF weighting to account for lenght of documents
from sklearn.feature_extraction.text import TfidfTransformer
tfidf_transformer = TfidfTransformer()
train_tfidf = tfidf_transformer.fit_transform(train_counts)
print 'Dimensions of vocabulary feature matrix are:'
print train_tfidf.shape
print 'But, its a sparse matrix: ' + str(type(train_tfidf))

"""
Explanation: Comparing classification models


Do some preprocessing on the text columns (subject, body, maybe to, cc, from)
Clean NaN's or remove rows of data with NaNs
Do stuff the Preprocess Text Azure module does for us (stopwords, etc)
Use scikit learn where possible


Do some feature construction using pandas & scikit learn
On subject, to, cc, from
Bag of words
TF/IDF


One-Hot Encode FolderId labels into their own boolean columns (1s & 0s)
Ignore better features for now, this is good enough for comparisions
Split data into training & test sets to be used for all ensemble members
For each classifier, train a model on the training data
Evaluate performance of model on test data, compare to Logistic Regression model

Constructing Subject Feature Matrix
End of explanation
"""


# Merge CC, To, From into one People column
df['CcRecipients'].fillna('', inplace=True)
df['ToRecipients'].fillna('', inplace=True)
df['Sender'].fillna('', inplace=True)
df['People'] = df['Sender'] + ';' + df['CcRecipients'] + ';' + df['ToRecipients']
df.head(10)

# Convert People to matrix representation
people_features = df['People'].str.get_dummies(sep=';')
print people_features.shape
people_features.head()

# Will need to store people vocabulary for feature construction during predictions
people_vocabulary = people_features.columns
print people_vocabulary[:2]
print len(people_vocabulary)

# Convert to csr_matrix and hstack with Subject feature matrix
import scipy
sparse_people_features = scipy.sparse.csr_matrix(people_features)
print people_features.shape
print sparse_people_features.shape

print sparse_people_features.shape
print train_tfidf.shape
feature_matrix = scipy.sparse.hstack([sparse_people_features, train_tfidf])
print feature_matrix.shape

"""
Explanation: Constructing CC, To, and From
End of explanation
"""


# Split into test and training data sets
from sklearn.model_selection import train_test_split
labels_train, labels_test, features_train, features_test = train_test_split(df['FolderId'], feature_matrix, test_size=0.20, random_state=42)
print labels_train.shape
print labels_test.shape
print features_train.shape
print features_test.shape

# Construct a list of classifiers
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier

names = [
    "Nearest Neighbors", 
    "Linear SVM", 
    "Decision Tree", 
    "Random Forest", 
    "Neural Net", 
    "AdaBoost",
]

candidate_classifiers = [
    KNeighborsClassifier(),
    SVC(kernel='linear', C=0.025),
    DecisionTreeClassifier(max_depth=5),
    RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
    MLPClassifier(alpha=1),
    AdaBoostClassifier(),
]

# Train and evaluate models, compare accuracy
from sklearn import metrics
for name, clf in zip(names, candidate_classifiers):
    model = clf.fit(features_train, labels_train)
    predictions = model.predict(features_test)
    print name + ": " + str(metrics.accuracy_score(labels_test, predictions))

# Construct a list of classifiers
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

dense_names = [
    "RBF SVM", 
#     "Gaussian Process", # Taking way too long
    "Naive Bayes",
#     "QDA" # Didn't work for classes with only one sample
]

candidate_dense_classifiers = [
    SVC(gamma=2, C=1),
#     GaussianProcessClassifier(1.0 * RBF(1.0), warm_start=True),
    GaussianNB(),
#     QuadraticDiscriminantAnalysis()
]

# Train and evaluate models using dense feature matrix, compare accuracy
from sklearn import metrics
dense_features_train = features_train.toarray()
dense_features_test = features_test.toarray()
for name, clf in zip (dense_names, candidate_dense_classifiers):
    model = clf.fit(dense_features_train, labels_train)
    predictions = model.predict(dense_features_test)
    print name + ": " + str(metrics.accuracy_score(labels_test, predictions))

"""
Explanation: Train models & compare accuracies
End of explanation
"""


