from __future__ import print_function  # Python 2/3 compatibility
import numpy as np
import pandas as pd

from IPython.display import Image

"""
Explanation: Objective
Build Multiple Models and Emsemble the Results
End of explanation
"""


train_df = pd.read_csv("data/train.tsv", sep="\t")

train_df.sample(10)

"""
Explanation: Load Data
End of explanation
"""


from sklearn.model_selection import train_test_split

X_train, X_valid, y_train, y_valid = train_test_split(train_df["review"], train_df["sentiment"], test_size=0.2)

print("Training Data: {}, Validation: {}".format(len(X_train), len(X_valid)))

"""
Explanation: Training process

Split the Overall Training examples into Training and Validation
Build the Models on Training Data
Score on Validation data
Choose the best model and submit to Kaggle

Caution:  If you do this enough times, you will be overfitting to the Validation data.  To avoid that it might be advisable to split into three ways like Train-Validation-Test and generate the final score on Test Data.
End of explanation
"""


from sklearn.feature_extraction.text import CountVectorizer

# The API is very similar to model building process.
# Step 1: Instantiate the Vectorizer or more generally called Transformer

vect = CountVectorizer(max_features=5000, binary=True, stop_words="english")

# Fit your Training Data
vect.fit(X_train)

# Transform your training and validation data
X_train_vect = vect.transform(X_train)
X_valid_vect = vect.transform(X_valid)

"""
Explanation: Vectorize Data (a.k.a. covert text to numbers)
Computers don't understand Texts, so we need to convert texts to numbers before we could do any math on it and see if we can build a system to classify a review as Positive or Negative.
Ways to vectorize data:

Bag of Words
TF-IDF
Word Embeddings (Word2Vec) 

Scikit-Learn has nice APIs for preprocessing and feature extraction modules.  In fact, these can be used even if you build your own models or use another libriary for model building process.
End of explanation
"""


from sklearn.linear_model import LogisticRegression

model_1 = LogisticRegression()
model_1.fit(X_train_vect, y_train)

# Training Accuracy
print("Training Accuracy: {:.3f}".format(model_1.score(X_train_vect, y_train)))
print("Validation Accuracy: {:.3f}".format(model_1.score(X_valid_vect, y_valid)))

"""
Explanation: Model - Logistic Regression

Logistic Regression
Naive Bayes
Neural Nets
Random Forest
Gradient Boosted Trees
...

Model 1 - Logisitc Regression
End of explanation
"""


from sklearn.naive_bayes import MultinomialNB

model_2 = MultinomialNB()
model_2.fit(X_train_vect, y_train)

# Training Accuracy
print("Training Accuracy: {:.3f}".format(model_2.score(X_train_vect, y_train)))
print("Validation Accuracy: {:.3f}".format(model_2.score(X_valid_vect, y_valid)))

"""
Explanation: Model 2 - Naive Bayes
End of explanation
"""


from sklearn.ensemble import RandomForestClassifier

model_3 = RandomForestClassifier(min_samples_leaf=3, n_estimators=25, n_jobs=-1)
model_3.fit(X_train_vect, y_train)

# Training Accuracy
print("Training Accuracy: {:.3f}".format(model_3.score(X_train_vect, y_train)))
print("Validation Accuracy: {:.3f}".format(model_3.score(X_valid_vect, y_valid)))

"""
Explanation: Model 3 - Random Forest
End of explanation
"""


from sklearn.ensemble import GradientBoostingClassifier

model_4 = RandomForestClassifier(min_samples_leaf=3, n_estimators=25, n_jobs=-1)
model_4.fit(X_train_vect, y_train)

# Training Accuracy
print("Training Accuracy: {:.3f}".format(model_4.score(X_train_vect, y_train)))
print("Validation Accuracy: {:.3f}".format(model_4.score(X_valid_vect, y_valid)))

"""
Explanation: Model 4 - Gradient Boosted Trees
End of explanation
"""


from sklearn.neural_network import MLPClassifier

model_5 = MLPClassifier(hidden_layer_sizes=(32,), max_iter=100)
model_5.fit(X_train_vect, y_train)

# Training Accuracy
print("Training Accuracy: {:.3f}".format(model_5.score(X_train_vect, y_train)))
print("Validation Accuracy: {:.3f}".format(model_5.score(X_valid_vect, y_valid)))

"""
Explanation: Model 5 - Neural Networks (CPU Only)
End of explanation
"""


## Pass

"""
Explanation: Neural Nets - Textbook Case of Overfitting.  Maybe the model is too powerful :-D
Model Tuning

All these models have tons of Parameters that could be tweaked to reduce overfitting
End of explanation
"""


from sklearn.ensemble import VotingClassifier

classifiers = [("Logistic Regression", model_1), 
               ("Naive Bayes", model_2), 
               ("Random Forest", model_3), 
               ("Gradient Boosted", model_4), 
               ("Neural Nets", model_5)]

classifiers

final_model = VotingClassifier(classifiers, n_jobs=-1)

# Unfortuately, have to run Fit Again on the ensembled model before using it
# Wish there was an option to not have to fit again
final_model.fit(X_train_vect, y_train)

# Drum Rolls - Accuracy on the final Model
print("Training Accuracy: {:.3f}".format(final_model.score(X_train_vect, y_train)))
print("Validation Accuracy: {:.3f}".format(final_model.score(X_valid_vect, y_valid)))

"""
Explanation: Finding it difficult to pick the Winning Model - Why not Average the Results

After all we collectively make the right decisions, on average.
End of explanation
"""


# Read in the Test Dataset
# Note that it's missing the Sentiment Column.  That's what we need to Predict
#
test_df = pd.read_csv("data/test.tsv", sep="\t")
test_df.head()

# Vectorize the Review Text

X_test = test_df.review
X_test_vect = vect.transform(X_test)

y_test_pred = final_model.predict(X_test_vect)

df = pd.DataFrame({
    "document_id": test_df.document_id,
    "sentiment": y_test_pred
})

df.to_csv("data/ensemble_submission1.csv", index=False)

"""
Explanation: Let's Update Kaggle Submission
Steps:

Load Test Dataset
Vectorize the Features (Review)
Predict the sentiment
Create the CSV file and update the submission
End of explanation
"""


