import os
import pandas as pd
import numpy as np
import scipy as sp
import seaborn as sns
import matplotlib.pyplot as plt
import json
from IPython.display import Image
from IPython.core.display import HTML

retval=os.chdir("..")

clean_data=pd.read_pickle('./clean_data/clean_data.pkl')

clean_data.head()

kept_cols=['helpful', 'num_sents', 'num_words', 'readability', 'neg_senti',
       'pos_senti', 'neu_senti', 'comp_senti', 'text_lemma',]

"""
Explanation: Model Building Part 3
Code for building the models
Author: Jimmy Charité
Email: jimmy.charite@gmail.com  
Following up with part one and two, I will combine the bag of words and the macro-level text summary statistics approach
End of explanation
"""


my_rand_state=0
test_size=0.25

from sklearn.model_selection import train_test_split

X = (clean_data[kept_cols].iloc[:,1:]).as_matrix()
y = (clean_data[kept_cols].iloc[:,0]).tolist()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, 
                                              random_state=my_rand_state)

"""
Explanation: Training and Testing Split
End of explanation
"""


from sklearn.preprocessing import StandardScaler

std_scale=StandardScaler()

"""
Explanation: Scaling
End of explanation
"""


from sklearn.feature_extraction.text import TfidfVectorizer

#set max_features to minimize training time
#also, I didn't apply LDA-based dimensionality reduction
tfidf=TfidfVectorizer(lowercase=False,max_features=200)

"""
Explanation: Text
End of explanation
"""


from sklearn.base import BaseEstimator, TransformerMixin

class ExtractText(BaseEstimator, TransformerMixin):
    """
    Separates the features by numerical and text
    """
    def __init__(self, text,n_text=-1):
        self.text = text
        self.n_text=n_text

    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        if(self.text):
            return X[:,self.n_text]
        else:
            return X[:,:self.n_text]

from sklearn.pipeline import FeatureUnion

"""
Explanation: Custom Feature Separator
End of explanation
"""


from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier

"""
Explanation: Classification Models
End of explanation
"""


nb_clf=GaussianNB()
priors=[None]

qda_clf=QuadraticDiscriminantAnalysis()
reg_param=[0.0, 0.25, 0.5, 0.75]

log_clf=LogisticRegression(penalty='l2')
C=[0.001 , 0.01, 10, 100,1000]

rf_clf=RandomForestClassifier()
n_estimators=[100,200]
max_features=[.1,.3,.5]

class_weight=['balanced']
class_weight.extend([{1: w} for w in [1, 2, 10]])

"""
Explanation: Although tuning is not necessary for Naive Bayes, I pass the default parameters of those models to GridSearchCV anyway so that I can do a direct pair-wise comparison with the other models across the different steps of cross-validation.  
In the interest of time, I didn't use the SVM classifier.
End of explanation
"""


from imblearn import pipeline #needed if mixing imblearn with sklearn classes
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold

"""
Explanation: Creating Pipelines
End of explanation
"""


n_jobs=4

n_folds=10
skfold = StratifiedKFold(n_splits=n_folds,random_state=my_rand_state, shuffle=False)

class DenseTransformer(BaseEstimator, TransformerMixin):

    def transform(self, X, y=None, **fit_params):
        return X.todense()

    def fit_transform(self, X, y=None, **fit_params):
        self.fit(X, y, **fit_params)
        return self.transform(X)

    def fit(self, X, y=None, **fit_params):
        return self

"""
Explanation: I plan on using imblearn classes for later iterations so I use it's pipeline in the beginning for convenience
End of explanation
"""


ft_union=FeatureUnion(transformer_list=[('text_pipe',pipeline.Pipeline([('extract',ExtractText(text=True)),
                                                                        ('tfidf',tfidf),
                                                                        ('to_dense', DenseTransformer())])),
                                        ('numb_pipe',pipeline.Pipeline([('extract',ExtractText(text=False)),
                                                                        ('scale',std_scale)]))])

"""
Explanation: Main Feature Union
End of explanation
"""


nb_clf_b = pipeline.Pipeline(steps=[('union',ft_union),('clf',nb_clf)])
nb_clf_est_b = GridSearchCV(estimator=nb_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__priors=priors))

"""
Explanation: Naive Bayes Estimators
End of explanation
"""


qda_clf_b = pipeline.Pipeline(steps=[('union',ft_union),('clf',qda_clf)])
qda_clf_est_b = GridSearchCV(estimator=qda_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__reg_param=reg_param))

"""
Explanation: QDA Estimators
End of explanation
"""


log_clf_b = pipeline.Pipeline(steps=[('union',ft_union),('clf',log_clf)])
log_clf_est_b = GridSearchCV(estimator=log_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__C=C,
                              clf__class_weight=class_weight))

"""
Explanation: Logistic Estimators
End of explanation
"""


rf_clf_b = pipeline.Pipeline(steps=[('union',ft_union),('clf',rf_clf)])
rf_clf_est_b = GridSearchCV(estimator=rf_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__n_estimators=n_estimators,
                              clf__max_features=max_features,
                              clf__class_weight=class_weight))

"""
Explanation: Random Forest Estimators
End of explanation
"""


from sklearn.externals import joblib

"""
Explanation: Fitting Estimators
End of explanation
"""


nb_clf_est_b.fit(X_train,y_train)
joblib.dump(nb_clf_est_b, './other_output/merged/nb_clf_est_b.pkl')

qda_clf_est_b.fit(X_train,y_train)
joblib.dump(qda_clf_est_b, './other_output/merged/qda_clf_est_b.pkl')

log_clf_est_b.fit(X_train,y_train)
joblib.dump(log_clf_est_b, './other_output/merged/log_clf_est_b.pkl')

rf_clf_est_b.fit(X_train,y_train)
joblib.dump(rf_clf_est_b, './other_output/merged/rf_clf_est_b.pkl')

"""
Explanation: Basic Estimators: no bag of words or PCA
End of explanation
"""


from sklearn.metrics import roc_curve, auc

nb_clf_est_b=joblib.load('./other_output/merged/nb_clf_est_b.pkl')
qda_clf_est_b=joblib.load('./other_output/merged/qda_clf_est_b.pkl')
log_clf_est_b=joblib.load('./other_output/merged/log_clf_est_b.pkl')
rf_clf_est_b=joblib.load('./other_output/merged/rf_clf_est_b.pkl')

nb_fpr, nb_tpr, _ = roc_curve(y_test, 
                    nb_clf_est_b.predict_proba(X_test)[:,1])
nb_roc_auc = auc(nb_fpr, nb_tpr)

qda_fpr, qda_tpr, _ = roc_curve(y_test, 
                    qda_clf_est_b.predict_proba(X_test)[:,1])
qda_roc_auc = auc(qda_fpr, qda_tpr)

log_fpr, log_tpr, _ = roc_curve(y_test, 
                    log_clf_est_b.predict_proba(X_test)[:,1])
log_roc_auc = auc(log_fpr, log_tpr)

rf_fpr, rf_tpr, _ = roc_curve(y_test, 
                    rf_clf_est_b.predict_proba(X_test)[:,1])
rf_roc_auc = auc(rf_fpr, rf_tpr)

plt.plot(nb_fpr, nb_tpr, color='cyan', linestyle='--',
         label='NB (area = %0.2f)' % nb_roc_auc, lw=2)

plt.plot(qda_fpr, qda_tpr, color='indigo', linestyle='--',
         label='QDA (area = %0.2f)' % qda_roc_auc, lw=2)

plt.plot(log_fpr, log_tpr, color='seagreen', linestyle='--',
         label='LOG (area = %0.2f)' % log_roc_auc, lw=2)

plt.plot(rf_fpr, rf_tpr, color='blue', linestyle='--',
         label='RF (area = %0.2f)' % rf_roc_auc, lw=2)

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='k',
         label='Luck')

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curves of Basic Models Using Bag of Words and Macro-Text Stats')
plt.legend(loc="lower right")
plt.savefig('./plots/ROC_Basic_BOW_MERGED.png', bbox_inches='tight')
plt.show()

"""
Explanation: Testing Estimators
End of explanation
"""


from scipy.stats import sem

len(y_test) #the sample is large enough that we can get away with 5% draws

y_test=np.array(y_test)

X_test=np.array(X_test)

#initialize 
n_bootstraps = 2000
rng_seed = 1
sample_percent=0.05
min_index=0
max_index=len(y_test)-1

draw_size=int(len(y_test)*sample_percent)
bootstrapped_scores = []
rng = np.random.RandomState(rng_seed)

for i in range(n_bootstraps):
    # bootstrap by sampling with replacement on 
    indices = rng.random_integers(min_index, max_index, draw_size)
    
    #calculate ROC from 
    rf_fpr, rf_tpr, _ = roc_curve(y_test[indices], 
                    rf_clf_est_b.predict_proba(X_test[indices,:])[:,1])
    rf_roc_auc = auc(rf_fpr, rf_tpr)
    
    #save
    bootstrapped_scores.append(rf_roc_auc)

import pickle

with open('./other_output/merged/rf_bootstrapped_scores.pkl', 'wb') as f:
    pickle.dump(bootstrapped_scores, f)

plt.hist(bootstrapped_scores, bins=50)
plt.title('Histogram of Bootstrapped AUC ROC of\nRandom Forest Model Using Bag of Words and Macro-Text Stats')
plt.savefig('./plots/ROC_Histogram_Basic_BOW_MERGED.png', bbox_inches='tight')
plt.show()

pd.DataFrame({'auc':bootstrapped_scores}).auc.describe()

"""
Explanation: Closer look at the variability of the best model
End of explanation
"""


with open('./other_output/rf_bootstrapped_scores.pkl', 'rb') as f:
    bootstrapped_scores_macrotext = pickle.load(f)

with open('./other_output/bow/rf_bootstrapped_scores.pkl', 'rb') as f:
    bootstrapped_scores_bow = pickle.load(f)

bs_auc=pd.DataFrame({'Macro-Text Only':bootstrapped_scores_macrotext,
                     'Bag of Words Only':bootstrapped_scores_bow,
                     'Combined':bootstrapped_scores})

bs_auc=bs_auc[['Macro-Text Only','Bag of Words Only','Combined']]

bs_auc.describe()

pl_axes=bs_auc.boxplot(return_type='axes')
x=pl_axes.axes.set_ylabel('AUC ROC')
x=pl_axes.axes.set_title('Comparison of Bootstrapped AUC ROC of Random Forest Model\nUsing Macro-Text Stats, Bag of Words, and Combined Approaches')
plt.savefig('./plots/BoxPlot_ROC_MacText_BOX.png', bbox_inches='tight')
plt.show()

"""
Explanation: Importing bootstrapped scores from parts 1 & 2 for side-by-side comparison
End of explanation
"""


