from __future__ import division
import numpy as np
import matplotlib.pyplot as plt

from operator import itemgetter
from tabulate import tabulate

from sklearn.preprocessing    import StandardScaler
from sklearn.decomposition    import PCA 
from sklearn.neighbors        import KNeighborsClassifier
from sklearn.pipeline         import Pipeline
from sklearn.grid_search      import GridSearchCV
from sklearn.grid_search      import RandomizedSearchCV
from sklearn.cross_validation import StratifiedKFold
from sklearn.cross_validation import ShuffleSplit
from sklearn.metrics          import confusion_matrix, classification_report

import sys, math, time

# private functions
sys.path.append('/home/george/Dropbox/MNIST/src') 
from MNIST_utilities import load_all_MNIST,        \
                            plot_confusion_matrix, \
                            print_imgs,            \
                            plot_learning_curve

%matplotlib inline

#%qtconsole

"""
Explanation: KNN
End of explanation
"""


# load MNIST here
trainX, trainY, testX, testY = \
            load_all_MNIST(portion=1.0)

"""
Explanation: Load the MNIST data
End of explanation
"""


# ... but in this case the test-set predictions were much worse
#scaler = StandardScaler()
#trainX = scaler.fit_transform(trainX)
#testX  = scaler.transform(testX)

"""
Explanation: Note that distance measures can be significantly affected by variable scale
End of explanation
"""


t0 = time.time()

knn   = KNeighborsClassifier(weights='distance', p=2)

pipe  = Pipeline(steps=[ ('knn', knn)])

#Parameters of pipelines can be set using ‘__’ separated parameter names:
search_grid = dict(knn__n_neighbors  = np.arange(1,8+1))

# ----------------------------------------------------------------------------
# you can't randomize if the number of grid points is less than the iterations
n_iter      = 100
grid_points = 1
for value in search_grid.itervalues():
    grid_points *= len(value)
print("Total points in the search grid: {}".format(grid_points))

if grid_points <= n_iter:
    estimator = GridSearchCV(estimator = pipe, param_grid = search_grid,
                             cv        = StratifiedKFold(y = trainY, n_folds = 5),
                             n_jobs=-1, pre_dispatch=10, verbose=1)
    
else:
    estimator = RandomizedSearchCV(estimator = pipe, param_distributions = search_grid, n_iter = n_iter,
                                   cv        = StratifiedKFold(y = trainY, n_folds = 5),
                                   n_jobs=-1, pre_dispatch=10, verbose=1)
# ----------------------------------------------------------------------------


estimator.fit(trainX, trainY)

print("\ntime in minutes {0:.2f}".format((time.time()-t0)/60))

"""
Explanation: Perform a grid search to find the best KNN parameters
End of explanation
"""


# what proportion of parameter combinations
# had an accuracy below 98% (anything 98% or below is not a contender)
# --------------------------------------------------------------------

mean_score_list = [score.mean_validation_score for score in estimator.grid_scores_]
print("\nProportion of scores below 98%: {0:.2f}\n". \
      format(sum(np.array(mean_score_list)<0.98)/len(mean_score_list)))
    

# what do the top 10 parameter combinations look like?
# ----------------------------------------------------

for score in sorted(estimator.grid_scores_, key=itemgetter(1), reverse=True)[:10]:
    print score

"""
Explanation: Analyze the results of the grid search
End of explanation
"""


target_names     = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]

predicted_values = estimator.predict(testX)
y_true, y_pred   = testY, predicted_values

#print(classification_report(y_true, y_pred, target_names=target_names))

cm = confusion_matrix(y_true, y_pred)  

print(cm)
model_accuracy = sum(cm.diagonal())/len(testY)
model_misclass = 1 - model_accuracy
print("\nModel accuracy: {0}, model misclass rate: {1}".format(model_accuracy, model_misclass))

plot_confusion_matrix(cm, target_names)

"""
Explanation: Predict the test data
End of explanation
"""


print_imgs(images           = testX, 
           actual_labels    = y_true, 
           predicted_labels = y_pred,
           starting_index   = np.random.randint(0, high=testY.shape[0]-36, size=1)[0],
           size             = 6)

"""
Explanation: Print a random sample of predictions
End of explanation
"""


t0 = time.time()

parm_list = ""
for i, param in enumerate(estimator.best_params_):
    if i % 3 == 0: parm_list += "\n"
    param_val = estimator.best_params_[param]
    parm_list += param + "=" + str(param_val) + " "


plot_learning_curve(estimator = estimator.best_estimator_, 
                    title     = "KNN" + parm_list, 
                    X         = trainX, 
                    y         = trainY, 
                    ylim      = (0.85, 1.01), 
                    cv        = ShuffleSplit(n            = trainX.shape[0], 
                                             n_iter       = 5, 
                                             test_size    = 0.2, 
                                             random_state = 0), 
                    n_jobs      = -1)

plt.show()

print("\ntime in minutes {0:.2f}".format((time.time()-t0)/60))

"""
Explanation: Learning Curves

do we predict the training data well? (flat red line hugs the 1.0 line)
does the prediction improve with more data? (green line increases from left to right)
End of explanation
"""


