clf = sklearn.linear_model.SGDClassifier(loss='log', penalty='elasticnet')

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, 0.5*settings.class_priors + 0.5*clf.predict_proba(X_test))))

settings.class_priors.shape

clf.predict_proba(X_test).shape

"""
Explanation: Stochastic Gradient Descent Classifier
End of explanation
"""


clf = sklearn.linear_model.SGDClassifier(loss='log', penalty='elasticnet', n_iter=100)

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, 0.5*settings.class_priors + 0.5*clf.predict_proba(X_test))))

"""
Explanation: Maybe we can do better if we run for longer
End of explanation
"""


clf = sklearn.linear_model.SGDClassifier(loss='log', penalty='elasticnet', n_iter=1000)

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, 0.5*settings.class_priors + 0.5*clf.predict_proba(X_test))))

"""
Explanation: That was worse!
How about even longer?
End of explanation
"""


import sklearn.ensemble

clf = sklearn.ensemble.RandomForestClassifier(n_estimators=1000, max_depth=20, min_samples_leaf=5)

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

"""
Explanation: Okay, forget trying to do gradient descent.
Random forest
End of explanation
"""


np.unique(y_test)

np.unique(y_test)

CM = sklearn.metrics.confusion_matrix(y_test, clf.predict(X_test), labels=np.unique(y))

plt.matshow(CM)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

holoviews.Image(CM)

CM.sum(0)

CM.sum(1)

CM1 = (CM+0.0) / CM.sum(1)

plt.matshow(CM1)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

def confusion_matrix_from_proba(y_true, y_pred, labels=None):
    y_true = np.array(y_true)
    if labels is None:
        labels = np.union1d(y_true,np.arange(y_pred.shape[1]))
    n_classes = len(labels)
    M = np.zeros((n_classes,n_classes))
    for i in range(n_classes):
        li = (y_true == i)
        M[i,:] = np.mean(y_pred[li,:],0)
    return M

CM2 = confusion_matrix_from_proba(y_test, clf.predict_proba(X_test), labels=np.unique(y))

CM2.shape

plt.matshow(CM2)
plt.title('Confusion matrix v2')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

"""
Explanation: Plot confusion matrix
End of explanation
"""


import scipy.cluster.hierarchy

Y = scipy.cluster.hierarchy.distance.pdist(CM, metric='euclidean')
Z = scipy.cluster.hierarchy.linkage(Y, method='single')
ax = scipy.cluster.hierarchy.dendrogram(Z, show_contracted=True)

Y = scipy.cluster.hierarchy.distance.pdist(CM1, metric='euclidean')
Z = scipy.cluster.hierarchy.linkage(Y, method='single')
ax = scipy.cluster.hierarchy.dendrogram(Z, show_contracted=True)

Y = scipy.cluster.hierarchy.distance.pdist(CM2, metric='euclidean')
Z = scipy.cluster.hierarchy.linkage(Y, method='single')
ax = scipy.cluster.hierarchy.dendrogram(Z, show_contracted=True)

"""
Explanation: Determine hierarchy
End of explanation
"""


clf = sklearn.linear_model.LogisticRegression(random_state=42)

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

"""
Explanation: Logistic Regression
End of explanation
"""


t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(
    sklearn.preprocessing.StandardScaler().fit_transform(XF.squeeze(0)), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

"""
Explanation: Forgot to scale, and that is important for Logistic Regression!
End of explanation
"""


import sklearn.naive_bayes

clf = sklearn.naive_bayes.GaussianNB()

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

"""
Explanation: Now we're about as good as the Random Forest!
Naive Bayes
End of explanation
"""


t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(
    sklearn.preprocessing.StandardScaler().fit_transform(XF.squeeze(0)), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

"""
Explanation: Well that sucked. Maybe better with a Z-score?
End of explanation
"""


clf = sklearn.svm.LinearSVC()

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(
    sklearn.preprocessing.StandardScaler().fit_transform(XF.squeeze(0)), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

"""
Explanation: Nope. Naive Bayes just plain sucks.
Support Vector Classifier
Linear SVC
End of explanation
"""


cv = sklearn.cross_validation.StratifiedShuffleSplit(y)

clf = sklearn.linear_model.LogisticRegression(random_state=42)

print('Cross-validating')
results = []
for train, test in cv:
    sc = sklearn.preprocessing.StandardScaler()
    sc.fit(XF.squeeze(0)[train,:])
    clf.fit(sc.transform(XF.squeeze(0)[train,:]), np.array(y)[train])
    res = sklearn.metrics.log_loss(np.array(y)[test], clf.predict_proba(sc.transform(XF.squeeze(0)[test,:])))
    print(res)
    results.append(res)

t0 = time.time()
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(XF.squeeze(0), y, test_size=0.5, random_state=42)
clf.fit(X_train, y_train)

t1 = time.time()
total = t1-t0
print("Time={}".format(total))

print("Accuracy={}".format(clf.score(X_test, y_test)))
print("Logloss={}".format(sklearn.metrics.log_loss(y_test, clf.predict_proba(X_test))))

sklearn.grid_search.GridSearchCV(estimator, param_grid, scoring=None)

"""
Explanation: No predict probabilities opiton with SVC 
Results so far
We have very similar results for LogisticRegression and the RandomForest. However, LogisticRegression could still be optimised with a grid search.
Let's do proper CV
End of explanation
"""


