"""
# <span style= color:Aquamarine>Introduction
While the MNIST Dataset , has many good notebooks , this notebooks seems to explain some concepts which might be useful for beginners . Have a look!!
"""

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
%matplotlib inline 
import matplotlib as mpl 
import matplotlib.pyplot as plt 
import seaborn as sns
sns.set_style("darkgrid")
mpl.rc("axes",labelsize=16)
mpl.rc("xtick",labelsize=14)
mpl.rc("ytick",labelsize=14)

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))


train=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
test=pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
train.info()
test.info()
train.head()
train_label=train.label
train_data=train.drop("label",axis=1)
print(train_label)
print(train_data)
test.head()
from xgboost import XGBClassifier
xgb=XGBClassifier()
smldata=train_data[0:200]
smllabel=train_label[0:200]
smldata2=train_data[200:400]
smllabel2=train_label[200:400]
xgb.fit(smldata,smllabel)
initial_predictions=xgb.predict(smldata2)
print(initial_predictions[0:5])
print(train_label[200:205])
from sklearn.metrics import mean_absolute_error
init_diff=mean_absolute_error(smllabel,initial_predictions)
print(np.sqrt(init_diff))
train_all=train_data.to_numpy()
some_digit= train_data.loc[36000]
some_digit_image=some_digit.values.reshape(28,28)
plt.imshow(some_digit_image,cmap='Greys_r',interpolation="nearest")
plt.axis("off")
plt.show()
train_data.shape
"""
# <span style= color:Green>Plotting Digits 
"""
def plot_digit(data):
    image=data.values.reshape(28,28)
    plt.imshow(image,cmap='Greys_r',interpolation='nearest')
    plt.axis("off")
    
plot_digit(train_data.loc[1])
print(type(train_data.loc[1]))
def plot_digits(instances, images_per_row=10, **options):
    flx=instances.to_numpy()
    size = 28
    images_per_row = min(len(instances), images_per_row)
    images = [instance.reshape(size,size) for instance in flx]
    n_rows = (len(instances) - 1) // images_per_row + 1
    row_images = []
    n_empty = n_rows * images_per_row - len(instances)
    images.append(np.zeros((size, size * n_empty)))
    for row in range(n_rows):
        rimages = images[row * images_per_row : (row + 1) * images_per_row]
        row_images.append(np.concatenate(rimages, axis=1))
    image = np.concatenate(row_images, axis=0)
    plt.imshow(image, cmap = "Greys_r", **options)
    plt.axis("off")
plt.figure(figsize=(10,10))
plot_digits(train_data[5:69],images_per_row=10)
shuffle_index=np.random.permutation(42000)
train_shuffle,label_shuffle=train_data.loc[shuffle_index],train_label.loc[shuffle_index]
train_5=(label_shuffle==5)
"""
## <span style=color:Purple>Using SGD Classifier
"""
from sklearn.linear_model import SGDClassifier
sgd_clf=SGDClassifier(max_iter=5,tol=-np.infty,random_state=17)
sgd_clf.fit(train_shuffle,train_5)
sgd_clf.predict([some_digit])
from sklearn.model_selection import cross_val_score
cross_val_score(sgd_clf,train_shuffle,train_5,cv=3,scoring="accuracy")
from sklearn.model_selection import StratifiedKFold
from sklearn.base import clone
skfolds=StratifiedKFold(n_splits=3)

for train_index,test_index in skfolds.split(train_shuffle, train_5):
    clone_clf=clone(sgd_clf)
    train_shuffle_folds=train_shuffle.loc[train_index]
    train_5_folds=train_5.loc[train_index]
    Test_shuffle_folds=train_shuffle.loc[test_index]
    Test_5_folds=train_5.loc[test_index]
    
    clone_clf.fit(train_shuffle_folds,train_5_folds)
    pred=clone_clf.predict(Test_shuffle_folds)
    n_correct=sum(pred==Test_5_folds)
    print(n_correct/len(pred))
from sklearn.base import BaseEstimator 
class Never5Estimator(BaseEstimator):
    def fit(self,X,y=None):
        pass
    def predict(self,X):
        return np.zeros((len(X),1),dtype=bool)

never_5_clf=Never5Estimator()
cross_val_score(never_5_clf,train_shuffle,train_5,cv=3,scoring="accuracy")
from sklearn.model_selection import cross_val_predict
train_pred=cross_val_predict(sgd_clf,train_shuffle,train_5,cv=3)
"""
# <span style= color:skyblue >Confusion Matrix

"""
from sklearn.metrics import confusion_matrix
confusion_matrix(train_5,train_pred)
from sklearn.metrics import precision_score, recall_score
precision_score(train_5,train_pred)
2809/(2809+712)
recall_score(train_5,train_pred)
2809/(2809+986)
from sklearn.metrics import f1_score
f1_score(train_5,train_pred)
scores=sgd_clf.decision_function([some_digit])
scores
threshold=0
some_digit_pred=(scores >threshold)
some_digit_pred
threshold=140000
some_digit_pred=(scores >threshold)
some_digit_pred
scores=cross_val_predict(sgd_clf,train_shuffle,train_5,cv=3,method='decision_function')
scores.shape
"""
# <span style=color:Coral>Precision Recall Curve
Precision-Recall is a useful measure of success of prediction when the classes are very imbalanced. In information retrieval, precision is a measure of result relevancy, while recall is a measure of how many truly relevant results are returned.

The precision-recall curve shows the tradeoff between precision and recall for different threshold. A high area under the curve represents both high recall and high precision, where high precision relates to a low false positive rate, and high recall relates to a low false negative rate. High scores for both show that the classifier is returning accurate results (high precision), as well as returning a majority of all positive results (high recall).

A system with high recall but low precision returns many results, but most of its predicted labels are incorrect when compared to the training labels. A system with high precision but low recall is just the opposite, returning very few results, but most of its predicted labels are correct when compared to the training labels. An ideal system with high precision and high recall will return many results, with all results labeled correctly.

![image.png](attachment:image.png)
"""
from sklearn.metrics import precision_recall_curve
precisions, recalls , thresholds =precision_recall_curve(train_5,scores)
print(precisions)
print(thresholds)
def plot_precision_recall_vs_thresholds(precisions,recalls,thresholds):
    plt.plot(thresholds,precisions[:-1],"b--",label="Precision",linewidth=2)
    plt.plot(thresholds,recalls[:-1],"g-",label="Recall",linewidth=2)
    plt.xlabel("Threshold",fontsize=16)
    plt.legend(loc="upper right",fontsize=16)
    plt.ylim([0,1])
    
plt.figure(figsize=(14,8))
plot_precision_recall_vs_thresholds(precisions,recalls,thresholds)
plt.xlim([-800000,700000])
plt.show()
(train_pred==(scores >0)).all()
train_pred_90=(scores > 170000)
precision_score(train_5,train_pred_90)
recall_score(train_5,train_pred_90)
"""
# <span style= color:Thistle>Precision vs Recall Plot
"""
def plot_precision_vs_recall(precisions,recalls):
    plt.plot(recalls,precisions,"b-",linewidth=2)
    plt.xlabel('Recall',fontsize=14)
    plt.ylabel('Precision',fontsize=14)
    plt.axis([0,1,0,1])
    
plt.figure(figsize=(14,8))
plot_precision_vs_recall(precisions,recalls)
plt.title("Precision vs Recall Plot")
plt.show()
from sklearn.metrics import roc_curve
fpr,tpr, thresholds=roc_curve(train_5,scores)

"""
# <span style= color:Maroon>Recieving Operator Characterstic Curve 
AUC - ROC curve is a performance measurement for classification problem at various thresholds settings. ROC is a probability curve and AUC represents degree or measure of separability. It tells how much model is capable of distinguishing between classes. Higher the AUC, better the model is at predicting 0s as 0s and 1s as 1s. By analogy, Higher the AUC, better the model is at distinguishing between patients with disease and no disease.
The ROC curve is plotted with TPR against the FPR where TPR is on y-axis and FPR is on the x-axis.
![image.png](attachment:image.png)
![image.png](attachment:image.png)
![image.png](attachment:image.png)
"""
def plot_roc_curve(fpr,tpr,label=None):
    plt.plot(fpr,tpr,linewidth=2,label=label)
    plt.plot([0,1],[0,1],'r--')
    plt.axis([0,1,0,1])
    plt.xlabel('False Positive Rate',fontsize=16)
    plt.ylabel('True Postive Rate',fontsize=16)
    
plt.figure(figsize=(14,8))
plot_roc_curve(fpr,tpr)
plt.title('Recieving Operator Characterstic Curve')
plt.show()
from sklearn.metrics import roc_auc_score
roc_auc_score(train_5,scores)
from xgboost import XGBClassifier
xgb_clf=XGBClassifier(n_estimators=10,random_state=17)
proba_xgb=cross_val_predict(xgb_clf,train_shuffle,train_5,cv=3,method="predict_proba")
scores_xgb=proba_xgb[:,1]
fpr_xgb,tpr_xgb,thresholds_xgb=roc_curve(train_5,scores_xgb)
plt.figure(figsize=(14,8))
plt.plot(fpr,tpr,"b:",linewidth=2,label="SGD")
plot_roc_curve(fpr_xgb,tpr_xgb,"XGB")
plt.legend(loc="lower right",fontsize=16)
plt.title('Recieving Operator Characterstic Curve')
plt.show()
roc_auc_score(train_5,scores_xgb)
xgb_pred=cross_val_predict(xgb_clf,train_shuffle,train_5,cv=3)
precision_score(train_5,xgb_pred)
recall_score(train_5,xgb_pred)
from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
train_scaled=scaler.fit_transform(train_data)
test_scaled=scaler.fit_transform(test)
sgd_clf.fit(train_data,train_label)
sgd_clf.predict([some_digit])
some_digit_scores=sgd_clf.decision_function([some_digit])
some_digit_scores
np.argmax(some_digit_scores)
sgd_clf.classes_[1]
xgb_clf.fit(train_all,train_label)
some_digit=train_all[15030]
xgb_clf.predict(train_all[34:36])
cross_val_score(sgd_clf,train_data, train_label,cv=3,scoring="accuracy")
from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
train_scaled=scaler.fit_transform(train_data.astype(np.float64))
cross_val_score(sgd_clf,train_scaled,train_label,cv=3,scoring="accuracy")
predictions=cross_val_predict(sgd_clf,train_scaled,train_label,cv=3)
conf_mx=confusion_matrix(train_label,predictions)  
"""
# <span style= color:Navy>Confusion Matrix Plot

"""
def plot_confusion_matrix(matrix):
    fig=plt.figure(figsize=(8,8))
    ax=fig.add_subplot(111)
    cax=ax.matshow(matrix)
    fig.clorbar(cax)
plt.matshow(conf_mx,cmap="Greens_r")
plt.show()
row_sums=conf_mx.sum(axis=1,keepdims=True)
norm_conf_mx=conf_mx/row_sums
np.fill_diagonal(norm_conf_mx,0)
plt.matshow(norm_conf_mx,cmap="Greens_r")
plt.show()
cl_a,cl_b=3,5
X_aa=train_data[(train_label==cl_a) & (predictions == cl_a)]
X_ab=train_data[(train_label==cl_a) & (predictions == cl_b)]
X_ba=train_data[(train_label==cl_b) & (predictions == cl_a)]
X_bb=train_data[(train_label==cl_b) & (predictions == cl_b)]

plt.figure(figsize=(8,8))
plt.subplot(221); plot_digits(X_aa[:25], images_per_row=5)
plt.subplot(222); plot_digits(X_ab[:25], images_per_row=5)
plt.subplot(223); plot_digits(X_ba[:25], images_per_row=5)
plt.subplot(224); plot_digits(X_bb[:25], images_per_row=5)
plt.show()
"""
## <span style= color:Aquamarine>Using KNN Classifier 
"""
from sklearn.neighbors import KNeighborsClassifier

train_large=(train_label >=7)
train_odd=(train_label % 2==1)
train_multilabel=np.c_[train_large,train_odd]

knn_clf=KNeighborsClassifier()
knn_clf.fit(train_data,train_multilabel)
knn_clf.predict([some_digit])
"""
# <span style= color:Orange>Adding Noise to Images 
"""
noise = np.random.randint(0, 100, (len(train), 784))
X_train_mod = train_all + noise
noise = np.random.randint(0, 100, (len(test), 784))
test_all=test.to_numpy()
X_test_mod = test + noise
y_train_mod = train
y_test_mod = test
X_test_mod.info()
some_index=5500
plt.subplot(121); plot_digit(X_test_mod.loc[some_index])
plt.subplot(122); plot_digit(y_test_mod.loc[some_index])
plt.show()