# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
# **Import all Packages**
"""
import numpy as np
import pandas as pd
import tensorflow as tf
import keras
from keras import backend as K
%matplotlib inline
import matplotlib.pyplot as plt
from matplotlib import pyplot
from statistics import mean
import glob
import os
import cv2
import sys
"""
# **Load the Dataset**
"""
# Import CIFAR 10 dataset
from keras.datasets import mnist

# Load Dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()

# Summarize Loaded Dataset
print('Train data: X=%s, y=%s' % (x_train.shape, y_train.shape))
print('Test data: X=%s, y=%s' % (x_test.shape, y_test.shape))

# Plot first nine images
for i in range(9):
	# define subplot
	pyplot.subplot(330 + 1 + i)
	# plot raw pixel data
	pyplot.imshow(x_train[i])
# show the figure
pyplot.show()
y_train
x_train
"""
# **Reshape Dataset to Have a Single Channel**
"""
x_train = x_train.reshape((x_train.shape[0], 28, 28, 1))
x_test = x_test.reshape((x_test.shape[0], 28, 28, 1))

# Check again Summarize & Shape Dataset
print('Check shape Training Data: X=%s, y=%s' % (x_train.shape, y_train.shape))
print('Check shape Testing Data: X=%s, y=%s' % (x_test.shape, y_test.shape))
"""
# **Normalize Input**
"""
#x_train2 = x_train/255
#x_test2 = x_test/255
x_train2 = x_train/255
x_test2 = x_test/255

# Convert class labels to one-hot encoded
y_train2 = keras.utils.to_categorical(y_train)
y_test2 = keras.utils.to_categorical(y_test)

# Print labels
y_train2
"""
# **Import Necessary CNN Building Blocks**
"""
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation, Dropout
from keras.layers.advanced_activations import LeakyReLU
"""
# **Define/Create CNN Architecture**
"""
# Create a model function
def make_model():
    model = Sequential()

    model.add(Conv2D(filters = 16, kernel_size = (3, 3), padding='same', input_shape=(28, 28, 1)))
    model.add(LeakyReLU(0.1))
    model.add(Conv2D(filters = 32, kernel_size = (3, 3), padding='same'))
    model.add(LeakyReLU(0.1))
    
    model.add(MaxPooling2D((2, 2)))
    model.add(Dropout(0.25))
    
    model.add(Conv2D(filters = 32, kernel_size = (3, 3), padding='same'))
    model.add(LeakyReLU(0.1))
    model.add(Conv2D(filters = 64, kernel_size = (3, 3), padding='same'))
    model.add(LeakyReLU(0.1))

    model.add(MaxPooling2D((2, 2)))
    model.add(Dropout(0.25))
    
    model.add(Flatten())
    
    model.add(Dense(256))
    model.add(LeakyReLU(0.1))

    model.add(Dropout(0.5))
    
    model.add(Dense(10))
    model.add(Activation("softmax"))
    
    return model
# Summarize a model
model = make_model()
model.summary()
"""
# **Define the loss function, the optimizer and the metrics**
"""
INIT_LR = 1e-2 # Learning rate, is default schedule in all Keras Optimizers
BATCH_SIZE = 32 # Batch size is a hyperparameter of gradient descent that controls the number of training samples
EPOCHS = 80 # Number of epochs

# Create object of deep learning model
model = make_model()

# Define the loss function, the optimizer and the metrics
model.compile(
    loss='categorical_crossentropy',  
    optimizer=keras.optimizers.Adamax(lr=INIT_LR), 
    metrics=['accuracy']  # report accuracy during training
)

# Scheduler of learning rate (decay with epochs)
def lr_scheduler(epoch):
    return INIT_LR * 0.9 ** epoch

# Callback for printing of actual learning rate used by optimizer
class LrHistory(keras.callbacks.Callback):
    def on_epoch_begin(self, epoch, logs={}):
        print("Learning rate:", K.get_value(model.optimizer.lr))
"""
# **Fit the Deep Learning Mode**
"""
import tensorflow_addons as tfa
tqdm_callback = tfa.callbacks.TQDMProgressBar()

model.fit(
    x_train2, y_train2,  # Prepared data
    batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    callbacks=[keras.callbacks.LearningRateScheduler(lr_scheduler), 
               LrHistory(), 
               tqdm_callback],
    shuffle=True,
    verbose=0,
    initial_epoch=0
)
"""
# **Predict New Samples**
"""
# Predict using testing data without labels/classes
y_pred_test = model.predict(x_test2)
y_pred_test_classes = np.argmax(y_pred_test, axis=1) # Change to normal classes
y_pred_test_classes
# Create the same format for actual classes
y_actual_test_classes = np.argmax(y_test2, axis=1) # Change to normal classes
y_actual_test_classes
##########################################################
# MULTI-CLASS CONFUSION MATRIX FOR EACH CLASS
##########################################################

from sklearn.metrics import multilabel_confusion_matrix
import math

# Actual and predicted classes
lst_actual_class = y_actual_test_classes
lst_predicted_class = y_pred_test_classes


# Class = digit labels 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
lst_classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # Must in order

# Compute multi-class confusion matrix
arr_out_matrix = multilabel_confusion_matrix(lst_actual_class, lst_predicted_class, labels=lst_classes)

# Temp store results
store_sens = [];
store_spec = [];
store_acc = [];
store_bal_acc = [];
store_prec = [];
store_fscore = [];
store_mcc = [];

# Loop for each taget label
for no_class in range(len(lst_classes)):
    arr_data = arr_out_matrix[no_class];
    print("Predicted Performance of Digit Label/Class: {0}".format(no_class));

    tp = arr_data[1][1]
    fp = arr_data[0][1]
    tn = arr_data[0][0]
    fn = arr_data[1][0]

    sensitivity = round(tp/(tp+fn), 3);
    specificity = round(tn/(tn+fp), 3);
    accuracy = round((tp+tn)/(tp+fp+tn+fn), 3);
    balanced_accuracy = round((sensitivity+specificity)/2, 3);
    precision = round(tp/(tp + fp), 3)
    fscore = round((2 * ((precision * sensitivity) / (precision + sensitivity))), 3)
    mcc = round((((tp * tn)-(fp * fn))/ math.sqrt((tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))),3)

    store_sens.append(sensitivity);
    store_spec.append(specificity);
    store_acc.append(accuracy);
    store_bal_acc.append(balanced_accuracy);
    store_prec.append(precision);
    store_fscore.append(fscore);
    store_mcc.append(mcc);
    
    print("TP={0}, FP={1}, TN={2}, FN={3}".format(tp, fp, tn, fn));
    print("Sensitivity: {0}".format(sensitivity));
    print("Specificity: {0}".format(specificity));
    print("Accuracy: {0}".format(accuracy));
    print("Balanced Accuracy: {0}".format(balanced_accuracy));
    print("Precision: {0}".format(precision));
    print("F1-Score: {0}".format(fscore))
    print("MCC: {0}\n".format(mcc))
##########################################################
# OVERALL - FINAL PERFORMANCE PREDICTION 
##########################################################

print("Overall Performance Prediction:");
print("Sensitivity: {0}%".format(round(mean(store_sens)*100, 4)));
print("Specificity: {0}%".format(round(mean(store_spec)*100, 4)));
print("Accuracy: {0}%".format(round(mean(store_acc)*100, 4)));
print("Balanced Accuracy: {0}%".format(round(mean(store_bal_acc)*100, 4)));
print("Precision: {0}%".format(round(mean(store_prec)*100, 4)));
print("F1-Score: {0}%".format(round(mean(store_fscore)*100, 4)))
print("MCC: {0}\n".format(round(mean(store_mcc), 4)))