import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import pickle

plt.style.use('fivethirtyeight')
# plt.rcParams['font.family'] = 'serif'
plt.rcParams['font.serif'] = 'Helvetica'
plt.rcParams['font.monospace'] = 'Consolas'
plt.rcParams['font.size'] = 16
plt.rcParams['axes.labelsize'] = 16
plt.rcParams['axes.labelweight'] = 'bold'
plt.rcParams['xtick.labelsize'] = 14
plt.rcParams['ytick.labelsize'] = 14
plt.rcParams['legend.fontsize'] = 16
plt.rcParams['figure.titlesize'] = 20
plt.rcParams['lines.linewidth'] = 2

%matplotlib inline

# for auto-reloading external modules
%load_ext autoreload
%autoreload 2

"""
Explanation: Data Exploration
This is a notebook to explore the pickle files saved out by the convert_data.py script. We'll sanity check all the pickle files, by loading in the image files and displaying them with their labels.
End of explanation
"""


# Set up the file directory and names
DIR = '../input/'
X_TRAIN = DIR + 'train-images-idx3-ubyte.pkl'
Y_TRAIN = DIR + 'train-labels-idx1-ubyte.pkl'
X_TEST = DIR + 't10k-images-idx3-ubyte.pkl'
Y_TEST = DIR + 't10k-labels-idx1-ubyte.pkl'

print('Loading pickle files')
X_train = pickle.load( open( X_TRAIN, "rb" ) )
y_train = pickle.load( open( Y_TRAIN, "rb" ) )
X_test = pickle.load( open( X_TEST, "rb" ) )
y_test = pickle.load( open( Y_TEST, "rb" ) )

n_train = X_train.shape[0]
n_test = X_test.shape[0]

print('Train images shape {}, labels shape {}'.format(X_train.shape, y_train.shape))
print('Test images shape {}, labels shape {}'.format(X_test.shape, y_test.shape))

"""
Explanation: Loading pickle files
The convert_data.py converts the ubyte format input files into numpy arrays.  These arrays are then saved out as pickle files to be quickly loaded later on. The shape of the numpy arrays for images and labels are:

Images: (N, rows, cols)
Labels: (N, 1)
End of explanation
"""


# Check a few training values at random as a sanity check
def show_label_images(X, y):
    '''Shows random images in a grid'''
    
    num = 9
    
    images = np.random.randint(0, X.shape[0], num)
    print('Showing training image indexes {}'.format(images))

    fig, axes = plt.subplots(3,3, figsize=(6,6))
    for idx, val in enumerate(images):
        r, c = divmod(idx, 3)
        axes[r][c].imshow(X[images[idx]])
        axes[r][c].annotate('Label: {}'.format(y[val]), xy=(1, 1))
        axes[r][c].xaxis.set_visible(False)
        axes[r][c].yaxis.set_visible(False)
        
show_label_images(X_train, y_train)

"""
Explanation: Sample training images with labels
Let's show a few of the training images with the corresponding labels, so we can sanity check that the labels match the numbers, and the images themselves look like actual digits.
End of explanation
"""


# Now do the same for the training dataset
show_label_images(X_test, y_test)

# # Training label distribution
y_train_df = pd.DataFrame(y_train, columns=['class'])
y_train_df.plot.hist(legend=False)
hist_df = pd.DataFrame(y_train_df['class'].value_counts(normalize=True))
hist_df.index.name = 'class'
hist_df.columns = ['train']

"""
Explanation: Sample test images with labels
Now we can check the test images and labels by picking a few random ones, and making sure the images look reasonable and  they match their labels.
End of explanation
"""


# Test label distribution
y_test_df = pd.DataFrame(y_test, columns=['class'])
y_test_df.plot.hist(legend=False, bins=10)
test_counts = y_test_df['class'].value_counts(normalize=True)
hist_df['test'] = test_counts

"""
Explanation: The class distribution is pretty evenly split between the classes. 1 is the most popular class with 11.24% of instances, and at the other end 5 is the least frequent class, with 9.04% of instances
End of explanation
"""


hist_df['diff'] = np.abs(hist_df['train'] - hist_df['test'])
hist_df.sort_values('diff', ascending=False)['diff'].plot.bar()

"""
Explanation: The distribution looks very similar between training and test datasets.
End of explanation
"""


# Final quick check of datatypes
assert X_train.dtype == np.uint8
assert y_train.dtype == np.uint8
assert X_test.dtype == np.uint8
assert y_test.dtype == np.uint8

"""
Explanation: The largest difference is 0.0040% in the number 2 class.
End of explanation
"""


