%matplotlib inline

import seaborn as sns
import numpy as np
import pandas as pd

from keras.models import Sequential # 1 neural network pls = 1 model
from keras.layers.core import Dense, Activation, Flatten, Dropout
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils
from keras.preprocessing import text

from sklearn.cross_validation import train_test_split

"""
Explanation: Imports
End of explanation
"""


img_rows, img_cols = 28, 28
in_shape = (img_rows, img_cols, 1)
batch_size = 256
nb_classes = 10
nb_epoch = 3

"""
Explanation: Define Constants
img_rows, img_cols: Input image dimensions in pixels
in_shape: For keras (tensorflow), the dimensions of our input for a 2D convolutional layer
batch_size: Number of images used in each minibatch
nb_classes: One class per digit
nb_epoch: Number of times the whole data is used to learn
End of explanation
"""


train = pd.read_csv('train.csv').values
test  = pd.read_csv('test.csv').values

"""
Explanation: Data
Here we are given a set of training and testing data. We will use the training data to train the neural network, and the testing data as submission to kaggle
End of explanation
"""


X_train = train[:, 1:].reshape(train.shape[0], img_rows, img_cols, 1)

X_train.shape

"""
Explanation: Training data
Inputs (X)
First we need reshape our data to be used by a Tensorflow CNN. This follows the format: (nb_of_samples, img_width, img_heigh, nb_of_color_channels)
End of explanation
"""


X_train = X_train.astype('float32')
X_train /= 255

"""
Explanation: Next we need to make sure everything is a float. Then divide it by 255 to normalize the values between [0;1] rather than [0;255]
End of explanation
"""


Y_train = y_train = train[:, 0]

Y_train[10]

Y_train = np_utils.to_categorical(y_train, nb_classes)

Y_train[10]

"""
Explanation: Outputs (Y)
For the outputs, we need to get the zero'th column of train. Next we will use np_utils to one-hot-encode out vector into a binary class matrix.
[image: 28,28 ] = [0,0,0,1,0,0,0,0,0,0]
End of explanation
"""


X_test = test.reshape(test.shape[0], img_rows, img_cols, 1)
X_test = X_test.astype('float32')
X_test /= 255

"""
Explanation: Testing data
Inputs (X)
For the testing data, we are not given outputs. This means that we only need to follow the reshape procedure of the training inputs (X)
End of explanation
"""


print("X_train shape: {}".format(X_train.shape))
print("Y_train shape: {}".format(Y_train.shape))
print("X_test shape : {}".format(X_test.shape))

"""
Explanation: Check shapes of all training and testing data.
End of explanation
"""


sns.heatmap(X_train[3].reshape(28,28), cmap='gray')

"""
Explanation: Visualize data
Below we are going to visualize on of the inputs (X) for our training data to make sure it looks correct.
End of explanation
"""


model = Sequential()

"""
Explanation: Model
Next we are going to define our convolutional neural network, we are going to use three convolution layers and two fully connected layers
Create sequential model object
End of explanation
"""


model.add(Convolution2D(16, 3, 3, border_mode='same', input_shape=in_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.summary()

model.add(Convolution2D(32, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.summary()

model.add(Convolution2D(32, 6, 4, border_mode='same', activation='relu'))

model.summary()

"""
Explanation: Add convolution layers
End of explanation
"""


model.add(Flatten())  # this converts our 2D feature maps to 1D feature vectors
model.summary()

model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(0.5)) # helps prevent overfitting
model.summary()

model.add(Dense(nb_classes)) # nb_classes should be 10 for mnist
model.add(Activation('softmax'))

"""
Explanation: Add fully connected layers
End of explanation
"""


model.summary()

"""
Explanation: The .summary() allows us to peak inside our model. This allows you to see the overall structure of your model
End of explanation
"""


model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=["accuracy"])

"""
Explanation: Compile model
This step allows you to tell the model what optimizer you want (the thing that controls learning rate), what type of loss to use, and all other sorts of things
End of explanation
"""


model.fit(X_train, Y_train, # specify training data
          batch_size=batch_size, # use this many images per mini-batch - memory dependent - 256
          nb_epoch=nb_epoch, # go through my training data this number of times - 3
          validation_split=.2, # use 20% of the training data as validation data
          verbose=True # please print things 
         )

"""
Explanation: Train model!
loss: the loss of the training data
acc: the accuracy of the training data
val_loss: the loss of the validation set
val_acc: the accuracy of the validation data
End of explanation
"""


Y_pred = model.predict(X_test)

Y_pred[0]

model.save_weights('mnist_weights.h5')

model.load_weights('mnist_weights.h5')

"""
Explanation: Predict test data
End of explanation
"""


