# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""




# Convolutional Neural Networks with KERAS
"""
"""
In this practice, we are going to implement two convolutional neural networks to recognize the digits of the MNIST reference data set. In addition, we will also train a simple super-resolution model. Specifically, the following three points will be implemented:

1. A convolutional neural network of one layer
2. A deep convolutional neural network with x layers
3. A super-resolution model

In all three cases, the Keras library will be used for model implementation, compilation, and training. Next we will also use Keras to predict the image classification of the test set.

"""
"""
# 0 Initialization and load data

"""
"""
The following code loads the necessary packages for the practice and also reads the data that we will use to train the neural network. The MNIST Dataset corresponds to images of digits from 0 to 9 of size 28x28 pixels.
"""
import numpy as np
import keras
from keras.datasets import mnist
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, Activation, Dropout, MaxPooling2D
import matplotlib.pyplot as plt

# Download the MNIST dataset and partition train / test
train = pd.read_csv("../input/digit-recognizer/train.csv") 
test = pd.read_csv("../input/digit-recognizer/test.csv") 

x_train_orig = np.array(train.iloc[:, :-1].values)
y_train_orig = np.array(train.iloc[:, 1].values)
x_test_orig = np.array(test.iloc[:, :-1].values)
y_test_orig = np.array(test.iloc[:, 1].values)

# Checking an example
first_image = x_train_orig[0]
first_image = np.array(first_image, dtype='float')
pixels = first_image.reshape((28, 28))
plt.imshow(pixels, cmap='gray')
plt.show()
# Shape
x_train_orig[0].shape
"""
## 1. Convolutional Neural Network with one layer

"""
"""
Next we will implement a convolutional neural network of one layer and we will do the training and test on the MNIST dataset. We have 60,000 images to train and 10,000 to test.
"""
"""
### 1.1 Pre-processing data

The first step in training a neural network is to pre-process the training and test data to match the input of the neural network.

"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">

<strong> Adjust dimensions in images (X input): </strong> Adjust the size of the training and test data using 4 dimensions (the last dimension has to be 1 to indicate that the images are in gray scale). TIP: use the number of training and test data and the size of the images.
</div>
"""
"""
<div style="background-color: #C7FCE5; border-color: #61FEBA; border-left: 5px solid #61FEBA; padding: 0.5em;">
We have 'reshaped' x_train_orig and x_test_orig since our CNN will only accept a four-dimensional vector. Also, we set our tuple consisting of:
     <ul>
         <li> the 60,000 images that we have in training camp, </li>
         <li> 28 the height of the image, </li>
         <li> 28 the width of the image and </li>
         <li> 1 which will be the number of channels. (1 for grayscale and 3 for RGB colors) </li>
     </ul>
</div>
"""
print('Training data shape : ', x_train_orig.shape, y_train_orig.shape)

print('Testing data shape : ', x_test_orig.shape, y_test_orig.shape)

nCols, nDims = 784, 1
input_shape = (nRows, nCols, nDims)
x_train = x_train_orig.reshape(x_train_orig.shape, nCols, nDims, 1)
x_test = x_test_orig.reshape(x_test_orig.shape, nCols, nDims, 1)
print(nRows, nCols, nDims, 1)
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong> Coding the output (Y output): </strong> Coding the values of the output tags into a one-hot vector. For example, the output vector for an image containing a 5 would be: [0., 0., 0., 0., 0., 1., 0., 0., 0., 0.]. TIP: Keras.utils to_categorical function can be used.
</div>
"""
x_train = x_train / 255
x_test = x_test / 255

num_classes = 10
y_train = to_categorical(y_train_orig, num_classes)
y_test = to_categorical(y_test_orig, num_classes)
y_train[5]
"""
### 1.2 Model Creation


We are going to use a Keras Sequential model since it is very easy to use. These types of models allow us to build the model layer by layer. Specific:

- The first layer we will add will be a convolutional layer with the following properties:
     - Number of kernels (neurons) first layer: 64 neurons
     - Kernels size: 3x3
     - Activation of kernels: ReLU
- Next we will add a Flatten layer to connect the output of the convolutional layer with the input of a dense layer.
- Finally, we will add a dense output layer, and therefore it will have as many neurons as we want to predict classes. The activation of this last layer will be Softmax. The final prediction of the model will then be the class with the highest probability.
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Model construction</strong>
</div>
"""
num_classes = 10

model = Sequential()

model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=(28,28,1)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(num_classes, activation='softmax'))
"""
### 1.3 Build the model

Once the model is defined, it must be compiled so that Keras prepares the training. For this we are going to use the ADAM optimization algorithm, the cost function "categorical_crossentropy" and the "accuracy" metric
"""
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
"""
### 1.4 Model training

We now train the model. To do this, we will make the model see each image 9 different times, and we will use the test set to validate the process.
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Training the model</strong> 
</div>
"""
#Entrenamiento del modelo
n_epochs_onelayer = 9
mfit_onelayer = model.fit(x_train, y_train,
                              epochs=n_epochs_onelayer,
                              verbose=1,
                              validation_data=(x_test, y_test))

score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
"""
### 1.5  Accuracy evolution
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
Visualize the evolution of accuracy in the training and test set according to the times.
</div>
"""
# plot del training loss y el accuracy
def plot_prediction(n_epochs, mfit):

    # Plot training & validation loss values
    plt.plot(mfit.history['loss'])
    plt.plot(mfit.history['val_loss'])
    title = 'Model loss (' + str(n_epochs) + ' epochs)'
    plt.title(title)
    plt.ylabel('Loss')
    plt.xlabel('Epoch')
    plt.legend(['Train', 'Test'], loc='upper left')
    plt.show()
    
    return plt
plot_prediction(n_epochs_onelayer, mfit_onelayer)
mfit_onelayer
"""
### 1.6 Prediction

Finally we can make the prediction for four of the images in the test set and see if the results are correct or not
"""
# Hacemos la predicción para las 4 primeras imágenes del set de test
print(model.predict(x_test[:4]))

# Mostramos el ground truth para las primeras 4 imágenes
y_test[:4]
"""
## 2. Deep CNN + Dropout 
"""
"""
In the previous exercise we have implemented a single layer convolutional network. Now we are going to implement a deep convolutional neural network and we will see how this translates into better performance in the results. We will initialize and pre-process the data first.
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Coding the Y output tags</strong> 
</div>
"""
y_train = to_categorical(y_train_orig)
y_test = to_categorical(y_test_orig)
"""
In this case we will use a Keras Sequential model again that will consist of:
- Two convolution layers of 32 and 64 kernels respectively of 3x3 size, and with a reluctance activation function
- A MaxPooling layer with a size of 2x2
- A Dropout layer with a rate = 0.25
- A Flatten layer
- A dense layer with 128 neurons and a brilliant activation function
- A Dropout layer with a rate = 0.5
- A dense layer with softmax activation function
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Development</strong> 
</div>
"""
def creaModeloRedNeuronalProfunda():
    num_classes = 10

    model = Sequential()

    ##TODO: Añadir las capas
    model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=(28,28,1)))
    model.add(Conv2D(64, (3, 3), padding='valid', activation='relu'))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))
    
    return model

"""
We then compile, train and evaluate the model. For compilation we will use the cost function "categorical_crossentropy" and the metric "accuracy" again, but in this case we will use the Adadelta optimizer.

We will train the model for 12 periods, using a batch_size of 128, and the test set to validate. Finally we will also use the test set to evaluate the model.
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Training</strong> 
</div>
"""
batch_size = 128
n_epochs = 12 

deep_model = creaModeloRedNeuronalProfunda()

deep_model.compile(optimizer='adadelta',
                   loss='categorical_crossentropy',
                   metrics=['accuracy'])

deep_model.summary()
mfit = deep_model.fit(x_train, y_train, batch_size=batch_size, epochs=n_epochs, verbose=1, 
                   validation_data=(x_test, y_test))


# Evaluation in test
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])

# Visualizción de la evolución de la métrica accuracy
plot_prediction(n_epochs, mfit)
# Prediction
print(model.predict(x_test[:4]))
y_test[:4]
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Analysis</strong>
</div>
"""
"""
By looking at the graphs of the Accuracy measurement of a '* normal convolutional network *' and a deep '* CNN + Dropout *' we can assure that ** this second ** method (CNN + Dropout) has a higher ** accuracy ** comparing it to the first one. In addition, it greatly minimizes the cost function (** loss function **), obtaining much smaller * loss * values. The two layers *** dropout *** probability ** 0.25 and 0.5 ** contribute to the model ** with much more outstanding results, allowing a noticeable increase in the generalizability of the network ** and consequently better results are obtained. On the other hand, ** CNN + Dropout training is much faster ** for each epoch (epoch).

Increasing the number of intermediate layers can be a possible improvement of the model and an increase in the processing and classification capacity, but we must take into account and adjust the possible over-specialization that this may imply.
"""
"""
## 3. CNN applying for super resolution (encoder-decoder)
"""
"""
In this exercise we will build a convolutional neural network such that, given low resolution images, it allows us to obtain the same images but with a higher resolution. To do this, from the same MNIST dataset we will create low-resolution images with which we will train our model.
"""
"""
First we reduce the resolution of the images:
"""
x_train = x_train_orig[:, ::2, ::2]
x_test = x_test_orig[:, ::2, ::2]
"""
And we normalize the pixel values and adjust the dimensions of the data:
"""
# Normalizamos los valores de los píxeles
x_train = x_train / 255.0
x_test = x_test / 255.0

y_train = x_train_orig / 255.0
y_test = x_test_orig / 255.0

# Ajustamos las dimensiones de los datos
x = np.expand_dims(x_train, axis=3)
y = np.expand_dims(y_train, axis=3)

x_test_final = np.expand_dims(x_test, axis=3)
y_test_final = np.expand_dims(y_test, axis=3)
y_test.shape

y.shape
"""
### 3.1 Model creation and training

"""
"""
Next we will create and train the model with the following characteristics:

- A convolutional layer of 32 kernels of 3x3 size, and with a relu activation function.
- A MaxPooling layer with a size of 2x2.
- Another convolutional layer of 32 kernels of 3x3 size, and with relu activation function.
- A deconvolutionary layer with 32 kernels, size 3x3, stride 2x2, and relu activation function.
- A last deconvolutionary layer with a single kernel of size 3x3, stride 2x2, and activation function sigmoid.

All convolutional and deconvolutionary layers have to have the padding parameter "sames", so that the size of the images is not affected in these layers.
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Model creation</strong>
</div>
"""
def creaModeloRedNeuronalConvolucional():
    num_classes = 10

    model = Sequential()

    # Add layer  
    model.add(Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(14, 14, 1)))    
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
    # deconvolutional layer
    model.add(keras.layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding='same', activation='relu'))
    model.add(keras.layers.Conv2DTranspose(1, (3, 3), strides=(2, 2), padding='same', activation='sigmoid'))
    
    return model
# Model creation
convolutional_network = creaModeloRedNeuronalConvolucional()

convolutional_network.summary()
#Entrenar y compilar el modelo
convolutional_network.compile(loss='binary_crossentropy',
              optimizer='Adadelta',
              metrics=['accuracy'])
n_epochs = 12 
convolutional_network.fit(x, y, epochs=n_epochs, validation_data=(x_test_final, y_test_final))
"""
### 3.2 Predicción de algunas imágenes del conjunto de test
"""
"""
<div style="background-color: #EDF7FF; border-color: #7C9DBF; border-left: 5px solid #7C9DBF; padding: 0.5em;">
<strong>Ejercicio [1 pto.]:</strong> Visualizar tres imágenes al azar del conjunto de test. Mostrar la versión original de la imagen, la versión con resolución reducida, y la predicción del modelo.
</div>
"""
# Predicción de tres imágenes del conjunto de test
n_images = 3
idx_images = np.random.randint(x_test.shape[0], size=n_images)

images = np.expand_dims(np.stack([x_test[i] for i in idx_images]), axis=3)

pred = convolutional_network.predict(images)

# Ajustamos las dimensiones de las imágenes predichas al formato adecuado y desnormalizamos los píxeles
pred = np.squeeze(pred)
pred = pred * 255.0
def drawImage(i, pred, alto, ancho):
  #Visualizamos la imagen i del dataset
  first_image = pred[i]
  first_image = np.array(first_image, dtype='float')
  pixels = first_image.reshape((alto, ancho))
  plt.imshow(pixels, cmap='gray')
  plt.show()
#Visualizamos la primera imagen 
drawImage(0, images, 14, 14)
drawImage(0, pred, 28, 28)

"""
Here we can observe first the reduced image and secondly the image predicted from our neural network model. Next we will do the same with the other 2 remaining examples.
"""
#Visualizamos la segunda imagen 
drawImage(1, images, 14, 14)
drawImage(1, pred, 28, 28)

#Visualizamos la tercera imagen 
drawImage(2, images, 14, 14)
drawImage(2, pred, 28, 28)