IMAGE_SIZE = (180,202) # The dimensions to which all images found will be resized.
BATCH_SIZE = 16
NUMBER_EPOCHS = 8

TENSORBOARD_DIRECTORY = "../logs/simple_model/tensorboard"
TRAIN_DIRECTORY = "../data/train/"
VALID_DIRECTORY = "../data/valid/"
WEIGHTS_DIRECTORY = "../weights/"
TEST_DIRECTORY = "../data/test/"

NUMBER_TRAIN_SAMPLES = 20000
NUMBER_VALIDATION_SAMPLES = 5000
NUMBER_TEST_SAMPLES = 2500

PRECOMPUTED_DIRECTORY = "../precomputed/vgg16/"

"""
Explanation: VGG16 pre-trained model
In order to explore tranfer learning we are going to use a VGG16 model pre-trained with the ImageNet dataset. The model we are going to use are the ones provided by keras. We are not going to used the pre-trained top layers, instead we are going to define our small Fully Connected Network. This FCN is going to be trained to try to use the features calculated by the pre-trained layers with the new dataset of "cats and dogs"
Common configuration
End of explanation
"""


from tensorflow.python.client import device_lib
def get_available_gpus():
    local_device_protos = device_lib.list_local_devices()
    return [x.name for x in local_device_protos if x.device_type == 'GPU']
    
get_available_gpus()

import tensorflow as tf
# Creates a graph.
with tf.device('/gpu:0'):
  a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')
  b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')
  c = tf.matmul(a, b)
# Creates a session with log_device_placement set to True.
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
# Runs the op.
print(sess.run(c))

"""
Explanation: Check that we are using the GPU:
End of explanation
"""


from keras.applications.vgg16 import VGG16

# create the base pre-trained model
base_model = VGG16(weights='imagenet', include_top=False)

"""
Explanation: Model
Model definition
End of explanation
"""


base_model.summary()

"""
Explanation: Base Model arquitecture
We have the following model arquitecture:
End of explanation
"""


from keras.layers import Dense, Dropout, GlobalAveragePooling2D

# add a global spatial average pooling layer
x = base_model.output

x = GlobalAveragePooling2D()(x)

# let's add a fully-connected layer
x = Dense(64, activation='relu')(x)
x = Dropout(0.3)(x)

# and a logistic layer 
predictions = Dense(2, activation='softmax')(x)

from keras.models import Model
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)

"""
Explanation: Complete model with FCN Classifier on top
End of explanation
"""


TRAINABLE_LAST_LAYERS = 0

assert TRAINABLE_LAST_LAYERS >= 0

# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
if TRAINABLE_LAST_LAYERS == 0:
    for layer in base_model.layers:
        layer.trainable = False
    print(len(base_model.layers))
else:
    for layer in base_model.layers[:-TRAINABLE_LAST_LAYERS]:
        layer.trainable = False
    print(len(base_model.layers[:-TRAINABLE_LAST_LAYERS]))

model.summary()

import pandas as pd
df = pd.DataFrame(([layer.name, layer.trainable] for layer in model.layers), columns=['layer', 'trainable'])
df

"""
Explanation: Set the non trainable layers
We are going to set all the vgg16 layers as non trainables:
End of explanation
"""


from keras.callbacks import EarlyStopping
from keras.callbacks import TensorBoard

# Early stop in case of getting worse
early_stop = EarlyStopping(monitor = 'val_loss', patience = 3, verbose = 0)

#TensorBoard
# run tensorboard with tensorboard --logdir=/full_path_to_your_logs
#tensorboard_path = TENSORBOARD_DIRECTORY
#tensorboard_logger = TensorBoard(log_dir=tensorboard_path, histogram_freq=0, write_graph=False, write_images=False)
#print('Logging basic info to be used by TensorBoard to {}. To see this log run:'.format(tensorboard_path))
#print('tensorboard --logdir={}'.format(tensorboard_path))

callbacks = [early_stop]#, tensorboard_logger]

"""
Explanation: Training the top layer
Keras callbacks
End of explanation
"""


OPTIMIZER_LEARNING_RATE = 1e-2
OPTIMIZER_DECAY = 1e-4
OPTIMIZER_MOMENTUM = 0.89
OPTIMIZER_NESTEROV_ENABLED = False

from keras.optimizers import SGD

optimizer = SGD(lr=OPTIMIZER_LEARNING_RATE, 
          decay=OPTIMIZER_DECAY, 
          momentum=OPTIMIZER_MOMENTUM, 
          nesterov=OPTIMIZER_NESTEROV_ENABLED)

"""
Explanation: Model optimizer
End of explanation
"""


model.compile(loss='categorical_crossentropy', 
              optimizer=optimizer, 
              metrics=["accuracy"])

"""
Explanation: Model compilation
End of explanation
"""


from keras.preprocessing.image import ImageDataGenerator

## train generator with shuffle but no data augmentation
train_datagen = ImageDataGenerator(rescale = 1./255,
                                   shear_range=0.2,
                                   zoom_range=0.2,
                                   horizontal_flip=True)

train_batch_generator =  train_datagen.flow_from_directory(TRAIN_DIRECTORY, 
                                                 target_size = IMAGE_SIZE,
                                                 class_mode = 'categorical', 
                                                 batch_size = BATCH_SIZE)

"""
Explanation: Model Training
Train data generator
End of explanation
"""


from keras.preprocessing.image import ImageDataGenerator

## train generator with shuffle but no data augmentation
validation_datagen = ImageDataGenerator(rescale = 1./255)

valid_batch_generator =  validation_datagen.flow_from_directory(VALID_DIRECTORY, 
                                                 target_size = IMAGE_SIZE,
                                                 class_mode = 'categorical', 
                                                 batch_size = BATCH_SIZE)

"""
Explanation: Validation data generator
End of explanation
"""


# fine-tune the model
hist = model.fit_generator(
        train_batch_generator,
        steps_per_epoch=NUMBER_TRAIN_SAMPLES/BATCH_SIZE,
        epochs=NUMBER_EPOCHS,  # epochs: Integer, total number of iterations on the data.
        validation_data=valid_batch_generator,
        validation_steps=NUMBER_VALIDATION_SAMPLES/BATCH_SIZE,
        callbacks=callbacks,
        verbose=2)

"""
Explanation: Model fitting
End of explanation
"""


(val_classes, trn_classes, val_labels, trn_labels, 
    val_filenames, filenames, test_filenames) = get_all_classes()

import pickle

file = PRECOMPUTED_DIRECTORY + '/classes_and_filenames.dat'

# Saving the objects:
with open(file, 'wb') as file:  # Python 2: open(..., 'w')
    pickle.dump([val_classes, trn_classes, val_labels, trn_labels, 
    val_filenames, filenames, test_filenames], file)
    

"""
Explanation: Filenames and labels
Save and load classes and filenames:
End of explanation
"""


from keras.callbacks import EarlyStopping
from keras.callbacks import TensorBoard

# Early stop in case of getting worse
early_stop = EarlyStopping(monitor = 'val_loss', patience = 3, verbose = 0)

#TensorBoard
# run tensorboard with tensorboard --logdir=/full_path_to_your_logs
tensorboard_path = TENSORBOARD_DIRECTORY
tensorboard_logger = TensorBoard(log_dir=tensorboard_path, histogram_freq=0, write_graph=False, write_images=False)
print('Logging basic info to be used by TensorBoard to {}. To see this log run:'.format(tensorboard_path))
print('tensorboard --logdir={}'.format(tensorboard_path))

callbacks = [early_stop, tensorboard_logger]

"""
Explanation: Keras callbacks
We are going to define two callbacks that are going to be called in the training. EarlyStopping to stop the training if its not getting better. And a tensorboard callback to log information to be used by tensorboard.
End of explanation
"""


OPTIMIZER_LEARNING_RATE = 1e-2
OPTIMIZER_DECAY = 1e-4  # LearningRate = LearningRate * 1/(1 + decay * epoch)
OPTIMIZER_MOMENTUM = 0.89
OPTIMIZER_NESTEROV_ENABLED = False

from keras.optimizers import SGD

optimizer = SGD(lr=OPTIMIZER_LEARNING_RATE, 
          decay=OPTIMIZER_DECAY, 
          momentum=OPTIMIZER_MOMENTUM, 
          nesterov=OPTIMIZER_NESTEROV_ENABLED)

"""
Explanation: Model Optimizer
End of explanation
"""


model.compile(loss='categorical_crossentropy', 
              optimizer=optimizer, \
              metrics=["accuracy"])

"""
Explanation: Compile the model
End of explanation
"""


from keras.preprocessing.image import ImageDataGenerator

## train generator with shuffle but no data augmentation
train_datagen = ImageDataGenerator(rescale = 1./255)

train_batch_generator =  train_datagen.flow_from_directory(TRAIN_DIRECTORY, 
                                                 target_size = IMAGE_SIZE,
                                                 class_mode = 'categorical', 
                                                 batch_size = BATCH_SIZE)

"""
Explanation: Training
Train data generator
End of explanation
"""


from keras.preprocessing.image import ImageDataGenerator

## train generator with shuffle but no data augmentation
validation_datagen = ImageDataGenerator(rescale = 1./255)

valid_batch_generator =  validation_datagen.flow_from_directory(VALID_DIRECTORY, 
                                                 target_size = IMAGE_SIZE,
                                                 class_mode = 'categorical', 
                                                 batch_size = BATCH_SIZE)

"""
Explanation: Validation data generator
End of explanation
"""


# fine-tune the model
hist = model.fit_generator(
        train_batch_generator,
        steps_per_epoch=NUMBER_TRAIN_SAMPLES/BATCH_SIZE,
        epochs=NUMBER_EPOCHS,  # epochs: Integer, total number of iterations on the data.
        validation_data=valid_batch_generator,
        validation_steps=NUMBER_VALIDATION_SAMPLES/BATCH_SIZE,
        callbacks=callbacks,
        verbose=2)

"""
Explanation: Model fitting
End of explanation
"""


import matplotlib.pyplot as plt

# summarize history for accuracy
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(hist.history['acc']); plt.plot(hist.history['val_acc']);
plt.title('model accuracy'); plt.ylabel('accuracy');
plt.xlabel('epoch'); plt.legend(['train', 'valid'], loc='upper left');

# summarize history for loss
plt.subplot(1, 2, 2)
plt.plot(hist.history['loss']); plt.plot(hist.history['val_loss']);
plt.title('model loss'); plt.ylabel('loss');
plt.xlabel('epoch'); plt.legend(['train', 'valid'], loc='upper left');
plt.show()

"""
Explanation: Training plots
End of explanation
"""


############
# load weights
############
model_save_path = WEIGHTS_DIRECTORY + 'vgg16_pretrained_v2.h5'
print("Loading weights from: {}".format(model_save_path))
model.load_weights(model_save_path)

from keras.preprocessing.image import ImageDataGenerator

## train generator with shuffle but no data augmentation
validation_datagen = ImageDataGenerator(rescale = 1./255)

test_batch_generator =  validation_datagen.flow_from_directory(TEST_DIRECTORY, 
                                                 target_size = IMAGE_SIZE,
                                                 class_mode = 'categorical', 
                                                 batch_size = BATCH_SIZE)

model.evaluate_generator(test_batch_generator,
                         steps = NUMBER_TEST_SAMPLES/BATCH_SIZE)

"""
Explanation: Plot a few examples
Evaluate the model
End of explanation
"""


from keras.preprocessing.image import ImageDataGenerator

## train generator with shuffle but no data augmentation
test_datagen = ImageDataGenerator(rescale = 1./255)

test_batch_generator = test_datagen.flow_from_directory(
                                TEST_DIRECTORY,
                                target_size = IMAGE_SIZE,
                                batch_size=1,
                                shuffle = False, # Important !!!
                                classes = None,
                                class_mode = None)

test_batch_generator.classes.shape

import pickle
test_classes_file = open("../results/vgg16_true.pickle", "wb" )
pickle.dump( test_batch_generator.classes, test_classes_file )

true_values = test_batch_generator.classes

len(test_batch_generator.filenames)

test_filenames = open("../results/vgg16_filenames.pickle", "wb" )
pickle.dump( test_batch_generator.filenames, test_filenames )

import numpy as np

pred = []

for i in range(int(NUMBER_TEST_SAMPLES)):
    X = next(test_batch_generator)  # get the next batch
    #print(X.shape)
    pred1 = model.predict(X, batch_size = 1, verbose = 0) #predict on a batch
    pred = pred + pred1.tolist()

probabilities = np.array(pred)
print(probabilities.shape)
assert probabilities.shape == (NUMBER_TEST_SAMPLES, 2)

test_filenames = open("../results/vgg16_probabilities.pickle", "wb")
pickle.dump( probabilities, test_filenames )

probabilities[0]

predictions=np.argmax(probabilities,1)

test_filenames = open("../results/vgg16_predictions.pickle", "wb" )
pickle.dump( predictions, test_filenames )

predictions[0]

import matplotlib.pyplot as plt

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

import itertools
from sklearn.metrics import confusion_matrix

class_names = ['cat', 'dog']
cnf_matrix = confusion_matrix(true_values, predictions)
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names,
                      title='Confusion matrix')
plt.show()

from numpy.random import random, permutation
#1. A few correct labels at random
correct = np.where(predictions==true_values)[0]

idx = permutation(correct)[:4]
#plots_idx(idx, probs[idx])

len(correct)

from scipy import ndimage
from PIL import Image
import matplotlib.pyplot as plt

im = ndimage.imread("../data/test/" + test_batch_generator.filenames[idx[0]])
image = Image.fromarray(im)
plt.imshow(image)
plt.title(probabilities[idx[0]])
plt.show()

im = ndimage.imread("../data/test/" + test_batch_generator.filenames[idx[1]])
image = Image.fromarray(im)
plt.imshow(image)
plt.title(probabilities[idx[1]])
plt.show()

from numpy.random import random, permutation
#1. A few correct labels at random
correct = np.where(predictions != true_values)[0]

idx = permutation(correct)[:4]
#plots_idx(idx, probs[idx])

im = ndimage.imread("../data/test/" + test_batch_generator.filenames[idx[0]])
image = Image.fromarray(im)
plt.imshow(image)
plt.title(probabilities[idx[0]])
plt.show()

im = ndimage.imread("../data/test/" + test_batch_generator.filenames[idx[1]])
image = Image.fromarray(im)
plt.imshow(image)
plt.title(probabilities[idx[1]])
plt.show()

"""
Explanation: Test the model
End of explanation
"""


