# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.6
#     language: python
#     name: python3
# ---

# # CASE STUDY - convolutional neural networks

# !pip install joblib

# +
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import csv
import joblib
import time
from collections import Counter
import numpy as np
import pandas as pd
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
import matplotlib.pyplot as plt

plt.style.use('seaborn')
# %matplotlib inline
# -

from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.decomposition import PCA

# Synopsis
# ----------
#
# You were hired at AAVAIL to be a member of a data science team that works closely together.  Some of your first projects
# are meant to help marketing with customer retention and to investigate market specific trends. There are also some
# projects relating to user comments that are getting off the ground.  However, you will also be working alongside
# the deep-learning specialists that maintain the core product at AAVAIL---its audio and visual manipulation models.
#
# Because the team meets regularly all new data science hires are expected to go through a series deep-learning tutorials
# to ensure that they can contribute to conversations about the core product.   The first in this series is the following
# tutorial on CNNs.  You will be guided through the following parts.
#
#   1. Environment setup
#   2. Model scaffolding using Keras
#   3. Logging and Model serialization
#   4. Model iteration

# ## Fashion MNIST
#
# >One project that the data science team at AAVAIL has been tasked with is ensuring that the video feeds are in fact news video feeds.  There are people that are performing quality assurance on these feeds, but eventually the data science team will need to build a service that samples a number of frames from a video, then identifies objects in the images, flagging for review any feeds that may be different.
#
# A solid benchmark dataset for this task is the Fashion MNIST dataset.  
#
# * training set - 60,000 images
# * test set - 10,000 images
# * images are 28 pixels x 28 pixels
# * classes: T-shirt/top, Trouser, Pullover, Dress, Coat, Sandal, Shirt, Sneaker, Bag, Ankle boot

# ## PART 1 - environment setup

## check hardware availability
from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())

# +
### load the data  
fashion_mnist = keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data() 
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
num_classes = len(class_names)

## Normalize pixel values to be between 0 and 1
train_images, test_images = train_images / 255.0, test_images / 255.0

X_train = train_images
X_test = test_images
y_train = train_labels
y_test = test_labels
# -

# ### QUESTION 1
#
# Visualize a sample of the images to QA the data set (You should scroll through several pages of images) and print a summary of the data.  Also create a base model to compare your neural network to.

## (visualization code)
fig = plt.figure(figsize=(10, 8), facecolor="white")
offset = 25
for i in range(25):
    ax = fig.add_subplot(5, 5, i+1)
    ax.set_xticks([])
    ax.set_yticks([])
    ax.grid(False)
    ax.imshow(train_images[i+offset], cmap=plt.cm.binary)
    ax.set_xlabel(class_names[train_labels[i+offset]])

# +
## (Summarize the data)

print("X_train: {}".format(X_train.shape))
class_info = list(sorted(Counter(y_train).items()))
print("num classes: {}, classes:{}".format(len(class_info), [i[0] for i in class_info]))
print("class samples: {}".format([i[1] for i in class_info]))
print("class balance: {}".format([round(i[1]/X_train.shape[0], 2) for i in class_info]))
# -

# ### QUESTION 2
#
# Build a base model.  Construct a pipeline that uses PCA into a classic machine learning model.

# +
## (base model)

X_train_flat = np.array([i.flatten() for i in train_images])
X_test_flat = np.array([i.flatten() for i in test_images])

pipe = Pipeline(steps=[
    ("pca", PCA(n_components=25)),
    ("rf", RandomForestClassifier(random_state=42, class_weight="balanced"))
])

## create model directory
if not os.path.isdir("models"):
    os.mkdir("models")
    
# grid search
param_grid = {
    'rf__n_estimators': [10, 50, 100],
    'rf__max_depth': [8, 9, 10]
}

saved_model = os.path.join("models",'pca-rf.joblib')
if not os.path.exists(saved_model):
    time_start = time.time()
    grid = GridSearchCV(pipe, param_grid=param_grid, cv=3, iid=False, n_jobs=-1)
    grid.fit(X_train_flat, y_train)
    joblib.dump(grid, saved_model)
    print("train time", time.strftime('%H:%M:%S', time.gmtime(time.time()-time_start)))
    
else:
    print("...loading {} from file".format(saved_model))
    grid = joblib.load(saved_model)

## print best parameters
print(grid.best_params_)
# -

y_pred = grid.predict(X_test_flat)
print("-->".join(pipe.named_steps.keys()))
print(classification_report(y_test, y_pred,target_names=np.array(class_names)))

# ## PART 2 -  model scaffolding using Keras
#
# Create a function that returns a model using ``keras.models.Sequential()`` and ensure that you pass ``activation_function`` as an argument.  Instaintiate a version of the model and print the summary.  This function is just meant to return a simple multilayer perceptron network.  At a minimum the function code should contain:
#
# ```python
# model = keras.Sequential()
# model.add(keras.layers.Flatten(input_shape=(28, 28)))
# model.add(keras.layers.Dense(128, activation='relu'))
# model.add(keras.layers.Dense(10, activation='softmax'))
# ```

# ### QUESTION 3
#
# Build a simple MLP

# +
keras.backend.clear_session()

def build_mlp(input_shape=(28, 28), n_units=128, activation_fn="relu", n_outputs=10):
    """
    build multilayer perceptron with keras 
    """
    
    model = keras.Sequential()
    model.add(keras.layers.Flatten(input_shape=input_shape))
    model.add(keras.layers.Dense(n_units, activation=activation_fn))
    model.add(keras.layers.Dense(n_outputs, activation='softmax'))
    
    return model

print(X_train.shape)

mlp = build_mlp(n_outputs=num_classes, input_shape=X_train.shape[1:])
mlp.summary()
# -

# ### QUESTION 4
#
# Create another version of your neural network.  This time you should build a proper CNN.  Remember that one pattern to consider starting from is alternating ``Con2D`` and ``MaxPooling2D`` layers.  This is often followed by a couple of ``Dense`` layers.  Recall that the the output of every Conv2D and MaxPooling2D layer is a 3D tensor of shape (height, width, channels). The output of the last ``Dense`` layer should correspond to the number of classes and generally uses a 'softmax' activation.  Use `model.summary()` to ensure a cohesive architecture.

# +
keras.backend.clear_session()

def build_cnn(filters=[28, 64, 128], kernel_size=(3, 3), activation_fn='relu', input_shape=(28, 28, 1), n_outputs=10, dropout=None):
    """
    build a simple cnn
    """
    
    model = keras.Sequential()
    
    if not dropout:
        dropout = [False,False]
    
    model.add(keras.layers.Conv2D(filters=filters[0], kernel_size=kernel_size, activation=activation_fn, input_shape=input_shape))
    model.add(keras.layers.MaxPooling2D((2, 2)))
    
    if dropout[0]:
        model.add(tf.keras.layers.Dropout(rate=0.3))
        
    model.add(keras.layers.Conv2D(filters=filters[1], kernel_size=kernel_size, activation=activation_fn))
    model.add(keras.layers.MaxPooling2D((2, 2)))
    
    if dropout[1]:
        model.add(tf.keras.layers.Dropout(rate=0.3))
        
    model.add(keras.layers.Conv2D(filters=filters[2], kernel_size=kernel_size, activation=activation_fn))
    
    model.add(keras.layers.Flatten())
    model.add(keras.layers.Dense(128, activation=activation_fn))
    model.add(keras.layers.Dense(n_outputs, activation='softmax'))
    
    return model

X_train_3D = X_train[..., np.newaxis]
X_test_3D = X_test[..., np.newaxis]

print(X_train_3D.shape)

cnn = build_cnn(filters=[28, 64, 128], input_shape=X_train_3D.shape[1:], n_outputs=len(class_info))
cnn.summary()


# -

# ## PART 3 - logging and Model serialization
#
# You can use a trained model without having to retrain it.  Your can also continue training a model to pick-up training where you left off.  The `tf.keras.callbacks.ModelCheckpoint` callback allows to continually save the model both during and at the end of training.  For long running models this is ideal in case the training is interrupted.  Otherwise you can 
# used `model.save()` and `model.load()`.  In this part you will create a function that accomplished a few things at once.  Here is some pseudocode that you could work from.
# ```python
# def train_network(model_name,model,loss_fn, optimizer='adam'):
#
#     if not os.path.exists(saved model):
#         model.compile(optimizer=optimizer,
#                       loss=loss_fn,
#                       metrics=['accuracy'])
#         model.fit(X_train,
#                   y_train,
#                   batch_size=64,
#                   epochs=10,
#                   validation_data=(X_test, y_test))
#         
#         create_log_file()
#         model.save(saved_model)
#     else:
#         print("... loading saved model")
#         model = keras.models.load_model(saved_model)
#         
#     return(model)    
# ```    
# The two things you are trying to accomplish with this function are:
#
# 1. save your models so that each iteration only needs to be run once
# 2. save the specifics of your model in a log file 
#
#   * optimizer 
#   * loss_fn 
#   * test_loss
#   * test_accuracy
#   * any notes

def train_dnn(model, model_name, loss_fn, X_train, y_train, X_test, y_test, optimizer="adam", 
              batch_size=32, n_epochs=10):
    """
    compile, train and save deep neural network
    """
    
    # create save directory
    save_dir = "models"
    if not os.path.isdir(save_dir):
        os.mkdir(save_dir)
        
    saved_model = os.path.join(save_dir, "{}.h5".format(model_name))
    
    if not os.path.exists(saved_model):
        # compile the model
        model.compile(optimizer=optimizer, loss=loss_fn, metrics=["accuracy"])
        
        # using callbacks during training
        checkpoint_cb = keras.callbacks.ModelCheckpoint(saved_model, save_best_only=True)
        
        # fit model to the train set, use test set for validation
        history = model.fit(X_train, y_train, batch_size=batch_size, epochs=n_epochs,
                            validation_data=(X_test, y_test),
                            callbacks=[checkpoint_cb])
        
        model = keras.models.load_model(saved_model) # rollback to best model
        
        # evaluate final model
        test_loss, test_acc = model.evaluate(X_test, y_test, verbose=2)
        
        # save a log file
        log_file = os.path.join(save_dir, "{}.log".format(model_name))
        with open(log_file, "w") as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(["loss_function",loss_fn])
            writer.writerow(["optimizer", optimizer])
            writer.writerow(["test_loss",test_loss])
            writer.writerow(["test_acc",test_acc])
            
    else:
        print("... loading saved model")
        model = keras.models.load_model(saved_model) # load model
        
    return model


# ## PART 4 - model iteration

# +
## (build and train a MLP)
model_mlp = build_mlp(n_outputs=num_classes, input_shape=X_train.shape[1:])

model_mlp = train_dnn(model=model_mlp, model_name="simple_mlp", 
                      loss_fn="sparse_categorical_crossentropy", 
                      X_train=X_train, y_train=y_train, 
                      X_test=X_test, y_test=y_test, 
                      batch_size=32, n_epochs=10)

# -

# !cat ./models/simple_mlp.log

# +
## (build and train a CNN)

## add the channel dimensions to your data
#X_train_1 = np.expand_dims(X_train, -1)
#X_test_1 = np.expand_dims(X_test, -1)

X_train_3D = X_train[..., np.newaxis]
X_test_3D = X_test[..., np.newaxis]

model_cnn = build_cnn(filters=[28, 64, 128], input_shape=X_train_3D.shape[1:],
                      n_outputs=len(class_info))

model_cnn = train_dnn(model=model_cnn, model_name="simple_cnn", 
                      loss_fn="sparse_categorical_crossentropy", 
                      X_train=X_train_3D, y_train=y_train, 
                      X_test=X_test_3D, y_test=y_test, 
                      batch_size=32, n_epochs=5)

# -

# !cat ./models/simple_cnn.log

StarcoderdataJupyterScriptsDedupFiltered