# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Transfer Learning
#
# Transfer learning is, in essence, the application of a neural net trained on one task to a new task. Because convolutional neural nets are almost exclusively used to examine visual data, they are particularly good at transfer learning. 
#
# A network trained to recognize dogs and cats in photographs will learn some features that are universal to processing photographs, and so it may be a good starting point for a network that detects frogs and birds in photographs. A network trained to detect cancer in radiographs will learn some things that are universal to processing radiographs, and so might be a good starting point for a network that detects foregin bodies, or bone fractures. 
#
# In this lab we'll use transfer learning to re-train a neural network to perform a new task. 

# +
# this should look familiar from the previous labs. 
# We could choose any of the pre-built nets here instead. 
from keras.applications.mobilenet_v2 import MobileNetV2, preprocess_input, decode_predictions
from keras.layers import Dense, GlobalAveragePooling2D, Dropout
from keras.models import Model, Sequential
from keras.preprocessing.image import ImageDataGenerator, image as keras_image
from keras.utils import to_categorical
import matplotlib.pyplot as plt
import numpy as np
from scipy import ndimage

# This is a new built in dataset that we haven't seen before.
# It is 60,000 (50k training, 10k test) small (32x32) RGB images 
# classified into 100 classes:
from keras.datasets import cifar100
(x_train, y_train), (x_test, y_test) = cifar100.load_data(label_mode='fine')

# Some global factors that will be kept constant. 
batch_size = 32
num_classes = 100
image_size = 96
epochs = 5

# FOR DISPLAY PURPOSES
unprocessed_training_images = x_train
unprocessed_training_labels = y_train

# Because the network we're fine-tuning (MobileNetV2) has several pooling layers
# the smallest image it can process is 96x96, these images are 32x32. To fix this
# we are manually rescaling all the images using scipy. We are also applying the 
# MobileNetV2 preprocess_input function here. 
def adjust_input_image(rgb_data):
    adjusted = preprocess_input(rgb_data)
    
    # Scales width and height by 3, leaves color channels at original scale
    adjusted = ndimage.zoom(adjusted, (3, 3, 1), order=0)

    return adjusted

x_train = np.array([adjust_input_image(x) for x in x_train])
x_test = np.array([adjust_input_image(x) for x in x_test])

# And we still need to one-hot encode the labels as usual
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
# -

# We can see that the preprocessing function changes the pixel values by scaling some of them between -1 and 1, this is not nessesarally typical, and if we use matplotlibs imshow on the preprocessed image-set, it will sometimes look a little funky as a result:

plt.imshow(x_train[0])
plt.show()
plt.imshow(unprocessed_training_images[0])
plt.show()

# Lets also take alook at a few of the examples from cifar100. This is a challenging dataset, the images are small and therefore low resolution, and some are pretty obscure—the 'cloud' and two 'sea' images below lack detail. The 'aquarium fish' and 'rose' and 'squirrel' are tough as well. 

# + code_folding=[1]
# Some of these are genuinely challenging... 
label_map = [
    'apple', # id 0
    'aquarium_fish',
    'baby',
    'bear',
    'beaver',
    'bed',
    'bee',
    'beetle',
    'bicycle',
    'bottle',
    'bowl',
    'boy',
    'bridge',
    'bus',
    'butterfly',
    'camel',
    'can',
    'castle',
    'caterpillar',
    'cattle',
    'chair',
    'chimpanzee',
    'clock',
    'cloud',
    'cockroach',
    'couch',
    'crab',
    'crocodile',
    'cup',
    'dinosaur',
    'dolphin',
    'elephant',
    'flatfish',
    'forest',
    'fox',
    'girl',
    'hamster',
    'house',
    'kangaroo',
    'computer_keyboard',
    'lamp',
    'lawn_mower',
    'leopard',
    'lion',
    'lizard',
    'lobster',
    'man',
    'maple_tree',
    'motorcycle',
    'mountain',
    'mouse',
    'mushroom',
    'oak_tree',
    'orange',
    'orchid',
    'otter',
    'palm_tree',
    'pear',
    'pickup_truck',
    'pine_tree',
    'plain',
    'plate',
    'poppy',
    'porcupine',
    'possum',
    'rabbit',
    'raccoon',
    'ray',
    'road',
    'rocket',
    'rose',
    'sea',
    'seal',
    'shark',
    'shrew',
    'skunk',
    'skyscraper',
    'snail',
    'snake',
    'spider',
    'squirrel',
    'streetcar',
    'sunflower',
    'sweet_pepper',
    'table',
    'tank',
    'telephone',
    'television',
    'tiger',
    'tractor',
    'train',
    'trout',
    'tulip',
    'turtle',
    'wardrobe',
    'whale',
    'willow_tree',
    'wolf',
    'woman',
    'worm'
]

plt.figure(figsize=(15, 10))
for i in range(25):
    ax = plt.subplot(5, 5, i+1)
    ax.axis('off')
    ax.set_title(label_map[unprocessed_training_labels[i][0]])
    plt.imshow(unprocessed_training_images[i])

plt.tight_layout()
plt.show()


# -

# This should look familiar from previous labs.
def plot_training_history(history, model):
    figure = plt.figure()

    plt.subplot(1, 2, 1)
    plt.plot(history.history['accuracy'])
    plt.plot(history.history['val_accuracy'])
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.tight_layout()

    plt.subplot(1, 2, 2)
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.tight_layout()

    figure.tight_layout()
    plt.show()


# Lets quickly loook a little closer at MobileNetV2, so we can think carefully about which layers to freeze and how to apply transfer learning. 

# +
base_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(96, 96, 3))

# Layers go from 0-154
#   initial=0-9
#   block1 = 10-18
#   block2 = 19-27
#   block3 = 28-36
#   block4 = 37-45
#   block5 = 46-54
#   block6 = 55-63
#.  block7 = 64-72
#   block8 = 73-81
#.  block9 = 81-90
#   block10= 91-98
#   block11= 99-107
#.  block12= 108-116
#.  block13= 117-125
#   block14= 126-134
#.  block15= 135-143
#.  block16= 144-151
#.  final_conv = 152, 
#   conv_bn 153
#   out_relu = 154
# for i, layer in enumerate(base_model.layers):
#     print(i, layer.name)
    
base_model.summary()
# -

# # Transfer Learning Tips and Pitfalls
#
# One of the challenges with the Cifar100 dataset is that the images are very small. Roughly 10x smaller than the images that MobileNetV2 was originally trained on. This means 1) there is just less input information, and if we look at some of the Cifar images I'm sure you'll agree that some are quite difficult to identify, even for us humans and 2) the kernals that extracted useful information from larger images might not apply as well to smaller ones. 
#
# Additionally, different hyperparameters can dramatically impact the effectiveness of transfer learning. Some combinations will be very effective, others will lead to overfitting, others won't learn anything at all. With transfer learning it is important to consider the similarity of the task being trained, as well as the amount of data available. 
#
# If we have a small amount of data, we'll be more likely to overfit during training. If the task is significantly different than the task the original net trained on, the learned kernels may not extract information that is useful for the new task. 

# +
# The first way we'll do transfer learning REALLY saves time, but
# only works if you don't want to fine tune any of the layers 
# from the original network. What we can do instead is memorize
# the output of the base network, since it won't be changing,
# and then repeatedly train on those outputs and the training labels

# Like before, we grab a pretrained model with include_top=False
base_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(96, 96, 3))

# Unlike before, we're going to just run the images through this base layer once
# This takes awhile, we're essentially doing a round of evaluation on both datasets.
# And we'll save them incase we want to experiment with different models to transfer
# on top of these.
training_features = base_model.predict(x_train)
np.savez('MobileNetV2_features_train', features=training_features)

test_features = base_model.predict(x_test)
np.savez('MobileNetV2_features_test', features=test_features)

# +
# If you have not run the above cell during this session, uncomment these lines to load the features.
# training_features = np.load('MobileNetV2_features_train.npz')['features']
# test_features = np.load('MobileNetV2_features_test.npz')['features']

# Also unlike before, we're going to build a complete model
# in the normal way, but define the input shape based on the 
# features predicted by base_model!
model = Sequential()
model.add(GlobalAveragePooling2D(input_shape=training_features.shape[1:]))
model.add(Dropout(rate=0.3))
model.add(Dense(units=100, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(training_features, y_train, batch_size=batch_size, epochs=20, validation_split=0.2, verbose=False)

plot_training_history(history, model)
loss, accuracy = model.evaluate(test_features, y_test, verbose=False)
print(f'Test loss: {loss:.3}')
print(f'Test accuracy: {accuracy:.3}')


# -

# It's worth noting that cifar100 is a very tricky dataset. The images are small, 100 classes is several, and 100 classes amongst rougly 60k examples is only on average 600 samples per class. In fact 28% would have put us in the top 20 on a semi-recent Kaggle competition, and 50% in the top ten: 
#
# https://www.kaggle.com/c/ml2016-7-cifar-100/leaderboard
#
# Boom, we're a top ten Kaggel competitor just like that. Though, looking at validation accuracy and loss, it's not clear we're going to improve much further. It may be worth trying to do transfer learning from a different pre-trained model.

# # Ignoring Some Layers and Fine Tuning
#
# We can use the same idea, but allow some of the layers of the original network to retrain as well. For example, MobileNetV2 is split into 16 blocks. If we think the final few convolutional layers are too specific to the task our original net was trained on (the ImageNet dataset in this case) then we can chop off some of these blocks and get the trained features from an earlier block, and use THOSE as our training_features. Furthermore, in our new model we can have the same convoluitonal layers as the original mode, and train those. This is called fine-tuning.
#
# Unfortunately, doing this the way we have done above is very hard in Keras, especially with merge layers which MobileNetV2 has. Instead, we have to "freeze" the layers by setting them to be untrainable manually, then "train" the entire network with some layers of the layers frozen, especially the early ones. This is a significant perfomance penalty because we have to recompute the output of all the frozen layers each time, instead of simply memorizing it. 
#
# The alternative is to manually construct a model that matches the architecture we're seeking, which can be challenging for the more complex architectures. 

# +
# One way to do transfer learning is called "fine tuning". This is where you take an existing network
# and freeze some of the layers. In addition to freezing some layers, you'll have to add at a minimum
# a new output layer that matches the number of classes for the new task. 

# Typically the earlier layers are frozen, because these are likely to have learned universal/simple features.
# The later layers have learned combinations of those features and are increasingly specific to the task
# that the network was trained for. 

def transfer_from_mobilenet(optimizer, freeze_first_n_layers):
    base_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(96, 96, 3))

    old_top = base_model.output
    old_top = GlobalAveragePooling2D()(old_top)
    old_top = Dropout(rate=0.2)(old_top)
    new_top = Dense(num_classes, activation='softmax')(old_top)
    
    model = Model(inputs=base_model.input, outputs=new_top)

    for layer in model.layers[:freeze_first_n_layers]:
        layer.trainable = False

    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

    history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.2, verbose=True)
    
    return history, model


# +
from itertools import product
from keras.optimizers import Adam

# These experiments took *several hours* to run originally.
# Saving the model after an experiment is a good idea, then you
# can experiment further with promising models, without rerunning
# entier experiments.
save_directory = 'saved_models/'

# More agressive optimizers like adam can result in overfitting
# We should watch for that in our results.
optimizers = [
    'sgd',
    Adam(lr=0.0001)  # Slower than normal learning rate. 
]

# Note: experimentally freezing only the first 2 blocks
# resulted in much slower performance, though for SGD freezing 
# only the first block did achieve 50% accuracy on test data 
freeze_layer_counts = [
    134,   # Up through block 14 frozen
    143    # Up through block 15 frozen
    
    # If we were to set a value here to 154, it would freeze all the original layers
    # which can be quite successful, but we can make training much faster using a 
    # different tactic to freeze all the original layers as demonstrated above
]

# This takes a LOOOOONNNNNNNGGGGG time, don't re-run it unless you're prepared to wait. 
for optimizer, freeze_first_n_layers in product(optimizers, freeze_layer_counts):
    saved_name = f'{optimizer}_{freeze_first_n_layers}'
    print(f'Transfer learning with {optimizer}, {freeze_first_n_layers}', "Will save as: ", saved_name)
    print()

    history, model = transfer_from_mobilenet(optimizer, freeze_first_n_layers)
    model.save(save_directory + saved_name)
    plot_training_history(history, model)
    
    loss, accuracy = model.evaluate(x_test, y_test, verbose=False)
    print(f'Test loss: {loss:.3}')
    print(f'Test accuracy: {accuracy:.3}')
    

# +
history, model = transfer_from_mobilenet('adam', 151) # Leave the final 2 conv layers alone, freeze all the bottleneck blocks
model.save(save_directory + saved_name)
plot_training_history(history, model)

loss, accuracy = model.evaluate(x_test, y_test, verbose=False)
print(f'Test loss: {loss:.3}')
print(f'Test accuracy: {accuracy:.3}')
# -

# Hmm. That's not what we were hoping for was it? 
#
# Why do we think this transfer attempt wasn't very successful?

# +
# lets try unfreezing a lot more. We have a reasonable amount of training data, so lets try!

# WARNING, THIS TOOK ~2 hours on my MacbookPro
history, model = transfer_from_mobilenet('adam', 0) # unfreeze it all!
model.save(save_directory + "adam_train_all")
plot_training_history(history, model)

# +
# That's a little more like it! Lets keep going... 
history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.2, verbose=True)
plot_training_history(history, model)

loss, accuracy = model.evaluate(x_test, y_test, verbose=False)
print(f'Test loss: {loss:.3}')
print(f'Test accuracy: {accuracy:.3}')
# -

# Taken together, it looks like we're still learning, even if it's a bit erratic. It takes a long time to train, and we are still overfitting, even while we keep improving on the validation data... This would be a good time to apply data augmentation to synthetically increase our dataset. 





StarcoderdataJupyterScriptsDedupFiltered