import warnings
warnings.filterwarnings('ignore')

%matplotlib inline
%pylab inline

import matplotlib.pylab as plt
import numpy as np

from distutils.version import StrictVersion

import sklearn
print(sklearn.__version__)

assert StrictVersion(sklearn.__version__ ) >= StrictVersion('0.18.1')

import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
print(tf.__version__)

assert StrictVersion(tf.__version__) >= StrictVersion('1.1.0')

import keras
print(keras.__version__)

assert StrictVersion(keras.__version__) >= StrictVersion('2.0.0')

"""
Explanation: Data Augmentation

https://keras.io/preprocessing/image/
https://datascience.stackexchange.com/questions/5224/how-to-prepare-augment-images-for-neural-network
http://machinelearningmastery.com/image-augmentation-deep-learning-keras/
End of explanation
"""


# !curl -O https://raw.githubusercontent.com/DJCordhose/speed-limit-signs/master/data/speed-limit-signs.zip

# !rm -r speed-limit-signs
# https://docs.python.org/3/library/zipfile.html
from zipfile import ZipFile
zip = ZipFile(r'speed-limit-signs.zip')
zip.extractall('.')

"""
Explanation: Load Original Data Set
End of explanation
"""


from keras.preprocessing.image import ImageDataGenerator

# ImageDataGenerator?

train_dir = 'speed-limit-signs'
generated_train_dir = 'generated'

datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range=15,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.4,
    horizontal_flip=False,
    vertical_flip=False,    
    fill_mode='nearest')

# train_dir = 'speed-limit-signs/0'
train_dir = 'speed-limit-signs/5'
number_of_transformations = 10

"""
Explanation: Exploring Keras ImageDataGenerator
End of explanation
"""


!rm -r generated
!mkdir generated

import os
from keras.preprocessing import image
import matplotlib
import matplotlib.pyplot as plt

datagen.flow_from_directory(
        train_dir,
        target_size=(64, 64),
        batch_size=32,
        save_to_dir=generated_train_dir
)

fnames = [os.path.join(train_dir, fname) for fname in os.listdir(train_dir)]

img_path = fnames[0] # We pick one image to "augment"

img = image.load_img(img_path, target_size=(64, 64)) # Read the image and resize it

x = image.img_to_array(img) # Convert it to a Numpy array with shape (64, 64, 3)

x = x.reshape((1,) + x.shape) # Reshape it to (1, 64, 64, 3)

# plt.figure(figsize=(15, 15))

# The .flow() command below generates batches of randomly transformed images.
# It will loop indefinitely, so we need to `break` the loop at some point!
i = 1
for batch in datagen.flow(x):
    plt.figure(i)
#     pillow image: http://pillow.readthedocs.io/en/3.4.x/reference/Image.html
    img = image.array_to_img(batch[0])
    fname = 'generated/augmented_{index}.png'.format(index=i)
    img.save(fname)
    plt.imshow(img)
#     plt.subplot(8, 8, i)  # A grid of 8 rows x 8 columns
    plt.axis('off')
    if i % number_of_transformations == 0:
        break
    i += 1
plt.show()

!ls -lh generated/

"""
Explanation: Ten Samples of first 120 sign
End of explanation
"""


def traverse(root_dir, 
             category_callback=lambda category_dir, category: print(category_dir, category),
             image_callback=lambda path, directory, basename: print(path, directory, basename)):
    directories = [d for d in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, d))]
    for d in directories:
        label_dir = os.path.join(data_dir, d)
        category_callback(label_dir, d)
        file_names = [f for f in os.listdir(label_dir) if f.endswith(".ppm")]
        for f in file_names:
            path = os.path.join(label_dir, f)
            basename = os.path.splitext(f)[0]
            image_callback(path, d, basename)

ROOT_PATH = "./"
data_dir = os.path.join(ROOT_PATH, "speed-limit-signs")
# traverse(data_dir)

def augment_image(input_path, output_dir, output_name, number_of_transformations=10, plot=False):
    img = image.load_img(input_path, target_size=(64, 64))
    x = image.img_to_array(img)
    x = x.reshape((1,) + x.shape) # Reshape it to (1, 64, 64, 3)
    i = 1
    for batch in datagen.flow(x):
        if plot:
            plt.figure(i)
    #     pillow image: http://pillow.readthedocs.io/en/3.4.x/reference/Image.html
        img = image.array_to_img(batch[0])
        if plot:
            plt.imshow(img)
    #     plt.subplot(8, 8, i)  # A grid of 8 rows x 8 columns
            plt.axis('off')
        else:
            output_path = '{dir}/{name}_{index}.png'.format(index=i, dir=output_dir, name=output_name)
            img.save(output_path)

        if i % number_of_transformations == 0:
            break
        i += 1
    if plot:
        plt.show()

generated_train_dir = 'augmented-signs'
!rm -r augmented-signs
!mkdir augmented-signs

traverse(data_dir, 
         category_callback=lambda category_dir, category: os.makedirs(os.path.join(generated_train_dir, category)) if not os.path.exists(os.path.join(generated_train_dir, category)) else None,
         image_callback=lambda path, category, basename: augment_image(path, os.path.join(generated_train_dir, category), basename)
        )        

!ls -lh augmented-signs

!ls -lh augmented-signs/0

!rm augmented-signs.zip
!zip -r augmented-signs.zip augmented-signs >/dev/null

!ls -lh

!curl --upload-file augmented-signs.zip https://transfer.sh/augmented-signs.zip

"""
Explanation: Hands-On
Optimize with ImageDataGenerator Parameters

Change parameters used to generate the 10 variaions above
Can you as a human still recognize the imgae? If not changes might be too drastic.
Change number of variations changing variable number_of_transformations
Change sign category by changing variable train_dir


Now that we have good parameters, we can generate variations of all Signs
don't execute this on Azure Notebooks, will take forever
End of explanation
"""


