# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/fangyiyu/Modified_MNIST/blob/master/tensorflow_v2.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="kwkM8UqSWBkX" outputId="ca8cc478-7ee4-4a16-9bc1-92b0727757ed" colab={"base_uri": "https://localhost:8080/", "height": 35}
import tensorflow as tf
tf.__version__

# + id="b6bLpp_Ce5Ok"
# Import libraries
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from __future__ import absolute_import, division, print_function
import tensorflow as tf
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten, LSTM, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
import warnings
warnings.filterwarnings("ignore")

# + id="wvU8p9mSXO49" outputId="488b95d1-0b57-4ae6-a3d4-d69e51ea2008" colab={"base_uri": "https://localhost:8080/", "height": 377}
# !wget -O comp551w18-modified-mnist.zip https://www.dropbox.com/sh/dhu4cu8l8e32cvl/AACSbGE9X6P7an61STwwr8R0a?dl=0

# + id="Tq3COEzCXVRT" outputId="e49e4ea2-88ea-4a68-d481-432d629f2efc" colab={"base_uri": "https://localhost:8080/", "height": 85}
# Run two times
# !unzip comp551w18-modified-mnist.zip

# + id="0JxbV7CbZ83M" outputId="77ef907d-75cb-4b41-f3f0-b3792ce057ff" colab={"base_uri": "https://localhost:8080/", "height": 51}
# !ls

# + id="u7w5IXiMaIcF"
x = np.loadtxt("train_x.csv", delimiter=",")# Load from text
y = np.loadtxt("train_y.csv", delimiter=",")
x = x.reshape(-1, 64, 64) # Reshape
y = y.reshape(-1, 1)

# + id="-hW8k0DSaVUb" outputId="3a5293f6-f1d8-4901-d031-e68a68cc0992" colab={"base_uri": "https://localhost:8080/", "height": 1000}
for i in range(5):
    plt.figure()
    plt.imshow(x[i])

# + [markdown] id="oMO7avD4aX3P"
# This project is using Modified MNIST handwritten digits. The dataset contains 50000 examples for training and 10000 for testing. The digits will be size-normalized and centered in a fixed-size image(64*64 pixels) with values from 0 to 255. 

# + [markdown] id="XDHJ9lH4vJE5"
# # 1. Data Preprocessing

# + id="ZXrGKvndvH-n"
# Split dataset into test and train
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=7)

# + id="RqjLg05NvN17"
# MNIST dataset parameters.
num_classes = 10 # 0 to 9 digits
num_features = 4096 # 64*64

# Convert to float32.
x_train, x_test = np.array(x_train, np.float32), np.array(x_test, np.float32)
# Flatten images to 1-D vector of 4096 features (64*264).
x_train, x_test = x_train.reshape([-1, num_features]), x_test.reshape([-1, num_features])
# Normalize images value from [0, 255] to [0, 1].
x_train, x_test = x_train / 255., x_test / 255.

# + id="he-1IfWkvQQC"
y_test = y_test.reshape(len(y_test), )
y_test = y_test.astype(int)

# + id="7j3wnZlnvQyh"
y_train = y_train.reshape(len(y_train), )
y_train = y_train.astype(int)

# + [markdown] id="y4gpuaaubEsQ"
# # 1. Logistic Regression
#
# To feed in the LR model, each image will be converted to float 32, normalized to [0,1] and flattened to a 1-D array of 4096 features(64*64).

# + id="p1TrgOsAa2T3"
# Training parameters.
learning_rate = 0.01
training_steps = 500
batch_size = 128
display_step = 50

# + id="2ME3mk7AcJ08"
# Use tf.data API to shuffle and batch data.
train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
train_data = train_data.repeat().shuffle(5000).batch(batch_size).prefetch(1)

# + id="nxDDBNBVcOAs"
# Weight of shape [4096, 10], the 64*64 image features, and total number of classes.
W = tf.Variable(tf.ones([num_features, num_classes]), name="weight")
# Bias of shape [10], the total number of classes.
b = tf.Variable(tf.zeros([num_classes]), name="bias")


# + id="AuMCJcJNcVcp"
# Logistic regression (Wx + b).
def logistic_regression(x):
    # Apply softmax to normalize the logits to a probability distribution.
    return tf.nn.softmax(tf.matmul(x, W) + b)

# Cross-Entropy loss function.
def cross_entropy(y_pred, y_true):
    # Encode label to a one hot vector.
    y_true = tf.one_hot(y_true, depth=num_classes)
    # Clip prediction values to avoid log(0) error.
    y_pred = tf.clip_by_value(y_pred, 1e-9, 1.)
    # Compute cross-entropy.
    return tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred),1))

# Accuracy metric.
def accuracy(y_pred, y_true):
    # Predicted class is the index of highest score in prediction vector (i.e. argmax).
    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))
    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# Stochastic gradient descent optimizer.
optimizer = tf.optimizers.SGD(learning_rate)


# + id="LlHckfM-cmrV"
# Optimization process. 
def run_optimization(x, y):
    # Wrap computation inside a GradientTape for automatic differentiation.
    with tf.GradientTape() as g:
        pred = logistic_regression(x)
        loss = cross_entropy(pred, y)

    # Compute gradients.
    gradients = g.gradient(loss, [W, b])
    
    # Update W and b following gradients.
    optimizer.apply_gradients(zip(gradients, [W, b]))


# + id="VYAqJBU6crA4" outputId="9334bec0-2548-46b8-807b-dbf5a5a85631" colab={"base_uri": "https://localhost:8080/", "height": 187}
# Run training for the given number of steps.
for step, (batch_x, batch_y) in enumerate(train_data.take(training_steps), 1):
    # Run the optimization to update W and b values.
    run_optimization(batch_x, batch_y)
    
    if step % display_step == 0:
        pred = logistic_regression(batch_x)
        loss = cross_entropy(pred, batch_y)
        acc = accuracy(pred, batch_y)
        print("step: %i, loss: %f, accuracy: %f" % (step, loss, acc))

# + id="ahPWjoxgfZdS" outputId="b85d4384-c1bd-4a2d-c94b-33920b326761" colab={"base_uri": "https://localhost:8080/", "height": 34}
# Test model on validation set.
pred = logistic_regression(x_test)
print("Test Accuracy: %f" % accuracy(pred, y_test))

# + id="JTJACcN_feQJ" outputId="4d69df2e-aed9-450c-9088-8d8c4b92831d" colab={"base_uri": "https://localhost:8080/", "height": 1000}
# Predict 5 images from validation set.
n_images = 5
test_images = x_test[:n_images]
predictions = logistic_regression(test_images)

# Display image and model prediction.
for i in range(n_images):
    plt.imshow(np.reshape(test_images[i], [64, 64]))
    plt.show()
    print("Model prediction: %i" % np.argmax(predictions.numpy()[i]))

# + [markdown] id="mln_TS_ygGgA"
# # 2. Convolutional Neural Networks
#
# Unlike Logistic Regression and Recurrent Neural Networks, an advantage of CNN is that the input data doesn't need to be flattened, thus we will reshape the data to its origial shape to fit the model.

# + id="ortym2B6f0f9"
# Modified MNIST dataset parameters.
num_classes = 10 # total classes (0-9 digits).

# Training parameters.
learning_rate = 0.01
training_steps = 500
batch_size = 128
display_step = 50
dropout = 0.7

# Network parameters.
num_input = 4096
conv1_filters = 16 # number of filters for 1st conv layer.
conv2_filters = 32 # number of filters for 2nd conv layer.
conv3_filters = 128 # number of filters for 3nd conv layer.
fc1_units = 1024 # number of neurons for 1st fully-connected layer.

# + id="oZX_6Yg9hz1R" outputId="95f8af98-33d8-41ae-af22-6504b63e98e8" colab={"base_uri": "https://localhost:8080/", "height": 34}
(x_train.shape,x_test.shape,y_train.shape,y_test.shape)


# + id="a2tso1o8h6fY"
# Create some wrappers for simplicity.
def conv2d(x, W, b, strides=1):
    # Conv2D wrapper, with bias and relu activation.
    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')
    x = tf.nn.bias_add(x, b)
    return tf.nn.relu(x)

def maxpool2d(x, k=2):
    # MaxPool2D wrapper.
    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')


# + id="nrzqtNJeh_QE"
# Store layers weight & bias

# A random value generator to initialize weights.
random_normal = tf.initializers.RandomNormal()

weights = {
    # Conv Layer 1: 5x5 conv, 1 input, 16 filters 
    'wc1': tf.Variable(random_normal([5, 5, 1, conv1_filters])),
    # Conv Layer 2: 3x3 conv, 16 inputs, 32 filters.
    'wc2': tf.Variable(random_normal([3, 3, conv1_filters, conv2_filters])),
    # Conv Layer 3: 3x3 conv, 32 inputs, 128 filters
    'wc3': tf.Variable(random_normal([3, 3, conv2_filters, conv3_filters])),
    # FC Layer 1: 8*8*128 inputs, 1024 units.
    'wd1': tf.Variable(random_normal([8*8*128, fc1_units])),
    # FC Out Layer: 1024 inputs, 10 units (total number of classes)
    'out': tf.Variable(random_normal([fc1_units, num_classes]))
}

biases = {
    'bc1': tf.Variable(tf.zeros([conv1_filters])),
    'bc2': tf.Variable(tf.zeros([conv2_filters])),
    'bc3': tf.Variable(tf.zeros([conv3_filters])),
    'bd1': tf.Variable(tf.zeros([fc1_units])),
    'out': tf.Variable(tf.zeros([num_classes]))
}


# + id="JnCo3n1uiNcw"
# Create model
def conv_net(x):
    
    # Input shape: [-1, 64, 64, 1]. A batch of 64x64x1 images.
    x = tf.reshape(x, [-1, 64, 64, 1])

    # Convolution Layer. Output shape: [-1, 64, 64, 16].
    conv1 = conv2d(x, weights['wc1'], biases['bc1'])
    
    # Max Pooling (down-sampling). Output shape: [-1, 32, 32, 16].
    conv1 = maxpool2d(conv1, k=2)

    # Convolution Layer. Output shape: [-1, 32, 32, 32].
    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])
    
    # Max Pooling (down-sampling). Output shape: [-1, 16, 16, 32].
    conv2 = maxpool2d(conv2, k=2)

    # Convolution Layer. Output shape: [-1, 16, 16, 128].
    conv3 = conv2d(conv2, weights['wc3'], biases['bc3'])
    
    # Max Pooling (down-sampling). Output shape: [-1, 8, 8, 128].
    conv3 = maxpool2d(conv3, k=2)

    # Reshape conv3 output to fit fully connected layer input, Output shape: [-1, 8*8*128].
    fc1 = tf.reshape(conv3, [-1, weights['wd1'].get_shape().as_list()[0]])
    
    # Fully connected layer, Output shape: [-1, 1024].
    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
    # Apply ReLU to fc1 output for non-linearity.
    fc1 = tf.nn.relu(fc1)
    # Apply Dropout
    fc1 = tf.nn.dropout(fc1, dropout)

    # Fully connected layer, Output shape: [-1, 10].
    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
    # Apply softmax to normalize the logits to a probability distribution.
    return tf.nn.softmax(out)


# + id="QcWsFy_Xivfy"
# Cross-Entropy loss function.
def cross_entropy(y_pred, y_true):
    # Encode label to a one hot vector.
    y_true = tf.one_hot(y_true, depth=num_classes)
    # Clip prediction values to avoid log(0) error.
    y_pred = tf.clip_by_value(y_pred, 1e-9, 1.)
    # Compute cross-entropy.
    return tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred)))

# Accuracy metric.
def accuracy(y_pred, y_true):
    # Predicted class is the index of highest score in prediction vector (i.e. argmax).
    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))
    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)

# ADAM optimizer.
optimizer = tf.optimizers.Adam(learning_rate)


# + id="BTDL_vMkizAq"
# Optimization process. 
def run_optimization(x, y):
    # Wrap computation inside a GradientTape for automatic differentiation.
    with tf.GradientTape() as g:
        pred = conv_net(x)
        loss = cross_entropy(pred, y)
        
    # Variables to update, i.e. trainable variables.
    trainable_variables = list(weights.values()) + list(biases.values())

    # Compute gradients.
    gradients = g.gradient(loss, trainable_variables)
    
    # Update W and b following gradients.
    optimizer.apply_gradients(zip(gradients, trainable_variables))


# + id="kJ6LyIOri1cb" outputId="c41c29c1-1f87-4c2b-ddb8-66c63cd08b0b" colab={"base_uri": "https://localhost:8080/", "height": 187}
# Run training for the given number of steps.
for step, (batch_x, batch_y) in enumerate(train_data.take(training_steps), 1):
    # Run the optimization to update W and b values.
    run_optimization(batch_x, batch_y)
    
    if step % display_step == 0:
        pred = conv_net(batch_x)
        loss = cross_entropy(pred, batch_y)
        acc = accuracy(pred, batch_y)
        print("step: %i, loss: %f, accuracy: %f" % (step, loss, acc))

# + id="ezmU1tTei4WH" outputId="af33bd7b-fa88-4d55-b190-ac27aa258041" colab={"base_uri": "https://localhost:8080/", "height": 34}
# Test model on validation set.
pred = conv_net(x_test)
print("Test Accuracy: %f" % accuracy(pred, y_test))


# + [markdown] id="qcPkKoc9xNkF"
# # 3. Gated CNN
# The convolutional layer in GCN used a gating mechanism to allow the network to control what information should be propagated the hierarchy of layers.

# + id="dt9bys4yzAXx"
# Create some wrappers for simplicity.
def gconv2d(x, W, b, V, c, num, strides=1):
    # Conv2D wrapper, with bias and gates.
    A = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')
    B = tf.nn.conv2d(x, V, strides=[1, strides, strides, 1], padding='SAME')
    A = tf.nn.bias_add(A, b)
    A = tf.nn.bias_add(B, c)
    return tf.math.multiply(A, tf.math.sigmoid(B))


# + id="B6Pi-ApFzDvL"
# Store layers weight & bias

# A random value generator to initialize weights.
random_normal = tf.initializers.RandomNormal()

weights = {
    # Conv Layer 1: 5x5 conv, 1 input, 16 filters 
    'wc1': tf.Variable(random_normal([5, 5, 1, conv1_filters])),
    'wv1': tf.Variable(random_normal([5, 5, 1, conv1_filters])),
    'c1': tf.Variable(5.),
    # Conv Layer 2: 3x3 conv, 16 inputs, 32 filters.
    'wc2': tf.Variable(random_normal([3, 3, conv1_filters, conv2_filters])),
    'wv2': tf.Variable(random_normal([3, 3, conv1_filters, conv2_filters])),
    'c2': tf.Variable(3.),
    # Conv Layer 3: 3x3 conv, 32 inputs, 128 filters
    'wc3': tf.Variable(random_normal([3, 3, conv2_filters, conv3_filters])),
    'wv3': tf.Variable(random_normal([3, 3, conv2_filters, conv3_filters])),
    'c3': tf.Variable(3.),
    # FC Layer 1: 8*8*128 inputs, 1024 units.
    'wd1': tf.Variable(random_normal([8*8*128, fc1_units])),
    # FC Out Layer: 1024 inputs, 10 units (total number of classes)
    'out': tf.Variable(random_normal([fc1_units, num_classes]))
}

biases = {
    'bc1': tf.Variable(tf.zeros([conv1_filters])),
    'bv1': tf.Variable(tf.zeros([conv1_filters])),
    'bc2': tf.Variable(tf.zeros([conv2_filters])),
    'bv2': tf.Variable(tf.zeros([conv2_filters])),
    'bc3': tf.Variable(tf.zeros([conv3_filters])),
    'bv3': tf.Variable(tf.zeros([conv3_filters])),
    'bd1': tf.Variable(tf.zeros([fc1_units])),
    'out': tf.Variable(tf.zeros([num_classes]))
}


# + id="R4cpoX9rzKSF"
# Create model
def gconv_net(x):
    
    # Input shape: [-1, 64, 64, 1]. A batch of 64x64x1 images.
    x = tf.reshape(x, [-1, 64, 64, 1])

    # Convolution Layer. Output shape: [-1, 64, 64, 16].
    conv1 = gconv2d(x, weights['wc1'], biases['bc1'], weights['wv1'], biases['bv1'], weights['c1'])
    #conv1 = conv2d(x, weights['wc1'], biases['bc1'])
    # Max Pooling (down-sampling). Output shape: [-1, 32, 32, 16].
    conv1 = maxpool2d(conv1, k=2)

    # Convolution Layer. Output shape: [-1, 32, 32, 32].
    conv2 = gconv2d(conv1, weights['wc2'], biases['bc2'], weights['wv2'], biases['bv2'], weights['c2'])
    #conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])
    # Max Pooling (down-sampling). Output shape: [-1, 16, 16, 32].
    conv2 = maxpool2d(conv2, k=2)

    # Convolution Layer. Output shape: [-1, 16, 16, 128].
    conv3 = gconv2d(conv2, weights['wc3'], biases['bc3'], weights['wv3'], biases['bv3'], weights['c3'])
    #conv3 = conv2d(conv2, weights['wc3'], biases['bc3'])
    # Max Pooling (down-sampling). Output shape: [-1, 8, 8, 128].
    conv3 = maxpool2d(conv3, k=2)

    # Reshape conv3 output to fit fully connected layer input, Output shape: [-1, 8*8*128].
    fc1 = tf.reshape(conv3, [-1, weights['wd1'].get_shape().as_list()[0]])
    
    # Fully connected layer, Output shape: [-1, 1024].
    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
    # Apply ReLU to fc1 output for non-linearity.
    fc1 = tf.nn.relu(fc1)
    # Apply Dropout
    fc1 = tf.nn.dropout(fc1, dropout)

    # Fully connected layer, Output shape: [-1, 10].
    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
    # Apply softmax to normalize the logits to a probability distribution.
    return tf.nn.softmax(out)


# + id="pSflc5DxzPh-"
# Cross-Entropy loss function.
def cross_entropy(y_pred, y_true):
    # Encode label to a one hot vector.
    y_true = tf.one_hot(y_true, depth=num_classes)
    # Clip prediction values to avoid log(0) error.
    y_pred = tf.clip_by_value(y_pred, 1e-9, 1.)
    # Compute cross-entropy.
    return tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred)))

# Accuracy metric.
def accuracy(y_pred, y_true):
    # Predicted class is the index of highest score in prediction vector (i.e. argmax).
    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))
    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)

# ADAM optimizer.
optimizer = tf.optimizers.Adam(learning_rate)


# + id="IY4D3F-szSnm"
# Optimization process. 
def run_optimization(x, y):
    # Wrap computation inside a GradientTape for automatic differentiation.
    with tf.GradientTape() as g:
        pred = gconv_net(x)
        loss = cross_entropy(pred, y)
        
    # Variables to update, i.e. trainable variables.
    trainable_variables = list(weights.values()) + list(biases.values())

    # Compute gradients.
    gradients = g.gradient(loss, trainable_variables)
    
    # Update W and b following gradients.
    optimizer.apply_gradients((grad, var) for (grad, var) in zip(gradients, trainable_variables) if grad is not None)


# + id="7k5jMt5IzU8A" outputId="e76af344-1202-4144-ccae-aa7ca4307ddc" colab={"base_uri": "https://localhost:8080/", "height": 187}
# Run training for the given number of steps.
for step, (batch_x, batch_y) in enumerate(train_data.take(training_steps), 1):
    # Run the optimization to update W and b values.
    run_optimization(batch_x, batch_y)
    
    if step % display_step == 0:
        pred = gconv_net(batch_x)
        loss = cross_entropy(pred, batch_y)
        acc = accuracy(pred, batch_y)
        print("step: %i, loss: %f, accuracy: %f" % (step, loss, acc))

# + [markdown] id="JLMtrFYxYClv"
# # 4. Recurrent Neural Networks_LSTM
#
# To classify images using a recurrent neural network, we consider every image row as a sequence of pixels. Because Modified MNIST image shape is 64*64px, we will then handle 64 sequences of 64 timesteps for every sample.

# + id="q1iYww8G9nSf"
y_train_r = np_utils.to_categorical(y_train, num_classes)
y_test_r = np_utils.to_categorical(y_test, num_classes)

# + id="zV4E7yNL-Apw" outputId="28a3a218-759a-4f3e-a832-2ce308c20ba1" colab={"base_uri": "https://localhost:8080/", "height": 34}
(x_train.shape,x_test.shape,y_train_r.shape,y_test_r.shape)

# + id="iXy9l-88Xyq7" outputId="28b6dc37-55b9-4258-aff2-60989618b24e" colab={"base_uri": "https://localhost:8080/", "height": 221}
# Hyper parameters
batch_size = 128
nb_epoch = 30
# Parameters for MNIST dataset
img_rows, img_cols = 64, 64
nb_classes = 10
# Parameters for LSTM network
nb_lstm_outputs = 30
nb_time_steps = img_rows
dim_input_vector = img_cols
input_shape = (64, 64)

x_train=x_train.reshape(x_train.shape[0],64,64)
x_test=x_test.reshape(x_test.shape[0],64,64)

model_LSTM = Sequential()
model_LSTM.add(LSTM(nb_lstm_outputs, input_shape=input_shape))
model_LSTM.add(Dense(10, activation='softmax'))
model_LSTM.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
model_LSTM.summary()

# + id="Vn_ouHEpX60p" outputId="22ac8843-0c9c-4c57-b476-f2f6380a8651" colab={"base_uri": "https://localhost:8080/", "height": 1000}
# Train the model
history = model_LSTM.fit(x_train, 
                    y_train_r, 
                    epochs=nb_epoch, 
                    batch_size=batch_size, 
                    shuffle=True,
                    validation_data=(x_test, y_test_r),
                    verbose=1)

# + [markdown] id="n8vhbTos-yNB"
# # 5. CNN_VGG
# Apart from a vanilla CNN model, I also used a more sophisticated model-VGG and got a decent performance.

# + id="lSEa6ViJ_FAh"
x_train, x_test = x_train.reshape([-1,64,64,1]), x_test.reshape([-1,64,64,1])

# + id="oJ_8BWJm-oxD"
# Data augmentation
input_shape = (64,64,1)
datagen = ImageDataGenerator(
    rotation_range=20, 
    width_shift_range=0.1,
    shear_range=0.2,
    height_shift_range=0.1,
    zoom_range=0.2
    )

datagen.fit(x_train)


# + id="AD2Sq-b7_vZn"
# Build model
def VGG_modified(input_shape, opt = tf.keras.optimizers.Adam(amsgrad=True)):
    model = Sequential()
    
    model.add(Conv2D(32, kernel_size=(3, 3), padding="same", activation='relu', input_shape=input_shape,
                            data_format="channels_last"))
    model.add(Conv2D(32, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(32, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))

    model.add(Conv2D(64, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(64, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(64, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))

    model.add(Conv2D(128, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(128, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(128, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))

    model.add(Conv2D(256, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(256, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(Conv2D(256, kernel_size=(3, 3), padding="same", activation='relu'))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))

    model.add(Flatten())
    model.add(Dense(512, activation='relu'))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(10, activation='softmax'))
    
    #sgd=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
    return model


# + id="gb6mo9RJ_vzV" outputId="c2aa48f5-0cfb-490f-b8ba-dc5781019d8e" colab={"base_uri": "https://localhost:8080/", "height": 598}
model = VGG_modified((64,64,1))
history = model.fit_generator(datagen.flow(x_train, y_train_r, batch_size=64),
                              validation_data=(x_test, y_test_r),
                              epochs = 15)

# + id="PM9v7L_vlfS5"


StarcoderdataJupyterScriptsDedupFiltered