# -*- coding: utf-8 -*-
"""ReNS experiments - CIFAR10

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1byZ4xTfCK2x1Rhkxpl-Vv4sqA-bo4bis

# SETUP
"""

#@title Insatlling Pyorch

# !pip install torch
# !pip install torchvision

#@title Import Dependencies
import numpy as np
import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
from tqdm import tqdm
from typing import Optional, Union, Tuple, List, Sequence, Iterable
import math
from scipy.spatial.distance import euclidean
from torch.nn.modules.utils import _pair
from torchvision import models
from sklearn.metrics import jaccard_score
import matplotlib.pyplot as plt
from models.models import RegularAutoEncoder, ModulatedAutoEncoder, PseudoRecAutoEncoder


"""# TRAINING"""

batch_size = 32
num_epochs = 5

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

# Load MNIST data.
train_data = dsets.CIFAR10(root = './data', train = True,
                        transform = transform, download = True)

test_data = dsets.CIFAR10(root = './data', train = False,
                       transform = transform)

train_gen = torch.utils.data.DataLoader(dataset = train_data,
                                             batch_size = batch_size,
                                             shuffle = True)

test_gen = torch.utils.data.DataLoader(dataset = test_data,
                                      batch_size = batch_size,
                                      shuffle = False)

reflexor_size = 500
image_size = 32
channels = 3

# net = recurrentLayer(784, 784, 10, 5, 10, 0)
net1 = RegularAutoEncoder(channels * image_size ** 2, channels * image_size ** 2, reflexor_size)
net2 = ModulatedAutoEncoder(channels * image_size ** 2, channels * image_size ** 2, reflexor_size)
net3 = PseudoRecAutoEncoder(channels * image_size ** 2, channels * image_size ** 2, reflexor_size)



lr = .0001 # size of step
loss_function = nn.MSELoss()

# Unnormalize the image to display it
def img_fix(img):
  return np.transpose((img / 2 + 0.5).numpy(), (1, 2, 0))

# Commented out IPython magic to ensure Python compatibility.
train_losses = [[],[],[]]
test_losses = [[],[],[]]

real_imgs = [[],[],[]]
reconstructed_imgs = [[],[],[]]

param_counts = np.ones(3)

steps = [[],[],[]]

for num, net in enumerate([net1, net2, net3]):

  optimizer = torch.optim.Adam( net.parameters(), lr=lr)
  param_counts[num] = (sum(p.numel() for p in net.parameters() if p.requires_grad))

  for epoch in range(num_epochs):
    for i ,(images,labels) in enumerate(train_gen):
      #images = Variable(images.view(-1,28*28))
      labels = Variable(images.view(-1,3 * image_size ** 2))

      optimizer.zero_grad()
      outputs = net(images)
      loss = loss_function(outputs, labels)
      loss.backward()
      optimizer.step()

      if (i+1) % 300 == 0:
        temp_loss = loss.item()
        print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'
                   %(epoch+1, num_epochs, i+1, len(train_data)//batch_size, temp_loss))
        dupe = Variable(outputs[0].data, requires_grad=False)
        # plt.imshow(img_fix(images[0]))
        # plt.show()
        # plt.imshow(img_fix(dupe.view(3, image_size, image_size)))
        # plt.show()
        train_losses[num].append(temp_loss)
        steps[num].append((50000 * epoch) + ((i + 1) * batch_size))

        real_imgs[num].append(img_fix(images[0]))
        reconstructed_imgs[num].append(img_fix(dupe.view(3, image_size, image_size)))

        # Test Data

        score = 0
        total = 0
        for images,labels in test_gen:
          #images = Variable(images.view(-1,784))

          output = net(images)
          score += loss_function(output, images.view(-1, 3 * image_size ** 2)).item()
        test_losses[num].append((score))

plt.plot(steps[0], train_losses[0], label= "Baseline")
plt.plot(steps[1], train_losses[1], label= "Modulated")
plt.plot(steps[2], train_losses[2], label= "Recurrent with Modulation")
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.title('Training loss history')
plt.legend()
plt.show()

plt.plot(steps[0], test_losses[0], label= "Baseline")
plt.plot(steps[1], test_losses[1], label= "Modulated")
plt.plot(steps[2], test_losses[2], label= "Recurrent with Modulation")
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.title('Testing loss history')
plt.legend()
plt.show()

for num,count in enumerate(param_counts):
  param_counts[num] /= 1000


plt.bar(["Base", "Modulated", "ReNS"], param_counts)
plt.xlabel('Model')
plt.ylabel('# of thousands of Parameters')
plt.show()

from mpl_toolkits.axes_grid1 import ImageGrid

num_smaples = len(real_imgs[0])


for num in [0,1,2]:
  fig = plt.figure(figsize=(20.,20.))
  grid = ImageGrid(fig, 111,  # similar to subplot(111)
                  nrows_ncols=(2, num_smaples),  # creates 2x2 grid of axes
                  axes_pad=0.1,  # pad between axes in inch.
                  )

  for ax, im in zip(grid, real_imgs[num]+reconstructed_imgs[num]):
      # Iterating over the grid returns the Axes.
      ax.imshow(im)
      ax.axis("off")

  plt.show()

[((45, 13, 46, 63), 'torchvision.datasets.CIFAR10', 'dsets.CIFAR10', (), '', True, 'import torchvision.datasets as dsets\n'), ((48, 12, 49, 45), 'torchvision.datasets.CIFAR10', 'dsets.CIFAR10', (), '', True, 'import torchvision.datasets as dsets\n'), ((51, 12, 53, 60), 'torch.utils.data.DataLoader', 'torch.utils.data.DataLoader', (), '', False, 'import torch\n'), ((55, 11, 57, 54), 'torch.utils.data.DataLoader', 'torch.utils.data.DataLoader', (), '', False, 'import torch\n'), ((64, 7, 64, 96), 'models.models.RegularAutoEncoder', 'RegularAutoEncoder', ({(64, 26, 64, 52): 'channels * image_size ** 2', (64, 54, 64, 80): 'channels * image_size ** 2', (64, 82, 64, 95): 'reflexor_size'}, {}), '(channels * image_size ** 2, channels * image_size ** 2,\n    reflexor_size)', False, 'from models.models import RegularAutoEncoder, ModulatedAutoEncoder, PseudoRecAutoEncoder\n'), ((65, 7, 65, 98), 'models.models.ModulatedAutoEncoder', 'ModulatedAutoEncoder', ({(65, 28, 65, 54): 'channels * image_size ** 2', (65, 56, 65, 82): 'channels * image_size ** 2', (65, 84, 65, 97): 'reflexor_size'}, {}), '(channels * image_size ** 2, channels * image_size ** 2,\n    reflexor_size)', False, 'from models.models import RegularAutoEncoder, ModulatedAutoEncoder, PseudoRecAutoEncoder\n'), ((66, 7, 66, 98), 'models.models.PseudoRecAutoEncoder', 'PseudoRecAutoEncoder', ({(66, 28, 66, 54): 'channels * image_size ** 2', (66, 56, 66, 82): 'channels * image_size ** 2', (66, 84, 66, 97): 'reflexor_size'}, {}), '(channels * image_size ** 2, channels * image_size ** 2,\n    reflexor_size)', False, 'from models.models import RegularAutoEncoder, ModulatedAutoEncoder, PseudoRecAutoEncoder\n'), ((71, 16, 71, 28), 'torch.nn.MSELoss', 'nn.MSELoss', ({}, {}), '()', True, 'import torch.nn as nn\n'), ((84, 15, 84, 25), 'numpy.ones', 'np.ones', ({(84, 23, 84, 24): '3'}, {}), '(3)', True, 'import numpy as np\n'), ((130, 0, 130, 54), 'matplotlib.pyplot.plot', 'plt.plot', (), '', True, 'import matplotlib.pyplot as plt\n'), ((131, 0, 131, 55), 'matplotlib.pyplot.plot', 'plt.plot', (), '', True, 'import matplotlib.pyplot as plt\n'), ((132, 0, 132, 71), 'matplotlib.pyplot.plot', 'plt.plot', (), '', True, 'import matplotlib.pyplot as plt\n'), ((133, 0, 133, 23), 'matplotlib.pyplot.xlabel', 'plt.xlabel', ({(133, 11, 133, 22): '"""Iteration"""'}, {}), "('Iteration')", True, 'import matplotlib.pyplot as plt\n'), ((134, 0, 134, 18), 'matplotlib.pyplot.ylabel', 'plt.ylabel', ({(134, 11, 134, 17): '"""Loss"""'}, {}), "('Loss')", True, 'import matplotlib.pyplot as plt\n'), ((135, 0, 135, 34), 'matplotlib.pyplot.title', 'plt.title', ({(135, 10, 135, 33): '"""Training loss history"""'}, {}), "('Training loss history')", True, 'import matplotlib.pyplot as plt\n'), ((136, 0, 136, 12), 'matplotlib.pyplot.legend', 'plt.legend', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((137, 0, 137, 10), 'matplotlib.pyplot.show', 'plt.show', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((139, 0, 139, 53), 'matplotlib.pyplot.plot', 'plt.plot', (), '', True, 'import matplotlib.pyplot as plt\n'), ((140, 0, 140, 54), 'matplotlib.pyplot.plot', 'plt.plot', (), '', True, 'import matplotlib.pyplot as plt\n'), ((141, 0, 141, 70), 'matplotlib.pyplot.plot', 'plt.plot', (), '', True, 'import matplotlib.pyplot as plt\n'), ((142, 0, 142, 23), 'matplotlib.pyplot.xlabel', 'plt.xlabel', ({(142, 11, 142, 22): '"""Iteration"""'}, {}), "('Iteration')", True, 'import matplotlib.pyplot as plt\n'), ((143, 0, 143, 18), 'matplotlib.pyplot.ylabel', 'plt.ylabel', ({(143, 11, 143, 17): '"""Loss"""'}, {}), "('Loss')", True, 'import matplotlib.pyplot as plt\n'), ((144, 0, 144, 33), 'matplotlib.pyplot.title', 'plt.title', ({(144, 10, 144, 32): '"""Testing loss history"""'}, {}), "('Testing loss history')", True, 'import matplotlib.pyplot as plt\n'), ((145, 0, 145, 12), 'matplotlib.pyplot.legend', 'plt.legend', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((146, 0, 146, 10), 'matplotlib.pyplot.show', 'plt.show', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((152, 0, 152, 52), 'matplotlib.pyplot.bar', 'plt.bar', ({(152, 8, 152, 37): "['Base', 'Modulated', 'ReNS']", (152, 39, 152, 51): 'param_counts'}, {}), "(['Base', 'Modulated', 'ReNS'], param_counts)", True, 'import matplotlib.pyplot as plt\n'), ((153, 0, 153, 19), 'matplotlib.pyplot.xlabel', 'plt.xlabel', ({(153, 11, 153, 18): '"""Model"""'}, {}), "('Model')", True, 'import matplotlib.pyplot as plt\n'), ((154, 0, 154, 42), 'matplotlib.pyplot.ylabel', 'plt.ylabel', ({(154, 11, 154, 41): '"""# of thousands of Parameters"""'}, {}), "('# of thousands of Parameters')", True, 'import matplotlib.pyplot as plt\n'), ((155, 0, 155, 10), 'matplotlib.pyplot.show', 'plt.show', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((163, 8, 163, 37), 'matplotlib.pyplot.figure', 'plt.figure', (), '', True, 'import matplotlib.pyplot as plt\n'), ((164, 9, 167, 19), 'mpl_toolkits.axes_grid1.ImageGrid', 'ImageGrid', (), '', False, 'from mpl_toolkits.axes_grid1 import ImageGrid\n'), ((174, 2, 174, 12), 'matplotlib.pyplot.show', 'plt.show', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((41, 5, 41, 26), 'torchvision.transforms.ToTensor', 'transforms.ToTensor', ({}, {}), '()', True, 'import torchvision.transforms as transforms\n'), ((42, 5, 42, 59), 'torchvision.transforms.Normalize', 'transforms.Normalize', ({(42, 26, 42, 41): '(0.5, 0.5, 0.5)', (42, 43, 42, 58): '(0.5, 0.5, 0.5)'}, {}), '((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))', True, 'import torchvision.transforms as transforms\n'), ((108, 15, 108, 61), 'torch.autograd.Variable', 'Variable', (), '', False, 'from torch.autograd import Variable\n')]