# As usual, a bit of setup

import time, os, json
import numpy as np
import matplotlib.pyplot as plt

from cs231n.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array

%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

def rel_error(x, y):
  """ returns relative error """
  return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

"""
Explanation: I wanted to implement a GAN but my derivatives didn't work after a lot of try, so I gave up. If you are available in Monday or Tuesday can we have a look at them?
Generative Adverserial Networks
In this notebook, I will try to implement the idea of a GAN, for capturing unimodal and bimodal distributions.
End of explanation
"""


inputVector = np.random.uniform(-100.,100.,(1000000,1))

"""
Explanation: As our input vector we will always use a uniform random distrubiton, but instead of using between 0-1 we will use 0-100 range to make life a little bit easier for our Generator network.
End of explanation
"""


from cs231n.classifiers.neural_net import * #we will use the 2layer FCN from HW1
g_inpsize = 1   #we will just take 1 real number uniform distrubtion
g_hidsize = 50 #
g_outsize = 1   #we will output a real number from our data distribution

Generator = GenNet(g_inpsize, g_hidsize, g_outsize) #it is gonna perform cross entropy loss instead of softmax,
                                                         #since our labels are just real or fake, 
                                                         #it fits our purposes

d_inpsize = 1 #first check without batch normaliziton
d_hidsize = 50
d_outsize = 1 #again we are just gonna output real or fake

Discriminator = DiscNet(d_inpsize, d_hidsize, d_outsize) #we will use slightly modified version of TwoLayerNet
                                                            #we just added sigmoid at the output layer since all of
                                                        #our outputs needs to be in range [0,1]

"""
Explanation: Generator Network
We will use a basic 2 layer FCN for our generator and discriminator, since our dataset is basically one dimensional it should be enough
End of explanation
"""


from cs231n.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array

D = 1
x = np.random.rand(200, D)
y = np.ones((200,1))
W1 = np.random.randn(D, d_hidsize)
W2 = np.random.randn(d_hidsize, d_hidsize)
W3 = np.random.randn(d_hidsize, d_outsize)
b1 = np.random.randn(d_hidsize)
b2 = np.random.randn(d_hidsize)
b3 = np.random.randn(d_outsize)

fx = lambda x: Discriminator.loss(x, y)[0]
fW1 = lambda W1: Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fW2 = lambda W2: Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fW3 = lambda W3: Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fb1 = lambda b1: Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fb2 = lambda b2: Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fb3 = lambda b3: Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]

num_grad = lambda x,y: eval_numerical_gradient(x,y, verbose=False, h=1e-6)

dx_num = num_grad(fx, x)
dW1_num = num_grad(fW1, W1)
dW2_num = num_grad(fW2, W2)
dW3_num = num_grad(fW3, W3)
db1_num = num_grad(fb1, b1)
db2_num = num_grad(fb2, b2)
db3_num = num_grad(fb3, b3)

loss, grads = Discriminator.loss(x, y, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)
dx, dW1, dW2, dW3, db1, db2, db3 = grads['X'], grads['W1'], grads['W2'], grads['W3'], grads['b1'], grads['b2'], grads['b3']
print 'dx error: ', rel_error(dx_num, dx)
print 'dW1 error: ', rel_error(dW1_num, dW1)
print 'dW2 error: ', rel_error(dW2_num, dW2)
print 'dW3 error: ', rel_error(dW3_num, dW3)
print 'db1 error: ', rel_error(db1_num, db1)
print 'db2 error: ', rel_error(db2_num, db2)
print 'db3 error: ', rel_error(db3_num, db3)

D = 1
x = np.random.randn(200, D)
W1 = np.random.randn(D, g_hidsize)
W2 = np.random.randn(g_hidsize, g_hidsize)
W3 = np.random.randn(g_hidsize, g_outsize)
b1 = np.random.randn(g_hidsize)
b2 = np.random.randn(g_hidsize)
b3 = np.random.randn(g_outsize)

fx = lambda x: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fW1 = lambda W1: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fW2 = lambda W2: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fW3 = lambda W3: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fb1 = lambda b1: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fb2 = lambda b2: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]
fb3 = lambda b3: Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)[0]

num_grad = lambda x,y: eval_numerical_gradient(x,y, verbose=False)

dx_num = num_grad(fx, x)
dW1_num = num_grad(fW1, W1)
dW2_num = num_grad(fW2, W2)
dW3_num = num_grad(fW3, W3)
db1_num = num_grad(fb1, b1)
db2_num = num_grad(fb2, b2)
db3_num = num_grad(fb3, b3)

loss, grads = Generator.loss(x, Discriminator, W1=W1, W2=W2, W3=W3, b1=b1, b2=b2, b3=b3)
dx, dW1, dW2, dW3, db1, db2, db3 = grads['X'], grads['W1'], grads['W2'], grads['W3'], grads['b1'], grads['b2'], grads['b3']

print 'dx error: ', rel_error(dx_num, dx)
print 'dW1 error: ', rel_error(dW1_num, dW1)
print 'dW2 error: ', rel_error(dW2_num, dW2)
print 'dW3 error: ', rel_error(dW3_num, dW3)
print 'db1 error: ', rel_error(db1_num, db1)
print 'db2 error: ', rel_error(db2_num, db2)
print 'db3 error: ', rel_error(db3_num, db3)

"""
Explanation: Gradient Check
End of explanation
"""


N = 1000000
mean = 0.0
std = 1.0

data = np.random.normal(mean, std, (N,1))

_ = plt.hist(data, 100, normed=1)
_ = plt.hist(inputVector[np.abs(inputVector)<4], 100, normed=1)

g_inpsize = 1   #we will just take 1 real number uniform distrubtion
g_hidsize = 50 #
g_outsize = 1   #we will output a real number from our data distribution

Generator = GenNet(g_inpsize, g_hidsize, g_outsize) #it is gonna perform cross entropy loss instead of softmax,
                                                         #since our labels are just real or fake, 
                                                         #it fits our purposes

d_inpsize = 1 #first check without batch normaliziton
d_hidsize = 50
d_outsize = 1 #again we are just gonna output real or fake

Discriminator = DiscNet(d_inpsize, d_hidsize, d_outsize) #we will use slightly modified version of TwoLayerNet
                                                            #we just added sigmoid at the output layer since all of
                                                        #our outputs needs to be in range [0,1]

Trainer = GANTrainer(Generator, Discriminator, data, update_rule='adam',
           num_epochs=5,
           batch_size=100,
           optim_config={
             'learning_rate': 2e-5,
           },
           lr_decay=0.995,
           verbose=True, print_every=10000)

Trainer.train()

plt.plot([x[0] for x in Trainer.loss_history])
plt.plot([x[1] for x in Trainer.loss_history])
plt.plot([x[2] for x in Trainer.loss_history])

"""
Explanation: Let us start with a standart distribution as our dataset
End of explanation
"""


