import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

"""
Explanation: Building a neural network with TensorFlow
In this module we are going to build a neural network for regression. Regression is the prediction of a real-valued number given some inputs.
End of explanation
"""


n_observations = 1000
xs = np.linspace(-3.0, 3.0, n_observations)
ys = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)

plt.scatter(xs, ys, alpha=0.15, marker = '+')
plt.show()
# alpha makes the points transparent and marker changes it from dots to +'s

"""
Explanation: Let's generate some data, in this case, a noisy sine wave as plotted below
End of explanation
"""


X = tf.placeholder(tf.float32, name = 'X')
Y = tf.placeholder(tf.float32, name = 'Y')

sess = tf.InteractiveSession()

n = tf.random_normal([1000]).eval()
n_ = tf.random_normal([1000], stddev = 0.1).eval()

plt.hist(n) # plt.hist(n, 20) gives answer with 20 buckets
plt.hist(n_) # We need initial values much closer to 0 for initializing the weights

"""
Explanation: We are going to use placeholders from now on. Placeholder for X and Y are as follows
End of explanation
"""


W = tf.Variable(tf.random_normal([1], stddev=0.1), name = 'weight')
B = tf.Variable(0.0, name = 'bias')

"""
Explanation: We need two parameters, weight W and bias B for our model
End of explanation
"""


# Perceptron model (or Linear regression)
Y_ = X*W + B  

def distance(y, y_):
    return tf.abs(y-y_)

# cost = distance(Y_, tf.sin(X))

# Mean absolute error
cost = tf.reduce_mean(distance(Y_, Y))

optimizer = tf.train.GradientDescentOptimizer(learning_rate = 0.01).minimize(cost)

"""
Explanation: We need to define model, and a cost function
End of explanation
"""


n_iterations = 100
sess.run(tf.initialize_all_variables())
for _ in range(n_iterations):
    sess.run(optimizer, feed_dict = {X:xs, Y:ys})
    training_cost = sess.run(cost, feed_dict = {X:xs, Y:ys})
    
    # This is how to print the values mid execution
    print training_cost, sess.run(W), sess.run(B)

"""
Explanation: Now we have defined the variables, we need to run the code
Before we run the code, we must also run the variables using tf.initialize_all_variables() to give an initial value to W and B.
End of explanation
"""


n_iterations = 100
sess.run(tf.initialize_all_variables())
for _ in range(n_iterations):
    ids = np.random.permutation([x for x in range(0, len(xs))])
    batch_size = 100
    i = 0
    while i < len(ids):
        sess.run(optimizer, feed_dict = {X:xs[i:i+batch_size], Y: ys[i:i+batch_size]})
        i += batch_size
        training_cost = sess.run(cost, feed_dict = {X:xs[i:i+batch_size], Y: ys[i:i+batch_size]})
        
        print training_cost, sess.run(W), sess.run(B)

"""
Explanation: We'll now try doing the same thing with mini-batch learning
End of explanation
"""


def train(X, Y, Y_, n_iterations = 100, batch_size = 100, learning_rate = 0.02):
    cost = tf.reduce_mean(distance(Y,Y_))
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
    fig, ax = plt.subplots(1, 1)
    ax.scatter(xs, ys, alpha= 0.1, marker='+')
    
    ax.set_xlim([-4, 4])
    ax.set_ylim([-2, 2])
    
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        
        prev_training_cost = 0.0
        for it_i in range(n_iterations):
            idxs = np.random.permutation(range(len(xs)))
            n_batches = len(idxs) // batch_size
            for batch_i in range(n_batches):
                idxs_i = idxs[batch_i*batch_size: (batch_i+1)*batch_size]
                sess.run(optimizer, feed_dict = {X:xs[idxs_i], Y:ys[idxs_i]})
                
            training_cost = sess.run(cost, feed_dict={X:xs, Y:ys})
            
            if it_i%10 == 0:
                ys_pred = Y_.eval(feed_dict = {X:xs}, session = sess)
                ax.plot(xs, ys_pred)
                print training_cost
        
    fig.show()
    plt.draw()

"""
Explanation: Higher dimensional approximation to Sine wave.
First let us a function for training our neural net
End of explanation
"""


Y_ = tf.Variable(tf.random_normal([1]), name = 'bias')

for pow_i in range(0, 4):
    W = tf.Variable(tf.random_normal([1], stddev = 0.1, name = 'weight_%d' % pow_i))
    Y_ = tf.add(tf.mul(tf.pow(X, pow_i), W), Y_)

train(X, Y, Y_)

"""
Explanation: Let's model Y as a polynomial
End of explanation
"""


n_neurons = 10
W = tf.Variable(tf.random_normal([1, n_neurons]), name = 'W')
B = tf.Variable(tf.random_normal([n_neurons], dtype=tf.float32), name = 'bias')

h = tf.nn.tanh(tf.matmul(tf.expand_dims(X, 1), W) +  B, name = 'h')
Y_ = tf.reduce_sum(h)

train(X, Y, Y_)

W.get_shape()

"""
Explanation: Introducing non-linearities
End of explanation
"""


