# imports
%matplotlib inline
# %pylab osx
import os
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx
plt.style.use('ggplot')

"""
Explanation: Taken in part from the course Creative applications of deep learning with tensorflow

Regression to a noisy sine wave
L1 minimization with SGD
Linear regression iterations
Regression by a cubic polynomial
Non linear activation
Simple network with a non linear activation
Going deeper
End of explanation
"""


#---------------------------------------------
# Create the data set: Sine wave with noise
#--------------------------------------------
n_observations = 1000
xs             = np.linspace(-3, 3, n_observations)
ys             = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)
#plt.scatter(xs, ys, alpha=0.15, marker='+')


"""
Explanation: <a name="regression-1d"></a>
Regression to a sine wave
Creating the dataset
End of explanation
"""


# L1 cost function
def distance(p1, p2):
    return tf.abs(p1 - p2)

def train(X, Y, Y_pred, n_iterations=100, batch_size=200, learning_rate=0.02):
    cost      = tf.reduce_mean(distance(Y_pred, Y))  # cost ==> mean L1 distance for alls samples
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)


    # Plot the true data distribution
    fig, ax = plt.subplots(1, 1)
    ax.scatter(xs, ys, alpha=0.15, marker='+')
    ax.set_xlim([-4, 4])
    ax.set_ylim([-2, 2])
    with tf.Session() as sess:
        # Here we tell tensorflow that we want to initialize all
        # the variables in the graph so we can use them
        # This will set `W` and `b` to their initial random normal value.
        sess.run(tf.initialize_all_variables())

        # We now run a loop over epochs
        prev_training_cost = 0.0
        for it_i in range(n_iterations):
            idxs = np.random.permutation(range(len(xs)))
            n_batches = len(idxs) // batch_size
            for batch_i in range(n_batches):
                idxs_i = idxs[batch_i * batch_size: (batch_i + 1) * batch_size]
                sess.run(optimizer, feed_dict={X: xs[idxs_i], Y: ys[idxs_i]})

            training_cost = sess.run(cost, feed_dict={X: xs, Y: ys})

            if it_i % 10 == 0:
                ys_pred = Y_pred.eval(feed_dict={X: xs}, session=sess)
                ax.plot(xs, ys_pred, 'k', alpha=it_i / float(n_iterations))
                
                # getting the values as numpy array
                #    w =  sess.run(W, feed_dict={X: xs, Y: ys})
                #    b =  sess.run(B, feed_dict={X: xs, Y: ys})
                print ' iteration: {:3} Cost: {} '.format(it_i,training_cost)
                                                                       
                                                                        

    fig.show()
    plt.draw()
    



"""
Explanation: <a name="L1-SGD"></a>
The training procedure: L1 minimization with batch stochadtic GD
End of explanation
"""


# Reset default graph
tf.reset_default_graph()

# Declare variables 
# placeholders to hold input data
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')

# Variables for internal weights
W         = tf.Variable(tf.random_normal([1], dtype=tf.float32, stddev=0.1), name='weight')
B         = tf.Variable(tf.constant([1], dtype=tf.float32), name='bias')
Y_pred    = X * W + B

train(X,Y,Y_pred)

"""
Explanation: <a name="regression-1d-sine"></a>
Linear regression
End of explanation
"""


# Reset default graph
tf.reset_default_graph()

# Declare variables 
# placeholders to hold input data
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')

# Variables for internal weights
B         = tf.Variable(tf.constant([1], dtype=tf.float32), name='bias')
Y_pred    = tf.Variable(tf.random_normal([1]), name='bias')

for pow_i in range(0, 4):
    # Instantiate weight for each monomial
    W = tf.Variable(
        tf.random_normal([1], stddev=0.1), name='weight_%d' % pow_i)
    Y_pred = tf.add(tf.mul(tf.pow(X, pow_i), W), Y_pred)


train(X,Y,Y_pred)

"""
Explanation: <a name="cubic-regression"></a>
Cubic polynomial regression
End of explanation
"""


# Reset default graph
tf.reset_default_graph()

sess = tf.InteractiveSession()
x = np.linspace(-6,6,1000)
plt.plot(x, tf.nn.tanh(x).eval(), label='tanh')
plt.plot(x, tf.nn.sigmoid(x).eval(), label='sigmoid')
plt.plot(x, tf.nn.relu(x).eval(), label='relu')
plt.legend(loc='lower right')
plt.xlim([-6, 6])
plt.ylim([-2, 2])
plt.xlabel('Input')
plt.ylabel('Output')
plt.grid('on')

"""
Explanation: <a name="non-linear-activation"></a> 
Non linear activation
End of explanation
"""


X         = tf.placeholder(tf.float32, name='X')
Y         = tf.placeholder(tf.float32, name='Y')     

#---------------------------------------------------------------------------
# Create inner layer of 10 neurons followed by non-linear activation
#  W  : 1 x n_neurons         martix
#  b  : n_neurons             vector
#
#  X  : n_vals                Input vector of values
#  Y  : n_vals                Input noisy values of sin(X)
#
# For a single input value x_i we do:
#      h_k(x_i)= h(x_i * w_k + b_k) , k=1,...,n_neuorns
# In vector notations
#  (1)    h       = x_i * W + b
# where h is the non-linear activation function.
#  (2)   y_pred_i = h_1 + h_2  + .... + h_n_neurons
#
# From single input/output value to a vector
# -------------------------------------------
# Consider now a vector of input valuse: X = (x_1,...,x_n_vals)^T
# Let us expand this vector to a column matrix XM = f.expand_dims(X, 1)
# Then,instead of the single vector in (1) we have a matrix of n_vals x n_neurons entries
#       H = matmul(XM,W) 
# which we sum, row by row, as in (2) to get the prediction vector Y_PRED
#---------------------------------------------------------------------------
n_neurons = 10
W         = tf.Variable(tf.random_normal([1, n_neurons]), name='W')
b         = tf.Variable(tf.constant(0, dtype=tf.float32, shape=[n_neurons]), name='b')
h         = tf.nn.tanh(tf.matmul(tf.expand_dims(X, 1), W) + b, name='h')
Y_pred    = tf.reduce_sum(h, 1)

#with tf.Session() as sess:
#    sess.run(tf.initialize_all_variables())
#    print 'Shape of H 111 x 10',sess.run(h, feed_dict={X: xs[0:111], Y: ys[0:111]}).shape


# And retrain w/ our new Y_pred
train(X, Y, Y_pred)

"""
Explanation: <a name="net-with-non-linear-activation"></a> 
Simple network with tanh non linear activation
End of explanation
"""


# Define a single hidden layer with activation function
# Creating variables with scopes help in further debugging
def linear(X, n_input, n_output, activation=None, scope=None):
    with tf.variable_scope(scope or "linear"):
        # Create/return variable with a given scope
        W = tf.get_variable(
            name='W',
            shape=[n_input, n_output],
            initializer=tf.random_normal_initializer(mean=0.0, stddev=0.1))
        b = tf.get_variable(
            name='b',
            shape=[n_output],
            initializer=tf.constant_initializer())
        h = tf.matmul(X, W) + b
        if activation is not None:
            h = activation(h)
        return h

# first clear the graph
from tensorflow.python.framework import ops
ops.reset_default_graph()

# let's get the current graph
g = tf.get_default_graph()

# See the names of any operations in the graph
print 'Empty graph: ',[op.name for op in tf.get_default_graph().get_operations()]

# let's create a new network
X = tf.placeholder(tf.float32, name='X')
h = linear(X, 1, 10, scope='layer1')

with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    print 's',sess.run(h, feed_dict={X: np.expand_dims(xs[:111],1)}).shape

# See the names of any operations in the graph
print 'Graph of first layer with 2 input and 10 neurons: '
for op in [op.name for op in tf.get_default_graph().get_operations()]:
    print '  ',op

ops.reset_default_graph()
g  = tf.get_default_graph()

n_observations = 1000
xs             = np.linspace(-3, 3, n_observations)
ys             = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)
xs             = xs.reshape(n_observations,1)
ys             = ys.reshape(n_observations,1)


X         = tf.placeholder(tf.float32, shape=[None,1],name='X')
Y         = tf.placeholder(tf.float32, shape=[None,1],name='Y')     

n_inputs  = 1
n_neurons = 10
h1        = linear(X,   n_inputs,   n_neurons, activation=tf.nn.tanh, scope='layer1')
h2        = linear(h1,  n_neurons,  n_neurons, activation=tf.nn.tanh, scope='layer2')
Y_pred    = linear(h2, n_neurons, 1, scope='layer3')

#Y_pred = linear(h2, 10,  1, scope='layer3')

with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    print 'Dimension of prediction vector is: ',sess.run(Y_pred, feed_dict={X: xs[0:111]}).shape


# And retrain w/ our new Y_pred
train(X, Y, Y_pred,n_iterations=500, batch_size=30, learning_rate=0.055)

|

"""
Explanation: <a name="going-deeper"></a> 
Going deeper
End of explanation
"""


