import tensorflow as tf

    rate = tf.train.exponential_decay(starter_learning, global_step, 500, 0.9) #exponential learning rate decay
    #rate = starter_learning

    tvars = tf.trainable_variables() #list of trainable variables
    Npar= flatten(tvars).get_shape()[1] #total number of paramters in the network

    print('there are:', Npar,'parameters in the network')

    optimizer  = tf.train.AdamOptimizer(learning_rate = rate)   #Initialize Adam optimizer

    grads_var = optimizer.compute_gradients(cost, tvars ) #Get gradients layer by layer. Note that this function returns the pair (grads, var)
    grads = [grads_var[i][0] for i in range(len(grads_var))] #extract the gradients

    min   = optimizer.apply_gradients(grads_and_vars= grads_var, global_step= global_step)    #Apply the gradients to look for critical points

    gradients_and_par = [] #store gradients and training paramters for different epochs
