'Applying gradients aand tune hyperparams with YellowFin.
Args:
grads_and_vars: List of (gradient, variable) pairs as returned by
compute_gradients().
global_step: Optional Variable to increment by one after the
variables have been updated.
name:  Optional name for the returned operation. Default to the
name passed to the Optimizer constructor.
Returns:
(A group of operations)
Variable Update with Momentum ops,
YellowFin ops(Curvature, Variance, Distance) ops,
SingleStep and lr_mu tuning ops,
Step increment ops.'

def apply_gradients(self, grads_and_vars, global_step=None, name=None):

    (self._grad, self._vars) = zip(*[(g, t) for (g, t) in grads_and_vars if (g is not None)])
    with tf.variable_scope('apply_updates'):
        if (self._clip_thresh_var is not None):
            (self._grads_clip, self._grads_norm) = tf.clip_by_global_norm(self._grad, self._clip_thresh_var)
            apply_grad_op = self._momentum_optimizer.apply_gradients(zip(self._grads_clip, self._vars), global_step=global_step)
        else:
            apply_grad_op = self._momentum_optimizer.apply_gradients(zip(self._grad, self._vars), global_step=global_step)
    with tf.variable_scope('prepare_yellowFin_variables'):
        prepare_variables_op = self._prepare_variables()
    with tf.variable_scope('yellowfin'):
        with tf.control_dependencies([prepare_variables_op]):
            yellowfin_op = self._yellowfin()
    with tf.control_dependencies([yellowfin_op]):
        self._increment_step_op = tf.assign_add(self._step, 1).op
    return tf.group(apply_grad_op, prepare_variables_op, yellowfin_op, self._increment_step_op)
