'Apply the learning rule to all the layers and update the states.
Arguments:
param_list (list): a list of tuples of the form ((param, grad), state),
corresponding to parameters, grads, and states of layers to be updated
epoch (int): the current epoch, needed for the Schedule object.'

def optimize(self, layer_list, epoch):

    self.t = (self.t + 1)
    l = ((self.learning_rate * self.be.sqrt((1 - (self.beta_2 ** self.t)))) / (1 - (self.beta_1 ** self.t)))
    param_list = get_param_list(layer_list)
    scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)
    for ((param, grad), states) in param_list:
        param.rounding = self.stochastic_round
        if (len(states) == 0):
            states.extend([self.be.zeros_like(grad) for i in range(2)])
        grad = (grad / self.be.bsz)
        grad = self.clip_value(grad, self.gradient_clip_value)
        (m, v) = states
        m[:] = ((m * self.beta_1) + ((1.0 - self.beta_1) * grad))
        v[:] = ((v * self.beta_2) + (((1.0 - self.beta_2) * grad) * grad))
        param[:] = self.clip_value((param - (((scale_factor * l) * m) / (self.be.sqrt(v) + self.epsilon))), self.param_clip_value)
