'Apply the learning rule to all the layers and update the states.
Arguments:
layer_list (list): a list of Layer objects to optimize.
epoch (int): the current epoch, needed for the Schedule object.'

def optimize(self, layer_list, epoch):

    lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
    param_list = get_param_list(layer_list)
    scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)
    for ((param, grad), states) in param_list:
        param.rounding = self.stochastic_round
        if ((len(states) == 0) and (self.momentum_coef != 0)):
            states.append(self.be.zeros_like(grad))
        grad = (grad / self.be.bsz)
        grad = self.clip_value(grad, self.gradient_clip_value)
        if (self.momentum_coef == 0):
            param[:] = ((((- lrate) * scale_factor) * grad) + ((1 - (lrate * self.wdecay)) * param))
            param = self.clip_value(param, self.param_clip_value)
        else:
            grad = ((scale_factor * grad) + (self.wdecay * param))
            velocity = states[0]
            velocity[:] = ((self.momentum_coef * velocity) - (lrate * grad))
            if self.nesterov:
                param[:] = self.clip_value(((param + (self.momentum_coef * velocity)) - (lrate * grad)), self.param_clip_value)
            else:
                param[:] = self.clip_value((param + velocity), self.param_clip_value)
