def METHOD_NAME(self):
    """Learning rate decay functions from:
          https://openreview.net/pdf?id=BJYwwY9ll pg. 4"""
    # Use linear warmup for the initial part.
    if self.warmup_steps > 0 and self.num_steps <= self.warmup_steps:
        return self.max_lr * float(self.num_steps) / \
            float(self.warmup_steps)
    # If the learning rate is constant, just return the initial value.
    if self.decay_style == 'constant':
        return self.max_lr
    # For any steps larger than `self.decay_steps`, use `self.min_lr`.
    if self.num_steps > self.decay_steps:
        return self.min_lr
    # If we are done with the warmup period, use the decay style.
    num_steps_ = self.num_steps - self.warmup_steps
    decay_steps_ = self.decay_steps - self.warmup_steps
    decay_ratio = float(num_steps_) / float(decay_steps_)
    assert decay_ratio >= 0.0
    assert decay_ratio <= 1.0
    delta_lr = self.max_lr - self.min_lr
    if self.decay_style == 'linear':
        coeff = (1.0 - decay_ratio)
    elif self.decay_style == 'cosine':
        coeff = 0.5 * (math.cos(math.pi * decay_ratio) + 1.0)
    else:
        raise Exception('{} decay style is not supported.'.format(
            self.decay_style))
    return self.min_lr + coeff * delta_lr