|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import math |
|
import numpy as np |
|
|
|
|
|
def adjust_learning_rate(optimizer, epoch, args): |
|
"""Decay the learning rate with half-cycle cosine after warmup""" |
|
if epoch < args.warmup_epochs: |
|
lr = args.lr * epoch / args.warmup_epochs |
|
else: |
|
lr = args.min_lr + (args.lr - args.min_lr) * 0.5 * \ |
|
(1. + math.cos(math.pi * (epoch - args.warmup_epochs) / (args.epochs - args.warmup_epochs))) |
|
for param_group in optimizer.param_groups: |
|
if "lr_scale" in param_group: |
|
param_group["lr"] = lr * param_group["lr_scale"] |
|
else: |
|
param_group["lr"] = lr |
|
return lr |
|
|
|
|
|
def cosine_scheduler(base_value, final_value, epochs, niter_per_ep, warmup_epochs=0, |
|
start_warmup_value=0, warmup_steps=-1): |
|
warmup_schedule = np.array([]) |
|
warmup_iters = warmup_epochs * niter_per_ep |
|
if warmup_steps > 0: |
|
warmup_iters = warmup_steps |
|
print("Set warmup steps = %d" % warmup_iters) |
|
if warmup_epochs > 0: |
|
warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters) |
|
|
|
iters = np.arange(epochs * niter_per_ep - warmup_iters) |
|
schedule = np.array( |
|
[final_value + 0.5 * (base_value - final_value) * (1 + math.cos(math.pi * i / (len(iters)))) for i in iters]) |
|
|
|
schedule = np.concatenate((warmup_schedule, schedule)) |
|
|
|
assert len(schedule) == epochs * niter_per_ep |
|
return schedule |