class_docstr: Cosine annealing learning rate scheduler with learning rate warmup. A linear warmup schedule will be applied.
  
  Args:
      optimizer (:class:`torch.optim.Optimizer`): Wrapped optimizer.
      total_steps (int): Number of total training steps.
      warmup_steps (int, optional): Number of warmup steps, defaults to 0.
      eta_min (int, optional): Minimum learning rate, defaults to 0.
      last_epoch (int, optional): The index of last epoch, defaults to -1. When last_epoch=-1,
          the schedule is started from the beginning or When last_epoch=-1, sets initial lr as lr.
class_docstr_tok: ["Cosine","annealing","learning","rate","scheduler","with","learning","rate","warmup",".","A","linear","warmup","schedule","will","be","applied",".","Args",":","optimizer","(",":","class",":","`","torch.optim.Optimizer","`",")",":","Wrapped","optimizer",".","total_steps","(","int",")",":","Number","of","total","training","steps",".","warmup_steps","(","int",",","optional",")",":","Number","of","warmup","steps",",","defaults","to","0.","eta_min","(","int",",","optional",")",":","Minimum","learning","rate",",","defaults","to","0.","last_epoch","(","int",",","optional",")",":","The","index","of","last","epoch",",","defaults","to","-1",".","When","last_epoch=-1",",","the","schedule","is","started","from","the","beginning","or","When","last_epoch=-1",",","sets","initial","lr","as","lr","."]
code: def __init__(
      self,
      optimizer,
      total_steps: int,
      warmup_steps: int = 0,
      eta_min: float = 0.0,
      last_epoch: int = -1,
  ):
      base_scheduler = _CosineAnnealingLR(
          optimizer,
          total_steps - warmup_steps,
          eta_min=eta_min,
          last_epoch=last_epoch,
      )
      super().__init__(
          optimizer, warmup_steps, base_scheduler, last_epoch=last_epoch
      )
  
file_path: ColossalAI/colossalai/nn/lr_scheduler/cosine.py
imports: from torch.optim.lr_scheduler import CosineAnnealingLR
  from .delayed import DelayerScheduler, WarmupDelayerScheduler, WarmupScheduler
label_desc: Use image node_id 1 to create a new CosineAnnealingWarmupLR object from inherited base classes: WarmupScheduler with example: obj = CosineAnnealingWarmupLR(optimizer, total_steps, warmup_steps, eta_min, last_epoch)