class_docstr: Flat and cosine annealing learning rate scheduler with learning rate warmup. A linear warmup schedule will be
  applied, and then the learning rate will be a fixed value before starting decay.
  
  Args:
      optimizer (:class:`torch.optim.Optimizer`): Wrapped optimizer.
      total_steps (int): Number of total training steps.
      warmup_steps (int, optional): Number of warmup steps, defaults to 0.
      pct_start (float, optional): Percent of steps before starting learning rate decay, defaults to -0.72.
      eta_min (int, optional): Minimum learning rate, defaults to 0.
      last_epoch (int, optional): The index of last epoch, defaults to -1. When last_epoch=-1,
          the schedule is started from the beginning or When last_epoch=-1, sets initial lr as lr.
class_docstr_tok: ["Flat","and","cosine","annealing","learning","rate","scheduler","with","learning","rate","warmup",".","A","linear","warmup","schedule","will","be","applied",",","and","then","the","learning","rate","will","be","a","fixed","value","before","starting","decay",".","Args",":","optimizer","(",":","class",":","`","torch.optim.Optimizer","`",")",":","Wrapped","optimizer",".","total_steps","(","int",")",":","Number","of","total","training","steps",".","warmup_steps","(","int",",","optional",")",":","Number","of","warmup","steps",",","defaults","to","0.","pct_start","(","float",",","optional",")",":","Percent","of","steps","before","starting","learning","rate","decay",",","defaults","to","-0.72.","eta_min","(","int",",","optional",")",":","Minimum","learning","rate",",","defaults","to","0.","last_epoch","(","int",",","optional",")",":","The","index","of","last","epoch",",","defaults","to","-1",".","When","last_epoch=-1",",","the","schedule","is","started","from","the","beginning","or","When","last_epoch=-1",",","sets","initial","lr","as","lr","."]
code: def __init__(
      self,
      optimizer,
      total_steps: int,
      warmup_steps: int = 0,
      pct_start: float = 0.72,
      eta_min: int = 0,
      last_epoch: int = -1,
      **kwargs,
  ):
      if not (0.0 <= pct_start <= 1.0):
          raise ValueError(
              f"pct_start must >= 0.0 and <= 1.0, got {pct_start}"
          )
      flat_steps = int((total_steps - warmup_steps) * pct_start)
      anneal_steps = total_steps - warmup_steps - flat_steps
      base_scheduler = _CosineAnnealingLR(
          optimizer, anneal_steps, eta_min=eta_min
      )
      super().__init__(
          optimizer,
          warmup_steps,
          flat_steps,
          base_scheduler,
          last_epoch=last_epoch,
      )
  
file_path: ColossalAI/colossalai/nn/lr_scheduler/cosine.py
imports: from torch.optim.lr_scheduler import CosineAnnealingLR
  from .delayed import DelayerScheduler, WarmupDelayerScheduler, WarmupScheduler
label_desc: Use image node_id 1 to create a new FlatAnnealingWarmupLR object from inherited base classes: WarmupDelayerScheduler with example: obj = FlatAnnealingWarmupLR(optimizer, total_steps, warmup_steps, pct_start, eta_min, last_epoch)