text: import math
  from bisect import bisect_right
  from torch.optim.lr_scheduler import _LRScheduler
  
  
  class EpochBaseLR(_LRScheduler):
  
      def __init__(self, optimizer, milestones, lrs, last_epoch=-1, ):
          if len(milestones) + 1 != len(lrs):
              raise ValueError('The length of milestones must equal to the '
                               ' length of lr + 1. Got {} and {} separately', len(milestones) + 1, len(lrs))
          if not list(milestones) == sorted(milestones):
              raise ValueError('Milestones should be a list of'
                               ' increasing integers. Got {}', milestones)
  
          self.milestones = milestones
          self.lrs = lrs
          super(EpochBaseLR, self).__init__(optimizer, last_epoch)
  
      def get_lr(self):
          return self.lrs[bisect_right(self.milestones, self.last_epoch)]
  
      def step(self, epoch=None):
          if epoch is None:
              epoch = self.last_epoch + 1
          self.last_epoch = epoch
          lr = self.get_lr()
  
          for g in self.optimizer.param_groups:
              g['lr'] = lr
  
  
  class WarmupMultiStepLR(_LRScheduler):
      def __init__(
              self,
              optimizer,
              milestones,
              gamma=0.1,
              warmup_factor=1.0 / 3,
              warmup_iters=500,
              warmup_method="linear",
              last_epoch=-1,
      ):
          if not list(milestones) == sorted(milestones):
              raise ValueError(
                  "Milestones should be a list of" " increasing integers. Got {}",
                  milestones,
              )
  
          if warmup_method not in ("constant", "linear"):
              raise ValueError(
                  "Only 'constant' or 'linear' warmup_method accepted"
                  "got {}".format(warmup_method)
              )
          self.milestones = milestones
          self.gamma = gamma
          self.warmup_factor = warmup_factor
          self.warmup_iters = warmup_iters
          self.warmup_method = warmup_method
          super(WarmupMultiStepLR, self).__init__(optimizer, last_epoch)
  
      def get_lr(self):
          warmup_factor = 1
          if self.last_epoch < self.warmup_iters:
              if self.warmup_method == "constant":
                  warmup_factor = self.warmup_factor
              elif self.warmup_method == "linear":
                  alpha = self.last_epoch / self.warmup_iters
                  warmup_factor = self.warmup_factor * (1 - alpha) + alpha
          return [
              base_lr
              * warmup_factor
              * self.gamma ** bisect_right(self.milestones, self.last_epoch)
              for base_lr in self.base_lrs
          ]
  
  
  class CosineAnnealingWarmRestarts(_LRScheduler):
      r"""Set the learning rate of each parameter group using a cosine annealing
      schedule, where :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}`
      is the number of epochs since the last restart and :math:`T_{i}` is the number
      of epochs between two warm restarts in SGDR:
  
      .. math::
          \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})(1 +
          \cos(\frac{T_{cur}}{T_{i}}\pi))
  
      When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`.
      When :math:`T_{cur}=0`(after restart), set :math:`\eta_t=\eta_{max}`.
  
      It has been proposed in
      `SGDR: Stochastic Gradient Descent with Warm Restarts`_.
  
      Args:
          optimizer (Optimizer): Wrapped optimizer.
          T_0 (int): Number of iterations for the first restart.
          T_mult (int, optional): A factor increases :math:`T_{i}` after a restart. Default: 1.
          eta_min (float, optional): Minimum learning rate. Default: 0.
          last_epoch (int, optional): The index of last epoch. Default: -1.
  
      .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:
          https://arxiv.org/abs/1608.03983
      """
  
      def __init__(self, optimizer, T_0, T_mult=1, eta_min=0, last_epoch=-1):
          if T_0 <= 0 or not isinstance(T_0, int):
              raise ValueError("Expected positive integer T_0, but got {}".format(T_0))
          if T_mult < 1 or not isinstance(T_mult, int):
              raise ValueError("Expected integer T_mult >= 1, but got {}".format(T_mult))
          self.T_0 = T_0
          self.T_i = T_0
          self.T_mult = T_mult
          self.eta_min = eta_min
          super(CosineAnnealingWarmRestarts, self).__init__(optimizer, last_epoch)
          self.T_cur = self.last_epoch
  
      def get_lr(self):
          return [self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * self.T_cur / self.T_i)) / 2
                  for base_lr in self.base_lrs]
  
      def step(self, epoch=None):
          """Step could be called after every batch update
  
          Example:
              >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
              >>> iters = len(dataloader)
              >>> for epoch in range(20):
              >>>     for i, sample in enumerate(dataloader):
              >>>         inputs, labels = sample['inputs'], sample['labels']
              >>>         scheduler.step(epoch + i / iters)
              >>>         optimizer.zero_grad()
              >>>         outputs = net(inputs)
              >>>         loss = criterion(outputs, labels)
              >>>         loss.backward()
              >>>         optimizer.step()
  
          This function can be called in an interleaved way.
  
          Example:
              >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
              >>> for epoch in range(20):
              >>>     scheduler.step()
              >>> scheduler.step(26)
              >>> scheduler.step() # scheduler.step(27), instead of scheduler(20)
          """
          if epoch is None:
              epoch = self.last_epoch + 1
              self.T_cur = self.T_cur + 1
              if self.T_cur >= self.T_i:
                  self.T_cur = self.T_cur - self.T_i
                  self.T_i = self.T_i * self.T_mult
          else:
              if epoch < 0:
                  raise ValueError("Expected non-negative epoch, but got {}".format(epoch))
              if epoch >= self.T_0:
                  if self.T_mult == 1:
                      self.T_cur = epoch % self.T_0
                  else:
                      n = int(math.log((epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult))
                      self.T_cur = epoch - self.T_0 * (self.T_mult ** n - 1) / (self.T_mult - 1)
                      self.T_i = self.T_0 * self.T_mult ** (n)
              else:
                  self.T_i = self.T_0
                  self.T_cur = epoch
          self.last_epoch = math.floor(epoch)
          for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
              param_group['lr'] = lr
  
  
  class CosineAnnealingWarmUp(_LRScheduler):
      r"""Set the learning rate of each parameter group using a cosine annealing
      schedule, where :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}`
      is the number of epochs since the last restart and :math:`T_{i}` is the number
      of epochs between two warm restarts in SGDR:
  
      .. math::
          \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})(1 +
          \cos(\frac{T_{cur}}{T_{i}}\pi))
  
      When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`.
      When :math:`T_{cur}=0`(after restart), set :math:`\eta_t=\eta_{max}`.
  
      It has been proposed in
      `SGDR: Stochastic Gradient Descent with Warm Restarts`_.
  
      Args:
          optimizer (Optimizer): Wrapped optimizer.
          T_0 (int): Number of iterations for the first restart.
          T_epoch (int, optional): A factor increases :math:`T_{i}` before CosineAnnealing. Default: 1.
          eta_min (float, optional): Minimum learning rate. Default: 0.
          last_epoch (int, optional): The index of last epoch. Default: -1.
  
      .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:
          https://arxiv.org/abs/1608.03983
      """
  
      def __init__(self, optimizer, T_0, T_end=1, warmup_factor=1.0 / 3, last_epoch=-1):
          if T_0 <= 0 or not isinstance(T_0, int):
              raise ValueError("Expected positive integer T_0, but got {}".format(T_0))
          if T_end < 1 or not isinstance(T_end, int):
              raise ValueError("Expected integer T_mult >= 1, but got {}".format(T_end))
          self.T_0 = T_0
          # self.T_i = T_0
          self.T_end = T_end
          self.warmup_factor = warmup_factor
          self.T_cur = last_epoch
          super(CosineAnnealingWarmUp, self).__init__(optimizer, last_epoch)
  
      def get_lr(self):
          if self.T_cur < self.T_0:
              # print('divide', self.T_cur / self.T_0)
              return [self.warmup_factor * base_lr + base_lr * (1 - self.warmup_factor) * self.T_cur / self.T_0
                      for base_lr in self.base_lrs]
          else:
              return [base_lr * (1 + math.cos(math.pi * (self.T_cur - self.T_0) / (self.T_end - self.T_0))) / 2
                      for base_lr in self.base_lrs]
  
      def step(self, epoch=None):
          """Step could be called after every batch update
  
          Example:
              >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
              >>> iters = len(dataloader)
              >>> for epoch in range(20):
              >>>     for i, sample in enumerate(dataloader):
              >>>         inputs, labels = sample['inputs'], sample['labels']
              >>>         scheduler.step(epoch + i / iters)
              >>>         optimizer.zero_grad()
              >>>         outputs = net(inputs)
              >>>         loss = criterion(outputs, labels)
              >>>         loss.backward()
              >>>         optimizer.step()
  
          This function can be called in an interleaved way.
  
          Example:
              >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
              >>> for epoch in range(20):
              >>>     scheduler.step()
              >>> scheduler.step(26)
              >>> scheduler.step() # scheduler.step(27), instead of scheduler(20)
          """
          if epoch is None:
              epoch = 0
          else:
              if epoch < 0:
                  raise ValueError("Expected non-negative epoch, but got {}".format(epoch))
              self.T_cur = epoch
          self.last_epoch = epoch
          # self.last_epoch = epoch
          for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
              # print('param_group', lr)
              param_group['lr'] = lr
  