class_docstr: Anneals the learning rate in each parameter group to a fixed value.
  
  This learning rate scheduler is meant to be used with Stochastic Weight
  Averaging (SWA) method (see `torch.optim.swa_utils.AveragedModel`).
  
  Args:
      optimizer (torch.optim.Optimizer): wrapped optimizer
      swa_lrs (float or list): the learning rate value for all param groups
          together or separately for each group.
      annealing_epochs (int): number of epochs in the annealing phase
          (default: 10)
      annealing_strategy (str): "cos" or "linear"; specifies the annealing
          strategy: "cos" for cosine annealing, "linear" for linear annealing
          (default: "cos")
      last_epoch (int): the index of the last epoch (default: -1)
  
  The :class:`SWALR` scheduler can be used together with other
  schedulers to switch to a constant learning rate late in the training
  as in the example below.
  
  Example:
      >>> # xdoctest: +SKIP("Undefined variables")
      >>> loader, optimizer, model = ...
      >>> lr_lambda = lambda epoch: 0.9
      >>> scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer,
      >>>        lr_lambda=lr_lambda)
      >>> swa_scheduler = torch.optim.swa_utils.SWALR(optimizer,
      >>>        anneal_strategy="linear", anneal_epochs=20, swa_lr=0.05)
      >>> swa_start = 160
      >>> for i in range(300):
      >>>      for input, target in loader:
      >>>          optimizer.zero_grad()
      >>>          loss_fn(model(input), target).backward()
      >>>          optimizer.step()
      >>>      if i > swa_start:
      >>>          swa_scheduler.step()
      >>>      else:
      >>>          scheduler.step()
  
  .. _Averaging Weights Leads to Wider Optima and Better Generalization:
      https://arxiv.org/abs/1803.05407
class_docstr_tok: ["Anneals","the","learning","rate","in","each","parameter","group","to","a","fixed","value",".","This","learning","rate","scheduler","is","meant","to","be","used","with","Stochastic","Weight","Averaging","(","SWA",")","method","(","see","`","torch.optim.swa_utils.AveragedModel","`",")",".","Args",":","optimizer","(","torch.optim.Optimizer",")",":","wrapped","optimizer","swa_lrs","(","float","or","list",")",":","the","learning","rate","value","for","all","param","groups","together","or","separately","for","each","group",".","annealing_epochs","(","int",")",":","number","of","epochs","in","the","annealing","phase","(","default",":","10",")","annealing_strategy","(","str",")",":","``","cos","''","or","``","linear","''",";","specifies","the","annealing","strategy",":","``","cos","''","for","cosine","annealing",",","``","linear","''","for","linear","annealing","(","default",":","``","cos","''",")","last_epoch","(","int",")",":","the","index","of","the","last","epoch","(","default",":","-1",")","The",":","class",":","`","SWALR","`","scheduler","can","be","used","together","with","other","schedulers","to","switch","to","a","constant","learning","rate","late","in","the","training","as","in","the","example","below",".","Example",":",">",">",">","#","xdoctest",":","+SKIP","(","``","Undefined","variables","''",")",">",">",">","loader",",","optimizer",",","model","=","...",">",">",">","lr_lambda","=","lambda","epoch",":","0.9",">",">",">","scheduler","=","torch.optim.lr_scheduler.MultiplicativeLR","(","optimizer",",",">",">",">","lr_lambda=lr_lambda",")",">",">",">","swa_scheduler","=","torch.optim.swa_utils.SWALR","(","optimizer",",",">",">",">","anneal_strategy=","''","linear","''",",","anneal_epochs=20",",","swa_lr=0.05",")",">",">",">","swa_start","=","160",">",">",">","for","i","in","range","(","300",")",":",">",">",">","for","input",",","target","in","loader",":",">",">",">","optimizer.zero_grad","(",")",">",">",">","loss_fn","(","model","(","input",")",",","target",")",".backward","(",")",">",">",">","optimizer.step","(",")",">",">",">","if","i",">","swa_start",":",">",">",">","swa_scheduler.step","(",")",">",">",">","else",":",">",">",">","scheduler.step","(",")","..","_Averaging","Weights","Leads","to","Wider","Optima","and","Better","Generalization",":","https",":","\/\/arxiv.org\/abs\/1803.05407"]
code: def __init__(
      self,
      optimizer,
      swa_lr,
      anneal_epochs=10,
      anneal_strategy="cos",
      last_epoch=-1,
  ):
      swa_lrs = self._format_param(optimizer, swa_lr)
      for swa_lr, group in zip(swa_lrs, optimizer.param_groups):
          group["swa_lr"] = swa_lr
      if anneal_strategy not in ["cos", "linear"]:
          raise ValueError(
              "anneal_strategy must by one of 'cos' or 'linear', "
              f"instead got {anneal_strategy}"
          )
      elif anneal_strategy == "cos":
          self.anneal_func = self._cosine_anneal
      elif anneal_strategy == "linear":
          self.anneal_func = self._linear_anneal
      if not isinstance(anneal_epochs, int) or anneal_epochs < 0:
          raise ValueError(
              f"anneal_epochs must be equal or greater than 0, got {anneal_epochs}"
          )
      self.anneal_epochs = anneal_epochs
      super().__init__(optimizer, last_epoch)
  
imports: import itertools
  import math
  from copy import deepcopy
  import warnings
  import torch
  from torch.nn import Module
  from torch.optim.lr_scheduler import LRScheduler
  from torch.utils._foreach_utils import _get_foreach_kernels_supported_devices
  from torch.utils._foreach_utils import _group_tensors_by_device_and_dtype
label_desc: Use image node_id 1 to create a new SWALR object from inherited base classes: LRScheduler with example: obj = SWALR(optimizer, swa_lr, anneal_epochs, anneal_strategy, last_epoch)