code: def construct_optimizer(model):
      """Constructs the optimizer.
  
      Note that the momentum update in PyTorch differs from the one in Caffe2.
      In particular,
  
          Caffe2:
              V := mu * V + lr * g
              p := p - V
  
          PyTorch:
              V := mu * V + g
              p := p - lr * V
  
      where V is the velocity, mu is the momentum factor, lr is the learning rate,
      g is the gradient and p are the parameters.
  
      Since V is defined independently of the learning rate in PyTorch,
      when the learning rate is changed there is no need to perform the
      momentum correction by scaling V (unlike in the Caffe2 case).
      """
      # Split parameters into types and get weight decay for each type
      optim, wd, params = (
          cfg.OPTIM,
          cfg.OPTIM.WEIGHT_DECAY,
          [[], [], [], []],
      )
      for n, p in model.named_parameters():
          ks = [
              k
              for (k, x) in enumerate(["bn", "ln", "bias", ""])
              if x in n
          ]
          params[ks[0]].append(p)
      wds = [
          cfg.BN.CUSTOM_WEIGHT_DECAY
          if cfg.BN.USE_CUSTOM_WEIGHT_DECAY
          else wd,
          cfg.LN.CUSTOM_WEIGHT_DECAY
          if cfg.LN.USE_CUSTOM_WEIGHT_DECAY
          else wd,
          optim.BIAS_CUSTOM_WEIGHT_DECAY
          if optim.BIAS_USE_CUSTOM_WEIGHT_DECAY
          else wd,
          wd,
      ]
      param_wds = [
          {"params": p, "weight_decay": w}
          for (p, w) in zip(params, wds)
          if p
      ]
      # Set up optimizer
      if optim.OPTIMIZER == "sgd":
          if cfg.OPTIM.MTA:
              optimizer_fn = torch.optim._multi_tensor.SGD
          else:
              optimizer_fn = torch.optim.SGD
          return optimizer_fn(
              param_wds,
              lr=optim.BASE_LR,
              momentum=optim.MOMENTUM,
              weight_decay=wd,
              dampening=optim.DAMPENING,
              nesterov=optim.NESTEROV,
          )
      elif optim.OPTIMIZER == "adam":
          if cfg.OPTIM.MTA:
              optimizer_fn = torch.optim._multi_tensor.Adam
          else:
              optimizer_fn = torch.optim.Adam
          return optimizer_fn(
              param_wds,
              lr=optim.BASE_LR,
              betas=(optim.BETA1, optim.BETA2),
              weight_decay=wd,
          )
      elif optim.OPTIMIZER == "adamw":
          if cfg.OPTIM.MTA:
              optimizer_fn = torch.optim._multi_tensor.AdamW
          else:
              optimizer_fn = torch.optim.AdamW
          return optimizer_fn(
              param_wds,
              lr=optim.BASE_LR,
              betas=(optim.BETA1, optim.BETA2),
              weight_decay=wd,
          )
      else:
          raise NotImplementedError
  
file_path: pycls/pycls/core/optimizer.py
imports: import matplotlib.pyplot
  import numpy
  import torch
  from pycls.core.config import cfg
label_desc: Use image node_id 1 for calling a global function with example usage: construct_optimizer(model) and returns: optimizer_fn, optimizer_fn, optimizer_fn