code: def construct_optimizer(model, cfg):
      """
      Construct a stochastic gradient descent or ADAM optimizer with momentum.
      Details can be found in:
      Herbert Robbins, and Sutton Monro. "A stochastic approximation method."
      and
      Diederik P.Kingma, and Jimmy Ba.
      "Adam: A Method for Stochastic Optimization."
  
      Args:
          model (model): model to perform stochastic gradient descent
          optimization or ADAM optimization.
          cfg (config): configs of hyper-parameters of SGD or ADAM, includes base
          learning rate,  momentum, weight_decay, dampening, and etc.
      """
      if cfg.SOLVER.LAYER_DECAY > 0.0 and cfg.SOLVER.LAYER_DECAY < 1.0:
          optim_params = get_param_groups(model, cfg)
      elif cfg.SOLVER.LAYER_DECAY == 1.0:
          bn_parameters = []
          non_bn_parameters = []
          zero_parameters = []
          no_grad_parameters = []
          skip = {}
  
          if cfg.NUM_GPUS > 1:
              if hasattr(model.module, "no_weight_decay"):
                  skip = model.module.no_weight_decay()
          else:
              if hasattr(model, "no_weight_decay"):
                  skip = model.no_weight_decay()
  
          for name_m, m in model.named_modules():
              is_bn = isinstance(
                  m, torch.nn.modules.batchnorm._NormBase
              )
              for name_p, p in m.named_parameters(recurse=False):
                  name = "{}.{}".format(name_m, name_p).strip(".")
                  if not p.requires_grad:
                      no_grad_parameters.append(p)
                  elif is_bn:
                      bn_parameters.append(p)
                  elif any(k in name for k in skip):
                      zero_parameters.append(p)
                  elif cfg.SOLVER.ZERO_WD_1D_PARAM and (
                      len(p.shape) == 1 or name.endswith(".bias")
                  ):
                      zero_parameters.append(p)
                  else:
                      non_bn_parameters.append(p)
  
          optim_params = [
              {
                  "params": bn_parameters,
                  "weight_decay": cfg.BN.WEIGHT_DECAY,
                  "layer_decay": 1.0,
                  "apply_LARS": False,
              },
              {
                  "params": non_bn_parameters,
                  "weight_decay": cfg.SOLVER.WEIGHT_DECAY,
                  "layer_decay": 1.0,
                  "apply_LARS": cfg.SOLVER.LARS_ON,
              },
              {
                  "params": zero_parameters,
                  "weight_decay": 0.0,
                  "layer_decay": 1.0,
                  "apply_LARS": cfg.SOLVER.LARS_ON,
              },
          ]
          optim_params = [x for x in optim_params if len(x["params"])]
  
          # Check all parameters will be passed into optimizer.
          assert len(list(model.parameters())) == len(
              non_bn_parameters
          ) + len(bn_parameters) + len(zero_parameters) + len(
              no_grad_parameters
          ), "parameter size does not match: {} + {} + {} + {} != {}".format(
              len(non_bn_parameters),
              len(bn_parameters),
              len(zero_parameters),
              len(no_grad_parameters),
              len(list(model.parameters())),
          )
          print(
              "bn {}, non bn {}, zero {}, no grad {}".format(
                  len(bn_parameters),
                  len(non_bn_parameters),
                  len(zero_parameters),
                  len(no_grad_parameters),
              )
          )
      else:
          raise ValueError(
              "Layer decay should be in (0, 1], but is {}".format(
                  cfg.SOLVER.LAYER_DECAY
              )
          )
  
      if cfg.SOLVER.OPTIMIZING_METHOD == "sgd":
          optimizer = torch.optim.SGD(
              optim_params,
              lr=cfg.SOLVER.BASE_LR,
              momentum=cfg.SOLVER.MOMENTUM,
              weight_decay=cfg.SOLVER.WEIGHT_DECAY,
              dampening=cfg.SOLVER.DAMPENING,
              nesterov=cfg.SOLVER.NESTEROV,
          )
      elif cfg.SOLVER.OPTIMIZING_METHOD == "adam":
          optimizer = torch.optim.Adam(
              optim_params,
              lr=cfg.SOLVER.BASE_LR,
              betas=cfg.SOLVER.BETAS,
              weight_decay=cfg.SOLVER.WEIGHT_DECAY,
          )
      elif cfg.SOLVER.OPTIMIZING_METHOD == "adamw":
          optimizer = torch.optim.AdamW(
              optim_params,
              lr=cfg.SOLVER.BASE_LR,
              betas=cfg.SOLVER.BETAS,
              eps=1e-08,
              weight_decay=cfg.SOLVER.WEIGHT_DECAY,
          )
      elif cfg.SOLVER.OPTIMIZING_METHOD == "mt_adamw":
          optimizer = torch.optim._multi_tensor.AdamW(
              optim_params,
              lr=cfg.SOLVER.BASE_LR,
              betas=cfg.SOLVER.BETAS,
              eps=1e-08,
              weight_decay=cfg.SOLVER.WEIGHT_DECAY,
          )
      else:
          raise NotImplementedError(
              "Does not support {} optimizer".format(
                  cfg.SOLVER.OPTIMIZING_METHOD
              )
          )
      if cfg.SOLVER.LARS_ON:
          optimizer = LARS(
              optimizer=optimizer, trust_coefficient=0.001, clip=False
          )
      return optimizer
  
file_path: SlowFast/slowfast/models/optimizer.py
imports: import torch
  import slowfast.utils.lr_policy
label_desc: Use image node_id 1 for calling a global function with example usage: construct_optimizer(model, cfg) and returns: optimizer