code: def build_torch_optimizer(model, opt):
      """Builds the PyTorch optimizer.
  
      We use the default parameters for Adam that are suggested by
      the original paper https://arxiv.org/pdf/1412.6980.pdf
      These values are also used by other established implementations,
      e.g. https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer
      https://keras.io/optimizers/
      Recently there are slightly different values used in the paper
      "Attention is all you need"
      https://arxiv.org/pdf/1706.03762.pdf, particularly the value beta2=0.98
      was used there however, beta2=0.999 is still arguably the more
      established value, so we use that here as well
  
      Args:
        model: The model to optimize.
        opt. The dictionary of options.
  
      Returns:
        A ``torch.optim.Optimizer`` instance.
      """
      params = [p for p in model.parameters() if p.requires_grad]
      betas = [opt.adam_beta1, opt.adam_beta2]
      if opt.optim == "sgd":
          optimizer = optim.SGD(params, lr=opt.learning_rate)
      elif opt.optim == "adagrad":
          optimizer = optim.Adagrad(
              params,
              lr=opt.learning_rate,
              initial_accumulator_value=opt.adagrad_accumulator_init,
          )
      elif opt.optim == "adadelta":
          optimizer = optim.Adadelta(params, lr=opt.learning_rate)
      elif opt.optim == "adafactor":
          optimizer = AdaFactor(
              params,
              non_constant_decay=True,
              enable_factorization=True,
              weight_decay=0,
          )
      elif opt.optim == "adam":
          optimizer = optim.Adam(
              params, lr=opt.learning_rate, betas=betas, eps=1e-8
          )
      elif opt.optim == "sparseadam":
          dense = []
          sparse = []
          for name, param in model.named_parameters():
              if not param.requires_grad:
                  continue
              # TODO: Find a better way to check for sparse gradients.
              if "embed" in name:
                  sparse.append(param)
              else:
                  dense.append(param)
          optimizer = MultipleOptimizer(
              [
                  optim.Adam(
                      dense, lr=opt.learning_rate, betas=betas, eps=1e-8
                  ),
                  optim.SparseAdam(
                      sparse,
                      lr=opt.learning_rate,
                      betas=betas,
                      eps=1e-8,
                  ),
              ]
          )
      elif opt.optim == "fusedadam":
          optimizer = FusedAdam(
              params, lr=opt.learning_rate, betas=betas
          )
          try:
              import apex
          except ImportError:
              raise ImportError("Could not import apex")
          if opt.apex_opt_level in ["O0", "O1", "O2", "O3"]:
              # we use apex.amp
              loss_scale = (
                  "dynamic" if opt.loss_scale == 0 else opt.loss_scale
              )
              model, optimizer = apex.amp.initialize(
                  [model, model.generator],
                  optimizer,
                  opt_level=opt.apex_opt_level,
                  loss_scale=loss_scale,
                  keep_batchnorm_fp32=None,
              )
          else:
              if opt.model_dtype == "fp16":
                  # In this case use the old FusedAdam with
                  # FP16_optimizer wrapper
                  static_loss_scale = opt.loss_scale
                  dynamic_loss_scale = opt.loss_scale == 0
                  optimizer = apex.contrib.optimizers.FP16_Optimizer(
                      optimizer,
                      static_loss_scale=static_loss_scale,
                      dynamic_loss_scale=dynamic_loss_scale,
                  )
      elif opt.optim in [
          "adamw8bit",
          "pagedadamw8bit",
          "pagedadamw32bit",
      ]:
          try:
              os.environ["BITSANDBYTES_NOWELCOME"] = "1"
              import bitsandbytes as bnb
          except ImportError:
              raise ImportError(
                  "Install bitsandbytes to use bnb optimizers"
              )
          if opt.optim == "adamw8bit":
              optimizer = bnb.optim.AdamW8bit(
                  params,
                  lr=opt.learning_rate,
                  betas=betas,
                  eps=1e-8,
                  weight_decay=1e-2,
                  amsgrad=False,
                  optim_bits=8,
                  args=None,
                  min_8bit_size=1024,
                  percentile_clipping=100,
                  block_wise=True,
                  is_paged=False,
              )
          elif opt.optim == "pagedadamw8bit":
              optimizer = bnb.optim.PagedAdamW8bit(
                  params,
                  lr=opt.learning_rate,
                  betas=betas,
                  eps=1e-8,
                  weight_decay=1e-2,
                  amsgrad=False,
                  optim_bits=8,
                  args=None,
                  min_8bit_size=4096,
                  percentile_clipping=100,
                  block_wise=True,
              )
          elif opt.optim == "pagedadamw32bit":
              optimizer = bnb.optim.PagedAdamW32bit(
                  params,
                  lr=opt.learning_rate,
                  betas=betas,
                  eps=1e-8,
                  weight_decay=1e-2,
                  amsgrad=False,
                  optim_bits=32,
                  args=None,
                  min_8bit_size=4096,
                  percentile_clipping=100,
                  block_wise=True,
              )
          else:
              raise ValueError("Invalid optimizer type: " + opt.optim)
      else:
          raise ValueError("Invalid optimizer type: " + opt.optim)
  
      return optimizer
  
imports: import torch
  import torch.optim
  from torch.nn.utils import clip_grad_norm_
  import operator
  import functools
  from copy import copy
  from math import sqrt
  import types
  import os
  import importlib
  from onmt.utils.misc import fn_args
label_desc: Use image node_id 1 for calling a global function with example usage: build_torch_optimizer(model, opt) and returns: optimizer