code: def create_optimizer_from_params(
      named_parameters,
      learning_rate,
      t_total,
      warmup_steps,
      warmup_proportion,
      optimizer_epsilon=1e-8,
      optimizer_type="adam",
      verbose=False,
  ):
      # Prepare optimizer
      no_decay = [
          "bias",
          "LayerNorm.bias",
          "LayerNorm.weight",
          "adapter.down_project.weight",
          "adapter.up_project.weight",
          "weighted_sum.weights",
      ]
      if verbose:
          print("No optimizer decay for:")
          for n, p in named_parameters:
              if any(nd in n for nd in no_decay):
                  print(f"  {n}")
  
      used_named_parameters = [
          (n, p)
          for n, p in named_parameters
          if p.requires_grad and "weighted_sum.weights" not in n
      ]
      weighted_sum_params = [
          (n, p)
          for n, p in named_parameters
          if p.requires_grad and "weighted_sum.weights" in n
      ]
  
      optimizer_grouped_parameters = [
          {
              "params": [
                  p
                  for n, p in used_named_parameters
                  if not any(nd in n for nd in no_decay)
              ],
              "weight_decay": 0.01,
          },
          {
              "params": [
                  p
                  for n, p in used_named_parameters
                  if any(nd in n for nd in no_decay)
              ],
              "weight_decay": 0.0,
          },
          {
              "params": [p for n, p in weighted_sum_params],
              "weight_decay": 0.0,
              "lr": 0.01,
          },
      ]
  
      if optimizer_type == "adam":
          if verbose:
              print("Using AdamW")
          optimizer = transformers.AdamW(
              optimizer_grouped_parameters,
              lr=learning_rate,
              eps=optimizer_epsilon,
          )
      elif optimizer_type == "radam":
          if verbose:
              print("Using RAdam")
          optimizer = RAdam(
              optimizer_grouped_parameters,
              lr=learning_rate,
              eps=optimizer_epsilon,
          )
      else:
          raise KeyError(optimizer_type)
  
      warmup_steps = resolve_warmup_steps(
          t_total=t_total,
          warmup_steps=warmup_steps,
          warmup_proportion=warmup_proportion,
      )
      scheduler = transformers.get_linear_schedule_with_warmup(
          optimizer,
          num_warmup_steps=warmup_steps,
          num_training_steps=t_total,
      )
      optimizer_scheduler = OptimizerScheduler(
          optimizer=optimizer, scheduler=scheduler
      )
      return optimizer_scheduler
  
file_path: jiant/jiant/shared/model_setup.py
imports: import transformers
  import torch
  from jiant.ext.radam import RAdam
label_desc: Use image node_id 2 for calling a global function with example usage: create_optimizer_from_params(named_parameters, learning_rate, t_total, warmup_steps, warmup_proportion, optimizer_epsilon, optimizer_type, verbose) and returns: optimizer_scheduler