code: def create_optimizer(
      model: torch.nn.Module, opt_config: OptimizerConfig
  ) -> torch.optim.AdamW:
      """
      This long function is unfortunately doing something very simple and is being very defensive:
      We are separating out all parameters of the model into two buckets: those that will experience
      weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
      We are then returning the PyTorch optimizer object.
      """
  
      # separate out all parameters to those that will and won't experience regularizing weight decay
      decay = set()
      no_decay = set()
      allowlist_weight_modules = (torch.nn.Linear,)
      denylist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding)
      for mn, m in model.named_modules():
          for pn, _ in m.named_parameters():
              fpn = "%s.%s" % (mn, pn) if mn else pn  # full param name
              # random note: because named_modules and named_parameters are recursive
              # we will see the same tensors p many many times. but doing it this way
              # allows us to know which parent module any tensor p belongs to...
              if pn.endswith("bias"):
                  # all biases will not be decayed
                  no_decay.add(fpn)
              elif pn.endswith("weight") and isinstance(
                  m, allowlist_weight_modules
              ):
                  # weights of allowlist modules will be weight decayed
                  decay.add(fpn)
              elif pn.endswith("in_proj_weight"):
                  # MHA projection layer
                  decay.add(fpn)
              elif pn.endswith("weight") and isinstance(
                  m, denylist_weight_modules
              ):
                  # weights of denylist modules will NOT be weight decayed
                  no_decay.add(fpn)
              elif pn.endswith("pos_emb"):
                  # positional embedding shouldn't be decayed
                  no_decay.add(fpn)
  
      # validate that we considered every parameter
      param_dict = {pn: p for pn, p in model.named_parameters()}
      inter_params = decay & no_decay
      union_params = decay | no_decay
      assert (
          len(inter_params) == 0
      ), "parameters %s made it into both decay/no_decay sets!" % (
          str(inter_params),
      )
      assert len(param_dict.keys() - union_params) == 0, (
          "parameters %s were not separated into either decay/no_decay set!"
          % (str(param_dict.keys() - union_params),)
      )
  
      # create the pytorch optimizer object
      optim_groups = [
          {
              "params": [param_dict[pn] for pn in sorted(decay)],
              "weight_decay": opt_config.weight_decay,
          },
          {
              "params": [param_dict[pn] for pn in sorted(no_decay)],
              "weight_decay": 0.0,
          },
      ]
      optimizer = torch.optim.AdamW(
          optim_groups, lr=opt_config.learning_rate, betas=(0.9, 0.95)
      )
      return optimizer
  
imports: import math
  from dataclasses import dataclass
  from typing import Optional, Tuple
  import torch
  import torch.nn
  from torch.nn import functional
label_desc: Use image node_id 1 for calling a global function with example usage: create_optimizer(model, opt_config) and returns: optimizer