docstr: Setup the optimizer.
  
  We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
  Trainer's init through `optimizers`, or subclass and override this method in a subclass.
docstr_tok: ["Setup","the","optimizer",".","We","provide","a","reasonable","default","that","works","well",".","If","you","want","to","use","something","else",",","you","can","pass","a","tuple","in","the","Trainer","'s","init","through","`","optimizers","`",",","or","subclass","and","override","this","method","in","a","subclass","."]
code: def create_optimizer(self):
      """
      Setup the optimizer.
  
      We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
      Trainer's init through `optimizers`, or subclass and override this method in a subclass.
      """
      if is_sagemaker_mp_enabled():
          return super().create_optimizer()
      if self.sharded_ddp == ShardedDDPOption.SIMPLE:
          return super().create_optimizer()
  
      opt_model = self.model
  
      if self.args.lr_multi is not None:
          lr_multi_dict = {}
          for _dict in self.args.lr_multi.split("\\"):
              _key_val = _dict.split(":")
              lr_multi_dict[_key_val[0]] = float(_key_val[1])
  
      if self.optimizer is None:
          decay_parameters = get_parameter_names(
              opt_model, ALL_LAYERNORM_LAYERS
          )
          decay_parameters = [
              name for name in decay_parameters if "bias" not in name
          ]
          if self.args.lr_multi is not None:
              optimizer_grouped_parameters = [
                  {
                      "params": [
                          p
                          for n, p in opt_model.named_parameters()
                          if (
                              n in decay_parameters
                              and p.requires_grad
                              and not any(
                                  [
                                      _key in n
                                      for _key in lr_multi_dict.keys()
                                  ]
                              )
                          )
                      ],
                      "weight_decay": self.args.weight_decay,
                  },
                  {
                      "params": [
                          p
                          for n, p in opt_model.named_parameters()
                          if (
                              n not in decay_parameters
                              and p.requires_grad
                              and not any(
                                  [
                                      _key in n
                                      for _key in lr_multi_dict.keys()
                                  ]
                              )
                          )
                      ],
                      "weight_decay": 0.0,
                  },
              ]
              for _key in lr_multi_dict:
                  _key_decay = [
                      p
                      for n, p in opt_model.named_parameters()
                      if (
                          n in decay_parameters
                          and p.requires_grad
                          and _key in n
                      )
                  ]
                  _key_no_decay = [
                      p
                      for n, p in opt_model.named_parameters()
                      if (
                          n not in decay_parameters
                          and p.requires_grad
                          and _key in n
                      )
                  ]
                  if len(_key_decay) > 0:
                      optimizer_grouped_parameters.append(
                          {
                              "params": _key_decay,
                              "lr": self.args.learning_rate
                              * lr_multi_dict[_key],
                              "weight_decay": self.args.weight_decay,
                          },
                      )
                  if len(_key_no_decay) > 0:
                      optimizer_grouped_parameters.append(
                          {
                              "params": _key_no_decay,
                              "lr": self.args.learning_rate
                              * lr_multi_dict[_key],
                              "weight_decay": 0.0,
                          },
                      )
  
          else:
              optimizer_grouped_parameters = [
                  {
                      "params": [
                          p
                          for n, p in opt_model.named_parameters()
                          if (n in decay_parameters and p.requires_grad)
                      ],
                      "weight_decay": self.args.weight_decay,
                  },
                  {
                      "params": [
                          p
                          for n, p in opt_model.named_parameters()
                          if (
                              n not in decay_parameters
                              and p.requires_grad
                          )
                      ],
                      "weight_decay": 0.0,
                  },
              ]
  
          (
              optimizer_cls,
              optimizer_kwargs,
          ) = Trainer.get_optimizer_cls_and_kwargs(self.args)
  
          if self.sharded_ddp == ShardedDDPOption.SIMPLE:
              self.optimizer = OSS(
                  params=optimizer_grouped_parameters,
                  optim=optimizer_cls,
                  **optimizer_kwargs,
              )
          else:
              self.optimizer = optimizer_cls(
                  optimizer_grouped_parameters, **optimizer_kwargs
              )
              if optimizer_cls.__name__ == "Adam8bit":
                  import bitsandbytes
  
                  manager = (
                      bitsandbytes.optim.GlobalOptimManager.get_instance()
                  )
  
                  skipped = 0
                  for module in opt_model.modules():
                      if isinstance(module, nn.Embedding):
                          skipped += sum(
                              {
                                  p.data_ptr(): p.numel()
                                  for p in module.parameters()
                              }.values()
                          )
                          logger.info(
                              f"skipped {module}: {skipped/2**20}M params"
                          )
                          manager.register_module_override(
                              module, "weight", {"optim_bits": 32}
                          )
                          logger.debug(
                              f"bitsandbytes: will optimize {module} in fp32"
                          )
                  logger.info(f"skipped: {skipped/2**20}M params")
  
      return self.optimizer
  
imports: import os
  import torch
  import torch.nn
  from fairscale.optim import OSS
  from torch.utils.data import Sampler
  from transformers import Trainer
  from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
  from transformers.trainer_pt_utils import get_parameter_names
  from transformers.trainer_utils import ShardedDDPOption, has_length
  from transformers.utils import is_sagemaker_mp_enabled, logging
  from typing import List, Optional, Union, Dict, Any
label_desc: Use image node_id 2 for calling the LLaVATrainer obj's underlying member method code with example usage: obj.create_optimizer() and returns: self, super, super