code: def __init__(
      self,
      params,
      lr=1e-4,
      betas=(0.965, 0.99),
      rho=0.04,
      weight_decay=1e-1,
      *,
      maximize: bool = False,
      capturable: bool = False,
  ):
      if not 0.0 <= lr:
          raise ValueError("Invalid learning rate: {}".format(lr))
      if not 0.0 <= betas[0] < 1.0:
          raise ValueError(
              "Invalid beta parameter at index 0: {}".format(betas[0])
          )
      if not 0.0 <= betas[1] < 1.0:
          raise ValueError(
              "Invalid beta parameter at index 1: {}".format(betas[1])
          )
      if not 0.0 <= rho:
          raise ValueError(
              "Invalid rho parameter at index 1: {}".format(rho)
          )
      if not 0.0 <= weight_decay:
          raise ValueError(
              "Invalid weight_decay value: {}".format(weight_decay)
          )
      defaults = dict(
          lr=lr,
          betas=betas,
          rho=rho,
          weight_decay=weight_decay,
          maximize=maximize,
          capturable=capturable,
      )
      super().__init__(params, defaults)
  
file_path: bert-ultrafast/training/cramming/backend/optimizers/sophia.py
imports: import torch
  from torch import Tensor
  from torch.optim.optimizer import Optimizer
  from typing import List
label_desc: Use image node_id 1 to create a new Sophia object from inherited base classes: Optimizer with example: obj = Sophia(params, lr, betas, rho, weight_decay)