class_docstr: Implements Adafactor algorithm.
  
  It has been proposed in: `Adafactor: Adaptive Learning Rates with
  Sublinear Memory Cost`__.
  
  Arguments:
      params: iterable of parameters to optimize or dicts defining
          parameter groups
      lr: external learning rate (default: None)
      eps2: regularization constans for square gradient
          and parameter scale respectively (default: (1e-30, 1e-3))
      clip_threshold: threshold of root mean square of
          final gradient update (default: 1.0)
      decay_rate: coefficient used to compute running averages of square
          gradient (default: -0.8)
      beta1: coefficient used for computing running averages of gradient
          (default: None)
      weight_decay: weight decay (L2 penalty) (default: 0)
      scale_parameter: if true, learning rate is scaled by root mean square
          of parameter (default: True)
      relative_step: if true, time-dependent learning rate is computed
          instead of external learning rate (default: True)
      warmup_init: time-dependent learning rate computation depends on
          whether warm-up initialization is being used (default: False)
  
  Example:
      >>> import torch_optimizer as optim
      >>> optimizer = optim.Adafactor(model.parameters())
      >>> optimizer.zero_grad()
      >>> loss_fn(model(input), target).backward()
      >>> optimizer.step()
  
  __ https://arxiv.org/abs/1804.04235
  
  Note:
      Reference code: https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py  # noqa
class_docstr_tok: ["Implements","Adafactor","algorithm",".","It","has","been","proposed","in",":","`","Adafactor",":","Adaptive","Learning","Rates","with","Sublinear","Memory","Cost","`","__",".","Arguments",":","params",":","iterable","of","parameters","to","optimize","or","dicts","defining","parameter","groups","lr",":","external","learning","rate","(","default",":","None",")","eps2",":","regularization","constans","for","square","gradient","and","parameter","scale","respectively","(","default",":","(","1e-30",",","1e-3",")",")","clip_threshold",":","threshold","of","root","mean","square","of","final","gradient","update","(","default",":","1.0",")","decay_rate",":","coefficient","used","to","compute","running","averages","of","square","gradient","(","default",":","-0.8",")","beta1",":","coefficient","used","for","computing","running","averages","of","gradient","(","default",":","None",")","weight_decay",":","weight","decay","(","L2","penalty",")","(","default",":","0",")","scale_parameter",":","if","true",",","learning","rate","is","scaled","by","root","mean","square","of","parameter","(","default",":","True",")","relative_step",":","if","true",",","time-dependent","learning","rate","is","computed","instead","of","external","learning","rate","(","default",":","True",")","warmup_init",":","time-dependent","learning","rate","computation","depends","on","whether","warm-up","initialization","is","being","used","(","default",":","False",")","Example",":",">",">",">","import","torch_optimizer","as","optim",">",">",">","optimizer","=","optim.Adafactor","(","model.parameters","(",")",")",">",">",">","optimizer.zero_grad","(",")",">",">",">","loss_fn","(","model","(","input",")",",","target",")",".backward","(",")",">",">",">","optimizer.step","(",")","__","https",":","\/\/arxiv.org\/abs\/1804.04235","Note",":","Reference","code",":","https",":","\/\/github.com\/pytorch\/fairseq\/blob\/master\/fairseq\/optim\/adafactor.py","#","noqa"]
code: def _get_options(
      self, param_group: ParamGroup, param_shape: Tuple[int, ...]
  ) -> Tuple[bool, bool]:
      factored = len(param_shape) >= 2
      use_first_moment = param_group["beta1"] is not None
      return factored, use_first_moment
  
file_path: pytorch-optimizer/torch_optimizer/adafactor.py
imports: import math
  from typing import Any, Dict, Tuple
  import torch
  from torch.optim.optimizer import Optimizer
  from .types import OptFloat, OptLossClosure, Params, State
label_desc: Use image node_id 3 for calling the Adafactor obj's underlying member method code with example usage: obj._get_options(param_group, param_shape) and returns: factored,  use_first_moment