class_docstr: Implements Adafactor algorithm.
  This implementation is based on:
  `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost`
  (see https://arxiv.org/abs/1804.04235)
  Note that this optimizer internally adjusts the learning rate
  depending on the *scale_parameter*, *relative_step* and
  *warmup_init* options. To use a manual (external) learning rate
  schedule you should set `scale_parameter=False` and
  `relative_step=False`.
  Args:
      params (iterable): iterable of parameters to optimize or dicts defining
          parameter groups
      lr (float, optional): external learning rate (default: None)
      eps (tuple[float, float]): regularization constans for square gradient
          and parameter scale respectively (default: (1e-30, 1e-3))
      clip_threshold (float): threshold of root mean square of
          final gradient update (default: 1.0)
      decay_rate (float): coefficient used to compute running averages of square
          gradient (default: -0.8)
      beta1 (float): coefficient used for computing running averages of gradient
          (default: None)
      weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
      scale_parameter (bool): if True, learning rate is scaled by root mean square of
          parameter (default: True)
      relative_step (bool): if True, time-dependent learning rate is computed
          instead of external learning rate (default: True)
      warmup_init (bool): time-dependent learning rate computation depends on
          whether warm-up initialization is being used (default: False)
class_docstr_tok: ["Implements","Adafactor","algorithm",".","This","implementation","is","based","on",":","`","Adafactor",":","Adaptive","Learning","Rates","with","Sublinear","Memory","Cost","`","(","see","https",":","\/\/arxiv.org\/abs\/1804.04235",")","Note","that","this","optimizer","internally","adjusts","the","learning","rate","depending","on","the","*","scale_parameter","*",",","*","relative_step","*","and","*","warmup_init","*","options",".","To","use","a","manual","(","external",")","learning","rate","schedule","you","should","set","`","scale_parameter=False","`","and","`","relative_step=False","`",".","Args",":","params","(","iterable",")",":","iterable","of","parameters","to","optimize","or","dicts","defining","parameter","groups","lr","(","float",",","optional",")",":","external","learning","rate","(","default",":","None",")","eps","(","tuple","[","float",",","float","]",")",":","regularization","constans","for","square","gradient","and","parameter","scale","respectively","(","default",":","(","1e-30",",","1e-3",")",")","clip_threshold","(","float",")",":","threshold","of","root","mean","square","of","final","gradient","update","(","default",":","1.0",")","decay_rate","(","float",")",":","coefficient","used","to","compute","running","averages","of","square","gradient","(","default",":","-0.8",")","beta1","(","float",")",":","coefficient","used","for","computing","running","averages","of","gradient","(","default",":","None",")","weight_decay","(","float",",","optional",")",":","weight","decay","(","L2","penalty",")","(","default",":","0",")","scale_parameter","(","bool",")",":","if","True",",","learning","rate","is","scaled","by","root","mean","square","of","parameter","(","default",":","True",")","relative_step","(","bool",")",":","if","True",",","time-dependent","learning","rate","is","computed","instead","of","external","learning","rate","(","default",":","True",")","warmup_init","(","bool",")",":","time-dependent","learning","rate","computation","depends","on","whether","warm-up","initialization","is","being","used","(","default",":","False",")"]
code: def _rms(self, tensor):
      return tensor.norm(2) / (tensor.numel() ** 0.5)
  
file_path: NeMo/nemo/core/optim/adafactor.py
imports: import math
  import torch
  from torch.optim.optimizer import Optimizer
label_desc: Use image node_id 6 for calling the Adafactor obj's underlying member method code with example usage: obj._rms(tensor) and returns: unknown