class_docstr: Implements rectified adam as derived in the following paper:
  "On the Variance of the Adaptive Learning Rate and Beyond"
  (https://arxiv.org/abs/1908.03265)
  
  This code is mostly a direct copy-paste of the code provided by the authors here:
  https://github.com/LiyuanLucasLiu/RAdam/blob/master/radam.py
class_docstr_tok: ["Implements","rectified","adam","as","derived","in","the","following","paper",":","''","On","the","Variance","of","the","Adaptive","Learning","Rate","and","Beyond","''","(","https",":","\/\/arxiv.org\/abs\/1908.03265",")","This","code","is","mostly","a","direct","copy-paste","of","the","code","provided","by","the","authors","here",":","https",":","\/\/github.com\/LiyuanLucasLiu\/RAdam\/blob\/master\/radam.py"]
code: def __init__(
      self,
      params,
      lr=1e-3,
      betas=(0.9, 0.999),
      eps=1e-8,
      weight_decay=0,
  ):
      defaults = {
          "lr": lr,
          "betas": betas,
          "eps": eps,
          "weight_decay": weight_decay,
      }
      self.buffer = [[None, None, None] for ind in range(10)]
      PT_Optimizer.__init__(self, params, defaults)
  
file_path: pytext/pytext/optimizer/radam.py
imports: import math
  import torch
  from pytext.optimizer.optimizers import Optimizer
  from torch.optim import Optimizer
label_desc: Use image node_id 1 to create a new RAdam object from inherited base classes: Optimizer, PT_Optimizer with example: obj = RAdam(params, lr, betas, eps, weight_decay)