class_docstr: `MADGRAD Optimizer`: A Momentumized, Adaptive, Dual Averaged Gradient Method for Stochastic
  Optimization.
  Paper: https://arxiv.org/abs/2101.11075
  
  Implementation has been copied over from the original author
  (https://github.com/facebookresearch/madgrad/blob/master/madgrad/madgrad.py)
class_docstr_tok: ["`","MADGRAD","Optimizer","`",":","A","Momentumized",",","Adaptive",",","Dual","Averaged","Gradient","Method","for","Stochastic","Optimization",".","Paper",":","https",":","\/\/arxiv.org\/abs\/2101.11075","Implementation","has","been","copied","over","from","the","original","author","(","https",":","\/\/github.com\/facebookresearch\/madgrad\/blob\/master\/madgrad\/madgrad.py",")"]
code: def __init__(
      self,
      params,
      lr: float = 1e-2,
      momentum: float = 0.9,
      weight_decay: float = 0,
      eps: float = 1e-6,
      k: int = 0,
  ):
      if momentum < 0 or momentum >= 1:
          raise ValueError(
              f"Momentum {momentum} must be in the range [0,1]"
          )
      if lr <= 0:
          raise ValueError(f"Learning rate {lr} must be positive")
      if weight_decay < 0:
          raise ValueError(
              f"Weight decay {weight_decay} must be non-negative"
          )
      if eps < 0:
          raise ValueError("Eps must be non-negative")
  
      defaults = {
          "lr": lr,
          "eps": eps,
          "momentum": momentum,
          "weight_decay": weight_decay,
          "k": k,
      }
  
      self.momentum = momentum
  
      PT_Optimizer.__init__(self, params, defaults)
  
      self.initialize_state()
  
file_path: pytext/pytext/optimizer/madgrad.py
imports: import math
  from collections import defaultdict
  from typing import Optional
  import torch
  from torch.optim.optimizer import Optimizer
  from .optimizers import Optimizer
label_desc: Use image node_id 2 to create a new MADGRAD object from inherited base classes: Optimizer, PT_Optimizer with example: obj = MADGRAD(params, lr, momentum, weight_decay, eps, k)