class_docstr: NOTE: This class has been copied verbatim from the separate Dense and
  Sparse versions of Adam in Pytorch.
  
  Implements Adam algorithm with dense & sparse gradients.
  It has been proposed in Adam: A Method for Stochastic Optimization.
  
  Parameters
  ----------
  params : ``iterable``
      iterable of parameters to optimize or dicts defining parameter groups
  lr : ``float``, optional (default: 1e-3)
      The learning rate.
  betas : ``Tuple[float, float]``, optional (default: (0.9, 0.999))
      coefficients used for computing running averages of gradient
      and its square.
  eps : ``float``, optional, (default: 1e-8)
      A term added to the denominator to improve numerical stability.
docstr: Performs a single optimization step.
  
  Parameters
  ----------
  closure : ``callable``, optional.
      A closure that reevaluates the model and returns the loss.
docstr_tok: ["Performs","a","single","optimization","step",".","Parameters","--","--","--","--","--","closure",":","``","callable","``",",","optional",".","A","closure","that","reevaluates","the","model","and","returns","the","loss","."]
code: def step(self, closure=None):
      """
      Performs a single optimization step.
  
      Parameters
      ----------
      closure : ``callable``, optional.
          A closure that reevaluates the model and returns the loss.
      """
      loss = None
      if closure is not None:
          loss = closure()
  
      for group in self.param_groups:
          for p in group["params"]:
              if p.grad is None:
                  continue
              grad = p.grad.data
  
              state = self.state[p]
  
              # State initialization
              if len(state) == 0:
                  state["step"] = 0
                  # Exponential moving average of gradient values
                  state["exp_avg"] = torch.zeros_like(p.data)
                  # Exponential moving average of squared gradient values
                  state["exp_avg_sq"] = torch.zeros_like(p.data)
  
              state["step"] += 1
  
              exp_avg, exp_avg_sq = (
                  state["exp_avg"],
                  state["exp_avg_sq"],
              )
              beta1, beta2 = group["betas"]
  
              if grad.is_sparse:
                  grad = (
                      grad.coalesce()
                  )  # the update is non-linear so indices must be unique
                  grad_indices = grad._indices()
                  grad_values = grad._values()
                  size = grad.size()
  
                  def make_sparse(values):
                      constructor = grad.new
                      if grad_indices.dim() == 0 or values.dim() == 0:
                          return constructor().resize_as_(grad)
                      return constructor(grad_indices, values, size)
  
                  # Decay the first and second moment running average coefficient
                  #      old <- b * old + (1 - b) * new
                  # <==> old += (1 - b) * (new - old)
                  old_exp_avg_values = exp_avg._sparse_mask(
                      grad
                  )._values()
                  exp_avg_update_values = grad_values.sub(
                      old_exp_avg_values
                  ).mul_(1 - beta1)
                  exp_avg.add_(make_sparse(exp_avg_update_values))
                  old_exp_avg_sq_values = exp_avg_sq._sparse_mask(
                      grad
                  )._values()
                  exp_avg_sq_update_values = (
                      grad_values.pow(2)
                      .sub_(old_exp_avg_sq_values)
                      .mul_(1 - beta2)
                  )
                  exp_avg_sq.add_(make_sparse(exp_avg_sq_update_values))
  
                  # Dense addition again is intended, avoiding another _sparse_mask
                  numer = exp_avg_update_values.add_(old_exp_avg_values)
                  exp_avg_sq_update_values.add_(old_exp_avg_sq_values)
                  denom = exp_avg_sq_update_values.sqrt_().add_(
                      group["eps"]
                  )
                  del exp_avg_update_values, exp_avg_sq_update_values
  
                  bias_correction1 = 1 - beta1 ** state["step"]
                  bias_correction2 = 1 - beta2 ** state["step"]
                  step_size = (
                      group["lr"]
                      * math.sqrt(bias_correction2)
                      / bias_correction1
                  )
  
                  p.data.add_(
                      make_sparse(-step_size * numer.div_(denom))
                  )
  
              else:
                  # Decay the first and second moment running average coefficient
                  exp_avg.mul_(beta1).add_(1 - beta1, grad)
                  exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
                  denom = exp_avg_sq.sqrt().add_(group["eps"])
  
                  bias_correction1 = 1 - beta1 ** state["step"]
                  bias_correction2 = 1 - beta2 ** state["step"]
                  step_size = (
                      group["lr"]
                      * math.sqrt(bias_correction2)
                      / bias_correction1
                  )
  
                  p.data.addcdiv_(-step_size, exp_avg, denom)
  
      return loss
  
file_path: magnitude/pymagnitude/third_party/allennlp/training/optimizers.py
imports: from __future__ import division
  from __future__ import absolute_import
  import logging
  import re
  import math
  import torch
  from allennlp.common import Params, Registrable
label_desc: Use image node_id 2 for calling the DenseSparseAdam obj's underlying member method code with example usage: obj.step(closure) and returns: loss, constructor, constructor