class_docstr: Implements stochastic gradient descent (optionally with momentum).
  
  Nesterov momentum is based on the formula from `On the importance of initialization and momentum in deep learning`__.
  
  Args:
      lr(float): learning rate
      momentum(float, optional): momentum factor(default: 0)
      weight_decay(float, optional): weight decay(L2 penalty)(default: 0)
      dampening(float, optional): dampening for momentum(default: 0)
      nesterov(bool, optional): enables Nesterov momentum(default: False)
  
  Typical usage example:
      >> > from singa import opt
      >> > optimizer = opt.SGD(lr=0.1, momentum=0.9)
      >> > optimizer.update()
  
  __ http: // www.cs.toronto.edu / %7Ehinton / absps / momentum.pdf
  
  .. note::
      The implementation of SGD with Momentum / Nesterov subtly differs from
      Sutskever et. al. and implementations in some other frameworks.
  
      Considering the specific case of Momentum, the update can be written as
  
      .. math::
                v = 
  ho * v + g \
                p = p - lr * v
  
      where p, g, v and: math: `
  ho` denote the parameters, gradient,
      velocity, and momentum respectively.
  
      This is in contrast to Sutskever et. al. and
      other frameworks which employ an update of the form
  
      .. math::
           v = 
  ho * v + lr * g \
           p = p - v
  
      The Nesterov version is analogously modified.
class_docstr_tok: ["Implements","stochastic","gradient","descent","(","optionally","with","momentum",")",".","Nesterov","momentum","is","based","on","the","formula","from","`","On","the","importance","of","initialization","and","momentum","in","deep","learning","`","__",".","Args",":","lr","(","float",")",":","learning","rate","momentum","(","float",",","optional",")",":","momentum","factor","(","default",":","0",")","weight_decay","(","float",",","optional",")",":","weight","decay","(","L2","penalty",")","(","default",":","0",")","dampening","(","float",",","optional",")",":","dampening","for","momentum","(","default",":","0",")","nesterov","(","bool",",","optional",")",":","enables","Nesterov","momentum","(","default",":","False",")","Typical","usage","example",":",">",">",">","from","singa","import","opt",">",">",">","optimizer","=","opt.SGD","(","lr=0.1",",","momentum=0.9",")",">",">",">","optimizer.update","(",")","__","http",":","\/\/","www.cs.toronto.edu","\/","%","7Ehinton","\/","absps","\/","momentum.pdf","..","note",":",":","The","implementation","of","SGD","with","Momentum","\/","Nesterov","subtly","differs","from","Sutskever","et",".","al",".","and","implementations","in","some","other","frameworks",".","Considering","the","specific","case","of","Momentum",",","the","update","can","be","written","as","..","math",":",":","v","=","ho","*","v","+","g","\\","p","=","p","-","lr","*","v","where","p",",","g",",","v","and",":","math",":","`","ho","`","denote","the","parameters",",","gradient",",","velocity",",","and","momentum","respectively",".","This","is","in","contrast","to","Sutskever","et",".","al",".","and","other","frameworks","which","employ","an","update","of","the","form","..","math",":",":","v","=","ho","*","v","+","lr","*","g","\\","p","=","p","-","v","The","Nesterov","version","is","analogously","modified","."]
docstr: Performs a single optimization step.
  
  Args:
          param_name(String): the name of the param
          param_value(Tensor): param values to be update in-place
          grad(Tensor): param gradients; the values may be updated
                  in this function; cannot use it anymore
docstr_tok: ["Performs","a","single","optimization","step",".","Args",":","param_name","(","String",")",":","the","name","of","the","param","param_value","(","Tensor",")",":","param","values","to","be","update","in-place","grad","(","Tensor",")",":","param","gradients",";","the","values","may","be","updated","in","this","function",";","can","not","use","it","anymore"]
code: def apply(self, param_name, param_value, param_grad):
      """Performs a single optimization step.
  
      Args:
              param_name(String): the name of the param
              param_value(Tensor): param values to be update in-place
              grad(Tensor): param gradients; the values may be updated
                      in this function; cannot use it anymore
      """
      assert param_value.shape == param_grad.shape, (
          "shape mismatch",
          param_value.shape,
          param_grad.shape,
      )
      self.device_check(
          param_value,
          self.step_counter,
          self.lr_value,
          self.mom_value,
          self.dam_value,
          self.decay_value,
      )
  
      # derive dtype from input
      # assert param_value.dtype == self.dtype
  
      # TODO add branch operator
      # if self.decay_value != 0:
      if self.weight_decay.init_value != 0:
          singa.Axpy(
              self.decay_value.data, param_value.data, param_grad.data
          )
  
      if self.momentum.init_value != 0:
          if param_name not in self.moments:
              flag = param_value.device.graph_enabled()
              param_value.device.EnableGraph(False)
              self.moments[param_name] = tensor.zeros_like(param_value)
              param_value.device.EnableGraph(flag)
  
          buf = self.moments[param_name]
          buf *= self.mom_value
          alpha = 1.0 - self.dam_value
          singa.Axpy(alpha.data, param_grad.data, buf.data)
  
          if self.nesterov:
              singa.Axpy(self.mom_value.data, buf.data, param_grad.data)
          else:
              param_grad = buf
  
      minus_lr = 0.0 - self.lr_value
      singa.Axpy(minus_lr.data, param_grad.data, param_value.data)
  
imports: import time
  from src.tools import utils
  from singa import singa_wrap
  from singa import device
  from singa import tensor
  from singa import opt
  from singa import autograd
  from singa.opt import Optimizer
  from singa.opt import DecayScheduler
  from singa.opt import Constant
  import numpy
  import time
  import argparse
  from PIL import Image
  from torch.utils.data import DataLoader
label_desc: Use image node_id 2 for calling the MSSGD obj's underlying member method code with example usage: obj.apply(param_name, param_value, param_grad) without return types