class_docstr: An optimizer that wraps a _DistributedOptimizer, intercepting push-pull operations.
  This class enables overlapping gradient push-pull with both backward and forward propagation while maintaining
  correct dependencies. It can achieve even higher training performance than the default BytePS with proper system
  parameters. To understand the principles behind barrier crossing, check the paper
  https://dl.acm.org/citation.cfm?id=3359642
class_docstr_tok: ["An","optimizer","that","wraps","a","_DistributedOptimizer",",","intercepting","push-pull","operations",".","This","class","enables","overlapping","gradient","push-pull","with","both","backward","and","forward","propagation","while","maintaining","correct","dependencies",".","It","can","achieve","even","higher","training","performance","than","the","default","BytePS","with","proper","system","parameters",".","To","understand","the","principles","behind","barrier","crossing",",","check","the","paper","https",":","\/\/dl.acm.org\/citation.cfm","?","id=3359642"]
docstr: Performs a single optimization step using SGD optimizer on a parameter.
  Arguments:
      p: The parameter to be updated.
docstr_tok: ["Performs","a","single","optimization","step","using","SGD","optimizer","on","a","parameter",".","Arguments",":","p",":","The","parameter","to","be","updated","."]
code: def _sgd(self, p):
      """Performs a single optimization step using SGD optimizer on a parameter.
      Arguments:
          p: The parameter to be updated.
      """
      for group in self.param_groups:
          weight_decay = group["weight_decay"]
          momentum = group["momentum"]
          dampening = group["dampening"]
          nesterov = group["nesterov"]
  
          for gp in group["params"]:
              if (
                  self._get_parameter_name(p)
                  != self._get_parameter_name(gp)
                  or gp.shape != p.shape
              ):
                  continue
              self._logger.debug(
                  "{} is updating {}".format(
                      self._desc, self._get_parameter_name(p)
                  )
              )
              if p.grad is None:
                  continue
              d_p = p.grad.data
              if weight_decay != 0:
                  d_p.add_(weight_decay, p.data)
              if momentum != 0:
                  param_state = self.state[p]
                  if "momentum_buffer" not in param_state:
                      buf = param_state[
                          "momentum_buffer"
                      ] = torch.zeros_like(p.data)
                      buf.mul_(momentum).add_(d_p)
                  else:
                      buf = param_state["momentum_buffer"]
                      buf.mul_(momentum).add_(1 - dampening, d_p)
                  if nesterov:
                      d_p = d_p.add(momentum, buf)
                  else:
                      d_p = buf
              p.data.add_(-group["lr"], d_p)
              break
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from byteps.torch.compression import Compression
  from byteps.torch.ops import push_pull_async_inplace
  from byteps.torch.ops import poll, synchronize
  from byteps.torch.ops import init, shutdown
  from byteps.torch.ops import size, local_size, rank, local_rank
  import threading
  import logging
  import time
  import math
  import torch
  import byteps.torch
label_desc: Use image node_id 12 for calling the _CrossBarrier obj's underlying member method code with example usage: obj._sgd(p) without return types