class_docstr: An optimizer that wraps a _DistributedOptimizer, intercepting push-pull operations.
  This class enables overlapping gradient push-pull with both backward and forward propagation while maintaining
  correct dependencies. It can achieve even higher training performance than the default BytePS with proper system
  parameters. To understand the principles behind barrier crossing, check the paper
  https://dl.acm.org/citation.cfm?id=3359642
class_docstr_tok: ["An","optimizer","that","wraps","a","_DistributedOptimizer",",","intercepting","push-pull","operations",".","This","class","enables","overlapping","gradient","push-pull","with","both","backward","and","forward","propagation","while","maintaining","correct","dependencies",".","It","can","achieve","even","higher","training","performance","than","the","default","BytePS","with","proper","system","parameters",".","To","understand","the","principles","behind","barrier","crossing",",","check","the","paper","https",":","\/\/dl.acm.org\/citation.cfm","?","id=3359642"]
docstr: Construct a new ScheduledOptimizer, which uses byteps optimizer under the hood for averaging gradients
   across all workers.
  Args:
      model: The training model. BytePS uses the model object to register hooks.
      byteps_opt: Optimizer to use for averaging gradients and applying updates.
      num_steps: The maximum number of training steps. BytePS needs to know when to stop cross-iteration
      scheduling.
docstr_tok: ["Construct","a","new","ScheduledOptimizer",",","which","uses","byteps","optimizer","under","the","hood","for","averaging","gradients","across","all","workers",".","Args",":","model",":","The","training","model",".","BytePS","uses","the","model","object","to","register","hooks",".","byteps_opt",":","Optimizer","to","use","for","averaging","gradients","and","applying","updates",".","num_steps",":","The","maximum","number","of","training","steps",".","BytePS","needs","to","know","when","to","stop","cross-iteration","scheduling","."]
code: def __init__(self, model, byteps_opt, num_steps=10**6):
      """Construct a new ScheduledOptimizer, which uses byteps optimizer under the hood for averaging gradients
       across all workers.
      Args:
          model: The training model. BytePS uses the model object to register hooks.
          byteps_opt: Optimizer to use for averaging gradients and applying updates.
          num_steps: The maximum number of training steps. BytePS needs to know when to stop cross-iteration
          scheduling.
      """
      self._model = model
      self._opt = byteps_opt
      self._logger = logging.getLogger("CrossBarrier")
  
      self._logger.info("CrossBarrier is enabled.")
      self._logger.debug(
          "byteps size {}, rank {}".format(size(), rank())
      )
      self._desc = "rank {}".format(rank())
  
      # Track training steps
      self._step = 0
      self._final_step = num_steps
  
      # Use lock to block the forward propagation of each parameter.
      self._locks = {}
      for param_group in self.param_groups:
          for p in param_group["params"]:
              self._locks[p] = threading.Lock()
  
      if size() > 1:
          self._register_forward_hooks()
          self._register_hooks()
  
          # Poll whether the tensor push-pull is finished.
          self._event_queue = queue.Queue()
          self._poller = threading.Thread(target=self._poll, args=())
          self._poller.start()
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from byteps.torch.compression import Compression
  from byteps.torch.ops import push_pull_async_inplace
  from byteps.torch.ops import poll, synchronize
  from byteps.torch.ops import init, shutdown
  from byteps.torch.ops import size, local_size, rank, local_rank
  import threading
  import logging
  import time
  import math
  import torch
  import byteps.torch
label_desc: Use image node_id 1 to create a new _CrossBarrier object from inherited base classes: _DistributedOptimizer with example: obj = _CrossBarrier(model, byteps_opt, num_steps)