class_docstr: A light-weight wrapper class for torch.distributed API.
  Only a subset of functions are wrapped. Once the init_process_group
  is initialized, standard torch.distributed.* can be used directly
  so no need to wrap all the functions. We can keep adding wrappers as
  needed.
class_docstr_tok: ["A","light-weight","wrapper","class","for","torch.distributed","API",".","Only","a","subset","of","functions","are","wrapped",".","Once","the","init_process_group","is","initialized",",","standard","torch.distributed",".","*","can","be","used","directly","so","no","need","to","wrap","all","the","functions",".","We","can","keep","adding","wrappers","as","needed","."]
code: def __init__(
      self,
      backend,
      timeout,
      init_method,
      rank=-1,
      world_size=-1,
      name="torch",
  ):
      super(TorchBackend, self).__init__()
      self.has_all_reduce_coalesced = has_all_reduce_coalesced()
      self.has_coalescing_manager = has_coalescing_manager()
      self.all_gather_function = self.get_all_gather_function()
      self.reduce_scatter_function = self.get_reduce_scatter_function()
      self.initialized = True
      self.name = name
      # Future functionality to support ds.initialize() on a single GPU
      # The idea is to fake that dist backend is initialized even when
      # it is not so we can run on a single GPU without doing any init_process_group
      self.single_gpu_mode = True
      self.init_process_group(
          backend, timeout, init_method, rank, world_size
      )
  
imports: from deepspeed import utils
  from .utils import *
  from .backend import *
  from .comm import *
  import os
label_desc: Use image node_id 1 to create a new TorchBackend object from inherited base classes: Backend with example: obj = TorchBackend(backend, timeout, init_method, rank, world_size, name)