class_docstr: A light-weight wrapper class for torch.distributed API.
  Only a subset of functions are wrapped. Once the init_process_group
  is initialized, standard torch.distributed.* can be used directly
  so no need to wrap all the functions. We can keep adding wrappers as
  needed.
class_docstr_tok: ["A","light-weight","wrapper","class","for","torch.distributed","API",".","Only","a","subset","of","functions","are","wrapped",".","Once","the","init_process_group","is","initialized",",","standard","torch.distributed",".","*","can","be","used","directly","so","no","need","to","wrap","all","the","functions",".","We","can","keep","adding","wrappers","as","needed","."]
code: def __init__(self, backend, timeout, init_method, name="torch"):
      super(TorchBackend, self).__init__()
      self.torch_version_before_18 = older_torch()
      self.has_allgather_base = has_allgather_base()
      self.has_reduce_scatter_base = has_reduce_scatter_base()
      self.initialized = True
      self.name = name
      # Future functionality to support ds.initialize() on a single GPU
      # The idea is to fake that dist backend is initialized even when
      # it is not so we can run on a single GPU without doing any init_process_group
      self.single_gpu_mode = True
      self.init_process_group(backend, timeout, init_method)
  
imports: from deepspeed import utils
  from .utils import *
  from .backend import *
  from .comm import *
label_desc: Use image node_id 1 to create a new TorchBackend object from inherited base classes: Backend with example: obj = TorchBackend(backend, timeout, init_method, name)