class_docstr: Profile alpha and beta value for a given device list.
  
  Usage:
      # Note: the environment of execution is supposed to be
      # multi-process with multi-gpu in mpi style.
      >>> physical_devices = [0, 1, 4, 5]
      >>> ab_profiler = AlphaBetaProfiler(physical_devices)
      >>> ab_dict = profiler.alpha_beta_dict
      >>> print(ab_dict)
      {(0, 1): (1.9641406834125518e-05, 4.74049549614719e-12), (0, 4): (1.9506998360157013e-05, 6.97421973297474e-11), (0, 5): (2.293858677148819e-05, 7.129930361393644e-11),
       (1, 4): (1.9010603427886962e-05, 7.077968863788975e-11), (1, 5): (1.9807778298854827e-05, 6.928845708992215e-11), (4, 5): (1.8681809306144713e-05, 4.7522367291330524e-12),
       (1, 0): (1.9641406834125518e-05, 4.74049549614719e-12), (4, 0): (1.9506998360157013e-05, 6.97421973297474e-11), (5, 0): (2.293858677148819e-05, 7.129930361393644e-11),
       (4, 1): (1.9010603427886962e-05, 7.077968863788975e-11), (5, 1): (1.9807778298854827e-05, 6.928845708992215e-11), (5, 4): (1.8681809306144713e-05, 4.7522367291330524e-12)}
docstr: Args:
      physical_devices: A list of device id, each element inside it is the global rank of that device.
      alpha_beta_dict: A dict which maps a process group to alpha-beta value pairs.
      ctype: 'a' for all-reduce, 'b' for broadcast.
      warmup: Number of warmup iterations.
      repeat: Number of iterations to measure.
      latency_iters: Number of iterations to measure latency.
docstr_tok: ["Args",":","physical_devices",":","A","list","of","device","id",",","each","element","inside","it","is","the","global","rank","of","that","device",".","alpha_beta_dict",":","A","dict","which","maps","a","process","group","to","alpha-beta","value","pairs",".","ctype",":","'","a","'","for","all-reduce",",","'","b","'","for","broadcast",".","warmup",":","Number","of","warmup","iterations",".","repeat",":","Number","of","iterations","to","measure",".","latency_iters",":","Number","of","iterations","to","measure","latency","."]
code: def __init__(
      self,
      physical_devices: List[int],
      alpha_beta_dict: Dict[
          Tuple[int, int], Tuple[float, float]
      ] = None,
      ctype: str = "a",
      warmup: int = 5,
      repeat: int = 25,
      latency_iters: int = 5,
      homogeneous_tolerance: float = 0.1,
  ):
      """
      Args:
          physical_devices: A list of device id, each element inside it is the global rank of that device.
          alpha_beta_dict: A dict which maps a process group to alpha-beta value pairs.
          ctype: 'a' for all-reduce, 'b' for broadcast.
          warmup: Number of warmup iterations.
          repeat: Number of iterations to measure.
          latency_iters: Number of iterations to measure latency.
      """
      self.physical_devices = physical_devices
      self.ctype = ctype
      self.world_size = len(physical_devices)
      self.warmup = warmup
      self.repeat = repeat
      self.latency_iters = latency_iters
      self.homogeneous_tolerance = homogeneous_tolerance
      self.process_group_dict = None
      self._init_profiling()
      if alpha_beta_dict is None:
          self.alpha_beta_dict = self.profile_ab()
      else:
          self.alpha_beta_dict = alpha_beta_dict
  
file_path: ColossalAI/colossalai/device/alpha_beta_profiler.py
imports: import math
  import time
  from typing import Dict, List, Tuple
  import torch
  import torch.distributed
  from colossalai.logging import get_dist_logger
label_desc: Use image node_id 1 to create a new AlphaBetaProfiler object  with example: obj = AlphaBetaProfiler(physical_devices, alpha_beta_dict, ctype, warmup, repeat, latency_iters, homogeneous_tolerance)