


ranks = list(range(8))
      >>> r1, r2 = ranks[:4], ranks[4:]
      >>> # Note: every rank calls into new_group for every
      >>> # process group created, even if that rank is not
      >>> # part of the group.
      >>> process_groups = [torch.distributed.new_group(pids) for pids in [r1, r2]]
      >>> process_group = process_groups[0 if dist.get_rank() <= 3 else 1]
      >>> # Without Learnable Parameters
      >>> m = nn.BatchNorm3d(100, affine=False, process_group=process_group)
      >>> input = torch.randn(20, 100, 35, 45, 10)
      >>> output = m(input)



  >>> # network is nn.BatchNorm layer
  >>> sync_bn_network = nn.SyncBatchNorm.convert_sync_batchnorm(network, process_group)
  >>> # only single gpu per process is currently supported
  >>> ddp_sync_bn_network = torch.nn.parallel.DistributedDataParallel(
  >>>                         sync_bn_network,
  >>>                         device_ids=[args.local_rank],
