class_docstr: An implementation of 2D naive sync batch normalization.
  In PyTorch<=1.5, ``nn.SyncBatchNorm`` has incorrect gradient
  when the batch size on each worker is different.
  (e.g., when scale augmentation is used, or when it is applied to mask head).
  
  This is a slower but correct alternative to `nn.SyncBatchNorm`.
  
  Args:
      num_sync_devices (int): number of (local) devices to sync.
      global_sync (bool): sync across all devices (on all machines).
      args (list): other arguments.
  
  Note:
      This module computes overall statistics by using
      statistics of each worker with equal weight.  The result is true statistics
      of all samples (as if they are all on one worker) only when all workers
      have the same (N, H, W). This mode does not support inputs with zero batch size.
class_docstr_tok: ["An","implementation","of","2D","naive","sync","batch","normalization",".","In","PyTorch","<","=1.5",",","``","nn.SyncBatchNorm","``","has","incorrect","gradient","when","the","batch","size","on","each","worker","is","different",".","(","e.g.",",","when","scale","augmentation","is","used",",","or","when","it","is","applied","to","mask","head",")",".","This","is","a","slower","but","correct","alternative","to","`","nn.SyncBatchNorm","`",".","Args",":","num_sync_devices","(","int",")",":","number","of","(","local",")","devices","to","sync",".","global_sync","(","bool",")",":","sync","across","all","devices","(","on","all","machines",")",".","args","(","list",")",":","other","arguments",".","Note",":","This","module","computes","overall","statistics","by","using","statistics","of","each","worker","with","equal","weight",".","The","result","is","true","statistics","of","all","samples","(","as","if","they","are","all","on","one","worker",")","only","when","all","workers","have","the","same","(","N",",","H",",","W",")",".","This","mode","does","not","support","inputs","with","zero","batch","size","."]
code: def __init__(self, num_sync_devices=None, global_sync=True, **args):
      self.global_sync = global_sync
      if self.global_sync and num_sync_devices is not None:
          raise ValueError(
              f"Cannot set num_sync_devices separately when global_sync = {self.global_sync}"
          )
      if not self.global_sync and num_sync_devices is None:
          raise ValueError(
              f"num_sync_devices cannot be None when global_sync = {self.global_sync}"
          )
  
      if not self.global_sync:
          self.num_sync_devices = num_sync_devices
          if self.num_sync_devices > 0:
              assert du.get_local_size() % self.num_sync_devices == 0, (
                  du.get_local_size(),
                  self.num_sync_devices,
              )
              self.num_groups = (
                  du.get_local_size() // self.num_sync_devices
              )
          else:
              self.num_sync_devices = du.get_local_size()
              self.num_groups = 1
      super(NaiveSyncBatchNorm2d, self).__init__(**args)
  
file_path: pytorchvideo/pytorchvideo/layers/batch_norm.py
imports: import pytorchvideo.layers.distributed
  import torch
  import torch.distributed
  from fvcore.nn.distributed import differentiable_all_reduce
  from torch import nn
label_desc: Use image node_id 1 to create a new NaiveSyncBatchNorm2d object from inherited base classes: nn with example: obj = NaiveSyncBatchNorm2d(num_sync_devices, global_sync)