class_docstr: An implementation of 2D naive sync batch normalization.
  In PyTorch<=1.5, ``nn.SyncBatchNorm`` has incorrect gradient
  when the batch size on each worker is different.
  (e.g., when scale augmentation is used, or when it is applied to mask head).
  
  This is a slower but correct alternative to `nn.SyncBatchNorm`.
  
  Args:
      num_sync_devices (int): number of (local) devices to sync.
      global_sync (bool): sync across all devices (on all machines).
      args (list): other arguments.
  
  Note:
      This module computes overall statistics by using
      statistics of each worker with equal weight.  The result is true statistics
      of all samples (as if they are all on one worker) only when all workers
      have the same (N, H, W). This mode does not support inputs with zero batch size.
class_docstr_tok: ["An","implementation","of","2D","naive","sync","batch","normalization",".","In","PyTorch","<","=1.5",",","``","nn.SyncBatchNorm","``","has","incorrect","gradient","when","the","batch","size","on","each","worker","is","different",".","(","e.g.",",","when","scale","augmentation","is","used",",","or","when","it","is","applied","to","mask","head",")",".","This","is","a","slower","but","correct","alternative","to","`","nn.SyncBatchNorm","`",".","Args",":","num_sync_devices","(","int",")",":","number","of","(","local",")","devices","to","sync",".","global_sync","(","bool",")",":","sync","across","all","devices","(","on","all","machines",")",".","args","(","list",")",":","other","arguments",".","Note",":","This","module","computes","overall","statistics","by","using","statistics","of","each","worker","with","equal","weight",".","The","result","is","true","statistics","of","all","samples","(","as","if","they","are","all","on","one","worker",")","only","when","all","workers","have","the","same","(","N",",","H",",","W",")",".","This","mode","does","not","support","inputs","with","zero","batch","size","."]
code: def forward(self, input):
      if du.get_world_size() == 1 or not self.training:
          return super().forward(input)
  
      B, C = input.shape[0], input.shape[1]
  
      assert B > 0, "SyncBatchNorm does not support zero batch size."
  
      mean = torch.mean(input, dim=[0, 2, 3])
      meansqr = torch.mean(input * input, dim=[0, 2, 3])
  
      vec = torch.cat([mean, meansqr], dim=0)
      #  sync stats globally or locally
      if self.global_sync:
          vec = differentiable_all_reduce(vec) * (
              1.0 / dist.get_world_size()
          )
      else:
          vec = du.GroupGather.apply(
              vec, self.num_sync_devices, self.num_groups
          ) * (1.0 / self.num_sync_devices)
  
      mean, meansqr = torch.split(vec, C)
      var = meansqr - mean * mean
  
      invstd = torch.rsqrt(var + self.eps)
      scale = self.weight * invstd
      bias = self.bias - mean * scale
      scale = scale.reshape(1, -1, 1, 1)
      bias = bias.reshape(1, -1, 1, 1)
  
      self.running_mean += self.momentum * (
          mean.detach() - self.running_mean
      )
      self.running_var += self.momentum * (
          var.detach() - self.running_var
      )
  
      return input * scale + bias
  
file_path: pytorchvideo/pytorchvideo/layers/batch_norm.py
imports: import pytorchvideo.layers.distributed
  import torch
  import torch.distributed
  from fvcore.nn.distributed import differentiable_all_reduce
  from torch import nn
label_desc: Use image node_id 2 for calling the NaiveSyncBatchNorm2d obj's underlying member method code with example usage: obj.forward(input) and returns: unknown, super