class_docstr: All reduce used by SyncBatchNorm
class_docstr_tok: ["All","reduce","used","by","SyncBatchNorm"]
code: def forward(ctx, input):
      input_list = [
          torch.zeros_like(input) for k in range(dist.get_world_size())
      ]
      # Use allgather instead of allreduce since I don't trust in-place operations ..
      dist.all_gather(input_list, input, async_op=False)
      inputs = torch.stack(input_list, dim=0)
      return torch.sum(inputs, dim=0)
  
imports: import logging
  import torch
  import torch.distributed
  from torch import nn
  from torch.autograd.function import Function
  from torch.nn import functional
  from torch.nn import BatchNorm2d
  from ..utils.comm import get_world_size
label_desc: Use image node_id 1 for calling the AllReduce obj's underlying member method code with example usage: obj.forward(ctx, input) and returns: torch