class_docstr: AllGatherWithGradient
class_docstr_tok: ["AllGatherWithGradient"]
code: def backward(ctx, grad_output):
      reduction = torch.distributed.all_reduce(
          grad_output, async_op=True
      )
      reduction.wait()
  
      world_size = dist.get_world_size()
      N = grad_output.size(0)
      mini_batchsize = N // world_size
      cur_gpu = torch.distributed.get_rank()
      grad_output = grad_output[
          cur_gpu * mini_batchsize : (cur_gpu + 1) * mini_batchsize
      ]
      return grad_output
  
imports: import functools
  import logging
  import pickle
  import torch
  import torch.distributed
  from pytorchvideo.layers.distributed import cat_all_gather, get_local_process_group, get_local_rank, get_local_size, get_world_size, init_distributed_training
label_desc: Use image node_id 2 for calling the AllGatherWithGradient obj's underlying member method code with example usage: obj.backward(ctx, grad_output) and returns: grad_output