def METHOD_NAME(self):
    world_size = distributed.get_world_size()
    for p in self.net.parameters():
        group = distributed.new_group(ranks=list(range(world_size)))
        tensor = p.grad.data.cpu()
        distributed.all_reduce(tensor, op=distributed.reduce_op.SUM, group=group)
        tensor /= float(world_size)
        p.grad.data = tensor.to(self.device)