code: def test_all_reduce(self, device, dtype):
      cpu_tensors = [
          torch.zeros(128).uniform_().to(dtype=dtype)
          for i in range(nGPUs)
      ]
      expected = torch.zeros(128, dtype=dtype)
      for t in cpu_tensors:
          expected.add_(t)
  
      tensors = [cpu_tensors[i].cuda(i) for i in range(nGPUs)]
      nccl.all_reduce(tensors)
  
      for tensor in tensors:
          self.assertEqual(tensor, expected)
  
      # Test with tuple.
      tensors = tuple(cpu_tensors[i].cuda(i) for i in range(nGPUs))
      nccl.all_reduce(tensors)
  
      for tensor in tensors:
          self.assertEqual(tensor, expected)
  
      # Test with set.
      tensors = {cpu_tensors[i].cuda(i) for i in range(nGPUs)}
      nccl.all_reduce(tensors)
  
      for tensor in tensors:
          self.assertEqual(tensor, expected)
  
file_path: pytorch/test/distributed/test_nccl.py
imports: import sys
  import torch
  import torch.cuda.nccl
  import torch.cuda
  import torch.distributed
  from torch.testing._internal.common_utils import TestCase, run_tests, IS_WINDOWS, load_tests, TEST_WITH_ROCM, skip_but_pass_in_sandcastle_if, NoTest
  from torch.testing._internal.common_cuda import TEST_CUDA, TEST_MULTIGPU
  from torch.testing._internal.common_device_type import instantiate_device_type_tests, dtypes
  import re
label_desc: Use image node_id 4 for calling the TestNCCL obj's underlying member method code with example usage: obj.test_all_reduce(device, dtype) without return types