code: def all_gather_list(data):
      """
      Gather arbitrary data from all nodes into a list.
  
      Similar to `~torch.distributed.all_gather` but for arbitrary Python
      data. Note that *data* must be picklable.
  
      :param data:
          data from the local worker to be gathered on other workers
  
      :returns:
          a list containing [data1, data2, ...] of all workers
      """
      if not is_distributed():
          # fall back to just keeping things basic if we're not distributed
          return [data]
  
      # stolen shamelessly from fairseq
      # https://github.com/pytorch/fairseq/blob/c37250ab1c845919af721cd3f5c4cec2993aefe1/fairseq/distributed_utils.py#L116-L170
      rank = dist.get_rank()
      world_size = dist.get_world_size()
  
      enc = list(pickle.dumps(data))
      enc_size = len(enc)
  
      # find the sizes of all the serialized items
      sizes = torch.zeros(world_size, dtype=torch.long).cuda()
      sizes[rank] = enc_size
      dist.all_reduce(sizes)
  
      # need to know our positions
      sizes = sizes.cpu()
      positions = sizes.cumsum(dim=0)
  
      buffer_size = positions[-1].item()
      buffer = torch.cuda.ByteTensor(buffer_size).zero_()
  
      start = positions[rank] - enc_size
      end = positions[rank]
      buffer[start:end] = torch.ByteTensor(enc)
  
      dist.all_reduce(buffer)
  
      result = []
      for i in range(world_size):
          out_buffer = buffer[positions[i] - sizes[i] : positions[i]]
          try:
              result.append(pickle.loads(bytes(out_buffer.tolist())))
          except pickle.UnpicklingError:
              raise RuntimeError(
                  "There was an unpickling error in all_gather_list. This likely "
                  "means your workers got out of synchronization (e.g. one is "
                  "expecting to sync and another is not.)"
              )
  
      return result
  
file_path: ParlAI/parlai/utils/distributed.py
imports: import builtins
  import copy
  import os
  import pickle
  import contextlib
  import subprocess
  import socket
  import parlai.utils.logging
label_desc: Use image node_id 6 for calling a global function with example usage: all_gather_list(data) and returns: result, list