docstr: If multiple GPUs were involved in finding permutations, make sure everyone's in sync
code: def sync_permutations(cls, fx_graph):
      """If multiple GPUs were involved in finding permutations, make sure everyone's in sync"""
  
      if not torch.distributed.is_initialized():
          return fx_graph
  
      rank = torch.distributed.get_rank()
      world_size = torch.distributed.get_world_size()
      dist_store = torch.distributed.TCPStore(
          "127.0.0.1", cls.__tcpstore_port, world_size, rank == 0
      )
  
      if cls.__verbosity > 0:
          print(
              f"Syncing permutations found among world size {world_size}"
          )
  
      torch.distributed.barrier()
      for sibling_group_id in sorted(
          cls.__group_data["sibling_groups"].keys()
      ):
          src_rank = sibling_group_id % world_size
  
          if src_rank == rank:
              to_send = cls.__group_data[
                  "sibling_group_permutations"
              ].get(sibling_group_id, None)
              skip_reason = None
              if to_send is None:
                  skip_reason = cls.__group_data[
                      "skipped_sibling_groups"
                  ].get(sibling_group_id, None)
                  if skip_reason is None:
                      to_send = ""
                  else:
                      to_send = "skip"
              else:
                  to_send = ",".join(str(c) for c in to_send)
  
              dist_store.set(str(sibling_group_id), to_send)
              if skip_reason is not None:
                  dist_store.set(
                      f"skip {sibling_group_id}", skip_reason
                  )
  
              if cls.__verbosity > 1:
                  print(
                      f"{rank}: stored permutation for sibling group {sibling_group_id}",
                      force=True,
                  )
  
      torch.distributed.barrier()
      for sibling_group_id in sorted(
          cls.__group_data["sibling_groups"].keys()
      ):
          permutation = dist_store.get(str(sibling_group_id)).decode()
  
          if permutation == "skip":
              permutation = None
              skip_reason = dist_store.get(
                  f"skip {sibling_group_id}"
              ).decode()
              cls.skip_sibling_group(
                  fx_graph, sibling_group_id, skip_reason
              )
          else:
              if len(permutation) == 0:
                  permutation = None
              else:
                  permutation = [int(c) for c in permutation.split(",")]
  
          cls.__group_data["sibling_group_permutations"][
              sibling_group_id
          ] = permutation
  
          if cls.__verbosity > 1:
              print(
                  f"Got permutation for sibling group {sibling_group_id}"
              )
  
      torch.distributed.barrier()
      return fx_graph
  
imports: import os
  import torch
  import json
  import string
  import time
  import numpy
  import sys
  import builtins
  import io
label_desc: Use image node_id 22 for calling the Permutation obj's underlying member method code with example usage: obj.sync_permutations(cls, fx_graph) and returns: fx_graph, fx_graph