code: def launcher_main(slots_per_node, num_nodes, cross_rank):
      # Use subprocess to start one worker process per node.
      procs = []
      for local_rank in range(slots_per_node):
          rank = cross_rank * slots_per_node + local_rank
          cmd = [
              # Use the determined.launch.wrap_rank to wrap the worker process.
              # This ensures logs from each worker can be filtered by rank in the WebUI.
              "python3",
              "-m",
              "determined.launch.wrap_rank",
              str(rank),
              "--",
              # Re-invoke this script but as a worker.
              "python3",
              __file__,
              "worker",
              str(rank),
              str(local_rank),
          ]
          procs.append(subprocess.Popen(cmd))
  
      # A good launcher normally waits for all workers to finish, but cleans up and exits
      # nonzero immediately if any worker fails to prevent distributed training jobs from
      # hanging.  One way to do this by managing each worker process in a thread and sending
      # exit codes over a Queue as workers complete.
      q = queue.Queue()
  
      def wait_for_worker(proc):
          worker_exit = proc.wait()
          q.put((proc, worker_exit))
  
      threads = [
          threading.Thread(target=wait_for_worker, args=(proc,))
          for proc in procs
      ]
  
      for t in threads:
          t.start()
  
      first_failed_exit = 0
      for i in range(slots_per_node):
          proc, worker_exit = q.get()
          procs.remove(proc)
          if worker_exit != 0 and first_failed_exit == 0:
              # When the first worker crashes, preempt the others.
              first_failed_exit = worker_exit
              for proc in procs:
                  proc.kill()
  
      for t in threads:
          t.join()
  
      return first_failed_exit
  
file_path: determined/examples/tutorials/core_api/4_distributed.py
imports: import logging
  import pathlib
  import queue
  import subprocess
  import sys
  import threading
  import time
  import determined
label_desc: Use image node_id 4 for calling a global function with example usage: launcher_main(slots_per_node, num_nodes, cross_rank) and returns: first_failed_exit