code: def main(script: List[str]) -> int:
      info = det.get_cluster_info()
      assert info is not None, "must be run on-cluster"
      assert (
          info.task_type == "TRIAL"
      ), f'must be run with task_type="TRIAL", not "{info.task_type}"'
      experiment_config = det.ExperimentConfig(info.trial._config)
      determined.common.set_logger(experiment_config.debug_enabled())
  
      multi_machine = len(info.container_addrs) > 1
      check_deepspeed_version(multi_machine)
  
      # Hack: get the resources id from the environment.
      resources_id = os.environ.get("DET_RESOURCES_ID")
      assert (
          resources_id is not None
      ), "Unable to run with DET_RESOURCES_ID unset"
  
      # TODO: refactor websocket and profiling to to not use the cli_cert.
      cert = certs.default_load(info.master_url)
      certs.cli_cert = cert
  
      # The launch layer should provide the chief_ip to the training code, so that the training code
      # can function with a different launch layer in a different environment.  Inside Determined, the
      # easiest way to get the chief_ip is with container_addrs.
      chief_ip = info.container_addrs[0]
  
      # Chief IP is set as an environment variable to support nested launch layers
      os.environ["DET_CHIEF_IP"] = chief_ip
  
      # If the NCCL_SOCKET_IFNAME environment variable wasn't explicitly set by
      # the user in the experiment's YAML file, then set it to the distributed
      # network interface, if the value of "dtrain_network_interface" under
      # "task_container_defaults" has been set in the "master.yaml".
      if is_using_cuda() and not is_nccl_socket_ifname_env_var_set():
          dtrain_network_interface = os.environ.get(
              "DET_INTER_NODE_NETWORK_INTERFACE", None
          )
  
          if (
              dtrain_network_interface is not None
              and len(dtrain_network_interface) > 0
          ):
              os.environ[
                  "NCCL_SOCKET_IFNAME"
              ] = dtrain_network_interface
  
      # In some downstream training code, the hostfile is expected on all nodes so we create the
      # hostfile on all nodes here before opening the non-chief node subprocess.  Since we
      # use the allocation id to create the hostfile path, we register that path to an environment
      # variable for access downstream.  A filelock is used to ensure we do not have clashing writes
      # if the hostfile_path is on a shared filesystem.
      hostfile_path = get_hostfile_path(
          multi_machine, info.allocation_id
      )
  
      lock = str(hostfile_path) + ".lock"
      with filelock.FileLock(lock):
          master_address = create_hostlist_file(
              hostfile_path=hostfile_path,
              host_slot_counts=info.container_slot_counts,
              ip_addresses=info.container_addrs,
          )
  
      # All ranks will need to run sshd.
      run_sshd_command = create_sshd_cmd()
  
      if info.container_rank > 0:
          # Non-chief machines just run sshd.
  
          # Mark sshd containers as daemon containers that the master should kill when all non-daemon
          # containers (deepspeed launcher, in this case) have exited.
          api.post(
              info.master_url,
              path=f"/api/v1/allocations/{info.allocation_id}/resources/{resources_id}/daemon",
              cert=cert,
          )
  
          # Wrap it in a pid_server to ensure that we can't hang if a worker fails.
          # This is useful for deepspeed which does not have good error handling for remote processes
          # spun up by pdsh.
          pid_server_cmd = create_pid_server_cmd(
              info.allocation_id, len(info.slot_ids)
          )
  
          logger.debug(
              f"Non-chief [{info.container_rank}] training process launch "
              f"command: {run_sshd_command}."
          )
          p = subprocess.Popen(pid_server_cmd + run_sshd_command)
          with det.util.forward_signals(p):
              return p.wait()
  
      # We always need to set this variable to initialize the context correctly, even in the single
      # slot case.
      os.environ["USE_DEEPSPEED"] = "1"
  
      # The chief has several layers of wrapper processes:
      # - a top-level pid_server, which causes the whole container to exit if any local worker dies.
      # - deepspeed, which launches $slots_per_trial copies of the following layers:
      #     - a pid_client process to contact the local pid_server
      #     - wrap_rank, which redirects stdin/stdout to the local container
      #     - harness.py, which actually does the training for the worker
  
      pid_server_cmd = create_pid_server_cmd(
          info.allocation_id, len(info.slot_ids)
      )
  
      cmd = create_run_command(master_address, hostfile_path)
  
      pid_client_cmd = create_pid_client_cmd(info.allocation_id)
  
      log_redirect_cmd = create_log_redirect_cmd()
  
      harness_cmd = script
  
      logger.debug(
          f"chief worker calling deepspeed with args: {cmd[1:]} ..."
      )
  
      full_cmd = (
          pid_server_cmd
          + cmd
          + pid_client_cmd
          + log_redirect_cmd
          + harness_cmd
      )
  
      if not multi_machine:
          p = subprocess.Popen(full_cmd)
          with det.util.forward_signals(p):
              return p.wait()
  
      # Create the environment file that will be passed by deepspeed to individual ranks.
      create_deepspeed_env_file()
      # Set custom PDSH args:
      # * bypass strict host checking
      # * -p our custom port
      # * other args are default ssh args for pdsh
      os.environ["PDSH_SSH_ARGS"] = (
          "-o PasswordAuthentication=no -o StrictHostKeyChecking=no "
          f"-p {constants.DTRAIN_SSH_PORT} -2 -a -x %h"
      )
  
      # Chief worker also needs to run sshd when using pdsh and multi-machine training.
      sshd_process = subprocess.Popen(run_sshd_command)
  
      try:
          # Chief machine waits for every worker's sshd to be available.  All machines should be
          # close to in-step by now because all machines just finished synchronizing rendezvous
          # info.
          deadline = time.time() + 20
          for peer_addr in info.container_addrs:
              util.check_sshd(
                  peer_addr, deadline, constants.DTRAIN_SSH_PORT
              )
  
          p = subprocess.Popen(full_cmd)
          with det.util.forward_signals(p):
              return p.wait()
      finally:
          sshd_process.kill()
          sshd_process.wait()
  
imports: import argparse
  import logging
  import os
  import re
  import shlex
  import subprocess
  import sys
  import time
  from typing import Dict, List, Mapping, Optional
  import deepspeed
  import filelock
  from deepspeed.launcher.runner import DEEPSPEED_ENVIRONMENT_NAME
  from packaging import version
  import determined
  import determined.common
  from determined import constants, util
  from determined.common import api
  from determined.common.api import certs
label_desc: Use image node_id 13 for calling a global function with example usage: main(script) and returns: p, p, p