code: def test_horovod_chief(
      mock_time: mock.MagicMock,
      mock_check_sshd: mock.MagicMock,
      mock_cluster_info: mock.MagicMock,
      mock_popen: mock.MagicMock,
      nslots: int,
      nnodes: int,
      autohorovod: bool,
  ) -> None:
      info = test_util.make_mock_cluster_info(
          ["0.0.0.{i}" for i in range(nnodes)], 0, num_slots=nslots
      )
      experiment_config = info.trial._config
      mock_cluster_info.return_value = info
      mock_start_time = time.time()
      mock_time.return_value = mock_start_time
      hvd_args = ["ds1", "ds2"]
      script = ["s1", "s2"]
  
      pid_server_cmd = launch.horovod.create_hvd_pid_server_cmd(
          info.allocation_id, len(info.slot_ids)
      )
  
      hvd_cmd = horovod.create_run_command(
          host_slot_counts=info.container_slot_counts,
          ip_addresses=info.container_addrs,
          inter_node_network_interface=info.trial._inter_node_network_interface,
          optimizations=experiment_config["optimizations"],
          debug=False,
          optional_args=hvd_args,
      )
  
      worker_wrapper_cmd = launch.horovod.create_worker_wrapper_cmd(
          info.allocation_id
      )
  
      launch_cmd = (
          pid_server_cmd + hvd_cmd + worker_wrapper_cmd + script
      )
  
      mock_proc = mock.MagicMock()
      mock_proc.wait.return_value = 99
  
      mock_popen.return_value = mock_proc
  
      with test_util.set_resources_id_env_var():
          assert (
              launch.horovod.main(hvd_args, script, autohorovod) == 99
          )
  
      if autohorovod and nnodes == 1 and nslots == 1:
          # Single-slot --autohorovod: we should have just called the script directly.
          mock_popen.assert_has_calls([mock.call(script)])
          mock_check_sshd.assert_not_called()
      else:
          # Multi-slot or non --autohorovod: expect a full horovodrun command.
          mock_cluster_info.assert_called_once()
          assert os.environ["DET_CHIEF_IP"] == info.container_addrs[0]
          assert os.environ["USE_HOROVOD"] == "1"
  
          mock_popen.assert_has_calls([mock.call(launch_cmd)])
  
          assert mock_check_sshd.call_count == len(
              info.container_addrs[1:]
          )
          mock_check_sshd.assert_has_calls(
              [
                  mock.call(
                      addr,
                      mock_start_time + 20,
                      constants.DTRAIN_SSH_PORT,
                  )
                  for addr in info.container_addrs[1:]
              ]
          )
  
          mock_proc.wait.assert_called_once()
  
file_path: determined/harness/tests/launch/test_horovod.py
imports: import os
  import time
  from unittest import mock
  import pytest
  import determined.launch.horovod
  from determined import constants, horovod, launch
  from determined.common.api import certs
  from tests.launch import test_util
label_desc: Use image node_id 2 for calling a global function with example usage: test_horovod_chief(mock_time, mock_check_sshd, mock_cluster_info, mock_popen, nslots, nnodes, autohorovod) without return types