code: def test_dynamic_generator_reconstruction_nondeterministic(
      ray_start_cluster, too_many_returns, num_returns_type
  ):
      config = {
          "health_check_failure_threshold": 10,
          "health_check_period_ms": 100,
          "health_check_initial_delay_ms": 0,
          "max_direct_call_object_size": 100,
          "task_retry_delay_ms": 100,
          "object_timeout_milliseconds": 200,
          "fetch_warn_timeout_milliseconds": 1000,
      }
      cluster = ray_start_cluster
      # Head node with no resources.
      cluster.add_node(
          num_cpus=1,
          _system_config=config,
          enable_object_reconstruction=True,
          resources={"head": 1},
      )
      ray.init(address=cluster.address)
      # Node to place the initial object.
      node_to_kill = cluster.add_node(
          num_cpus=1, object_store_memory=10**8
      )
      cluster.wait_for_nodes()
  
      @ray.remote(num_cpus=1, resources={"head": 1})
      class FailureSignal:
          def __init__(self):
              return
  
          def ping(self):
              return
  
      @ray.remote(num_returns=num_returns_type)
      def dynamic_generator(failure_signal):
          num_returns = 10
          try:
              ray.get(failure_signal.ping.remote())
          except ray.exceptions.RayActorError:
              if too_many_returns:
                  num_returns += 1
              else:
                  num_returns -= 1
          for i in range(num_returns):
              yield np.ones(1_000_000, dtype=np.int8) * i
  
      @ray.remote
      def fetch(x):
          return
  
      failure_signal = FailureSignal.remote()
      gen = ray.get(dynamic_generator.remote(failure_signal))
      cluster.remove_node(node_to_kill, allow_graceful=False)
      ray.kill(failure_signal)
      refs = list(gen)
      if too_many_returns:
          for i, ref in enumerate(refs):
              assert np.array_equal(
                  np.ones(1_000_000, dtype=np.int8) * i, ray.get(ref)
              )
              del ref
      else:
          if num_returns_type == "dynamic":
              # If dynamic is specified, when the num_returns
              # is different, all previous refs are failed.
              with pytest.raises(ray.exceptions.RayTaskError):
                  for ref in refs:
                      ray.get(ref)
                      del ref
          else:
              # Otherwise, we can reconstruct the refs again.
              # We allow it because the refs could have already obtained
              # by the generator.
              for i, ref in enumerate(refs):
                  assert np.array_equal(
                      np.ones(1_000_000, dtype=np.int8) * i,
                      ray.get(ref),
                  )
                  del ref
      # TODO(swang): If the re-executed task returns a different number of
      # objects, we should throw an error for every return value.
      # for ref in refs:
      #     with pytest.raises(ray.exceptions.RayTaskError):
      #         ray.get(ref)
      del gen
      del refs
      if num_returns_type is None:
          # TODO(sang): For some reasons, it fails when "dynamic"
          # is used. We don't fix the issue because we will
          # remove this flag soon anyway.
          assert_no_leak()
  
imports: import pytest
  import numpy
  import sys
  import time
  import gc
  from unittest.mock import Mock
  import ray
  from ray.util.client.ray_client_helpers import ray_start_client_server_for_address
  from ray._private.client_mode_hook import enable_client_mode
  from ray.tests.conftest import call_ray_start_context
label_desc: Use image node_id 9 for calling a global function with example usage: test_dynamic_generator_reconstruction_nondeterministic(ray_start_cluster, too_many_returns, num_returns_type) without return types