code: def test_on_workers_recreated_callback(self):
      tune.register_env("env", lambda cfg: CartPoleCrashing(cfg))
  
      config = (
          APPOConfig()
          .environment("env")
          .callbacks(OnWorkersRecreatedCallbacks)
          .rollouts(num_rollout_workers=3)
          .fault_tolerance(recreate_failed_workers=True)
      )
  
      for _ in framework_iterator(config, frameworks=("tf2", "torch")):
          algo = config.build()
          original_worker_ids = algo.workers.healthy_worker_ids()
          for id_ in original_worker_ids:
              self.assertTrue(
                  algo._counters[f"worker_{id_}_recreated"] == 0
              )
          self.assertTrue(
              algo._counters["total_num_workers_recreated"] == 0
          )
  
          # After building the algorithm, we should have 2 healthy (remote) workers.
          self.assertTrue(len(original_worker_ids) == 3)
  
          # Train a bit (and have the envs/workers crash a couple of times).
          for _ in range(5):
              print(algo.train())
  
          # After training, the `on_workers_recreated` callback should have captured
          # the exact worker IDs recreated (the exact number of times) as the actor
          # manager itself. This confirms that the callback is triggered correctly,
          # always.
          new_worker_ids = algo.workers.healthy_worker_ids()
          self.assertTrue(len(new_worker_ids) == 3)
          for id_ in new_worker_ids:
              # num_restored = algo.workers.restored_actors_history[id_]
              self.assertTrue(
                  algo._counters[f"worker_{id_}_recreated"] > 1
              )
          algo.stop()
  
imports: import tempfile
  import unittest
  import ray
  from ray.rllib.algorithms.appo import APPOConfig
  from ray.rllib.algorithms.callbacks import DefaultCallbacks
  from ray.rllib.algorithms.ppo import PPOConfig
  from ray.rllib.examples.env.cartpole_crashing import CartPoleCrashing
  from ray.rllib.utils.test_utils import framework_iterator
  from ray import tune
label_desc: Use image node_id 3 for calling the TestCallbacks obj's underlying member method code with example usage: obj.test_on_workers_recreated_callback() without return types