rllib/agents/trainer.py
def try_recover_from_step_attempt(self) -> None:
        """Try to identify and remove any unhealthy workers.

        This method is called after an unexpected remote error is encountered
        from a worker during the call to `self.step_attempt()` (within
        `self.step()`). It issues check requests to all current workers and
        removes any that respond with error. If no healthy workers remain,
        an error is raised. Otherwise, tries to re-build the execution plan
        with the remaining (healthy) workers.
        """
        # Try to get our "main" WorkerSet (used for training sample collection).
        workers = getattr(self, "workers", None)
        if not isinstance(workers, WorkerSet):
            return

        # Search for failed workers and try to recover (restart) them.
        if self.config["recreate_failed_workers"] is True:
            workers.recreate_failed_workers()
        elif self.config["ignore_worker_failures"] is True:
            workers.remove_failed_workers()

        if not self.config.get("_disable_execution_plan_api") and callable(
            self.execution_plan
        ):
            logger.warning("Recreating execution plan after failure")
            self.train_exec_impl = self.execution_plan(
                workers, self.config, **self._kwargs_for_execution_plan()
            )
Try to identify and remove any unhealthy workers.

        This method is called after an unexpected remote error is encountered
        from a worker during the call to `self.step_attempt()` (within
        `self.step()`). It issues check requests to all current workers and
        removes any that respond with error. If no healthy workers remain,
        an error is raised. Otherwise, tries to re-build the execution plan
        with the remaining (healthy) workers.
        
Try to identify and remove any unhealthy workers.
This method is called after an unexpected remote error is encountered
from a worker during the call to `self.step_attempt()` (within
`self.step()`). It issues check requests to all current workers and
removes any that respond with error. If no healthy workers remain,
an error is raised. Otherwise, tries to re-build the execution plan
with the remaining (healthy) workers.
Try to identify and remove any unhealthy workers . This method is called after an unexpected remote error is encountered from a worker during the call to ` self . step_attempt () ` ( within ` self . step () ` ) . It issues check requests to all current workers and removes any that respond with error . If no healthy workers remain an error is raised . Otherwise tries to re - build the execution plan with the remaining ( healthy ) workers .
def try_recover_from_step_attempt(self) -> None:
        workers = getattr(self, "workers", None)
        if not isinstance(workers, WorkerSet):
            return
        if self.config["recreate_failed_workers"] is True:
            workers.recreate_failed_workers()
        elif self.config["ignore_worker_failures"] is True:
            workers.remove_failed_workers()
        if not self.config.get("_disable_execution_plan_api") and callable(
            self.execution_plan
        ):
            logger.warning("Recreating execution plan after failure")
            self.train_exec_impl = self.execution_plan(
                workers, self.config, **self._kwargs_for_execution_plan()
            )
def try_recover_from_step_attempt ( self ) -> None : workers = getattr ( self , "workers" , None ) if not isinstance ( workers , WorkerSet ) : return if self . config [ "recreate_failed_workers" ] is True : workers . recreate_failed_workers ( ) elif self . config [ "ignore_worker_failures" ] is True : workers . remove_failed_workers ( ) if not self . config . get ( "_disable_execution_plan_api" ) and callable ( self . execution_plan ) : logger . warning ( "Recreating execution plan after failure" ) self . train_exec_impl = self . execution_plan ( workers , self . config , ** self . _kwargs_for_execution_plan ( ) )
Try to identify and remove any unhealthy workers.
Try to identify and remove any unhealthy workers .
"""Try to identify and remove any unhealthy workers.

        This method is called after an unexpected remote error is encountered
        from a worker during the call to `self.step_attempt()` (within
        `self.step()`). It issues check requests to all current workers and
        removes any that respond with error. If no healthy workers remain,
        an error is raised. Otherwise, tries to re-build the execution plan
        with the remaining (healthy) workers.
        """ # Try to get our "main" WorkerSet (used for training sample collection). # Search for failed workers and try to recover (restart) them.
self