class_docstr: Encapsulates running threads and processes of a Launchpad Program.
class_docstr_tok: ["Encapsulates","running","threads","and","processes","of","a","Launchpad","Program","."]
docstr: Checks status of running workers, terminate runtime in case of errors.
  
  This REQUIRES holding self._mutex.
docstr_tok: ["Checks","status","of","running","workers",",","terminate","runtime","in","case","of","errors",".","This","REQUIRES","holding","self._mutex","."]
code: def _check_workers(self):
      """Checks status of running workers, terminate runtime in case of errors.
  
      This REQUIRES holding self._mutex.
      """
      has_workers = False
      for label in self._active_workers:
          still_active = []
          for worker in self._active_workers[label]:
              active = True
              if isinstance(worker, ThreadWorker):
                  if not worker.thread.is_alive():
                      worker.thread.join()
                      if not self._stop_counter:
                          try:
                              worker.future.result()
                          except BaseException as e:
                              if (
                                  not self._first_failure
                                  and not self._stop_counter
                              ):
                                  self._first_failure = e
                                  _log_and_print_with_color(
                                      logging.ERROR,
                                      "Node %s crashed: %s",
                                      worker,
                                      traceback.format_exc(),
                                  )
                      active = False
              elif isinstance(worker, subprocess.Popen):
                  try:
                      res = worker.wait(0)
                      active = False
                      if (
                          res
                          and not self._first_failure
                          and not self._stop_counter
                      ):
                          self._first_failure = RuntimeError(
                              f"One of the workers exited with code {res}."
                          )
                  except subprocess.TimeoutExpired:
                      pass
              else:
                  try:
                      # We can't obtain return code of external process, so clean
                      # termination is assumed.
                      res = worker.wait(0)
                      active = False
                  except psutil.TimeoutExpired:
                      pass
              if active:
                  has_workers = True
                  still_active.append(worker)
          self._active_workers[label] = still_active
      if has_workers and self._first_failure and not self._stop_counter:
          self._stop()
      elif not has_workers:
          self._disable_alarm()
  
imports: import atexit
  import collections
  from concurrent import futures
  import os
  import signal
  import subprocess
  import sys
  import threading
  import time
  import traceback
  from typing import Optional, Sequence, Text
  from absl import flags
  from absl import logging
  from absl.testing import absltest
  from launchpad import flags
  import psutil
  import termcolor
label_desc: Use image node_id 21 for calling the WorkerManager obj's underlying member method code with example usage: obj._check_workers() without return types