class_docstr: Builds ensembles out of runs that exist in the Backend
  
  This is used by EnsembleBuilderManager and created in a dask-client
  every time a run finishes and there is currently no EnsembleBuilder active.
docstr: Get a list of candidates from `runs`, garuanteeing at least one
  
  Applies a set of reductions in order of parameters to reach a set of final
  candidates.
  
  Expects at least one `dummies` run.
  
  Parameters
  ----------
  runs : Sequence[Run]
      The runs to evaluate candidates from.
  
  dummies: Run | Sequence[Run]
      The dummy run to base from
  
  better_than_dummy: bool = False
      Whether the run must be better than the best dummy run to be a candidate.
      In the case where there are no candidates left, the dummies will then be
      used.
  
  nbest : int | float | None
      The nbest models to select. If `int`, acts as an absolute limit.
      If `float`, acts as a percentage of available candidates.
  
  metrics : Sequence[Scorer] | None = None
      The metrics to consider, defaults to the builders
  
  Returns
  -------
  (candidates: list[Run], discarded: set[Run])
      A tuple of runs that are candidates and also those that didn't make it
docstr_tok: ["Get","a","list","of","candidates","from","`","runs","`",",","garuanteeing","at","least","one","Applies","a","set","of","reductions","in","order","of","parameters","to","reach","a","set","of","final","candidates",".","Expects","at","least","one","`","dummies","`","run",".","Parameters","--","--","--","--","--","runs",":","Sequence","[","Run","]","The","runs","to","evaluate","candidates","from",".","dummies",":","Run","|","Sequence","[","Run","]","The","dummy","run","to","base","from","better_than_dummy",":","bool","=","False","Whether","the","run","must","be","better","than","the","best","dummy","run","to","be","a","candidate",".","In","the","case","where","there","are","no","candidates","left",",","the","dummies","will","then","be","used",".","nbest",":","int","|","float","|","None","The","nbest","models","to","select",".","If","`","int","`",",","acts","as","an","absolute","limit",".","If","`","float","`",",","acts","as","a","percentage","of","available","candidates",".","metrics",":","Sequence","[","Scorer","]","|","None","=","None","The","metrics","to","consider",",","defaults","to","the","builders","Returns","--","--","--","-","(","candidates",":","list","[","Run","]",",","discarded",":","set","[","Run","]",")","A","tuple","of","runs","that","are","candidates","and","also","those","that","did","n't","make","it"]
code: def candidate_selection(
      self,
      runs: Sequence[Run],
      dummies: Run | list[Run],
      *,
      better_than_dummy: bool = False,
      nbest: int | float | None = None,
      metrics: Sequence[Scorer] | None = None,
  ) -> tuple[list[Run], set[Run]]:
      """Get a list of candidates from `runs`, garuanteeing at least one
  
      Applies a set of reductions in order of parameters to reach a set of final
      candidates.
  
      Expects at least one `dummies` run.
  
      Parameters
      ----------
      runs : Sequence[Run]
          The runs to evaluate candidates from.
  
      dummies: Run | Sequence[Run]
          The dummy run to base from
  
      better_than_dummy: bool = False
          Whether the run must be better than the best dummy run to be a candidate.
          In the case where there are no candidates left, the dummies will then be
          used.
  
      nbest : int | float | None
          The nbest models to select. If `int`, acts as an absolute limit.
          If `float`, acts as a percentage of available candidates.
  
      metrics : Sequence[Scorer] | None = None
          The metrics to consider, defaults to the builders
  
      Returns
      -------
      (candidates: list[Run], discarded: set[Run])
          A tuple of runs that are candidates and also those that didn't make it
      """
      metrics = metrics if metrics else self.metrics
  
      if isinstance(dummies, Run):
          dummies = [dummies]
  
      assert (
          len(dummies) > 0 and len(runs) > 0
      ), "At least 1 real run and dummy run"
  
      all_discarded: set[Run] = set()
  
      # We filter out all runs that don't have any predictions for the ensemble
      candidates, discarded = split(
          runs,
          by=lambda run: run.pred_path("ensemble").exists(),
      )
      all_discarded.update(discarded)
  
      if len(candidates) == 0:
          self.logger.debug(
              "No runs with predictions on ensemble set, using dummies"
          )
          return dummies, all_discarded
  
      for run in discarded:
          self.logger.warning(f"Have no ensemble predictions for {run}")
  
      # Get all the ones that have all metrics and tangible losses
      has_metrics = lambda r: all(m.name in r.losses for m in metrics)
      tangible_losses = lambda r: all(
          loss < np.inf for loss in r.losses.values()
      )
  
      candidates, discarded = split(
          candidates,
          by=lambda r: has_metrics(r) and tangible_losses(r),
      )
      all_discarded.update(discarded)
  
      if len(candidates) == 0:
          self.logger.debug("No runs with a usable loss, using dummies")
          return dummies, all_discarded
  
      if better_than_dummy:
          # In the single objective case, they must explicitly be better than dummy
          if len(metrics) == 1:
              metric = metrics[0]
  
              best_dummy = min(
                  dummies, key=lambda d: d.losses[metric.name]
              )
              self.logger.debug(
                  f"Using dummy {best_dummy} to filter candidates"
              )
  
              candidates, discarded = split(
                  candidates,
                  by=lambda r: r.losses[metric.name]
                  < best_dummy.losses[metric.name],
              )
  
          # In the multiobjective case, they must be better than at least one of the
          # best dummies, where there is a best dummy for each metric
          else:
              best_dummies = {
                  metric.name: min(
                      dummies, key=lambda d: d.losses[metric.name]
                  )
                  for metric in metrics
              }
              self.logger.debug(
                  f"Using dummies {best_dummies} to filter candidates"
              )
  
              candidates, discarded = split(
                  candidates,
                  by=lambda r: any(
                      r.losses[metric_name]
                      < best_dummy.losses[metric_name]
                      for metric_name, best_dummy in best_dummies.items()
                  ),
              )
  
          all_discarded.update(discarded)
  
          # If there are no real candidates left, use the dummies
          if len(candidates) == 0:
              self.logger.warning(
                  "No models better than random - using Dummy losses!"
                  f"\n\tModels besides current dummy model: {len(candidates)}"
                  f"\n\tDummy models: {len(dummies)}",
              )
              return dummies, all_discarded
  
      if nbest is not None:
          # We order the runs according to a roundrobin of the metrics
          # i.e. the 1st run will be best in metric[0],
          #      the 2nd run will be best in metric[1],
          #      the 3rd run will be second best in metric[0],
          #      the 4th run will be second best in metric[1],
          #       ... whoops, what would be the 5th run, third best in metric[0] was
          #           also the same as the 4th run. This was skipped
          #      the 5th run will be third best in metric[1]  **as we skipped above**
          #
          # With removing duplicates, this means if a run was already contained, it
          # will skip to the next member in the roundrobin fashion.
          sorted_candidates = [
              sorted(
                  candidates,
                  key=lambda r: (
                      r.losses.get(m.name, np.inf),
                      r.num_run,
                  ),
              )
              for m in metrics
          ]
          candidates = list(
              roundrobin(*sorted_candidates, duplicates=False)
          )
  
          # Determine how many to keep, always keeping one
          if isinstance(nbest, float):
              nkeep = int(len(candidates) * nbest)
          else:
              nkeep = nbest
  
          candidates, discarded = cut(candidates, nkeep)
          self.logger.info(
              f"Discarding {len(discarded)}/{len(candidates)} runs"
          )
  
          # Always preserve at least one, the best
          if len(candidates) == 0:
              candidates, discared = cut(discarded, 1)
              self.logger.warning(
                  "nbest too aggresive, using single best"
              )
  
          all_discarded.update(discarded)
  
      return candidates, all_discarded
  
imports: from __future__ import annotations
  from typing import Any, Iterable, Mapping, Sequence, Type, cast
  import logging.handlers
  import multiprocessing
  import numbers
  import os
  import pickle
  import time
  import traceback
  from itertools import accumulate
  from pathlib import Path
  import numpy
  import pandas
  import pynisher
  from sklearn.utils.validation import check_random_state
  from autosklearn.automl_common.common.utils.backend import Backend
  from autosklearn.data.validation import SUPPORTED_FEAT_TYPES
  from autosklearn.data.xy_data_manager import XYDataManager
  from autosklearn.ensemble_building.run import Run, RunID
  from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble
  from autosklearn.ensembles.ensemble_selection import EnsembleSelection
  from autosklearn.metrics import Scorer, calculate_loss, calculate_scores
  from autosklearn.util.disk import rmtree
  from autosklearn.util.functional import cut, findwhere, roundrobin, split
  from autosklearn.util.logging_ import get_named_client_logger
  from autosklearn.util.parallel import preload_modules
label_desc: Use image node_id 10 for calling the EnsembleBuilder obj's underlying member method code with example usage: obj.candidate_selection(runs, dummies) and returns: candidates,  all_discarded, dummies,  all_discarded, dummies,  all_discarded, dummies,  all_discarded