class_docstr: The ensemble analyzer allows for evaluating the performance of ensembles
  across datasets.
  
  The analysis is run in parallel and should, thus, not be used in a Jupyter
  notebook. Instead, consider using the `tsbench` CLI.
docstr: Estimates the performance of a list of models on a particular dataset.
  For this, actually trained models are sampled for each configuration.
  
  Args:
      models: The list of models to evaluate.
      dataset: The dataset to evaluate on.
      member_performances: The (predicted) performances of the provided models. Used to weigh
          the ensemble members. If not provided, uses the true performances.
      num_samples: The number of samples for estimating the performance.
  
  Returns:
      The expected performance of the ensemble.
docstr_tok: ["Estimates","the","performance","of","a","list","of","models","on","a","particular","dataset",".","For","this",",","actually","trained","models","are","sampled","for","each","configuration",".","Args",":","models",":","The","list","of","models","to","evaluate",".","dataset",":","The","dataset","to","evaluate","on",".","member_performances",":","The","(","predicted",")","performances","of","the","provided","models",".","Used","to","weigh","the","ensemble","members",".","If","not","provided",",","uses","the","true","performances",".","num_samples",":","The","number","of","samples","for","estimating","the","performance",".","Returns",":","The","expected","performance","of","the","ensemble","."]
code: def get_ensemble_performance(
      self,
      models: List[ModelConfig],
      dataset: DatasetConfig,
      member_performances: Optional[List[Performance]] = None,
      num_samples: int = 10,
  ) -> Performance:
      """
      Estimates the performance of a list of models on a particular dataset.
      For this, actually trained models are sampled for each configuration.
  
      Args:
          models: The list of models to evaluate.
          dataset: The dataset to evaluate on.
          member_performances: The (predicted) performances of the provided models. Used to weigh
              the ensemble members. If not provided, uses the true performances.
          num_samples: The number of samples for estimating the performance.
  
      Returns:
          The expected performance of the ensemble.
      """
      if member_performances is None:
          member_performances = [
              self.tracker.get_performance(Config(m, dataset))
              for m in models
          ]
  
      # First, we need to get the forecasts for all models
      forecasts = [
          self.tracker.get_forecasts(Config(m, dataset)) for m in models
      ]
  
      # Then, we want to construct min(#available_choices, 10) different ensembles by randomly
      # choosing models from the configurations without replacement.
      max_choices = np.prod([len(f) for f in forecasts])
      num_choices = min(max_choices, num_samples)
      pool = itertools.product(*[range(len(f)) for f in forecasts])
      model_combinations = random.sample(list(pool), k=num_choices)
  
      # Then, we evaluate each of the ensembles
      evaluations = []
      for combination in model_combinations:
          ensembled_forecast = ensemble_forecasts(
              [f[i] for i, f in zip(combination, forecasts)],
              self.ensemble_weighting,
              [p.ncrps.mean for p in member_performances],
          )
          evaluation = evaluate_forecasts(
              ensembled_forecast, dataset.data.test().evaluation()
          )
          evaluations.append(evaluation)
  
      # And eventually, we build the resulting performance object
      performance = Evaluation.performance(evaluations)
      performance.num_gradient_updates = self._combine_metrics(
          member_performances, lambda p: p.num_gradient_updates
      )
      performance.num_model_parameters = self._combine_metrics(
          member_performances, lambda p: p.num_model_parameters
      )
      performance.latency = self._combine_metrics(
          member_performances, lambda p: p.latency
      )
      performance.training_time = self._combine_metrics(
          member_performances, lambda p: p.training_time
      )
      return performance
  
imports: import itertools
  import math
  import random
  from pathlib import Path
  from typing import Callable, Dict, List, Optional, Tuple, Type
  import numpy
  import pandas
  from tqdm.auto import tqdm
  from tsbench.analysis.utils import loocv_split, num_fitting_processes, run_parallel
  from tsbench.config import Config, DATASET_REGISTRY, DatasetConfig, ModelConfig, TrainConfig
  from tsbench.config.model.models import SeasonalNaiveModelConfig
  from tsbench.constants import DEFAULT_DATA_PATH
  from tsbench.evaluations.metrics.performance import Metric, Performance
  from tsbench.evaluations.tracking import ModelTracker
  from tsbench.forecasts import ensemble_forecasts, EnsembleWeighting, evaluate_forecasts, Evaluation
label_desc: Use image node_id 5 for calling the EnsembleAnalyzer obj's underlying member method code with example usage: obj.get_ensemble_performance(models, dataset, member_performances, num_samples) and returns: performance