class_docstr: A result object obtained when fitting a model.
class_docstr_tok: ["A","result","object","obtained","when","fitting","a","model","."]
docstr: Evaluates the given predictors on the specified dataset, logs the
  resulting metrics and stores the forecasts in the given directory. The
  forecasts of predictor `i` are stored in the subdirectory `model_<i>`.
  
  Args:
      dataset_config: The configuration of the dataset to make predictions for.
      dataset_split: The split of the dataset for which to make predictions for.
      directory: The directory where to store forecasts.
      validation: Whether to log only the mean weighted quantile loss with a `val_` prefix.
docstr_tok: ["Evaluates","the","given","predictors","on","the","specified","dataset",",","logs","the","resulting","metrics","and","stores","the","forecasts","in","the","given","directory",".","The","forecasts","of","predictor","`","i","`","are","stored","in","the","subdirectory","`","model_","<","i",">","`",".","Args",":","dataset_config",":","The","configuration","of","the","dataset","to","make","predictions","for",".","dataset_split",":","The","split","of","the","dataset","for","which","to","make","predictions","for",".","directory",":","The","directory","where","to","store","forecasts",".","validation",":","Whether","to","log","only","the","mean","weighted","quantile","loss","with","a","`","val_","`","prefix","."]
code: def evaluate_predictors(
      self,
      dataset_config: DatasetConfig,
      dataset_split: DatasetSplit,
      directory: Path,
      validation: bool = False,
  ) -> None:
      """
      Evaluates the given predictors on the specified dataset, logs the
      resulting metrics and stores the forecasts in the given directory. The
      forecasts of predictor `i` are stored in the subdirectory `model_<i>`.
  
      Args:
          dataset_config: The configuration of the dataset to make predictions for.
          dataset_split: The split of the dataset for which to make predictions for.
          directory: The directory where to store forecasts.
          validation: Whether to log only the mean weighted quantile loss with a `val_` prefix.
      """
      length = self.config.max_time_series_length(  # pylint: disable=assignment-from-none
          dataset_config
      )
      if length is not None:
          # pylint: disable=invalid-unary-operand-type
          dataset = [
              {**item, "target": item["target"][-length:]}
              for item in dataset_split.gluonts()
          ]
      else:
          dataset = dataset_split.gluonts()
  
      for i, predictor in enumerate(self.predictors):
          logging.info(
              "Evaluating predictor %d/%d...",
              i + 1,
              len(self.predictors),
          )
  
          # Evaluate
          with _suppress_stdout_stderr():  # need to do this to suppress Prophet outputs
              prediction, latency = generate_forecasts(
                  predictor,
                  dataset,
                  num_samples=self.config.prediction_samples,
                  parallelize=self.config.prefers_parallel_predictions,
              )
          evaluation = evaluate_forecasts(
              prediction, dataset_split.evaluation()
          )
  
          # Log the summary and store the predictions
          eval_dir = directory / f"model_{i}"
          eval_dir.mkdir(parents=True, exist_ok=True)
  
          prediction.save(eval_dir)
          if not validation:
              for metric, value in evaluation.summary.items():
                  log_metric(metric, value)
              log_metric("latency", latency)
          else:
              log_metric(
                  "val_ncprs",
                  evaluation.summary["ncrps"],
              )
  
file_path: gluonts/src/gluonts/nursery/tsbench/src/tsbench/evaluations/training/evaluate.py
imports: import logging
  import os
  from dataclasses import dataclass
  from pathlib import Path
  from typing import List
  from gluonts.env import env
  from gluonts.model.predictor import Predictor
  from tsbench.config import DatasetConfig, ModelConfig
  from tsbench.config.dataset import DatasetSplit
  from tsbench.forecasts import evaluate_forecasts, generate_forecasts
  from .logging import log_metric
label_desc: Use image node_id 1 for calling the FitResult obj's underlying member method code with example usage: obj.evaluate_predictors(dataset_config, dataset_split, directory, validation) without return types