docstr: Fit, predict and compute the loss for cross-validation and
  holdout (both iterative and non-iterative)
docstr_tok: ["Fit",",","predict","and","compute","the","loss","for","cross-validation","and","holdout","(","both","iterative","and","non-iterative",")"]
code: def fit_predict_and_loss(self, iterative: bool = False) -> None:
      """Fit, predict and compute the loss for cross-validation and
      holdout (both iterative and non-iterative)
      """
      # Define beforehand for mypy
      additional_run_info: Optional[TYPE_ADDITIONAL_INFO] = None
  
      if iterative:
          if self.num_cv_folds == 1:
              for train_split, test_split in self.splitter.split(
                  self.X_train,
                  self.Y_train,
                  groups=self.resampling_strategy_args.get("groups"),
              ):
                  self.X_optimization = select(self.X_train, test_split)
                  self.Y_optimization = select(self.Y_train, test_split)
                  self.Y_actual_train = select(
                      self.Y_train, train_split
                  )
                  self._partial_fit_and_predict_iterative(
                      0,
                      train_indices=train_split,
                      test_indices=test_split,
                      add_model_to_self=True,
                  )
          else:
              # Test if the model allows for an iterative fit, if not,
              # call this method again without the iterative argument
              model = self._get_model(self.feat_type)
              if not model.estimator_supports_iterative_fit():
                  self.fit_predict_and_loss(iterative=False)
                  return
  
              self.partial = False
  
              converged = [False] * self.num_cv_folds
  
              Y_train_pred = [None] * self.num_cv_folds
              Y_optimization_pred = [None] * self.num_cv_folds
              Y_test_pred = [None] * self.num_cv_folds
              train_splits = [None] * self.num_cv_folds
  
              self.models = [
                  self._get_model(self.feat_type)
                  for i in range(self.num_cv_folds)
              ]
              iterations = [1] * self.num_cv_folds
              total_n_iterations = [0] * self.num_cv_folds
              # model.estimator_supports_iterative_fit -> true
              # After the if above, we know estimator support iterative fit
              model_max_iter = [
                  cast(IterativeComponent, model).get_max_iter()
                  for model in self.models
              ]
  
              if (
                  self.budget_type in ["iterations", "mixed"]
                  and self.budget is None
              ):
                  raise ValueError(
                      f"When budget type is {self.budget_type} the budget "
                      "can not be None"
                  )
  
              if (
                  self.budget_type in ["iterations", "mixed"]
                  and cast(float, self.budget) > 0
              ):
                  max_n_iter_budget = int(
                      np.ceil(
                          cast(float, self.budget)
                          / 100
                          * model_max_iter[0]
                      )
                  )
                  max_iter = min(model_max_iter[0], max_n_iter_budget)
              else:
                  max_iter = model_max_iter[0]
  
              models_current_iters = [0] * self.num_cv_folds
  
              Xt_array = [None] * self.num_cv_folds
              fit_params_array = [
                  {}
              ] * self.num_cv_folds  # type: List[Dict[str, Any]]
  
              y = _get_y_array(self.Y_train, self.task_type)
  
              # stores train loss(es) of each fold.
              train_losses = [dict()] * self.num_cv_folds
              # used as weights when averaging train losses.
              train_fold_weights = [np.NaN] * self.num_cv_folds
              # stores opt (validation) loss of each fold.
              opt_losses = [np.NaN] * self.num_cv_folds
              # weights for opt_losses.
              opt_fold_weights = [np.NaN] * self.num_cv_folds
  
              while not all(converged):
                  splitter = self.get_splitter(self.datamanager)
  
                  for i, (train_indices, test_indices) in enumerate(
                      splitter.split(
                          self.X_train,
                          y,
                          groups=self.resampling_strategy_args.get(
                              "groups"
                          ),
                      )
                  ):
                      if converged[i]:
                          continue
  
                      model = self.models[i]
  
                      if iterations[i] == 1:
                          self.Y_train_targets[train_indices] = select(
                              self.Y_train, train_indices
                          )
                          self.X_targets[i] = select(
                              self.X_train, test_indices
                          )
                          self.Y_targets[i] = select(
                              self.Y_train, test_indices
                          )
  
                          # Note: Be careful moving these into variables, caused a
                          # headache when trying to debug why things were breaking
                          Xt, fit_params = model.fit_transformer(
                              select(self.X_train, train_indices),
                              select(self.Y_train, train_indices),
                          )
                          Xt_array[i] = Xt
                          fit_params_array[i] = fit_params
  
                      n_iter = (
                          int(2 ** iterations[i] / 2)
                          if iterations[i] > 1
                          else 2
                      )
                      total_n_iterations[i] = (
                          total_n_iterations[i] + n_iter
                      )
  
                      model.iterative_fit(
                          Xt_array[i],
                          select(self.Y_train, train_indices),
                          n_iter=n_iter,
                          **fit_params_array[i],
                      )
  
                      (train_pred, opt_pred, test_pred) = self._predict(
                          model,
                          train_indices=train_indices,
                          test_indices=test_indices,
                      )
  
                      Y_train_pred[i] = train_pred
                      Y_optimization_pred[i] = opt_pred
                      Y_test_pred[i] = test_pred
                      train_splits[i] = train_indices
  
                      train_loss = self._loss(
                          select(self.Y_train, train_indices),
                          train_pred,
                          X_data=Xt_array[i],
                      )
                      train_losses[i] = train_loss
                      # Number of training data points for this fold.
                      # Used for weighting the average.
                      train_fold_weights[i] = len(train_indices)
  
                      # Compute validation loss of this fold and store it.
                      optimization_loss = self._loss(
                          self.Y_targets[i],
                          opt_pred,
                          X_data=self.X_targets[i],
                      )
                      opt_losses[i] = optimization_loss
                      # number of optimization data points for this fold.
                      # Used for weighting the average.
                      opt_fold_weights[i] = len(test_indices)
  
                      models_current_iters[i] = model.get_current_iter()
  
                      if (
                          model.configuration_fully_fitted()
                          or models_current_iters[i] >= max_iter
                      ):
                          converged[i] = True
  
                      iterations[i] = iterations[i] + 1
  
                  # Compute weights of each fold based on the number of samples
                  # in each fold.
                  train_fold_weights_percentage = [
                      w / sum(train_fold_weights)
                      for w in train_fold_weights
                  ]
                  opt_fold_weights_percentage = [
                      w / sum(opt_fold_weights)
                      for w in opt_fold_weights
                  ]
  
                  train_loss = {
                      metric.name: np.average(
                          [
                              train_losses[i][str(metric)]
                              for i in range(self.num_cv_folds)
                          ],
                          weights=train_fold_weights_percentage,
                      )
                      for metric in self.metrics
                  }
  
                  # if all_scoring_function is true, return a dict of opt_loss.
                  # Otherwise, return a scalar.
                  opt_loss = {}
                  for metric in opt_losses[0].keys():
                      opt_loss[metric] = np.average(
                          [
                              opt_losses[i][metric]
                              for i in range(self.num_cv_folds)
                          ],
                          weights=opt_fold_weights_percentage,
                      )
  
                  X_targets = self.X_targets
                  Y_targets = self.Y_targets
                  Y_train_targets = self.Y_train_targets
  
                  Y_optimization_pred_concat = concat_data(
                      Y_optimization_pred,
                      num_cv_folds=self.num_cv_folds,
                  )
                  X_targets = concat_data(
                      X_targets, num_cv_folds=self.num_cv_folds
                  )
                  Y_targets = concat_data(
                      Y_targets, num_cv_folds=self.num_cv_folds
                  )
  
                  if self.X_test is not None:
                      Y_test_preds = np.array(
                          [
                              Y_test_pred[i]
                              for i in range(self.num_cv_folds)
                              if Y_test_pred[i] is not None
                          ]
                      )
                      # Average the predictions of several models
                      if len(Y_test_preds.shape) == 3:
                          Y_test_preds = np.nanmean(
                              Y_test_preds, axis=0
                          )
                  else:
                      Y_test_preds = None
  
                  self.X_optimization = X_targets
                  self.Y_optimization = Y_targets
                  self.Y_actual_train = Y_train_targets
  
                  self.model = self._get_model(self.feat_type)
                  status = StatusType.DONOTADVANCE
                  if any(
                      [
                          model_current_iter == max_iter
                          for model_current_iter in models_current_iters
                      ]
                  ):
                      status = StatusType.SUCCESS
                  self.finish_up(
                      loss=opt_loss,
                      train_loss=train_loss,
                      opt_pred=Y_optimization_pred_concat,
                      test_pred=Y_test_preds,
                      additional_run_info=additional_run_info,
                      file_output=True,
                      final_call=all(converged),
                      status=status,
                  )
  
      else:
          self.partial = False
  
          Y_train_pred = [None] * self.num_cv_folds
          Y_optimization_pred = [None] * self.num_cv_folds
          Y_test_pred = [None] * self.num_cv_folds
          train_splits = [None] * self.num_cv_folds
  
          y = _get_y_array(self.Y_train, self.task_type)
  
          train_losses = []  # stores train loss of each fold.
          train_fold_weights = (
              []
          )  # used as weights when averaging train losses.
          opt_losses = []  # stores opt (validation) loss of each fold.
          opt_fold_weights = []  # weights for opt_losses.
  
          # TODO: mention that no additional run info is possible in this
          # case! -> maybe remove full CV from the train evaluator anyway and
          # make the user implement this!
          for i, (train_split, test_split) in enumerate(
              self.splitter.split(
                  self.X_train,
                  y,
                  groups=self.resampling_strategy_args.get("groups"),
              )
          ):
              # TODO add check that split is actually an integer array,
              # not a boolean array (to allow indexed assignement of
              # training data later).
  
              if self.budget_type is None:
                  (
                      train_pred,
                      opt_pred,
                      test_pred,
                      additional_run_info,
                  ) = self._partial_fit_and_predict_standard(
                      i,
                      train_indices=train_split,
                      test_indices=test_split,
                      add_model_to_self=self.num_cv_folds == 1,
                  )
              else:
                  (
                      train_pred,
                      opt_pred,
                      test_pred,
                      additional_run_info,
                  ) = self._partial_fit_and_predict_budget(
                      i,
                      train_indices=train_split,
                      test_indices=test_split,
                      add_model_to_self=self.num_cv_folds == 1,
                  )
  
              if (
                  additional_run_info is not None
                  and len(additional_run_info) > 0
                  and i > 0
              ):
                  raise TAEAbortException(
                      'Found additional run info "%s" in fold %d, '
                      "but cannot handle additional run info if fold >= 1."
                      % (additional_run_info, i)
                  )
  
              Y_train_pred[i] = train_pred
              Y_optimization_pred[i] = opt_pred
              Y_test_pred[i] = test_pred
              train_splits[i] = train_split
  
              X = select(self.X_train, train_split)
              y = select(self.Y_train_targets, train_split)
  
              train_loss = self._loss(y, train_pred, X_data=X)
              train_losses.append(train_loss)
              # number of training data points for this fold. Used for weighting
              # the average.
              train_fold_weights.append(len(train_split))
  
              # Compute validation loss of this fold and store it.
              optimization_loss = self._loss(
                  self.Y_targets[i],
                  opt_pred,
                  X_data=self.X_targets[i],
              )
              opt_losses.append(optimization_loss)
              # number of optimization data points for this fold. Used for weighting
              # the average.
              opt_fold_weights.append(len(test_split))
  
          # Compute weights of each fold based on the number of samples in each
          # fold.
          train_fold_weights = [
              w / sum(train_fold_weights) for w in train_fold_weights
          ]
          opt_fold_weights = [
              w / sum(opt_fold_weights) for w in opt_fold_weights
          ]
  
          train_loss = {
              metric.name: np.average(
                  [
                      train_losses[i][str(metric)]
                      for i in range(self.num_cv_folds)
                  ],
                  weights=train_fold_weights,
              )
              for metric in self.metrics
          }
  
          # if all_scoring_function is true, return a dict of opt_loss. Otherwise,
          # return a scalar.
          opt_loss = {}
          for metric_name in list(opt_losses[0].keys()) + [
              metric.name for metric in self.metrics
          ]:
              opt_loss[metric_name] = np.average(
                  [
                      opt_losses[i][metric_name]
                      for i in range(self.num_cv_folds)
                  ],
                  weights=opt_fold_weights,
              )
  
          X_targets = self.X_targets
          Y_targets = self.Y_targets
          Y_train_targets = self.Y_train_targets
  
          Y_optimization_pred = concat_data(
              Y_optimization_pred, num_cv_folds=self.num_cv_folds
          )
          X_targets = concat_data(
              X_targets, num_cv_folds=self.num_cv_folds
          )
          Y_targets = concat_data(
              Y_targets, num_cv_folds=self.num_cv_folds
          )
  
          if self.X_test is not None:
              Y_test_pred = np.array(
                  [
                      Y_test_pred[i]
                      for i in range(self.num_cv_folds)
                      if Y_test_pred[i] is not None
                  ]
              )
              # Average the predictions of several models
              if len(np.shape(Y_test_pred)) == 3:
                  Y_test_pred = np.nanmean(Y_test_pred, axis=0)
  
          self.X_optimization = X_targets
          self.Y_optimization = Y_targets
          self.Y_actual_train = Y_train_targets
  
          if self.num_cv_folds > 1:
              self.model = self._get_model(self.feat_type)
              # Bad style, but necessary for unit testing that self.model is
              # actually a new model
              self._added_empty_model = True
              # TODO check if there might be reasons for do-not-advance here!
              status = StatusType.SUCCESS
          elif (
              self.budget_type == "iterations"
              or self.budget_type == "mixed"
              and self.model.estimator_supports_iterative_fit()
          ):
              budget_factor = self.model.get_max_iter()
              # We check for budget being None in initialization
              n_iter = int(
                  np.ceil(
                      cast(float, self.budget) / 100 * budget_factor
                  )
              )
              model_current_iter = self.model.get_current_iter()
              if model_current_iter < n_iter:
                  status = StatusType.DONOTADVANCE
              else:
                  status = StatusType.SUCCESS
          else:
              if self.model.estimator_supports_iterative_fit():
                  model_max_iter = self.model.get_max_iter()
                  model_current_iter = self.model.get_current_iter()
                  if model_current_iter < model_max_iter:
                      status = StatusType.DONOTADVANCE
                  else:
                      status = StatusType.SUCCESS
              else:
                  status = StatusType.SUCCESS
  
          self.finish_up(
              loss=opt_loss,
              train_loss=train_loss,
              opt_pred=Y_optimization_pred,
              test_pred=Y_test_pred
              if self.X_test is not None
              else None,
              additional_run_info=additional_run_info,
              file_output=True,
              final_call=True,
              status=status,
          )
  
imports: from typing import Any, Dict, List, Optional, Sequence, Tuple, TypeVar, Union, cast
  import copy
  import json
  import logging
  import multiprocessing
  import warnings
  import numpy
  import pandas
  import pandas
  import scipy.sparse
  from ConfigSpace import Configuration
  from sklearn.base import BaseEstimator
  from sklearn.model_selection import BaseCrossValidator, KFold, PredefinedSplit, ShuffleSplit, StratifiedKFold, StratifiedShuffleSplit, train_test_split
  from sklearn.model_selection._split import BaseShuffleSplit, _RepeatedSplits
  from smac.tae import StatusType, TAEAbortException
  from autosklearn.automl_common.common.utils.backend import Backend
  from autosklearn.constants import CLASSIFICATION_TASKS, MULTILABEL_CLASSIFICATION, MULTIOUTPUT_REGRESSION, REGRESSION_TASKS
  from autosklearn.data.abstract_data_manager import AbstractDataManager
  from autosklearn.data.validation import SUPPORTED_FEAT_TYPES, SUPPORTED_TARGET_TYPES
  from autosklearn.evaluation.abstract_evaluator import TYPE_ADDITIONAL_INFO, AbstractEvaluator, _fit_and_suppress_warnings
  from autosklearn.evaluation.splitter import CustomStratifiedKFold, CustomStratifiedShuffleSplit
  from autosklearn.metrics import Scorer
  from autosklearn.pipeline.base import PIPELINE_DATA_DTYPE
  from autosklearn.pipeline.components.base import IterativeComponent, ThirdPartyComponents
  from autosklearn.util.logging_ import PicklableClientLogger
label_desc: Use image node_id 2 for calling the TrainEvaluator obj's underlying member method code with example usage: obj.fit_predict_and_loss(iterative) without return types