class_docstr: An error evaluator for objective functions based on cross-validation.
  
  This evaluator evaluates the objective function's statistical error, which comes from the
  randomness of dataset. This evaluator assumes that the objective function is the average of
  the cross-validation and uses the scaled variance of the cross-validation scores in the best
  trial at the moment as the statistical error.
class_docstr_tok: ["An","error","evaluator","for","objective","functions","based","on","cross-validation",".","This","evaluator","evaluates","the","objective","function","'s","statistical","error",",","which","comes","from","the","randomness","of","dataset",".","This","evaluator","assumes","that","the","objective","function","is","the","average","of","the","cross-validation","and","uses","the","scaled","variance","of","the","cross-validation","scores","in","the","best","trial","at","the","moment","as","the","statistical","error","."]
docstr: Evaluate the statistical error of the objective function based on cross-validation.
  
  Args:
      trials:
          A list of trials to consider. The best trial in ``trials`` is used to compute the
          statistical error.
  
      study_direction:
          The direction of the study.
  
  Returns:
      A float representing the statistical error of the objective function.
docstr_tok: ["Evaluate","the","statistical","error","of","the","objective","function","based","on","cross-validation",".","Args",":","trials",":","A","list","of","trials","to","consider",".","The","best","trial","in","``","trials","``","is","used","to","compute","the","statistical","error",".","study_direction",":","The","direction","of","the","study",".","Returns",":","A","float","representing","the","statistical","error","of","the","objective","function","."]
code: def evaluate(
      self,
      trials: list[FrozenTrial],
      study_direction: StudyDirection,
  ) -> float:
      """Evaluate the statistical error of the objective function based on cross-validation.
  
      Args:
          trials:
              A list of trials to consider. The best trial in ``trials`` is used to compute the
              statistical error.
  
          study_direction:
              The direction of the study.
  
      Returns:
          A float representing the statistical error of the objective function.
  
      """
      trials = [
          trial
          for trial in trials
          if trial.state == TrialState.COMPLETE
      ]
      assert len(trials) > 0
  
      if study_direction == StudyDirection.MAXIMIZE:
          best_trial = max(trials, key=lambda t: cast(float, t.value))
      else:
          best_trial = min(trials, key=lambda t: cast(float, t.value))
  
      best_trial_attrs = best_trial.system_attrs
      if _CROSS_VALIDATION_SCORES_KEY in best_trial_attrs:
          cv_scores = best_trial_attrs[_CROSS_VALIDATION_SCORES_KEY]
      else:
          raise ValueError(
              "Cross-validation scores have not been reported. Please call "
              "`report_cross_validation_scores(trial, scores)` during a trial and pass the "
              "list of scores as `scores`."
          )
  
      k = len(cv_scores)
      assert (
          k > 1
      ), "Should be guaranteed by `report_cross_validation_scores`."
      scale = 1 / k + 1 / (k - 1)
  
      var = scale * np.var(cv_scores)
      std = np.sqrt(var)
  
      return float(std)
  
file_path: optuna/optuna/terminator/erroreval.py
imports: from __future__ import annotations
  import abc
  from typing import cast
  import numpy
  from optuna._experimental import experimental_class
  from optuna.study import StudyDirection
  from optuna.trial import FrozenTrial
  from optuna.trial import Trial
  from optuna.trial._state import TrialState
label_desc: Use image node_id 1 for calling the CrossValidationErrorEvaluator obj's underlying member method code with example usage: obj.evaluate(trials, study_direction) and returns: float