code: def confusion_matrix_holdout(
      on_run=True, on_fold=True, on_rep=True, on_exp=True
  ):
      """Callback function to produce confusion matrices for holdout predictions at each stage of
      the Experiment
  
      Parameters
      ----------
      on_run: Boolean, default=True
          If False, skip making confusion matrices for individual Experiment runs
      on_fold: Boolean, default=True
          If False, skip making confusion matrices for individual Experiment folds
      on_rep: Boolean, default=True
          If False, skip making confusion matrices for individual Experiment repetitions
      on_exp: Boolean, default=True
          If False, skip making final confusion matrix for the Experiment
  
      Returns
      -------
      LambdaCallback
          An uninitialized :class:`LambdaCallback` to generate confusion matrices, produced by
          :func:`hyperparameter_hunter.callbacks.bases.lambda_callback`
  
      Notes
      -----
      Unlike :func:`hyperparameter_hunter.callbacks.recipes.confusion_matrix_oof`, this callback
      bypasses `lambda_callback`\'s ability to automatically aggregate stats returned by the "on..."
      functions. It does this simply by not returning values in the "on..." functions, and manually
      aggregating the stats in :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates`.
      This offers greater control over how your values are collected, but also requires additional
      overhead, namely, instantiating a dict to collect the values via :func:`_on_exp_start`.
      Note also that each of the "on..." functions must append their values to an explicitly named
      container in :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates` when using
      this method as opposed to :func:`hyperparameter_hunter.callbacks.recipes.confusion_matrix_oof`\'s
  
      If the size of this `lambda_callback` implementation is daunting, minimize the helper functions'
      docstrings. It's surprisingly simple"""
  
      def _on_exp_start(stat_aggregates):
          """Callback to instantiate container for Experiment values to be aggregated
  
          Parameters
          ----------
          stat_aggregates: Dict
              :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates`
          """
          stat_aggregates["confusion_matrix_holdout"] = dict(
              runs=[], folds=[], reps=[], final=None
          )
  
      def _on_run_end(stat_aggregates, data_holdout):
          """Callback to execute upon ending an Experiment's run. Note that parameters are
          named after Experiment attributes
  
          Parameters
          ----------
          stat_aggregates: Dict
              :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates`
          data_holdout: BaseDataset
              :attr:`hyperparameter_hunter.experiments.BaseCVExperiment.data_holdout`
          """
          stat_aggregates["confusion_matrix_holdout"]["runs"].append(
              _confusion_matrix(
                  data_holdout.target.fold, data_holdout.prediction.run
              )
          )
  
      def _on_fold_end(stat_aggregates, data_holdout):
          """Callback to execute upon ending an Experiment's fold. Note that parameters are
          named after Experiment attributes
  
          Parameters
          ----------
          stat_aggregates: Dict
              :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates`
          data_holdout: BaseDataset
              :attr:`hyperparameter_hunter.experiments.BaseCVExperiment.data_holdout`
          """
          stat_aggregates["confusion_matrix_holdout"]["folds"].append(
              _confusion_matrix(
                  data_holdout.target.fold, data_holdout.prediction.fold
              )
          )
  
      def _on_rep_end(stat_aggregates, data_holdout):
          """Callback to execute upon ending an Experiment's repetition. Note that parameters are
          named after Experiment attributes
  
          Parameters
          ----------
          stat_aggregates: Dict
              :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates`
          data_holdout: BaseDataset
              :attr:`hyperparameter_hunter.experiments.BaseCVExperiment.data_holdout`
          """
          stat_aggregates["confusion_matrix_holdout"]["reps"].append(
              # _confusion_matrix(data_holdout.target.final, data_holdout.prediction.rep)  # TODO: Should use this when collecting transformed targets
              _confusion_matrix(
                  data_holdout.target.d, data_holdout.prediction.rep
              )  # FLAG: TEMPORARY
          )
  
      def _on_exp_end(stat_aggregates, data_holdout):
          """Callback to execute upon ending an Experiment. Note that parameters are
          named after Experiment attributes
  
          Parameters
          ----------
          stat_aggregates: Dict
              :attr:`hyperparameter_hunter.experiments.BaseExperiment.stat_aggregates`
          data_holdout: BaseDataset
              :attr:`hyperparameter_hunter.experiments.BaseCVExperiment.data_holdout`
          """
          stat_aggregates["confusion_matrix_holdout"][
              "final"
          ] = _confusion_matrix(
              # data_holdout.target.final, data_holdout.prediction.final  # TODO: Should use this when collecting transformed targets
              data_holdout.target.d,
              data_holdout.prediction.final,  # FLAG: TEMPORARY
          )
  
      return lambda_callback(
          on_exp_start=_on_exp_start,
          on_run_end=_on_run_end if on_run else None,
          on_fold_end=_on_fold_end if on_fold else None,
          on_rep_end=_on_rep_end if on_rep else None,
          on_exp_end=_on_exp_end if on_exp else None,
          agg_name="confusion_matrix_holdout",
      )
  
file_path: hyperparameter_hunter/hyperparameter_hunter/callbacks/recipes.py
imports: from hyperparameter_hunter.callbacks.bases import lambda_callback
  from hyperparameter_hunter.data.data_chunks.target_chunks import TrainTargetChunk
  from hyperparameter_hunter.metrics import get_clean_prediction
  from copy import deepcopy
  import numpy
  from typing import List, Optional
  from sklearn.metrics import confusion_matrix
label_desc: Use image node_id 2 for calling a global function with example usage: confusion_matrix_holdout(on_run, on_fold, on_rep, on_exp) and returns: lambda_callback