docstr: MixIn class to manage metrics to record for each dataset type, and perform evaluations
  
  Parameters
  ----------
  metrics: Dict, List
      Specifies all metrics to be used by their id keys, along with a means to compute the
      metric. If list, all values must be strings that are attributes in
      :mod:`sklearn.metrics`. If dict, key/value pairs must be of the form:
      (<id>, <callable/None/str sklearn.metrics attribute>), where "id" is a str name for the
      metric. Its corresponding value must be one of: 1) a callable to calculate the metric,
      2) None if the "id" key is an attribute in `sklearn.metrics` and should be used to fetch
      a callable, 3) a string that is an attribute in `sklearn.metrics` and should be used to
      fetch a callable. Metric callable functions should expect inputs of form
      (target, prediction), and should return floats
  in_fold: List of strings, None, default=<all ids in `metrics`>
      Which metrics (from ids in `metrics`) should be recorded for in-fold data
  oof: List of strings, None, default=<all ids in `metrics`>
      Which metrics (from ids in `metrics`) should be recorded for out-of-fold data
  holdout: List of strings, None, default=<all ids in `metrics`>
      Which metrics (from ids in `metrics`) should be recorded for holdout data
  do_score: Boolean, default=True
      This is experimental. If False, scores will be neither calculated nor recorded for the
      duration of the experiment
  
  Notes
  -----
  For each kwarg in [`in_fold`, `oof`, `holdout`], the following must be true: if the value
  of the kwarg is a list, its contents must be a subset of `metrics`
docstr_tok: ["MixIn","class","to","manage","metrics","to","record","for","each","dataset","type",",","and","perform","evaluations","Parameters","--","--","--","--","--","metrics",":","Dict",",","List","Specifies","all","metrics","to","be","used","by","their","id","keys",",","along","with","a","means","to","compute","the","metric",".","If","list",",","all","values","must","be","strings","that","are","attributes","in",":","mod",":","`","sklearn.metrics","`",".","If","dict",",","key\/value","pairs","must","be","of","the","form",":","(","<","id",">",",","<","callable\/None\/str","sklearn.metrics","attribute",">",")",",","where","``","id","''","is","a","str","name","for","the","metric",".","Its","corresponding","value","must","be","one","of",":","1",")","a","callable","to","calculate","the","metric",",","2",")","None","if","the","``","id","''","key","is","an","attribute","in","`","sklearn.metrics","`","and","should","be","used","to","fetch","a","callable",",","3",")","a","string","that","is","an","attribute","in","`","sklearn.metrics","`","and","should","be","used","to","fetch","a","callable",".","Metric","callable","functions","should","expect","inputs","of","form","(","target",",","prediction",")",",","and","should","return","floats","in_fold",":","List","of","strings",",","None",",","default=","<","all","ids","in","`","metrics","`",">","Which","metrics","(","from","ids","in","`","metrics","`",")","should","be","recorded","for","in-fold","data","oof",":","List","of","strings",",","None",",","default=","<","all","ids","in","`","metrics","`",">","Which","metrics","(","from","ids","in","`","metrics","`",")","should","be","recorded","for","out-of-fold","data","holdout",":","List","of","strings",",","None",",","default=","<","all","ids","in","`","metrics","`",">","Which","metrics","(","from","ids","in","`","metrics","`",")","should","be","recorded","for","holdout","data","do_score",":","Boolean",",","default=True","This","is","experimental",".","If","False",",","scores","will","be","neither","calculated","nor","recorded","for","the","duration","of","the","experiment","Notes","--","--","-","For","each","kwarg","in","[","`","in_fold","`",",","`","oof","`",",","`","holdout","`","]",",","the","following","must","be","true",":","if","the","value","of","the","kwarg","is","a","list",",","its","contents","must","be","a","subset","of","`","metrics","`"]
code: def __init__(
      self,
      metrics,
      in_fold="all",
      oof="all",
      holdout="all",
      do_score=True,
  ):
      """MixIn class to manage metrics to record for each dataset type, and perform evaluations
  
      Parameters
      ----------
      metrics: Dict, List
          Specifies all metrics to be used by their id keys, along with a means to compute the
          metric. If list, all values must be strings that are attributes in
          :mod:`sklearn.metrics`. If dict, key/value pairs must be of the form:
          (<id>, <callable/None/str sklearn.metrics attribute>), where "id" is a str name for the
          metric. Its corresponding value must be one of: 1) a callable to calculate the metric,
          2) None if the "id" key is an attribute in `sklearn.metrics` and should be used to fetch
          a callable, 3) a string that is an attribute in `sklearn.metrics` and should be used to
          fetch a callable. Metric callable functions should expect inputs of form
          (target, prediction), and should return floats
      in_fold: List of strings, None, default=<all ids in `metrics`>
          Which metrics (from ids in `metrics`) should be recorded for in-fold data
      oof: List of strings, None, default=<all ids in `metrics`>
          Which metrics (from ids in `metrics`) should be recorded for out-of-fold data
      holdout: List of strings, None, default=<all ids in `metrics`>
          Which metrics (from ids in `metrics`) should be recorded for holdout data
      do_score: Boolean, default=True
          This is experimental. If False, scores will be neither calculated nor recorded for the
          duration of the experiment
  
      Notes
      -----
      For each kwarg in [`in_fold`, `oof`, `holdout`], the following must be true: if the value
      of the kwarg is a list, its contents must be a subset of `metrics`
      """
      self.metrics = format_metrics(metrics)
      self.do_score = do_score
  
      #################### ScoringMixIn-Only Mangled Attributes ####################
      self.__in_fold = in_fold if in_fold else []
      self.__oof = oof if oof else []
      self.__holdout = holdout if holdout else []
  
      self._validate_metrics_list_parameters()
      self.last_evaluation_results = dict(
          in_fold=None, oof=None, holdout=None
      )
  
imports: from collections import OrderedDict
  from contextlib import suppress
  import numpy
  import pandas
  from typing import Dict, Iterable, List, Tuple, Union
  from sklearn import metrics
  from sklearn.utils.multiclass import type_of_target, unique_labels
label_desc: Use image node_id 1 to create a new ScoringMixIn object from inherited base classes: object with example: obj = ScoringMixIn(metrics, in_fold, oof, holdout, do_score)