docstr: calculates AP/AUC scores for internal dataframe. called directly from evaluate_predictions during training for monitoring,
  or from score_test_df during inference (for single folds or aggregated test set). Loops over foreground classes
  and score_levels (typically 'roi' and 'patient'), gets scores and stores them. Optionally creates plots of
  prediction histograms and roc/prc curves.
  :param monitor_metrics: dict of dicts with all metrics of previous epochs.
  this function adds metrics for current epoch and returns the same object.
  :return: all_stats: list. Contains dicts with resulting scores for each combination of foreground class and
  score_level.
  :return: monitor_metrics
docstr_tok: ["calculates","AP\/AUC","scores","for","internal","dataframe",".","called","directly","from","evaluate_predictions","during","training","for","monitoring",",","or","from","score_test_df","during","inference","(","for","single","folds","or","aggregated","test","set",")",".","Loops","over","foreground","classes","and","score_levels","(","typically","'roi","'","and","'patient","'",")",",","gets","scores","and","stores","them",".","Optionally","creates","plots","of","prediction","histograms","and","roc\/prc","curves",".",":","param","monitor_metrics",":","dict","of","dicts","with","all","metrics","of","previous","epochs",".","this","function","adds","metrics","for","current","epoch","and","returns","the","same","object",".",":","return",":","all_stats",":","list",".","Contains","dicts","with","resulting","scores","for","each","combination","of","foreground","class","and","score_level",".",":","return",":","monitor_metrics"]
code: def return_metrics(self, monitor_metrics=None):
      """
      calculates AP/AUC scores for internal dataframe. called directly from evaluate_predictions during training for monitoring,
      or from score_test_df during inference (for single folds or aggregated test set). Loops over foreground classes
      and score_levels (typically 'roi' and 'patient'), gets scores and stores them. Optionally creates plots of
      prediction histograms and roc/prc curves.
      :param monitor_metrics: dict of dicts with all metrics of previous epochs.
      this function adds metrics for current epoch and returns the same object.
      :return: all_stats: list. Contains dicts with resulting scores for each combination of foreground class and
      score_level.
      :return: monitor_metrics
      """
      df = self.test_df
  
      all_stats = []
      for cl in list(self.cf.class_dict.keys()):
          cl_df = df[df.pred_class == cl]
  
          for score_level in self.cf.report_score_level:
              stats_dict = {}
              stats_dict["name"] = "fold_{} {} cl_{}".format(
                  self.cf.fold, score_level, cl
              )
  
              if score_level == "rois":
                  # kick out dummy entries for true negative patients. not needed on roi-level.
                  spec_df = cl_df[cl_df.det_type != "patient_tn"]
                  stats_dict["ap"] = get_roi_ap_from_df(
                      [
                          spec_df,
                          self.cf.min_det_thresh,
                          self.cf.per_patient_ap,
                      ]
                  )
                  # AUC not sensible on roi-level, since true negative box predictions do not exist. Would reward
                  # higher amounts of low confidence false positives.
                  stats_dict["auc"] = 0
                  stats_dict["roc"] = None
                  stats_dict["prc"] = None
  
                  # for the aggregated test set case, additionally get the scores for averaging over fold results.
                  if len(df.fold.unique()) > 1:
                      aps = []
                      for fold in df.fold.unique():
                          fold_df = spec_df[spec_df.fold == fold]
                          aps.append(
                              get_roi_ap_from_df(
                                  [
                                      fold_df,
                                      self.cf.min_det_thresh,
                                      self.cf.per_patient_ap,
                                  ]
                              )
                          )
                      stats_dict["mean_ap"] = np.mean(aps)
                      stats_dict["mean_auc"] = 0
  
              # on patient level, aggregate predictions per patient (pid): The patient predicted score is the highest
              # confidence prediction for this class. The patient class label is 1 if roi of this class exists in patient, else 0.
              if score_level == "patient":
                  spec_df = cl_df.groupby(["pid"], as_index=False).agg(
                      {
                          "class_label": "max",
                          "pred_score": "max",
                          "fold": "first",
                      }
                  )
  
                  if len(spec_df.class_label.unique()) > 1:
                      stats_dict["auc"] = roc_auc_score(
                          spec_df.class_label.tolist(),
                          spec_df.pred_score.tolist(),
                      )
                      stats_dict["roc"] = roc_curve(
                          spec_df.class_label.tolist(),
                          spec_df.pred_score.tolist(),
                      )
                  else:
                      stats_dict["auc"] = np.nan
                      stats_dict["roc"] = np.nan
  
                  if (spec_df.class_label == 1).any():
                      stats_dict["ap"] = average_precision_score(
                          spec_df.class_label.tolist(),
                          spec_df.pred_score.tolist(),
                      )
                      stats_dict["prc"] = precision_recall_curve(
                          spec_df.class_label.tolist(),
                          spec_df.pred_score.tolist(),
                      )
                  else:
                      stats_dict["ap"] = np.nan
                      stats_dict["prc"] = np.nan
  
                  # for the aggregated test set case, additionally get the scores for averaging over fold results.
                  if len(df.fold.unique()) > 1:
                      aucs = []
                      aps = []
                      for fold in df.fold.unique():
                          fold_df = spec_df[spec_df.fold == fold]
                          if len(fold_df.class_label.unique()) > 1:
                              aucs.append(
                                  roc_auc_score(
                                      fold_df.class_label.tolist(),
                                      fold_df.pred_score.tolist(),
                                  )
                              )
                          if (fold_df.class_label == 1).any():
                              aps.append(
                                  average_precision_score(
                                      fold_df.class_label.tolist(),
                                      fold_df.pred_score.tolist(),
                                  )
                              )
                      stats_dict["mean_auc"] = np.mean(aucs)
                      stats_dict["mean_ap"] = np.mean(aps)
  
              # fill new results into monitor_metrics dict. for simplicity, only one class (of interest) is monitored on patient level.
              if monitor_metrics is not None and not (
                  score_level == "patient"
                  and cl != self.cf.patient_class_of_interest
              ):
                  score_level_name = (
                      "patient"
                      if score_level == "patient"
                      else self.cf.class_dict[cl]
                  )
                  monitor_metrics[score_level_name + "_ap"].append(
                      stats_dict["ap"] if stats_dict["ap"] > 0 else None
                  )
                  if score_level == "patient":
                      monitor_metrics[score_level_name + "_auc"].append(
                          stats_dict["auc"]
                          if stats_dict["auc"] > 0
                          else None
                      )
  
              if self.cf.plot_prediction_histograms:
                  out_filename = os.path.join(
                      self.cf.plot_dir,
                      "pred_hist_{}_{}_{}_cl{}".format(
                          self.cf.fold,
                          "val" if "val" in self.mode else self.mode,
                          score_level,
                          cl,
                      ),
                  )
                  type_list = (
                      None
                      if score_level == "patient"
                      else spec_df.det_type.tolist()
                  )
                  plotting.plot_prediction_hist(
                      spec_df.class_label.tolist(),
                      spec_df.pred_score.tolist(),
                      type_list,
                      out_filename,
                  )
  
              all_stats.append(stats_dict)
  
              # analysis of the  hyper-parameter cf.min_det_thresh, for optimization on validation set.
              if self.cf.scan_det_thresh:
                  conf_threshs = list(np.arange(0.9, 1, 0.01))
                  pool = Pool(processes=10)
                  mp_inputs = [
                      [spec_df, ii, self.cf.per_patient_ap]
                      for ii in conf_threshs
                  ]
                  aps = pool.map(
                      get_roi_ap_from_df, mp_inputs, chunksize=1
                  )
                  pool.close()
                  pool.join()
                  self.logger.info(
                      "results from scanning over det_threshs:",
                      [[i, j] for i, j in zip(conf_threshs, aps)],
                  )
  
      if self.cf.plot_stat_curves:
          out_filename = os.path.join(
              self.cf.plot_dir,
              "{}_{}_stat_curves".format(self.cf.fold, self.mode),
          )
          plotting.plot_stat_curves(all_stats, out_filename)
  
      # get average stats over foreground classes on roi level.
      avg_ap = np.mean(
          [d["ap"] for d in all_stats if "rois" in d["name"]]
      )
      all_stats.append(
          {"name": "average_foreground_roi", "auc": 0, "ap": avg_ap}
      )
      if len(df.fold.unique()) > 1:
          avg_mean_ap = np.mean(
              [d["mean_ap"] for d in all_stats if "rois" in d["name"]]
          )
          all_stats[-1]["mean_ap"] = avg_mean_ap
          all_stats[-1]["mean_auc"] = 0
  
      # in small data sets, values of model_selection_criterion can be identical across epochs, wich breaks the
      # ranking of model_selector. Thus, pertube identical values by a neglectibale random term.
      for sc in self.cf.model_selection_criteria:
          if (
              "val" in self.mode
              and monitor_metrics[sc].count(monitor_metrics[sc][-1]) > 1
              and monitor_metrics[sc][-1] is not None
          ):
              monitor_metrics[sc][-1] += 1e-6 * np.random.rand()
  
      return all_stats, monitor_metrics
  
file_path: medicaldetectiontoolkit/evaluator.py
imports: import os
  import numpy
  import pandas
  from sklearn.metrics import roc_auc_score, average_precision_score
  from sklearn.metrics import roc_curve, precision_recall_curve
  import utils.model_utils
  import plotting
  from multiprocessing import Pool
label_desc: Use image node_id 3 for calling the Evaluator obj's underlying member method code with example usage: obj.return_metrics(monitor_metrics) and returns: all_stats,  monitor_metrics