docstr: Performs the matching of predicted boxes and ground truth boxes. Loops over list of matching IoUs and foreground classes.
  Resulting info of each prediction is stored as one line in an internal dataframe, with the keys:
  det_type: 'tp' (true positive), 'fp' (false positive), 'fn' (false negative), 'tn' (true negative)
  pred_class: foreground class which the object predicts.
  pid: corresponding patient-id.
  pred_score: confidence score [0, 1]
  fold: corresponding fold of CV.
  match_iou: utilized IoU for matching.
  :param results_list: list of model predictions. Either from train/val_sampling (patch processing) for monitoring with form:
  [[[results_0, ...], [pid_0, ...]], [[results_n, ...], [pid_n, ...]], ...]
  Or from val_patient/testing (patient processing), with form: [[results_0, pid_0], [results_1, pid_1], ...])
  :param monitor_metrics (optional):  dict of dicts with all metrics of previous epochs.
  :return monitor_metrics: if provided (during training), return monitor_metrics now including results of current epoch.
docstr_tok: ["Performs","the","matching","of","predicted","boxes","and","ground","truth","boxes",".","Loops","over","list","of","matching","IoUs","and","foreground","classes",".","Resulting","info","of","each","prediction","is","stored","as","one","line","in","an","internal","dataframe",",","with","the","keys",":","det_type",":","'tp","'","(","true","positive",")",",","'fp","'","(","false","positive",")",",","'fn","'","(","false","negative",")",",","'tn","'","(","true","negative",")","pred_class",":","foreground","class","which","the","object","predicts",".","pid",":","corresponding","patient-id",".","pred_score",":","confidence","score","[","0",",","1","]","fold",":","corresponding","fold","of","CV",".","match_iou",":","utilized","IoU","for","matching",".",":","param","results_list",":","list","of","model","predictions",".","Either","from","train\/val_sampling","(","patch","processing",")","for","monitoring","with","form",":","[","[","[","results_0",",","...","]",",","[","pid_0",",","...","]","]",",","[","[","results_n",",","...","]",",","[","pid_n",",","...","]","]",",","...","]","Or","from","val_patient\/testing","(","patient","processing",")",",","with","form",":","[","[","results_0",",","pid_0","]",",","[","results_1",",","pid_1","]",",","...","]",")",":","param","monitor_metrics","(","optional",")",":","dict","of","dicts","with","all","metrics","of","previous","epochs",".",":","return","monitor_metrics",":","if","provided","(","during","training",")",",","return","monitor_metrics","now","including","results","of","current","epoch","."]
code: def evaluate_predictions(self, results_list, monitor_metrics=None):
      """
      Performs the matching of predicted boxes and ground truth boxes. Loops over list of matching IoUs and foreground classes.
      Resulting info of each prediction is stored as one line in an internal dataframe, with the keys:
      det_type: 'tp' (true positive), 'fp' (false positive), 'fn' (false negative), 'tn' (true negative)
      pred_class: foreground class which the object predicts.
      pid: corresponding patient-id.
      pred_score: confidence score [0, 1]
      fold: corresponding fold of CV.
      match_iou: utilized IoU for matching.
      :param results_list: list of model predictions. Either from train/val_sampling (patch processing) for monitoring with form:
      [[[results_0, ...], [pid_0, ...]], [[results_n, ...], [pid_n, ...]], ...]
      Or from val_patient/testing (patient processing), with form: [[results_0, pid_0], [results_1, pid_1], ...])
      :param monitor_metrics (optional):  dict of dicts with all metrics of previous epochs.
      :return monitor_metrics: if provided (during training), return monitor_metrics now including results of current epoch.
      """
      # gets results_list = [[batch_instances_box_lists], [batch_instances_pids]]*n_batches
      # we want to evaluate one batch_instance (= 2D or 3D image) at a time.
  
      df_list_preds = []
      df_list_labels = []
      df_list_class_preds = []
      df_list_pids = []
      df_list_type = []
      df_list_match_iou = []
  
      self.logger.info("evaluating in mode {}".format(self.mode))
  
      if self.mode == "train" or self.mode == "val_sampling":
          # batch_size > 1, with varying patients across batch:
          # [[[results_0, ...], [pid_0, ...]], [[results_n, ...], [pid_n, ...]], ...]
          # -> [results_0, results_1, ..] , [pid_0, pid_1, ...]
          batch_elements_list = [
              [b_box_list]
              for item in results_list
              for b_box_list in item[0]
          ]
          pid_list = [pid for item in results_list for pid in item[1]]
      else:
          # patient processing, one element per batch = one patient.
          # [[results_0, pid_0], [results_1, pid_1], ...] -> [results_0, results_1, ..] , [pid_0, pid_1, ...]
          batch_elements_list = [item[0] for item in results_list]
          pid_list = [item[1] for item in results_list]
  
      for match_iou in self.cf.ap_match_ious:
          self.logger.info(
              "evaluating with match_iou: {}".format(match_iou)
          )
          for cl in list(self.cf.class_dict.keys()):
              for pix, pid in enumerate(pid_list):
                  len_df_list_before_patient = len(df_list_pids)
  
                  # input of each batch element is a list of boxes, where each box is a dictionary.
                  for bix, b_boxes_list in enumerate(
                      batch_elements_list[pix]
                  ):
                      b_tar_boxes = np.array(
                          [
                              box["box_coords"]
                              for box in b_boxes_list
                              if (
                                  box["box_type"] == "gt"
                                  and box["box_label"] == cl
                              )
                          ]
                      )
                      b_cand_boxes = np.array(
                          [
                              box["box_coords"]
                              for box in b_boxes_list
                              if (
                                  box["box_type"] == "det"
                                  and box["box_pred_class_id"] == cl
                              )
                          ]
                      )
                      b_cand_scores = np.array(
                          [
                              box["box_score"]
                              for box in b_boxes_list
                              if (
                                  box["box_type"] == "det"
                                  and box["box_pred_class_id"] == cl
                              )
                          ]
                      )
  
                      # check if predictions and ground truth boxes exist and match them according to match_iou.
                      if (
                          not 0 in b_cand_boxes.shape
                          and not 0 in b_tar_boxes.shape
                      ):
                          overlaps = mutils.compute_overlaps(
                              b_cand_boxes, b_tar_boxes
                          )
                          match_cand_ixs = np.argwhere(
                              np.max(overlaps, 1) > match_iou
                          )[:, 0]
                          non_match_cand_ixs = np.argwhere(
                              np.max(overlaps, 1) <= match_iou
                          )[:, 0]
                          match_gt_ixs = (
                              np.argmax(overlaps[match_cand_ixs, :], 1)
                              if not 0 in match_cand_ixs.shape
                              else np.array([])
                          )
                          non_match_gt_ixs = np.array(
                              [
                                  ii
                                  for ii in np.arange(
                                      b_tar_boxes.shape[0]
                                  )
                                  if ii not in match_gt_ixs
                              ]
                          )
                          unique, counts = np.unique(
                              match_gt_ixs, return_counts=True
                          )
  
                          # check for double assignments, i.e. two predictions having been assigned to the same gt.
                          # according to the COCO-metrics, only one prediction counts as true positive, the rest counts as
                          # false positive. This case is supposed to be avoided by the model itself by,
                          #  e.g. using a low enough NMS threshold.
                          if np.any(counts > 1):
                              double_match_gt_ixs = unique[
                                  np.argwhere(counts > 1)[:, 0]
                              ]
                              keep_max = []
                              double_match_list = []
                              for dg in double_match_gt_ixs:
                                  double_match_cand_ixs = (
                                      match_cand_ixs[
                                          np.argwhere(
                                              match_gt_ixs == dg
                                          )
                                      ]
                                  )
                                  keep_max.append(
                                      double_match_cand_ixs[
                                          np.argmax(
                                              b_cand_scores[
                                                  double_match_cand_ixs
                                              ]
                                          )
                                      ]
                                  )
                                  double_match_list += [
                                      ii for ii in double_match_cand_ixs
                                  ]
  
                              fp_ixs = np.array(
                                  [
                                      ii
                                      for ii in match_cand_ixs
                                      if (
                                          ii in double_match_list
                                          and ii not in keep_max
                                      )
                                  ]
                              )
  
                              match_cand_ixs = np.array(
                                  [
                                      ii
                                      for ii in match_cand_ixs
                                      if ii not in fp_ixs
                                  ]
                              )
  
                              df_list_preds += [
                                  ii for ii in b_cand_scores[fp_ixs]
                              ]
                              df_list_labels += [0] * fp_ixs.shape[0]
                              df_list_class_preds += [
                                  cl
                              ] * fp_ixs.shape[0]
                              df_list_pids += [pid] * fp_ixs.shape[0]
                              df_list_type += ["det_fp"] * fp_ixs.shape[
                                  0
                              ]
  
                          # matched:
                          if not 0 in match_cand_ixs.shape:
                              df_list_preds += [
                                  ii
                                  for ii in b_cand_scores[
                                      match_cand_ixs
                                  ]
                              ]
                              df_list_labels += [
                                  1
                              ] * match_cand_ixs.shape[0]
                              df_list_class_preds += [
                                  cl
                              ] * match_cand_ixs.shape[0]
                              df_list_pids += [
                                  pid
                              ] * match_cand_ixs.shape[0]
                              df_list_type += [
                                  "det_tp"
                              ] * match_cand_ixs.shape[0]
                          # rest fp:
                          if not 0 in non_match_cand_ixs.shape:
                              df_list_preds += [
                                  ii
                                  for ii in b_cand_scores[
                                      non_match_cand_ixs
                                  ]
                              ]
                              df_list_labels += [
                                  0
                              ] * non_match_cand_ixs.shape[0]
                              df_list_class_preds += [
                                  cl
                              ] * non_match_cand_ixs.shape[0]
                              df_list_pids += [
                                  pid
                              ] * non_match_cand_ixs.shape[0]
                              df_list_type += [
                                  "det_fp"
                              ] * non_match_cand_ixs.shape[0]
                          # rest fn:
                          if not 0 in non_match_gt_ixs.shape:
                              df_list_preds += [
                                  0
                              ] * non_match_gt_ixs.shape[0]
                              df_list_labels += [
                                  1
                              ] * non_match_gt_ixs.shape[0]
                              df_list_class_preds += [
                                  cl
                              ] * non_match_gt_ixs.shape[0]
                              df_list_pids += [
                                  pid
                              ] * non_match_gt_ixs.shape[0]
                              df_list_type += [
                                  "det_fn"
                              ] * non_match_gt_ixs.shape[0]
                      # only fp:
                      if (
                          not 0 in b_cand_boxes.shape
                          and 0 in b_tar_boxes.shape
                      ):
                          df_list_preds += [ii for ii in b_cand_scores]
                          df_list_labels += [0] * b_cand_scores.shape[0]
                          df_list_class_preds += [
                              cl
                          ] * b_cand_scores.shape[0]
                          df_list_pids += [pid] * b_cand_scores.shape[0]
                          df_list_type += [
                              "det_fp"
                          ] * b_cand_scores.shape[0]
                      # only fn:
                      if (
                          0 in b_cand_boxes.shape
                          and not 0 in b_tar_boxes.shape
                      ):
                          df_list_preds += [0] * b_tar_boxes.shape[0]
                          df_list_labels += [1] * b_tar_boxes.shape[0]
                          df_list_class_preds += [
                              cl
                          ] * b_tar_boxes.shape[0]
                          df_list_pids += [pid] * b_tar_boxes.shape[0]
                          df_list_type += [
                              "det_fn"
                          ] * b_tar_boxes.shape[0]
  
                  # empty patient with 0 detections needs patient dummy score, in order to not disappear from stats.
                  # filtered out for roi-level evaluation later. During training (and val_sampling),
                  # tn are assigned per sample independently of associated patients.
                  if len(df_list_pids) == len_df_list_before_patient:
                      df_list_preds += [0] * 1
                      df_list_labels += [0] * 1
                      df_list_class_preds += [cl] * 1
                      df_list_pids += [pid] * 1
                      df_list_type += [
                          "patient_tn"
                      ] * 1  # true negative: no ground truth boxes, no detections.
  
          df_list_match_iou += [match_iou] * (
              len(df_list_preds) - len(df_list_match_iou)
          )
  
      self.test_df = pd.DataFrame()
      self.test_df["pred_score"] = df_list_preds
      self.test_df["class_label"] = df_list_labels
      self.test_df["pred_class"] = df_list_class_preds
      self.test_df["pid"] = df_list_pids
      self.test_df["det_type"] = df_list_type
      self.test_df["fold"] = self.cf.fold
      self.test_df["match_iou"] = df_list_match_iou
      if monitor_metrics is not None:
          return self.return_metrics(monitor_metrics)
  
file_path: medicaldetectiontoolkit/evaluator.py
imports: import os
  import numpy
  import pandas
  from sklearn.metrics import roc_auc_score, average_precision_score
  from sklearn.metrics import roc_curve, precision_recall_curve
  import utils.model_utils
  import plotting
  from multiprocessing import Pool
label_desc: Use image node_id 2 for calling the Evaluator obj's underlying member method code with example usage: obj.evaluate_predictions(results_list, monitor_metrics) and returns: self