code: def calculate_metrics(output_per_batch, num_classes):
      """Calculates mAP and mAR from the detector's output.
  
      The procedure for calculating the average precision for class ``C`` is as
      follows (see `VOC mAP metric`_ for more details):
  
      Start by ranking all the predictions (for a given image and said class) in
      order of confidence.  Each of these predictions is marked as correct (true
      positive, when it has a IoU-threshold greater or equal to `iou_thresholds`)
      or incorrect (false positive, in the other case).  This matching is
      performed greedily over the confidence scores, so a higher-confidence
      prediction will be matched over another lower-confidence one even if the
      latter has better IoU.  Also, each prediction is matched at most once, so
      repeated detections are counted as false positives.
  
      We then integrate over the interpolated PR curve, thus obtaining the value
      for the class' average precision.  This interpolation makes sure the
      precision curve is monotonically decreasing; for this, we go through the
      precisions and make sure it's always decreasing.  The integration is
      performed over 101 fixed points over the curve (``[0.0, 0.01, ..., 1.0]``).
  
      Average the result among all the classes to obtain the final, ``mAP``,
      value.
  
      Args:
          output_per_batch (dict): Output of the detector to calculate mAP.
              Expects the following keys: ``bboxes``, ``classes``, ``scores``,
              ``gt_bboxes``, ``gt_classes``. Under each key, there should be a
              list of the results per batch as returned by the detector.
          num_classes (int): Number of classes on the dataset.
  
      Returns:
          (``np.ndarray``, ``ndarray``) tuple. The first value is an array of
          size (`num_classes`,), with the AP value per class, while the second
          one is an array for the AR.
  
      .. _VOC mAP metric:
          http://host.robots.ox.ac.uk/pascal/VOC/pubs/everingham10.pdf
      """
      iou_thresholds = np.linspace(
          0.50, 0.95, np.round((0.95 - 0.50) / 0.05) + 1
      )
      # 101 recall levels, same as COCO evaluation.
      rec_thresholds = np.linspace(
          0.00, 1.00, np.round((1.00 - 0.00) / 0.01) + 1
      )
  
      # List; first by class, then by example. Each entry is a tuple of ndarrays
      # of size (D_{c,i},), for tp/fp labels and for score, where D_{c,i} is the
      # number of detected boxes for class `c` on image `i`.
      tp_fp_labels_by_class = [[] for _ in range(num_classes)]
      num_examples_per_class = [0 for _ in range(num_classes)]
  
      # For each image, order predictions by score and classify each as a true
      # positive or a false positive.
      num_batches = len(output_per_batch["bboxes"])
      for idx in range(num_batches):
          # Get the results of the batch.
          classes = output_per_batch["classes"][idx]  # (D_{c,i},)
          bboxes = output_per_batch["bboxes"][idx]  # (D_{c,i}, 4)
          scores = output_per_batch["scores"][idx]  # (D_{c,i},)
  
          gt_classes = output_per_batch["gt_classes"][idx]
          gt_bboxes = output_per_batch["gt_bboxes"][idx]
  
          # Analysis must be made per-class.
          for cls in range(num_classes):
              # Get the bounding boxes of `cls` only.
              cls_bboxes = bboxes[classes == cls, :]
              cls_scores = scores[classes == cls]
              cls_gt_bboxes = gt_bboxes[gt_classes == cls, :]
  
              num_gt = cls_gt_bboxes.shape[0]
              num_examples_per_class[cls] += num_gt
  
              # Sort by score descending, so we prioritize higher-confidence
              # results when matching.
              sorted_indices = np.argsort(-cls_scores)
  
              # Whether the ground-truth has been previously detected.
              is_detected = np.zeros((num_gt, len(iou_thresholds)))
  
              # TP/FP labels for detected bboxes of (class, image).
              tp_fp_labels = np.zeros(
                  (len(sorted_indices), len(iou_thresholds))
              )
  
              if num_gt == 0:
                  # If no ground truth examples for class, all predictions must
                  # be false positives.
                  tp_fp_labels_by_class[cls].append(
                      (tp_fp_labels, cls_scores[sorted_indices])
                  )
                  continue
  
              # Get the IoUs for the class' bboxes.
              ious = bbox_overlap(cls_bboxes, cls_gt_bboxes)
  
              # Greedily assign bboxes to ground truths (highest score first).
              for bbox_idx in sorted_indices:
                  gt_match = np.argmax(ious[bbox_idx, :])
                  # TODO: Try to vectorize.
                  for iou_idx, iou_threshold in enumerate(
                      iou_thresholds
                  ):
                      if ious[bbox_idx, gt_match] >= iou_threshold:
                          # Over IoU threshold.
                          if not is_detected[gt_match, iou_idx]:
                              # And first detection: it's a true positive.
                              tp_fp_labels[bbox_idx, iou_idx] = True
                              is_detected[gt_match, iou_idx] = True
  
              tp_fp_labels_by_class[cls].append(
                  (tp_fp_labels, cls_scores[sorted_indices])
              )
  
      # Calculate average precision per class.
      ap_per_class = np.zeros((num_classes, len(iou_thresholds)))
      ar_per_class = np.zeros((num_classes, len(iou_thresholds)))
      for cls in range(num_classes):
          tp_fp_labels = tp_fp_labels_by_class[cls]
          num_examples = num_examples_per_class[cls]
  
          # Flatten the tp/fp labels into a single ndarray.
          labels, scores = zip(*tp_fp_labels)
          labels = np.concatenate(labels)
          scores = np.concatenate(scores)
  
          # Sort the tp/fp labels by decreasing confidence score and calculate
          # precision and recall at every position of this ranked output.
          sorted_indices = np.argsort(-scores)
          true_positives = labels[sorted_indices, :]
          false_positives = 1 - true_positives
  
          cum_true_positives = np.cumsum(true_positives, axis=0)
          cum_false_positives = np.cumsum(false_positives, axis=0)
  
          recall = cum_true_positives.astype(float) / num_examples
          precision = np.divide(
              cum_true_positives.astype(float),
              cum_true_positives + cum_false_positives,
          )
  
          # Find AP by integrating over PR curve, with interpolated precision.
          for iou_idx in range(len(iou_thresholds)):
              p = precision[:, iou_idx]
              r = recall[:, iou_idx]
  
              # Interpolate the precision. (Make it monotonically-increasing.)
              for i in range(len(p) - 1, 0, -1):
                  if p[i] > p[i - 1]:
                      p[i - 1] = p[i]
  
              ap = 0
              inds = np.searchsorted(r, rec_thresholds)
              for ridx, pidx in enumerate(inds):
                  if pidx >= len(r):
                      # Out of bounds, no recall higher than threshold for any of
                      # the remaining thresholds (as they're ordered).
                      break
  
                  ap += p[pidx] / len(rec_thresholds)
  
              ap_per_class[cls, iou_idx] = ap
              if len(r):
                  ar_per_class[cls, iou_idx] = r[-1]
              else:
                  ar_per_class[cls, iou_idx] = 0
  
      return ap_per_class, ar_per_class
  
file_path: luminoth/luminoth/eval.py
imports: import click
  import json
  import numpy
  import os
  import tensorflow
  import time
  from luminoth.datasets import get_dataset
  from luminoth.models import get_model
  from luminoth.utils.bbox_overlap import bbox_overlap
  from luminoth.utils.config import get_config
  from luminoth.utils.image_vis import image_vis_summaries
label_desc: Use image node_id 4 for calling a global function with example usage: calculate_metrics(output_per_batch, num_classes) and returns: ap_per_class,  ar_per_class