text: # A modification version from chainercv repository.
  # (See https://github.com/chainer/chainercv/blob/master/chainercv/evaluations/eval_detection_voc.py)
  from __future__ import division
  
  import os
  from collections import defaultdict
  import numpy as np
  from maskrcnn_benchmark.structures.bounding_box import BoxList
  from maskrcnn_benchmark.structures.boxlist_ops import boxlist_iou
  
  
  def do_voc_evaluation(dataset, predictions, output_folder, logger):
      # TODO need to make the use_07_metric format available
      # for the user to choose
      pred_boxlists = []
      gt_boxlists = []
      for image_id, prediction in enumerate(predictions):
          img_info = dataset.get_img_info(image_id)
          if len(prediction) == 0:
              continue
          image_width = img_info["width"]
          image_height = img_info["height"]
          prediction = prediction.resize((image_width, image_height))
          pred_boxlists.append(prediction)
  
          gt_boxlist = dataset.get_groundtruth(image_id)
          gt_boxlists.append(gt_boxlist)
      result = eval_detection_voc(
          pred_boxlists=pred_boxlists,
          gt_boxlists=gt_boxlists,
          iou_thresh=0.5,
          use_07_metric=True,
      )
      result_str = "mAP: {:.4f}\n".format(result["map"])
      for i, ap in enumerate(result["ap"]):
          #if i == 0:  # skip background
          #    continue
          result_str += "{:<16}: {:.4f}\n".format(
              dataset.map_class_id_to_class_name(i), ap
          )
      logger.info(result_str)
      if output_folder:
          with open(os.path.join(output_folder, "result.txt"), "w") as fid:
              fid.write(result_str)
      return result
  
  
  def eval_detection_voc(pred_boxlists, gt_boxlists, iou_thresh=0.5, use_07_metric=False):
      """Evaluate on voc dataset.
      Args:
          pred_boxlists(list[BoxList]): pred boxlist, has labels and scores fields.
          gt_boxlists(list[BoxList]): ground truth boxlist, has labels field.
          iou_thresh: iou thresh
          use_07_metric: boolean
      Returns:
          dict represents the results
      """
      assert len(gt_boxlists) == len(
          pred_boxlists
      ), "Length of gt and pred lists need to be same."
      prec, rec = calc_detection_voc_prec_rec(
          pred_boxlists=pred_boxlists, gt_boxlists=gt_boxlists, iou_thresh=iou_thresh
      )
      ap = calc_detection_voc_ap(prec, rec, use_07_metric=use_07_metric)
      return {"ap": ap, "map": np.nanmean(ap)}
  
  
  def calc_detection_voc_prec_rec(gt_boxlists, pred_boxlists, iou_thresh=0.5):
      """Calculate precision and recall based on evaluation code of PASCAL VOC.
      This function calculates precision and recall of
      predicted bounding boxes obtained from a dataset which has :math:`N`
      images.
      The code is based on the evaluation code used in PASCAL VOC Challenge.
     """
      n_pos = defaultdict(int)
      score = defaultdict(list)
      match = defaultdict(list)
      for gt_boxlist, pred_boxlist in zip(gt_boxlists, pred_boxlists):
          pred_bbox = pred_boxlist.bbox.numpy()
          pred_label = pred_boxlist.get_field("labels").numpy()
          pred_score = pred_boxlist.get_field("scores").numpy()
          gt_bbox = gt_boxlist.bbox.numpy()
          gt_label = gt_boxlist.get_field("labels").numpy()
          gt_difficult = gt_boxlist.get_field("difficult").numpy()
  
          for l in np.unique(np.concatenate((pred_label, gt_label)).astype(int)):
              pred_mask_l = pred_label == l
              pred_bbox_l = pred_bbox[pred_mask_l]
              pred_score_l = pred_score[pred_mask_l]
              # sort by score
              order = pred_score_l.argsort()[::-1]
              pred_bbox_l = pred_bbox_l[order]
              pred_score_l = pred_score_l[order]
  
              gt_mask_l = gt_label == l
              gt_bbox_l = gt_bbox[gt_mask_l]
              gt_difficult_l = gt_difficult[gt_mask_l]
  
              n_pos[l] += np.logical_not(gt_difficult_l).sum()
              score[l].extend(pred_score_l)
  
              if len(pred_bbox_l) == 0:
                  continue
              if len(gt_bbox_l) == 0:
                  match[l].extend((0,) * pred_bbox_l.shape[0])
                  continue
  
              # VOC evaluation follows integer typed bounding boxes.
              pred_bbox_l = pred_bbox_l.copy()
              pred_bbox_l[:, 2:] += 1
              gt_bbox_l = gt_bbox_l.copy()
              gt_bbox_l[:, 2:] += 1
              iou = boxlist_iou(
                  BoxList(pred_bbox_l, gt_boxlist.size),
                  BoxList(gt_bbox_l, gt_boxlist.size),
              ).numpy()
              gt_index = iou.argmax(axis=1)
              # set -1 if there is no matching ground truth
              gt_index[iou.max(axis=1) < iou_thresh] = -1
              del iou
  
              selec = np.zeros(gt_bbox_l.shape[0], dtype=bool)
              for gt_idx in gt_index:
                  if gt_idx >= 0:
                      if gt_difficult_l[gt_idx]:
                          match[l].append(-1)
                      else:
                          if not selec[gt_idx]:
                              match[l].append(1)
                          else:
                              match[l].append(0)
                      selec[gt_idx] = True
                  else:
                      match[l].append(0)
  
      n_fg_class = max(n_pos.keys()) + 1
      prec = [None] * n_fg_class
      rec = [None] * n_fg_class
  
      for l in n_pos.keys():
          score_l = np.array(score[l])
          match_l = np.array(match[l], dtype=np.int8)
  
          order = score_l.argsort()[::-1]
          match_l = match_l[order]
  
          tp = np.cumsum(match_l == 1)
          fp = np.cumsum(match_l == 0)
  
          # If an element of fp + tp is 0,
          # the corresponding element of prec[l] is nan.
          prec[l] = tp / (fp + tp)
          # If n_pos[l] is 0, rec[l] is None.
          if n_pos[l] > 0:
              rec[l] = tp / n_pos[l]
  
      return prec, rec
  
  
  def calc_detection_voc_ap(prec, rec, use_07_metric=False):
      """Calculate average precisions based on evaluation code of PASCAL VOC.
      This function calculates average precisions
      from given precisions and recalls.
      The code is based on the evaluation code used in PASCAL VOC Challenge.
      Args:
          prec (list of numpy.array): A list of arrays.
              :obj:`prec[l]` indicates precision for class :math:`l`.
              If :obj:`prec[l]` is :obj:`None`, this function returns
              :obj:`numpy.nan` for class :math:`l`.
          rec (list of numpy.array): A list of arrays.
              :obj:`rec[l]` indicates recall for class :math:`l`.
              If :obj:`rec[l]` is :obj:`None`, this function returns
              :obj:`numpy.nan` for class :math:`l`.
          use_07_metric (bool): Whether to use PASCAL VOC 2007 evaluation metric
              for calculating average precision. The default value is
              :obj:`False`.
      Returns:
          ~numpy.ndarray:
          This function returns an array of average precisions.
          The :math:`l`-th value corresponds to the average precision
          for class :math:`l`. If :obj:`prec[l]` or :obj:`rec[l]` is
          :obj:`None`, the corresponding value is set to :obj:`numpy.nan`.
      """
  
      n_fg_class = len(prec)
      ap = np.empty(n_fg_class)
      for l in range(n_fg_class):
          if prec[l] is None or rec[l] is None:
              ap[l] = np.nan
              continue
  
          if use_07_metric:
              # 11 point metric
              ap[l] = 0
              for t in np.arange(0.0, 1.1, 0.1):
                  if np.sum(rec[l] >= t) == 0:
                      p = 0
                  else:
                      p = np.max(np.nan_to_num(prec[l])[rec[l] >= t])
                  ap[l] += p / 11
          else:
              # correct AP calculation
              # first append sentinel values at the end
              mpre = np.concatenate(([0], np.nan_to_num(prec[l]), [0]))
              mrec = np.concatenate(([0], rec[l], [1]))
  
              mpre = np.maximum.accumulate(mpre[::-1])[::-1]
  
              # to calculate area under PR curve, look for points
              # where X axis (recall) changes value
              i = np.where(mrec[1:] != mrec[:-1])[0]
  
              # and sum (\Delta recall) * prec
              ap[l] = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
  
      return ap
  