text: from multiprocessing import Pool
  
  import mmcv
  import numpy as np
  from mmcv.utils import print_log
  from terminaltables import AsciiTable
  
  from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps
  from mmdet.core.evaluation.class_names import get_classes
  
  import pandas as pd
  
  def average_precision(recalls, precisions, mode='area'):
      """Calculate average precision (for single or multiple scales).
  
      Args:
          recalls (ndarray): shape (num_scales, num_dets) or (num_dets, )
          precisions (ndarray): shape (num_scales, num_dets) or (num_dets, )
          mode (str): 'area' or '11points', 'area' means calculating the area
              under precision-recall curve, '11points' means calculating
              the average precision of recalls at [0, 0.1, ..., 1]
  
      Returns:
          float or ndarray: calculated average precision
      """
      no_scale = False
      if recalls.ndim == 1:
          no_scale = True
          recalls = recalls[np.newaxis, :]
          precisions = precisions[np.newaxis, :]
      assert recalls.shape == precisions.shape and recalls.ndim == 2
      num_scales = recalls.shape[0]
      ap = np.zeros(num_scales, dtype=np.float32)
      if mode == 'area':
          zeros = np.zeros((num_scales, 1), dtype=recalls.dtype)
          ones = np.ones((num_scales, 1), dtype=recalls.dtype)
          mrec = np.hstack((zeros, recalls, ones))
          mpre = np.hstack((zeros, precisions, zeros))
          for i in range(mpre.shape[1] - 1, 0, -1):
              mpre[:, i - 1] = np.maximum(mpre[:, i - 1], mpre[:, i])
          for i in range(num_scales):
              ind = np.where(mrec[i, 1:] != mrec[i, :-1])[0]
              ap[i] = np.sum(
                  (mrec[i, ind + 1] - mrec[i, ind]) * mpre[i, ind + 1])
      elif mode == '11points':
          for i in range(num_scales):
              for thr in np.arange(0, 1 + 1e-3, 0.1):
                  precs = precisions[i, recalls[i, :] >= thr]
                  prec = precs.max() if precs.size > 0 else 0
                  ap[i] += prec
          ap /= 11
      else:
          raise ValueError(
              'Unrecognized mode, only "area" and "11points" are supported')
      if no_scale:
          ap = ap[0]
      return ap
  
  
  def tpfp_imagenet(det_bboxes,
                    gt_bboxes,
                    gt_bboxes_ignore=None,
                    default_iou_thr=0.5,
                    area_ranges=None):
      """Check if detected bboxes are true positive or false positive.
  
      Args:
          det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5).
          gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4).
          gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image,
              of shape (k, 4). Default: None
          default_iou_thr (float): IoU threshold to be considered as matched for
              medium and large bboxes (small ones have special rules).
              Default: 0.5.
          area_ranges (list[tuple] | None): Range of bbox areas to be evaluated,
              in the format [(min1, max1), (min2, max2), ...]. Default: None.
  
      Returns:
          tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of
              each array is (num_scales, m).
      """
      # an indicator of ignored gts
      gt_ignore_inds = np.concatenate(
          (np.zeros(gt_bboxes.shape[0], dtype=np.bool),
           np.ones(gt_bboxes_ignore.shape[0], dtype=np.bool)))
      # stack gt_bboxes and gt_bboxes_ignore for convenience
      gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore))
  
      num_dets = det_bboxes.shape[0]
      num_gts = gt_bboxes.shape[0]
      if area_ranges is None:
          area_ranges = [(None, None)]
      num_scales = len(area_ranges)
      # tp and fp are of shape (num_scales, num_gts), each row is tp or fp
      # of a certain scale.
      tp = np.zeros((num_scales, num_dets), dtype=np.float32)
      fp = np.zeros((num_scales, num_dets), dtype=np.float32)
      if gt_bboxes.shape[0] == 0:
          if area_ranges == [(None, None)]:
              fp[...] = 1
          else:
              det_areas = (det_bboxes[:, 2] - det_bboxes[:, 0]) * (
                  det_bboxes[:, 3] - det_bboxes[:, 1])
              for i, (min_area, max_area) in enumerate(area_ranges):
                  fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1
          return tp, fp
      ious = bbox_overlaps(det_bboxes, gt_bboxes - 1)
      gt_w = gt_bboxes[:, 2] - gt_bboxes[:, 0]
      gt_h = gt_bboxes[:, 3] - gt_bboxes[:, 1]
      iou_thrs = np.minimum((gt_w * gt_h) / ((gt_w + 10.0) * (gt_h + 10.0)),
                            default_iou_thr)
      # sort all detections by scores in descending order
      sort_inds = np.argsort(-det_bboxes[:, -1])
      for k, (min_area, max_area) in enumerate(area_ranges):
          gt_covered = np.zeros(num_gts, dtype=bool)
          # if no area range is specified, gt_area_ignore is all False
          if min_area is None:
              gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool)
          else:
              gt_areas = gt_w * gt_h
              gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area)
          for i in sort_inds:
              max_iou = -1
              matched_gt = -1
              # find best overlapped available gt
              for j in range(num_gts):
                  # different from PASCAL VOC: allow finding other gts if the
                  # best overlaped ones are already matched by other det bboxes
                  if gt_covered[j]:
                      continue
                  elif ious[i, j] >= iou_thrs[j] and ious[i, j] > max_iou:
                      max_iou = ious[i, j]
                      matched_gt = j
              # there are 4 cases for a det bbox:
              # 1. it matches a gt, tp = 1, fp = 0
              # 2. it matches an ignored gt, tp = 0, fp = 0
              # 3. it matches no gt and within area range, tp = 0, fp = 1
              # 4. it matches no gt but is beyond area range, tp = 0, fp = 0
              if matched_gt >= 0:
                  gt_covered[matched_gt] = 1
                  if not (gt_ignore_inds[matched_gt]
                          or gt_area_ignore[matched_gt]):
                      tp[k, i] = 1
              elif min_area is None:
                  fp[k, i] = 1
              else:
                  bbox = det_bboxes[i, :4]
                  area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
                  if area >= min_area and area < max_area:
                      fp[k, i] = 1
      return tp, fp
  
  
  def tpfp_default(det_bboxes,
                   gt_bboxes,
                   gt_bboxes_ignore=None,
                   iou_thr=0.5,
                   area_ranges=None,
                   class_index=None,
                   gt_bboxes_all_classes=None):
      """Check if detected bboxes are true positive or false positive.
  
      Args:
          det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5).
          gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4).
          gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image,
              of shape (k, 4). Default: None
          iou_thr (float): IoU threshold to be considered as matched.
              Default: 0.5.
          area_ranges (list[tuple] | None): Range of bbox areas to be evaluated,
              in the format [(min1, max1), (min2, max2), ...]. Default: None.
  
      Returns:
          tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of
              each array is (num_scales, m).
      """
      # an indicator of ignored gts
      gt_ignore_inds = np.concatenate(
          (np.zeros(gt_bboxes.shape[0], dtype=np.bool),
           np.ones(gt_bboxes_ignore.shape[0], dtype=np.bool)))
      # stack gt_bboxes and gt_bboxes_ignore for convenience
      gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore))
  
      num_dets = det_bboxes.shape[0]
      num_gts = gt_bboxes.shape[0]
      if area_ranges is None:
          area_ranges = [(None, None)]
      num_scales = len(area_ranges)
      # tp and fp are of shape (num_scales, num_gts), each row is tp or fp of
      # a certain scale
      tp = np.zeros((num_scales, num_dets), dtype=np.float32)
      fp = np.zeros((num_scales, num_dets), dtype=np.float32)
      fp_redundant = np.zeros((num_scales, num_dets), dtype=np.float32)
  
      gt_covered = []
      ious_max = -1 * np.ones(num_dets, dtype=np.float32)
      ious_argmax = -1 * np.ones(num_dets, dtype=np.float32)
      matched_classes = -1*np.ones(num_dets, dtype=np.float32)
  
  
      # if there is no gt bboxes in this image, then all det bboxes
      # within area range are false positives
      if gt_bboxes.shape[0] == 0:
          if area_ranges == [(None, None)]:
              fp[...] = 1
          else:
              det_areas = (det_bboxes[:, 2] - det_bboxes[:, 0]) * (
                  det_bboxes[:, 3] - det_bboxes[:, 1])
              for i, (min_area, max_area) in enumerate(area_ranges):
                  fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1
  
          # return tp, fp, fp_redundant, gt_covered, ious_max, ious_argmax, -1 * np.ones(num_dets, dtype=np.float32), [], []
  
      else:
          ious = bbox_overlaps(det_bboxes, gt_bboxes)
          # for each det, the max iou with all gts
          ious_max = ious.max(axis=1)
          # for each det, which gt overlaps most with it
          ious_argmax = ious.argmax(axis=1)
          # sort all dets in descending order by scores
          sort_inds = np.argsort(-det_bboxes[:, -1])
          for k, (min_area, max_area) in enumerate(area_ranges):
              gt_covered = np.zeros(num_gts, dtype=bool)
              # if no area range is specified, gt_area_ignore is all False
              if min_area is None:
                  gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool)
              else:
                  gt_areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0]) * (
                      gt_bboxes[:, 3] - gt_bboxes[:, 1])
                  gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area)
              for i in sort_inds:
                  if ious_max[i] >= iou_thr:
                      matched_gt = ious_argmax[i]
                      if not (gt_ignore_inds[matched_gt]
                              or gt_area_ignore[matched_gt]):
                          if not gt_covered[matched_gt]:
                              gt_covered[matched_gt] = True
                              tp[k, i] = 1
                          else:
                              fp[k, i] = 1
                              fp_redundant[k, i] = 1
                      # otherwise ignore this detected bbox, tp = 0, fp = 0
                  elif min_area is None:
                      fp[k, i] = 1
                  else:
                      bbox = det_bboxes[i, :4]
                      area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
                      if area >= min_area and area < max_area:
                          fp[k, i] = 1
  
      if gt_bboxes.shape[0] != 0:
          matched_classes[np.where(ious_max != 0)] = class_index
  
      # Check if det matches gt from other class
      if -1 in matched_classes and gt_bboxes_all_classes['bboxes'].shape[0] != 0:
          unmatched_inds = np.where(matched_classes == -1)
  
          ious_all_classes = bbox_overlaps(det_bboxes, gt_bboxes_all_classes['bboxes'])
          ious_max_all_classes = ious_all_classes.max(axis=1)[unmatched_inds]
          ious_argmax_all_classes = ious_all_classes.argmax(axis=1)[unmatched_inds]
  
          labels = gt_bboxes_all_classes['labels'][ious_argmax_all_classes]
          labels[np.where(ious_max_all_classes == 0)] = -1
          matched_classes[unmatched_inds] = labels
  
          new_iou_max = np.array(ious_max_all_classes, copy=True)
          new_iou_max[np.where(ious_max_all_classes == 0)] = -1
          ious_max[unmatched_inds] = new_iou_max
  
          new_ious_argmax = np.array(ious_argmax_all_classes, copy=True)
          new_ious_argmax[np.where(ious_max_all_classes == 0)] = -1
          # Fix matched_gt_index. Find index within each class
          for i in range(len(new_ious_argmax)):
              index_all_gts = new_ious_argmax[i]
              if index_all_gts != -1:
                  label = gt_bboxes_all_classes['labels'][index_all_gts]
                  new_ious_argmax[i] = np.where(gt_bboxes_all_classes['labels'][:index_all_gts] == label)[0].shape[0]
  
          ious_argmax[unmatched_inds] = new_ious_argmax
  
      gt_iou_max, gt_iou_argmax = [], []
      if det_bboxes.shape[0] != 0 and gt_bboxes.shape[0] != 0:
          gt_iou_max, gt_iou_argmax = ious.max(axis=0), ious.argmax(axis=0)
          unmatched_inds = np.where(gt_iou_max == 0)
          gt_iou_max[unmatched_inds] = -1
          gt_iou_argmax[unmatched_inds] = -1
  
      return tp, fp, fp_redundant, gt_covered, ious_max, ious_argmax, matched_classes, gt_iou_max, gt_iou_argmax
  
  
  
  def get_cls_results(det_results, annotations, class_id):
      """Get det results and gt information of a certain class.
  
      Args:
          det_results (list[list]): Same as `eval_map()`.
          annotations (list[dict]): Same as `eval_map()`.
          class_id (int): ID of a specific class.
  
      Returns:
          tuple[list[np.ndarray]]: detected bboxes, gt bboxes, ignored gt bboxes
      """
      cls_dets = [img_res[class_id] for img_res in det_results]
      cls_gts = []
      cls_gts_ignore = []
      for ann in annotations:
          gt_inds = ann['labels'] == class_id
          cls_gts.append(ann['bboxes'][gt_inds, :])
  
          if ann.get('labels_ignore', None) is not None:
              ignore_inds = ann['labels_ignore'] == class_id
              cls_gts_ignore.append(ann['bboxes_ignore'][ignore_inds, :])
          else:
              cls_gts_ignore.append(np.empty((0, 4), dtype=np.float32))
  
      return cls_dets, cls_gts, cls_gts_ignore
  
  
  def eval_map(det_results,
               annotations,
               scale_ranges=None,
               iou_thr=0.5,
               dataset=None,
               logger=None,
               tpfp_fn=None,
               nproc=4,
               model_name=None):
      """Evaluate mAP of a dataset.
  
      Args:
          det_results (list[list]): [[cls1_det, cls2_det, ...], ...].
              The outer list indicates images, and the inner list indicates
              per-class detected bboxes.
          annotations (list[dict]): Ground truth annotations where each item of
              the list indicates an image. Keys of annotations are:
  
              - `bboxes`: numpy array of shape (n, 4)
              - `labels`: numpy array of shape (n, )
              - `bboxes_ignore` (optional): numpy array of shape (k, 4)
              - `labels_ignore` (optional): numpy array of shape (k, )
          scale_ranges (list[tuple] | None): Range of scales to be evaluated,
              in the format [(min1, max1), (min2, max2), ...]. A range of
              (32, 64) means the area range between (32**2, 64**2).
              Default: None.
          iou_thr (float): IoU threshold to be considered as matched.
              Default: 0.5.
          dataset (list[str] | str | None): Dataset name or dataset classes,
              there are minor differences in metrics for different datsets, e.g.
              "voc07", "imagenet_det", etc. Default: None.
          logger (logging.Logger | str | None): The way to print the mAP
              summary. See `mmcv.utils.print_log()` for details. Default: None.
          tpfp_fn (callable | None): The function used to determine true/
              false positives. If None, :func:`tpfp_default` is used as default
              unless dataset is 'det' or 'vid' (:func:`tpfp_imagenet` in this
              case). If it is given as a function, then this function is used
              to evaluate tp & fp. Default None.
          nproc (int): Processes used for computing TP and FP.
              Default: 4.
  
      Returns:
          tuple: (mAP, [dict, dict, ...])
      """
      assert len(det_results) == len(annotations)
  
      num_imgs = len(det_results)
      num_scales = len(scale_ranges) if scale_ranges is not None else 1
      num_classes = len(det_results[0])  # positive class num
      area_ranges = ([(rg[0]**2, rg[1]**2) for rg in scale_ranges]
                     if scale_ranges is not None else None)
  
      pool = Pool(nproc)
      eval_results = []
      for i in range(num_classes):
          iou_thr_c = iou_thr if i!=0 else 0.7
          # get gt and det bboxes of this class
          cls_dets, cls_gts, cls_gts_ignore = get_cls_results(
              det_results, annotations, i)
          # choose proper function according to datasets to compute tp and fp
          if tpfp_fn is None:
              if dataset in ['det', 'vid']:
                  tpfp_fn = tpfp_imagenet
              else:
                  tpfp_fn = tpfp_default
          if not callable(tpfp_fn):
              raise ValueError(
                  f'tpfp_fn has to be a function or None, but got {tpfp_fn}')
  
          # compute tp and fp for each image with multiple processes
          tpfp = pool.starmap(
              tpfp_fn,
              zip(cls_dets, cls_gts, cls_gts_ignore,
                  [iou_thr_c for _ in range(num_imgs)],
                  [area_ranges for _ in range(num_imgs)],
                  [i for _ in range(num_imgs)],
                  annotations))
  
          # tp, fp = tuple(zip(*tpfp))
          # tpfp = [tpfp_fn(dets, gts, gts_ignore, iou_t, area_r) for dets, gts, gts_ignore, iou_t, area_r
          #         in zip(cls_dets, cls_gts, cls_gts_ignore, [iou_thr for _ in range(num_imgs)], [area_ranges for _ in range(num_imgs)])]
  
          # ## TODO: FIX names
          # tp, fp = tuple(zip(*[t[:2] for t in tpfp]))
          tp, fp, fp_redundant, gt_covered, dets_iou_max, dets_iou_argmax, dets_matched_class, gt_iou_max, gt_iou_argmax = tuple(zip(* tpfp))
          tp_list, fp_list = tp, fp
  
          # calculate gt number of each scale
          # ignored gts or gts beyond the specific scale are not counted
          num_gts = np.zeros(num_scales, dtype=int)
          for j, bbox in enumerate(cls_gts):
              if area_ranges is None:
                  num_gts[0] += bbox.shape[0]
              else:
                  gt_areas = (bbox[:, 2] - bbox[:, 0]) * (
                      bbox[:, 3] - bbox[:, 1])
                  for k, (min_area, max_area) in enumerate(area_ranges):
                      num_gts[k] += np.sum((gt_areas >= min_area)
                                           & (gt_areas < max_area))
          # sort all det bboxes by score, also sort tp and fp
          cls_dets = np.vstack(cls_dets)
          num_dets = cls_dets.shape[0]
          sort_inds = np.argsort(-cls_dets[:, -1])
          tp = np.hstack(tp)[:, sort_inds]
          fp = np.hstack(fp)[:, sort_inds]
          # calculate recall and precision with tp and fp
          tp = np.cumsum(tp, axis=1)
          fp = np.cumsum(fp, axis=1)
          eps = np.finfo(np.float32).eps
          recalls = tp / np.maximum(num_gts[:, np.newaxis], eps)
          precisions = tp / np.maximum((tp + fp), eps)
          # calculate AP
          if scale_ranges is None:
              recalls = recalls[0, :]
              precisions = precisions[0, :]
              num_gts = num_gts.item()
          mode = 'area' if dataset != 'voc07' else '11points'
          ap = average_precision(recalls, precisions, mode)
          eval_results.append({
              'num_gts': num_gts,
              'num_dets': num_dets,
              'recall': recalls,
              'precision': precisions,
              'ap': ap,
              'tp': tp_list,
              'fp':  fp_list,
              'fp_redundant': fp_redundant,
              'gt_covered':  gt_covered,
              'dets_iou_max': dets_iou_max, 
              'dets_iou_argmax': dets_iou_argmax,
              'dets_matched_class': dets_matched_class,
              'gts_iou_max': gt_iou_max, 
              'gts_iou_argmax': gt_iou_argmax
          })
      pool.close()
      if scale_ranges is not None:
          # shape (num_classes, num_scales)
          all_ap = np.vstack([cls_result['ap'] for cls_result in eval_results])
          all_num_gts = np.vstack(
              [cls_result['num_gts'] for cls_result in eval_results])
          mean_ap = []
          for i in range(num_scales):
              if np.any(all_num_gts[:, i] > 0):
                  mean_ap.append(all_ap[all_num_gts[:, i] > 0, i].mean())
              else:
                  mean_ap.append(0.0)
      else:
          aps = []
          for cls_result in eval_results:
              if cls_result['num_gts'] > 0:
                  aps.append(cls_result['ap'])
          mean_ap = np.array(aps).mean().item() if aps else 0.0
  
      df_summary = print_map_summary(
          mean_ap, eval_results, dataset, area_ranges, logger=logger, model_name=model_name)
  
      return mean_ap, eval_results, df_summary
  
  
  def print_map_summary(mean_ap,
                        results,
                        dataset=None,
                        scale_ranges=None,
                        logger=None,
                        model_name=None):
      """Print mAP and results of each class.
  
      A table will be printed to show the gts/dets/recall/AP of each class and
      the mAP.
  
      Args:
          mean_ap (float): Calculated from `eval_map()`.
          results (list[dict]): Calculated from `eval_map()`.
          dataset (list[str] | str | None): Dataset name or dataset classes.
          scale_ranges (list[tuple] | None): Range of scales to be evaluated.
          logger (logging.Logger | str | None): The way to print the mAP
              summary. See `mmcv.utils.print_log()` for details. Default: None.
      """
  
      if logger == 'silent':
          return
  
      if isinstance(results[0]['ap'], np.ndarray):
          num_scales = len(results[0]['ap'])
      else:
          num_scales = 1
  
      if scale_ranges is not None:
          assert len(scale_ranges) == num_scales
  
      num_classes = len(results)
  
      recalls = np.zeros((num_scales, num_classes), dtype=np.float32)
      aps = np.zeros((num_scales, num_classes), dtype=np.float32)
      num_gts = np.zeros((num_scales, num_classes), dtype=int)
      for i, cls_result in enumerate(results):
          if cls_result['recall'].size > 0:
              recalls[:, i] = np.array(cls_result['recall'], ndmin=2)[:, -1]
          aps[:, i] = cls_result['ap']
          num_gts[:, i] = cls_result['num_gts']
  
      if dataset is None:
          label_names = [str(i) for i in range(num_classes)]
      elif mmcv.is_str(dataset):
          label_names = get_classes(dataset)
      else:
          label_names = dataset
  
      if not isinstance(mean_ap, list):
          mean_ap = [mean_ap]
  
      header = ['class', 'gts', 'dets', 'recall', 'ap', 'TP', 'FP', 'FPRed', 'FN']
      for i in range(num_scales):
          if scale_ranges is not None:
              print_log(f'Scale range {scale_ranges[i]}', logger=logger)
          table_data = [header]
          for j in range(num_classes):
              tps = int(sum([np.sum(tps) for tps in results[j]['tp']]))
              fps = int(sum([np.sum(fps) for fps in results[j]['fp']]))
              fps_red = int(sum([np.sum(fps) for fps in results[j]['fp_redundant']]))
              row_data = [
                  label_names[j], num_gts[i, j], results[j]['num_dets'],
                  f'{recalls[i, j]*100:.2f}', f'{aps[i, j]*100:.2f}',
                  tps, fps, fps_red, num_gts[i, j] - tps
              ]
              table_data.append(row_data)
          table_data.append(['All', np.sum(num_gts), sum(t[2] for t in table_data[1:4]),
                             np.mean([float(t[3]) for t in table_data[1:4]]).round(2),
                             np.float32(mean_ap[i]*100).round(2),
                             sum(t[5] for t in table_data[1:4]),
                             sum(t[6] for t in table_data[1:4]),
                             sum(t[7] for t in table_data[1:4]),
                             sum(t[8] for t in table_data[1:4])
                             ])
          table = AsciiTable(table_data)
          table.inner_footing_row_border = True
          print_log('\n' + table.table, logger=logger)
  
          df_summary = pd.DataFrame(table_data[1:], columns=header)
          df_summary["model_name"] = [model_name for _ in range(4)]
          df_summary = df_summary.set_index('model_name')
          return df_summary
  