text: from __future__ import print_function
  
  import numpy as np
  import json
  import os
  import pdb
  import editdistance
  import torch
  from utils import labels_to_text
  
  
  def compute_overlap(a, b):
      """
      Parameters
      ----------
      a: (N, 4) ndarray of float
      b: (K, 4) ndarray of float
      Returns
      -------
      overlaps: (N, K) ndarray of overlap between boxes and query_boxes
      """
      area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])
  
      iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0])
      ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1])
  
      iw = np.maximum(iw, 0)
      ih = np.maximum(ih, 0)
  
      ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih
  
      ua = np.maximum(ua, np.finfo(float).eps)
  
      intersection = iw * ih
  
      return intersection / ua
  
  
  def _compute_ap(recall, precision):
      """ Compute the average precision, given the recall and precision curves.
      Code originally from https://github.com/rbgirshick/py-faster-rcnn.
      # Arguments
          recall:    The recall curve (list).
          precision: The precision curve (list).
      # Returns
          The average precision as computed in py-faster-rcnn.
      """
      # correct AP calculation
      # first append sentinel values at the end
      mrec = np.concatenate(([0.], recall, [1.]))
      mpre = np.concatenate(([0.], precision, [0.]))
  
      # compute the precision envelope
      for i in range(mpre.size - 1, 0, -1):
          mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
  
      # to calculate area under PR curve, look for points
      # where X axis (recall) changes value
      i = np.where(mrec[1:] != mrec[:-1])[0]
  
      # and sum (\Delta recall) * prec
      ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
      return ap
  
  
  def _get_detections(dataset, retinanet, score_threshold=0.05, max_detections=100, save_path=None):
      """ Get the detections from the retinanet using the generator.
      The result is a list of lists such that the size is:
          all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes]
      # Arguments
          dataset         : The generator used to run images through the retinanet.
          retinanet           : The retinanet to run on the images.
          score_threshold : The score confidence threshold to use.
          max_detections  : The maximum number of detections to use per image.
          save_path       : The path to save the images with visualized detections to.
      # Returns
          A list of lists containing the detections for each image in the generator.
      """
      all_detections = [[None for i in range(dataset.num_classes())] for j in range(len(dataset))]
      all_transcriptions = {}
      retinanet.eval()
      with torch.no_grad():
  
          for index in range(len(dataset)):
              data = dataset[index]
              scale = data['scale']
              # run network
              scores, labels, boxes,transcriptions = retinanet(data['img'].permute(2, 0, 1).cuda().float().unsqueeze(dim=0))
              scores = scores.cpu().numpy()
              labels = labels.cpu().numpy()
              boxes  = boxes.cpu().numpy()
              transcriptions  = transcriptions.cpu().numpy()
              
              # correct boxes for image scale
              boxes /= scale
  
              # select indices which have a score above the threshold
              indices = np.where(scores > score_threshold)[0]
              if indices.shape[0] > 0:
                  # select those scores
                  scores = scores[indices]
  
                  # find the order with which to sort the scores
                  scores_sort = np.argsort(-scores)[:max_detections]
  
                  # select detections
                  image_boxes      = boxes[indices[scores_sort], :]
                  image_scores     = scores[scores_sort]
                  image_labels     = labels[indices[scores_sort]]
                  transcriptions = transcriptions[indices[scores_sort]]
                  transcriptions=np.argmax(transcriptions,axis=-1)
                  image_detections = np.concatenate([image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1)], axis=1)
  
                  # copy detections to all_detections
                  all_transcriptions[index]={}
                  for label in range(dataset.num_classes()):
                      all_detections[index][label] = image_detections[image_detections[:, -1] == label, :-1]
                      all_transcriptions[index][label]=transcriptions[image_detections[:, -1] == label, :-1]
  
              else:
                  # copy detections to all_detections
                  for label in range(dataset.num_classes()):
                      all_detections[index][label] = np.zeros((0, 5))
  
              print('{}/{}'.format(index + 1, len(dataset)), end='\r')
  
      return all_detections,all_transcriptions
  
  
  def _get_annotations(generator):
      """ Get the ground truth annotations from the generator.
      The result is a list of lists such that the size is:
          all_detections[num_images][num_classes] = annotations[num_detections, 5]
      # Arguments
          generator : The generator used to retrieve ground truth annotations.
      # Returns
          A list of lists containing the annotations for each image in the generator.
      """
      all_annotations = [[None for i in range(generator.num_classes())] for j in range(len(generator))]
      all_transcriptions={}
      for i in range(len(generator)):
          # load the annotations
          annotations = generator.load_annotations(i)
  
          # copy detections to all_annotations
          all_transcriptions[i]={}
          for label in range(generator.num_classes()):
              all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy()
              all_transcriptions[i][label]=annotations[annotations[:, 4] == label, 5:].copy()
  
          print('{}/{}'.format(i + 1, len(generator)), end='\r')
  
      return all_annotations,all_transcriptions
  
  
  def evaluate(
      generator,
      retinanet,
      iou_threshold=0.5,
      score_threshold=0.05,
      max_detections=400,
      save_path=None
  ):
      """ Evaluate a given dataset using a given retinanet.
      # Arguments
          generator       : The generator that represents the dataset to evaluate.
          retinanet           : The retinanet to evaluate.
          iou_threshold   : The threshold used to consider when a detection is positive or negative.
          score_threshold : The score confidence threshold to use for detections.
          max_detections  : The maximum number of detections to use per image.
          save_path       : The path to save images with visualized detections to.
      # Returns
          A dict mapping class names to mAP scores.
      """
  
  
      # gather all detections and annotations
      all_detections,all_text_preds     = _get_detections(generator, retinanet, score_threshold=score_threshold, max_detections=max_detections, save_path=save_path)
      all_annotations,all_text_annots    = _get_annotations(generator)
      average_precisions = {}
      cers=[]
      for label in range(generator.num_classes()):
          false_positives = np.zeros((0,))
          true_positives  = np.zeros((0,))
          scores          = np.zeros((0,))
          num_annotations = 0.0
  
          for i in range(len(generator)):
              detections           = all_detections[i][label]
              annotations          = all_annotations[i][label]
              text_dets =              all_text_preds[i][label]
              text_annots = all_text_annots[i][label]
              num_annotations     += annotations.shape[0]
              detected_annotations = []
              for j in range(len(detections)):
              #for d in detections:
                  d = detections[j]
                  scores = np.append(scores, d[4])
  
                  if annotations.shape[0] == 0:
                      false_positives = np.append(false_positives, 1)
                      true_positives  = np.append(true_positives, 0)
                      continue
  
                  overlaps            = compute_overlap(np.expand_dims(d, axis=0), annotations)
                  assigned_annotation = np.argmax(overlaps, axis=1)
                  max_overlap         = overlaps[0, assigned_annotation]
  
                  if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
                      false_positives = np.append(false_positives, 0)
                      true_positives  = np.append(true_positives, 1)
                      pred_str=labels_to_text(text_dets[j].astype('int'),retinanet.module.alphabet)
                      lab_str=labels_to_text(text_annots[assigned_annotation[0]].astype('int'),retinanet.module.alphabet)
                      if len(lab_str)>0:
                          cer=float(editdistance.eval(pred_str,lab_str)/len(lab_str))
  
                      cers.append(cer)
                      print("CER",cer)
                      detected_annotations.append(assigned_annotation)
                  else:
                      false_positives = np.append(false_positives, 1)
                      true_positives  = np.append(true_positives, 0)
  
          # no annotations -> AP for this class is 0 (is this correct?)
          if num_annotations == 0:
              average_precisions[label] = 0, 0
              continue
          # sort by score
          indices         = np.argsort(-scores)
          false_positives = false_positives[indices]
          true_positives  = true_positives[indices]
  
          # compute false positives and true positives
          false_positives = np.cumsum(false_positives)
          true_positives  = np.cumsum(true_positives)
  
          # compute recall and precision
          recall    = true_positives / num_annotations
          precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)
  
          # compute average precision
          average_precision  = _compute_ap(recall, precision)
          average_precisions[label] = average_precision, num_annotations
      
      print('\nmAP:')
      for label in range(generator.num_classes()):
          label_name = generator.label_to_name(label)
          print('{}: {}'.format(label_name, average_precisions[label][0]))
      print("Per box CER",np.mean(cers)) 
      return average_precisions
  
  