text: # --------------------------------------------------------
  # Fast R-CNN
  # Copyright (c) 2015 Microsoft
  # Licensed under The MIT License [see LICENSE for details]
  # Written by Ross Girshick
  # --------------------------------------------------------
  
  from __future__ import print_function
  import os, pdb
  import xml.dom.minidom as minidom
  import numpy as np
  import scipy.sparse
  import scipy.io as sio
  import pickle as cp
  import subprocess
  from .imdb import imdb
  from .voc_eval import voc_eval
  #from fastRCNN.imdb import imdb
  #from fastRCNN.voc_eval import voc_eval
  
  class pascal_voc(imdb):
      def __init__(self, image_set, year, classes, maxNrRois, cacheDir, devkit_path=None):
          imdb.__init__(self, 'voc_' + year + '_' + image_set)
          self._year = year
          self._image_set = image_set
          self._maxNrRois = maxNrRois
          self._ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
          self._cacheDir = cacheDir
          self._devkit_path = self._get_default_path() if devkit_path is None \
                              else os.path.join(devkit_path, 'VOCdevkit' + self._year)
          self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year)
          self._classes = classes
                           #('__background__', # always index 0
                           # 'aeroplane', 'bicycle', 'bird', 'boat',
                           # 'bottle', 'bus', 'car', 'cat', 'chair',
                           # 'cow', 'diningtable', 'dog', 'horse',
                           # 'motorbike', 'person', 'pottedplant',
                           # 'sheep', 'sofa', 'train', 'tvmonitor')
          self._class_to_ind = dict(zip(self.classes, range(self.num_classes)))
          self._image_ext = '.jpg'
          self._image_index = self._load_image_set_index()
          # Default to roidb handler
          self._roidb_handler = self.selective_search_roidb
  
          # PASCAL specific config options
          self.config = {'cleanup'  : True,
                         'use_salt' : True,
                         'top_k'    : 2000}
  
          assert os.path.exists(self._devkit_path), \
                  'VOCdevkit path does not exist: {}'.format(self._devkit_path)
          assert os.path.exists(self._data_path), \
                  'Path does not exist: {}'.format(self._data_path)
  
      @property
      def cache_path(self):
          cache_path = self._cacheDir
          #cache_path = osp.abspath(osp.join(datasets.ROOT_DIR, 'data', 'cache'))
          if not os.path.exists(cache_path):
              os.makedirs(cache_path)
          return cache_path
  
      def image_path_at(self, i):
          """
          Return the absolute path to image i in the image sequence.
          """
          return self.image_path_from_index(self._image_index[i])
  
      def image_path_from_index(self, index):
          """
          Construct an image path from the image's "index" identifier.
          """
          image_path = os.path.join(self._data_path, 'JPEGImages',
                                    index + self._image_ext)
          assert os.path.exists(image_path), \
                  'Path does not exist: {}'.format(image_path)
          return image_path
  
      def _load_image_set_index(self):
          """
          Load the indexes listed in this dataset's image set file.
          """
          # Example path to image set file:
          # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt
          image_set_file = os.path.join(self._data_path, 'ImageSets', 'Main',
                                        self._image_set + '.txt')
          assert os.path.exists(image_set_file), \
                  'Path does not exist: {}'.format(image_set_file)
          with open(image_set_file) as f:
              image_index = [x.strip() for x in f.readlines()]
          return image_index
  
      def _get_default_path(self):
          """
          Return the default path where PASCAL VOC is expected to be installed.
          """
          return os.path.join(self._ROOT_DIR, 'data', 'pascalVoc', 'VOCdevkit' + self._year)
  
      def gt_roidb(self):
          """
          Return the database of ground-truth regions of interest.
  
          This function loads/saves from/to a cache file to speed up future calls.
          """
          cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
          if os.path.exists(cache_file):
              with open(cache_file, 'rb') as fid:
                  roidb = cp.load(fid)
              print ('{} gt roidb loaded from {}'.format(self.name, cache_file))
              return roidb
  
          gt_roidb = [self._load_pascal_annotation(index)
                      for index in self.image_index]
          with open(cache_file, 'wb') as fid:
              cp.dump(gt_roidb, fid, cp.HIGHEST_PROTOCOL)
          print ('wrote gt roidb to {}'.format(cache_file))
  
          return gt_roidb
  
      def selective_search_roidb(self):
          """
          Return the database of selective search regions of interest.
          Ground-truth ROIs are also included.
  
          This function loads/saves from/to a cache file to speed up future calls.
          """
          cache_file = os.path.join(self.cache_path,
                                    self.name + '_selective_search_roidb.pkl')
  
          if os.path.exists(cache_file):
              with open(cache_file, 'rb') as fid:
                  roidb = cp.load(fid, encoding='latin1')
              print ('{} ss roidb loaded from {}'.format(self.name, cache_file))
              return roidb
  
          if int(self._year) == 2007 or not self._image_set.startswith('test'):
              gt_roidb = self.gt_roidb()
              ss_roidb = self._load_selective_search_roidb(gt_roidb)
              roidb = imdb.merge_roidbs(gt_roidb, ss_roidb)
          else:
              roidb = self._load_selective_search_roidb(None)
  
          # Keep max of e.g. 2000 rois
          if type(self._maxNrRois) == int:
              print ("Only keep the first %d ROIs..." % self._maxNrRois)
              for i in range(self.num_images):
                  gt_overlaps = roidb[i]['gt_overlaps']
                  gt_overlaps = gt_overlaps.todense()[:self._maxNrRois]
                  gt_overlaps = scipy.sparse.csr_matrix(gt_overlaps)
                  roidb[i]['boxes'] = roidb[i]['boxes'][:self._maxNrRois, :]
                  roidb[i]['gt_classes'] = roidb[i]['gt_classes'][:self._maxNrRois]
                  roidb[i]['gt_overlaps'] = roidb[i]['gt_overlaps'] = gt_overlaps
  
          with open(cache_file, 'wb') as fid:
              cp.dump(roidb, fid, cp.HIGHEST_PROTOCOL)
          print ('wrote ss roidb to {}'.format(cache_file))
  
          return roidb
  
      def _load_selective_search_roidb(self, gt_roidb):
          filename = os.path.abspath(os.path.join(self._devkit_path, '..',
                                                   'selective_search_data',
                                                   self.name + '.mat'))
          assert os.path.exists(filename), \
                 'Selective search data not found at: {}'.format(filename)
          raw_data = sio.loadmat(filename)['boxes'].ravel()
  
          box_list = []
          for i in range(raw_data.shape[0]):
              box_list.append(raw_data[i][:, (1, 0, 3, 2)] - 1)
  
          return self.create_roidb_from_box_list(box_list, gt_roidb)
  
      def selective_search_IJCV_roidb(self):
          """
          Return the database of selective search regions of interest.
          Ground-truth ROIs are also included.
  
          This function loads/saves from/to a cache file to speed up future calls.
          """
          cache_file = os.path.join(self.cache_path,
                  '{:s}_selective_search_IJCV_top_{:d}_roidb.pkl'.
                  format(self.name, self.config['top_k']))
  
          if os.path.exists(cache_file):
              with open(cache_file, 'rb') as fid:
                  roidb = cp.load(fid)
              print ('{} ss roidb loaded from {}'.format(self.name, cache_file))
              return roidb
  
          gt_roidb = self.gt_roidb()
          ss_roidb = self._load_selective_search_IJCV_roidb(gt_roidb)
          roidb = imdb.merge_roidbs(gt_roidb, ss_roidb)
          with open(cache_file, 'wb') as fid:
              cp.dump(roidb, fid, cp.HIGHEST_PROTOCOL)
          print ('wrote ss roidb to {}'.format(cache_file))
  
          return roidb
  
      def _load_selective_search_IJCV_roidb(self, gt_roidb):
          IJCV_path = os.path.abspath(os.path.join(self.cache_path, '..',
                                                   'selective_search_IJCV_data',
                                                   'voc_' + self._year))
          assert os.path.exists(IJCV_path), \
                 'Selective search IJCV data not found at: {}'.format(IJCV_path)
  
          top_k = self.config['top_k']
          box_list = []
          for i in range(self.num_images):
              filename = os.path.join(IJCV_path, self.image_index[i] + '.mat')
              raw_data = sio.loadmat(filename)
              box_list.append((raw_data['boxes'][:top_k, :]-1).astype(np.uint16))
  
          return self.create_roidb_from_box_list(box_list, gt_roidb)
  
      def _load_pascal_annotation(self, index):
          """
          Load image and bounding boxes info from XML file in the PASCAL VOC
          format.
          """
          filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
          # print ('Loading: {}'.format(filename))
          def get_data_from_tag(node, tag):
              return node.getElementsByTagName(tag)[0].childNodes[0].data
  
          with open(filename) as f:
              data = minidom.parseString(f.read())
  
          objs = data.getElementsByTagName('object')
          num_objs = len(objs)
  
          boxes = np.zeros((num_objs, 4), dtype=np.uint16)
          gt_classes = np.zeros((num_objs), dtype=np.int32)
          overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
  
          # Load object bounding boxes into a data frame.
          for ix, obj in enumerate(objs):
              # Make pixel indexes 0-based
              x1 = float(get_data_from_tag(obj, 'xmin')) - 1
              y1 = float(get_data_from_tag(obj, 'ymin')) - 1
              x2 = float(get_data_from_tag(obj, 'xmax')) - 1
              y2 = float(get_data_from_tag(obj, 'ymax')) - 1
              cls = self._class_to_ind[
                      str(get_data_from_tag(obj, "name")).lower().strip()]
              boxes[ix, :] = [x1, y1, x2, y2]
              gt_classes[ix] = cls
              overlaps[ix, cls] = 1.0
  
          overlaps = scipy.sparse.csr_matrix(overlaps)
  
          return {'boxes' : boxes,
                  'gt_classes': gt_classes,
                  'gt_overlaps' : overlaps,
                  'flipped' : False}
  
      def _write_voc_results_file(self, all_boxes, output_dir):
          comp_id = 'comp4'
          if self.config['use_salt']:
              comp_id += '-{}'.format(os.getpid())
  
          for cls_ind, cls in enumerate(self.classes):
              if cls == '__background__':
                  continue
              print ('Writing {} VOC results file'.format(cls))
              filename = self._get_voc_results_file_template(output_dir).format(cls)
              with open(filename, 'wt') as f:
                  for im_ind, index in enumerate(self.image_index):
                      dets = all_boxes[cls_ind][im_ind]
                      if dets == []:
                          continue
                      # the VOCdevkit expects 1-based indices
                      for k in range(dets.shape[0]):
                          f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'.
                                  format(index, dets[k, -1],
                                         dets[k, 0] + 1, dets[k, 1] + 1,
                                         dets[k, 2] + 1, dets[k, 3] + 1))
          return comp_id
  
      def evaluate_detections(self, all_boxes, output_dir, boUsePythonImpl = True, use_07_metric = False):
          self._write_voc_results_file(all_boxes, output_dir)
          if not boUsePythonImpl:
              self._do_matlab_eval(comp_id, output_dir)
          else:
              self._do_python_eval(output_dir, use_07_metric)
  
      def _do_matlab_eval(self, comp_id, output_dir='output'):
          rm_results = self.config['cleanup']
  
          path = os.path.join(os.path.dirname(__file__),
                              'VOCdevkit-matlab-wrapper')
          cmd = 'cd {} && '.format(path)
          cmd += '{:s} -nodisplay -nodesktop '.format(datasets.MATLAB)
          cmd += '-r "dbstop if error; '
          cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\',{:d}); quit;"' \
                 .format(self._devkit_path, comp_id,
                         self._image_set, output_dir, int(rm_results))
          print('Running:\n{}'.format(cmd))
          status = subprocess.call(cmd, shell=True)
  
      def competition_mode(self, on):
          if on:
              self.config['use_salt'] = False
              self.config['cleanup'] = False
          else:
              self.config['use_salt'] = True
              self.config['cleanup'] = True
  
      #########################################################################
      # Python evaluation functions (copied from faster-RCNN)
      ##########################################################################
      def _get_voc_results_file_template(self, evalDir):
          if not os.path.exists(evalDir):
              os.makedirs(evalDir)
          filename = self._image_set + '_{:s}.txt'
          return os.path.join(evalDir, filename)
  
      def _do_python_eval(self, output_dir='output', use_07_metric=None):
          annopath = os.path.join(self._devkit_path, 'VOC' + self._year, 'Annotations', '{}.xml')
          imagesetfile = os.path.join(
              self._devkit_path,
              'VOC' + self._year,
              'ImageSets',
              'Main',
              self._image_set + '.txt')
          aps = []
          # The PASCAL VOC metric changed in 2010
          if use_07_metric == None:
              use_07_metric = True if int(self._year) < 2010 else False
  
          print ('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
          if not os.path.isdir(output_dir):
              os.mkdir(output_dir)
          for i, cls in enumerate(self._classes):
              if cls == '__background__':
                  continue
              filename = self._get_voc_results_file_template(output_dir).format(cls)
  
              rec, prec, ap = voc_eval(
                  filename, annopath, imagesetfile, cls, cachedir = output_dir, ovthresh=0.5,
                  use_07_metric=use_07_metric)
              aps += [ap]
              print('AP for {} = {:.4f}'.format(cls, ap))
              with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
                  cp.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
          print('Mean AP = {:.4f}'.format(np.mean(aps)))
          # print('~~~~~~~~')
          # print('Results:')
          # for ap in aps:
          #     print('{:.3f}'.format(ap))
          # print('{:.3f}'.format(np.mean(aps)))
          # print('~~~~~~~~')
          # print('')
          print('--------------------------------------------------------------')
          print('Results computed with the **unofficial** Python eval code.')
          print('Results should be very close to the official MATLAB eval code.')
          print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
          print('-- Thanks, The Management')
      print('--------------------------------------------------------------')
  
  if __name__ == '__main__':
      d = datasets.pascal_voc('trainval', '2007')
      res = d.roidb
      from IPython import embed; embed()
  