text: import numpy as np
  import datetime
  import time
  from collections import defaultdict
  # from . import mask as maskUtils
  import copy
  import matplotlib.pyplot as plt
  import scipy.io as sio
  
  class COCOeval: 
  	# Interface for evaluating detection on the Microsoft COCO dataset.
  	#
  	# The usage for CocoEval is as follows:
  	#  cocoGt=..., cocoDt=...       # load dataset and results
  	#  E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object
  	#  E.params.recThrs = ...;      # set parameters as desired
  	#  E.evaluate();                # run per image evaluation
  	#  E.accumulate();              # accumulate per image results
  	#  E.summarize();               # display summary metrics of results
  	# For example usage see evalDemo.m and http://mscoco.org/.
  	#
  	# The evaluation parameters are as follows (defaults in brackets):
  	#  imgIds     - [all] N img ids to use for evaluation
  	#  catIds     - [all] K cat ids to use for evaluation
  	#  iouThrs    - [.5:.05:.95] T=10 IoU thresholds for evaluation
  	#  recThrs    - [0:.01:1] R=101 recall thresholds for evaluation
  	#  areaRng    - [...] A=4 object area ranges for evaluation
  	#  maxDets    - [1 10 100] M=3 thresholds on max detections per image
  	#  iouType    - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints'
  	#  iouType replaced the now DEPRECATED useSegm parameter.
  	#  useCats    - [1] if true use category labels for evaluation
  	# Note: if useCats=0 category labels are ignored as in proposal scoring.
  	# Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified.
  	#
  	# evaluate(): evaluates detections on every image and every category and
  	# concats the results into the "evalImgs" with fields:
  	#  dtIds      - [1xD] id for each of the D detections (dt)
  	#  gtIds      - [1xG] id for each of the G ground truths (gt)
  	#  dtMatches  - [TxD] matching gt id at each IoU or 0
  	#  gtMatches  - [TxG] matching dt id at each IoU or 0
  	#  dtScores   - [1xD] confidence of each dt
  	#  gtIgnore   - [1xG] ignore flag for each gt
  	#  dtIgnore   - [TxD] ignore flag for each dt at each IoU
  	#
  	# accumulate(): accumulates the per-image, per-category evaluation
  	# results in "evalImgs" into the dictionary "eval" with fields:
  	#  params     - parameters used for evaluation
  	#  date       - date evaluation was performed
  	#  counts     - [T,R,K,A,M] parameter dimensions (see above)
  	#  precision  - [TxRxKxAxM] precision for every evaluation setting
  	#  recall     - [TxKxAxM] max recall for every evaluation setting
  	# Note: precision and recall==-1 for settings with no gt objects.
  	#
  	# See also coco, mask, pycocoDemo, pycocoEvalDemo
  	#
  	# Microsoft COCO Toolbox.      version 2.0
  	# Data, paper, and tutorials available at:  http://mscoco.org/
  	# Code written by Piotr Dollar and Tsung-Yi Lin, 2015.
  	# Licensed under the Simplified BSD License [see coco/license.txt]
  	def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'):
  		'''
  		Initialize CocoEval using coco APIs for gt and dt
  		:param cocoGt: coco object with ground truth annotations
  		:param cocoDt: coco object with detection results
  		:return: None
  		'''
  		if not iouType:
  			print('iouType not specified. use default iouType segm')
  		self.cocoGt   = cocoGt              # ground truth COCO API
  		self.cocoDt   = cocoDt              # detections COCO API
  		self.params   = {}                  # evaluation parameters
  		self.evalImgs = defaultdict(list)   # per-image per-category evaluation results [KxAxI] elements
  		self.eval     = {}                  # accumulated evaluation results
  		self._gts = defaultdict(list)       # gt for evaluation
  		self._dts = defaultdict(list)       # dt for evaluation
  		self.params = Params(iouType=iouType) # parameters
  		self._paramsEval = {}               # parameters for evaluation
  		self.stats = []                     # result summarization
  		self.ious = {}                      # ious between all gts and dts
  		if not cocoGt is None:
  			self.params.imgIds = sorted(cocoGt.getImgIds())
  			self.params.catIds = sorted(cocoGt.getCatIds())
  
  
  	def _prepare(self, id_setup):
  		'''
  		Prepare ._gts and ._dts for evaluation based on params
  		:return: None
  		'''
  		p = self.params
  		if p.useCats:
  			gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds))
  			dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds))
  		else:
  			gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds))
  			dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds))
  
  
  		# set ignore flag
  		for gt in gts:
  			gt['ignore'] = gt['ignore'] if 'ignore' in gt else 0
  			gt['ignore'] = 1 if (gt['height'] < self.params.HtRng[id_setup][0] or gt['height'] > self.params.HtRng[id_setup][1]) or \
  			                    (gt['vis_ratio'] < self.params.VisRng[id_setup][0] or gt['vis_ratio'] > self.params.VisRng[id_setup][1]) else gt['ignore']
  
  		self._gts = defaultdict(list)       # gt for evaluation
  		self._dts = defaultdict(list)       # dt for evaluation
  		for gt in gts:
  			self._gts[gt['image_id'], gt['category_id']].append(gt)
  		for dt in dts:
  			self._dts[dt['image_id'], dt['category_id']].append(dt)
  		self.evalImgs = defaultdict(list)   # per-image per-category evaluation results
  		self.eval     = {}                  # accumulated evaluation results
  
  	def evaluate(self, id_setup):
  		'''
  		Run per image evaluation on given images and store results (a list of dict) in self.evalImgs
  		:return: None
  		'''
  		tic = time.time()
  		# print('Running per image evaluation...')
  		p = self.params
  		# add backward compatibility if useSegm is specified in params
  		if not p.useSegm is None:
  			p.iouType = 'segm' if p.useSegm == 1 else 'bbox'
  			print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType))
  		# print('Evaluate annotation type *{}*'.format(p.iouType))
  		p.imgIds = list(np.unique(p.imgIds))
  		if p.useCats:
  			p.catIds = list(np.unique(p.catIds))
  		p.maxDets = sorted(p.maxDets)
  		self.params=p
  
  		self._prepare(id_setup)
  		# loop through images, area range, max detection number
  		catIds = p.catIds if p.useCats else [-1]
  
  		computeIoU = self.computeIoU
  
  		self.ious = {(imgId, catId): computeIoU(imgId, catId) \
  						for imgId in p.imgIds
  						for catId in catIds}
  
  		evaluateImg = self.evaluateImg
  		maxDet = p.maxDets[-1]
  		HtRng = self.params.HtRng[id_setup]
  		VisRng = self.params.VisRng[id_setup]
  		self.evalImgs = [evaluateImg(imgId, catId, HtRng, VisRng, maxDet)
  				 for catId in catIds
  				 for imgId in p.imgIds
  			 ]
  		self._paramsEval = copy.deepcopy(self.params)
  		toc = time.time()
  		# print('DONE (t={:0.2f}s).'.format(toc-tic))
  
  
  	def computeIoU(self, imgId, catId):
  		p = self.params
  		if p.useCats:
  			gt = self._gts[imgId,catId]
  			dt = self._dts[imgId,catId]
  		else:
  			gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]]
  			dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]]
  		if len(gt) == 0 and len(dt) ==0:
  			return []
  		inds = np.argsort([-d['score'] for d in dt], kind='mergesort')
  		dt = [dt[i] for i in inds]
  		if len(dt) > p.maxDets[-1]:
  			dt=dt[0:p.maxDets[-1]]
  
  
  		if p.iouType == 'segm':
  			g = [g['segmentation'] for g in gt]
  			d = [d['segmentation'] for d in dt]
  		elif p.iouType == 'bbox':
  			g = [g['bbox'] for g in gt]
  			d = [d['bbox'] for d in dt]
  		else:
  			raise Exception('unknown iouType for iou computation')
  
  
  		# compute iou between each dt and gt region
  		iscrowd = [int(o['ignore']) for o in gt]
  		ious = self.iou(d,g,iscrowd)
  		return ious
  
  	def iou( self, dts, gts, pyiscrowd ):
  		dts = np.asarray(dts)
  		gts = np.asarray(gts)
  		pyiscrowd = np.asarray(pyiscrowd)
  		ious = np.zeros((len(dts), len(gts)))
  		for j, gt in enumerate(gts):
  			gx1 = gt[0]
  			gy1 = gt[1]
  			gx2 = gt[0] + gt[2]
  			gy2 = gt[1] + gt[3]
  			garea = gt[2] * gt[3]
  			for i, dt in enumerate(dts):
  				dx1 = dt[0]
  				dy1 = dt[1]
  				dx2 = dt[0] + dt[2]
  				dy2 = dt[1] + dt[3]
  				darea = dt[2] * dt[3]
  
  				unionw = min(dx2,gx2)-max(dx1,gx1)
  				if unionw <= 0:
  					continue
  				unionh = min(dy2,gy2)-max(dy1,gy1)
  				if unionh <= 0:
  					continue
  				t = unionw * unionh
  				if pyiscrowd[j]:
  					unionarea = darea
  				else:
  					unionarea = darea + garea - t
  
  				ious[i, j] = float(t)/unionarea
  		return ious
  
  	def evaluateImg(self, imgId, catId, hRng, vRng, maxDet):
  		'''
  		perform evaluation for single category and image
  		:return: dict (single image results)
  		'''
  		# if imgId==273:
  		#     pass
  		p = self.params
  		if p.useCats:
  			gt = self._gts[imgId,catId]
  			dt = self._dts[imgId,catId]
  		else:
  			gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]]
  			dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]]
  		if len(gt) == 0 and len(dt) ==0:
  			return None
  
  		for g in gt:
  			if g['ignore']:
  				g['_ignore'] = 1
  			else:
  				g['_ignore'] = 0
  		# sort dt highest score first, sort gt ignore last
  		gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort')
  		gt = [gt[i] for i in gtind]
  		dtind = np.argsort([-d['score'] for d in dt], kind='mergesort')
  		# dtind = np.argsort([-d['score'] for d in dt])
  		dt = [dt[i] for i in dtind[0:maxDet]]
  		# exclude dt out of height range
  		# dt = [d for d in dt if d['height'] >= hRng[0] / self.params.expFilter and d['height'] < hRng[1] * self.params.expFilter]
  		# dtind = np.array([int(d['id'] - dt[0]['id']) for d in dt])
  		dtind = [i for i in range(len(dt)) if dt[i]['height'] >= hRng[0] / self.params.expFilter and dt[i]['height'] < hRng[1] * self.params.expFilter]
  		dt = [d for d in dt if d['height'] >= hRng[0] / self.params.expFilter and d['height'] < hRng[1] * self.params.expFilter]
  		# if np.max(dtind)>1000:
  		#     pass
  
  		# load computed ious
  		if len(dtind) > 0:
  			ious = self.ious[imgId, catId][dtind, :] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId]
  			ious = ious[:, gtind]
  		else:
  			ious = []
  
  		T = len(p.iouThrs)
  		G = len(gt)
  		D = len(dt)
  		gtm  = np.zeros((T,G))
  		dtm  = np.zeros((T,D))
  		gtIg = np.array([g['_ignore'] for g in gt])
  		dtIg = np.zeros((T,D))
  		if not len(ious)==0:
  			for tind, t in enumerate(p.iouThrs):
  				for dind, d in enumerate(dt):
  					# information about best match so far (m=-1 -> unmatched)
  					iou = min([t,1-1e-10])
  					bstOa = iou
  					bstg = -2
  					bstm = -2
  					for gind, g in enumerate(gt):
  						m = gtm[tind,gind]
  						# if this gt already matched, and not a crowd, continue
  						if m>0:
  							continue
  						# if dt matched to reg gt, and on ignore gt, stop
  						if bstm!=-2 and gtIg[gind] == 1:
  							break
  						# continue to next gt unless better match made
  						if ious[dind,gind] < bstOa:
  							continue
  						# if match successful and best so far, store appropriately
  						bstOa=ious[dind,gind]
  						bstg = gind
  						if gtIg[gind] == 0:
  							bstm = 1
  						else:
  							bstm = -1
  
  					# if match made store id of match for both dt and gt
  					if bstg ==-2:
  						continue
  					dtIg[tind,dind] = gtIg[bstg]
  					dtm[tind,dind]  = gt[bstg]['id']
  					if bstm == 1:
  						gtm[tind,bstg]     = d['id']
  
  		# store results for given image and category
  		return {
  				'image_id':     imgId,
  				'category_id':  catId,
  				'hRng':         hRng,
  				'vRng':         vRng,
  				'maxDet':       maxDet,
  				'dtIds':        [d['id'] for d in dt],
  				'gtIds':        [g['id'] for g in gt],
  				'dtMatches':    dtm,
  				'gtMatches':    gtm,
  				'dtScores':     [d['score'] for d in dt],
  				'gtIgnore':     gtIg,
  				'dtIgnore':     dtIg,
  			}
  
  	def accumulate(self, p = None):
  		'''
  		Accumulate per image evaluation results and store the result in self.eval
  		:param p: input params for evaluation
  		:return: None
  		'''
  		# print('Accumulating evaluation results...')
  		tic = time.time()
  		if not self.evalImgs:
  			print('Please run evaluate() first')
  		# allows input customized parameters
  		if p is None:
  			p = self.params
  		p.catIds = p.catIds if p.useCats == 1 else [-1]
  		T           = len(p.iouThrs)
  		R           = len(p.fppiThrs)
  		K           = len(p.catIds) if p.useCats else 1
  		M           = len(p.maxDets)
  		ys   = -np.ones((T,R,K,M)) # -1 for the precision of absent categories
  
  
  		# create dictionary for future indexing
  		_pe = self._paramsEval
  		catIds = [1] #_pe.catIds if _pe.useCats else [-1]
  		setK = set(catIds)
  		setM = set(_pe.maxDets)
  		setI = set(_pe.imgIds)
  		# get inds to evaluate
  		k_list = [n for n, k in enumerate(p.catIds)  if k in setK]
  
  		m_list = [m for n, m in enumerate(p.maxDets) if m in setM]
  		i_list = [n for n, i in enumerate(p.imgIds)  if i in setI]
  		I0 = len(_pe.imgIds)
  
  		# retrieve E at each category, area range, and max number of detections
  		for k, k0 in enumerate(k_list):
  			Nk = k0*I0
  			for m, maxDet in enumerate(m_list):
  				E = [self.evalImgs[Nk + i] for i in i_list]
  				E = [e for e in E if not e is None]
  				if len(E) == 0:
  					continue
  
  				dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E])
  
  				# different sorting method generates slightly different results.
  				# mergesort is used to be consistent as Matlab implementation.
  
  				inds = np.argsort(-dtScores, kind='mergesort')
  
  				dtm = np.concatenate([e['dtMatches'][:, 0:maxDet] for e in E], axis=1)[:, inds]
  				dtIg = np.concatenate([e['dtIgnore'][:, 0:maxDet] for e in E], axis=1)[:, inds]
  				gtIg = np.concatenate([e['gtIgnore'] for e in E])
  				npig = np.count_nonzero(gtIg == 0)
  				if npig == 0:
  					continue
  				tps = np.logical_and(dtm, np.logical_not(dtIg))
  				fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg))
  				inds = np.where(dtIg==0)[1]
  				tps = tps[:,inds]
  				fps = fps[:,inds]
  
  				tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float)
  				fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float)
  				for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):
  					tp = np.array(tp)
  					fppi = np.array(fp)/I0
  					nd = len(tp)
  					recall = tp / npig
  					q = np.zeros((R,))
  
  					# numpy is slow without cython optimization for accessing elements
  					# use python array gets significant speed improvement
  					recall = recall.tolist()
  					q = q.tolist()
  
  					for i in range(nd - 1, 0, -1):
  						if recall[i] < recall[i - 1]:
  							recall[i - 1] = recall[i]
  
  					inds = np.searchsorted(fppi, p.fppiThrs, side='right') - 1
  					try:
  						for ri, pi in enumerate(inds):
  							q[ri] = recall[pi]
  					except:
  						pass
  					ys[t,:,k,m] = np.array(q)
  		self.eval = {
  			'params': p,
  			'counts': [T, R, K, M],
  			'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
  			'TP':   ys,
  		}
  		toc = time.time()
  		# print('DONE (t={:0.2f}s).'.format( toc-tic))
  
  	def summarize(self,id_setup, res_file):
  		'''
  		Compute and display summary metrics for evaluation results.
  		Note this functin can *only* be applied on the default parameter setting
  		'''
  		def _summarize(iouThr=None, maxDets=100 ):
  			p = self.params
  			iStr = ' {:<18} {} @ {:<18} [ IoU={:<9} | height={:>6s} | visibility={:>6s} ] = {:0.2f}%'
  			titleStr = 'Average Miss Rate'
  			typeStr = '(MR)'
  			setupStr = p.SetupLbl[id_setup]
  			iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \
  				if iouThr is None else '{:0.2f}'.format(iouThr)
  			heightStr = '[{:0.0f}:{:0.0f}]'.format(p.HtRng[id_setup][0], p.HtRng[id_setup][1])
  			occlStr = '[{:0.2f}:{:0.2f}]'.format(p.VisRng[id_setup][0], p.VisRng[id_setup][1])
  
  			mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
  
  			# dimension of precision: [TxRxKxAxM]
  			s = self.eval['TP']
  			# IoU
  			if iouThr is not None:
  				t = np.where(iouThr == p.iouThrs)[0]
  				s = s[t]
  			mrs = 1-s[:,:,:,mind]
  
  			if len(mrs[mrs<2])==0:
  				mean_s = -1
  			else:
  				mean_s = np.log(mrs[mrs<2])
  				mean_s = np.mean(mean_s)
  				mean_s = np.exp(mean_s)
  			print(iStr.format(titleStr, typeStr,setupStr, iouStr, heightStr, occlStr, mean_s*100))
  			# res_file.write(iStr.format(titleStr, typeStr,setupStr, iouStr, heightStr, occlStr, mean_s*100))
  			res_file.write(str(mean_s * 100))
  			res_file.write('\n')
  			return mean_s
  
  		if not self.eval:
  			raise Exception('Please run accumulate() first')
  		_summarize(iouThr=.5,maxDets=1000)
  
  	def summarize_nofile(self,id_setup):
  		'''
  		Compute and display summary metrics for evaluation results.
  		Note this functin can *only* be applied on the default parameter setting
  		'''
  		def _summarize(iouThr=None, maxDets=100 ):
  			p = self.params
  
  			mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
  
  			# dimension of precision: [TxRxKxAxM]
  			s = self.eval['TP']
  			# IoU
  			if iouThr is not None:
  				t = np.where(iouThr == p.iouThrs)[0]
  				s = s[t]
  			mrs = 1-s[:,:,:,mind]
  
  			if len(mrs[mrs<2])==0:
  				mean_s = -1
  			else:
  				mean_s = np.log(mrs[mrs<2])
  				mean_s = np.mean(mean_s)
  				mean_s = np.exp(mean_s)
  			return mean_s
  
  		if not self.eval:
  			raise Exception('Please run accumulate() first')
  		return _summarize(iouThr=.5, maxDets=1000)
  
  	def __str__(self):
  		self.summarize()
  
  class Params:
  	'''
  	Params for coco evaluation api
  	'''
  	def setDetParams(self):
  		self.imgIds = []
  		self.catIds = []
  		# np.arange causes trouble.  the data point on arange is slightly larger than the true value
  
  		#self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True)
  		self.recThrs = np.linspace(0, 1, 100 + 1, endpoint=True)
  		self.fppiThrs = np.array([0.0100,    0.0178,    0.0316,    0.0562,    0.1000,    0.1778,    0.3162,    0.5623,    1.0000])
  		self.maxDets = [1000]
  		self.expFilter = 1.25
  		self.useCats = 1
  
  		self.iouThrs = np.array([0.5])  # np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)
  
  		# self.HtRng = [[50, 1e5 ** 2], [50,75], [50, 1e5 ** 2], [20, 1e5 ** 2]]
  		# self.VisRng = [[0.65, 1e5 ** 2], [0.65, 1e5 ** 2], [0.2,0.65], [0.2, 1e5 ** 2]]
  		# self.SetupLbl = ['Reasonable', 'Reasonable_small','Reasonable_occ=heavy', 'All']
  
  		# self.HtRng = [[50, 1e5 ** 2], [50, 75], [75, 100], [100, 1e5 ** 2]]
  		# self.VisRng = [[0.65, 1e5 ** 2], [0.65, 1e5 ** 2], [0.65, 1e5 ** 2], [0.65, 1e5 ** 2]]
  		# self.SetupLbl = ['Reasonable', 'small', 'middle', 'large']
  
  		#original definition
  		self.HtRng = [[50, 1e5 ** 2], [50, 1e5 ** 2], [50, 1e5 ** 2], [50, 1e5 ** 2]]
  
  		#big pedestrians
  		#self.HtRng = [[500, 50000], [500, 10000], [500, 10000], [500, 10000]]
  		print('Heights range: {}/{}/{}/{}'.format(self.HtRng[0], self.HtRng[1], self.HtRng[2], self.HtRng[3]))
  		
  		self.VisRng = [[0.65, 1e5 ** 2], [0.9, 1e5 ** 2], [0.65, 0.9], [0, 0.65]]
  		self.SetupLbl = ['Reasonable', 'bare', 'partial', 'heavy']
  
  
  	def __init__(self, iouType='segm'):
  		if iouType == 'segm' or iouType == 'bbox':
  			self.setDetParams()
  		else:
  			raise Exception('iouType not supported')
  		self.iouType = iouType
  		# useSegm is deprecated
  		self.useSegm = None