# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """TODO: Add a description here.""" from typing import List, Tuple, Dict, Literal import evaluate import datasets import numpy as np from seametrics.detection import PrecisionRecallF1Support from seametrics.fo_utils.utils import _fo_dets_to_metrics_dict from seametrics.fo_utils.utils import _add_batch _CITATION = """\ @InProceedings{coco:2020, title = {Microsoft {COCO:} Common Objects in Context}, authors={Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and James Hays and Pietro Perona and Deva Ramanan and Piotr Dollar and C. Lawrence Zitnick}, booktitle = {Computer Vision - {ECCV} 2014 - 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part {V}}, series = {Lecture Notes in Computer Science}, volume = {8693}, pages = {740--755}, publisher = {Springer}, year={2014} } """ _DESCRIPTION = """\ This evaluation metric is designed to give provide object detection metrics at different object size levels. It is based on a modified version of the commonly used COCO-evaluation metrics. """ _KWARGS_DESCRIPTION = """ Calculates object detection metrics given predicted and ground truth bounding boxes for a single image. Args: predictions: list of predictions to score. Each prediction should be a list containing the four co-ordinates that specify the bounding box. Co-ordinate format is as defined when instantiating the metric (parameter: bbox_type, defaults to xywh). references: list of reference for each prediction. Each prediction should be a list containing the four co-ordinates that specify the bounding box. Bounding box format should be the same as for the predictions. Returns: dict containing dicts for each specified area range with following items: 'range': specified area with [max_px_area, max_px_area] 'iouThr': min. IOU-threshold of a prediction with a ground truth box to be considered a correct prediction 'maxDets': maximum number of detections 'tp': number of true positive (correct) predictions 'fp': number of false positive (incorrect) predictions 'fn': number of false negative (missed) predictions 'duplicates': number of duplicate predictions 'precision': best possible score = 1, worst possible score = 0 large if few false positive predictions formula: tp/(fp+tp) 'recall' best possible score = 1, worst possible score = 0 large if few missed predictions formula: tp/(tp+fn) 'f1': best possible score = 1, worst possible score = 0 trades off precision and recall formula: 2*(precision*recall)/(precision+recall) 'support': number of ground truth bounding boxes considered in the evaluation, 'fpi': number of images with no ground truth but false positive predictions, 'nImgs': number of images considered in evaluation Examples: >>> import evaluate >>> from seametrics.fo_to_payload.utils import fo_to_payload >>> payload = fo_to_payload(..., models=model_list) >>> for model in payload["models"]: >>> module = evaluate.load("./detection_metric.py", iou_thresholds=0.9) >>> module.add_batch(payload) >>> result = module.compute() >>> print(result) {'all': { 'range': [0, 10000000000.0], 'iouThr': '0.00', 'maxDets': 100, 'tp': 1, 'fp': 3, 'fn': 1, 'duplicates': 0, 'precision': 0.25, 'recall': 0.5, 'f1': 0.3333333333333333, 'support': 2, 'fpi': 0, 'nImgs': 2 } } """ @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) class DetectionMetric(evaluate.Metric): def __init__( self, area_ranges_tuples: List[Tuple[str, List[int]]] = [("all", [0, 1e5 ** 2])], iou_threshold: float = 1e-10, class_agnostic: bool = True, bbox_format: str = "xywh", iou_type: Literal["bbox", "segm"] = "bbox", **kwargs ): super().__init__(**kwargs) area_ranges = [v for _, v in area_ranges_tuples] area_ranges_labels = [k for k, _ in area_ranges_tuples] metric_params = dict( iou_thresholds=[iou_threshold], area_ranges=area_ranges, area_ranges_labels=area_ranges_labels, class_agnostic=class_agnostic, iou_type=iou_type, box_format=bbox_format ) self.coco_metric = PrecisionRecallF1Support(**metric_params) def _info(self): return evaluate.MetricInfo( # This is the description that will appear on the modules page. module_type="metric", description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, # This defines the format of each prediction and reference features=datasets.Features( { 'predictions': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))), 'references': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))), } ), # Additional links to the codebase or references codebase_urls=["https://github.com/SEA-AI/metrics/tree/main", "https://github.com/cocodataset/cocoapi/tree/master"] ) def add_batch( self, data: dict, model: str = None ): """Add predictions and ground truths of a single image to update the metric. Args: data (dict): containing standard payload of data that should be evaluated format should be as returned by function `fo_to_payload()` in seametrics library model (str): should be one out of values given in data["models"] if not defined, defaults to data["models"][0], as only one model can be evaluated a time. """ # populate two empty lists in format suitable for hugging face metric # nothing is computed based on them but prevents huggingface error self, predictions,references = _add_batch(self, data, model) # prevents hugging face error, doesn't do a lot super(evaluate.Metric, self).add_batch( predictions=predictions, references=references ) def _compute( self, predictions, references ): """Returns the scores""" result = self.coco_metric.compute()["metrics"] return result