det-metrics.py

# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TODO: Add a description here."""

from typing import List, Tuple, Dict, Literal

import evaluate
import datasets
import numpy as np

from seametrics.detection import PrecisionRecallF1Support
from seametrics.fo_utils.utils import _fo_dets_to_metrics_dict
from seametrics.fo_utils.utils import _add_batch

_CITATION = """\
@InProceedings{coco:2020,
title = {Microsoft {COCO:} Common Objects in Context},
authors={Tsung{-}Yi Lin and
                  Michael Maire and
                  Serge J. Belongie and
                  James Hays and
                  Pietro Perona and
                  Deva Ramanan and
                  Piotr Dollar and
                  C. Lawrence Zitnick},
booktitle    = {Computer Vision - {ECCV} 2014 - 13th European Conference, Zurich,
                Switzerland, September 6-12, 2014, Proceedings, Part {V}},
series       = {Lecture Notes in Computer Science},
volume       = {8693},
pages        = {740--755},
publisher    = {Springer},
year={2014}
}
"""

_DESCRIPTION = """\
This evaluation metric is designed to give provide object detection metrics at different object size levels.
It is based on a modified version of the commonly used COCO-evaluation metrics.
"""


_KWARGS_DESCRIPTION = """
Calculates object detection metrics given predicted and ground truth bounding boxes for a single image.
Args:
    predictions: list of predictions to score. Each prediction should
        be a list containing the four co-ordinates that specify the bounding box.
        Co-ordinate format is as defined when instantiating the metric
        (parameter: bbox_type, defaults to xywh).
    references: list of reference for each prediction. Each prediction should
        be a list containing the four co-ordinates that specify the bounding box.
        Bounding box format should be the same as for the predictions.
Returns:
    dict containing dicts for each specified area range with following items:
        'range': specified area with [max_px_area, max_px_area]
        'iouThr': min. IOU-threshold of a prediction with a ground truth box
            to be considered a correct prediction
        'maxDets': maximum number of detections
        'tp': number of true positive (correct) predictions
        'fp': number of false positive (incorrect) predictions
        'fn': number of false negative (missed) predictions
        'duplicates': number of duplicate predictions
        'precision': best possible score = 1, worst possible score = 0
            large if few false positive predictions
            formula: tp/(fp+tp)
        'recall' best possible score = 1, worst possible score = 0
            large if few missed predictions 
            formula: tp/(tp+fn)
        'f1': best possible score = 1, worst possible score = 0
            trades off precision and recall
            formula: 2*(precision*recall)/(precision+recall)
        'support': number of ground truth bounding boxes considered in the evaluation,
        'fpi': number of images with no ground truth but false positive predictions,
        'nImgs': number of images considered in evaluation
Examples:
    >>> import evaluate
    >>> from seametrics.fo_to_payload.utils import fo_to_payload
    >>> payload = fo_to_payload(..., models=model_list)
    >>> for model in payload["models"]:
        >>> module = evaluate.load("./detection_metric.py", iou_thresholds=0.9)
        >>> module.add_batch(payload)
        >>> result = module.compute()
        >>> print(result)
            {'all': {
                'range': [0, 10000000000.0],
                'iouThr': '0.00',
                'maxDets': 100,
                'tp': 1,
                'fp': 3,
                'fn': 1,
                'duplicates': 0,
                'precision': 0.25,
                'recall': 0.5,
                'f1': 0.3333333333333333,
                'support': 2,
                'fpi': 0,
                'nImgs': 2
                }
            }
"""


@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class DetectionMetric(evaluate.Metric):
    def __init__(
            self, 
            area_ranges_tuples: List[Tuple[str, List[int]]] = [("all", [0, 1e5 ** 2])], 
            iou_threshold: float = 1e-10, 
            class_agnostic: bool = True,
            bbox_format: str = "xywh",
            iou_type: Literal["bbox", "segm"] = "bbox",
            **kwargs
        ):
        super().__init__(**kwargs)
        area_ranges = [v for _, v in area_ranges_tuples]
        area_ranges_labels = [k for k, _ in area_ranges_tuples]

        metric_params = dict(
            iou_thresholds=[iou_threshold],
            area_ranges=area_ranges,
            area_ranges_labels=area_ranges_labels,
            class_agnostic=class_agnostic,
            iou_type=iou_type,
            box_format=bbox_format
        )
        self.coco_metric = PrecisionRecallF1Support(**metric_params)

    def _info(self):
        return evaluate.MetricInfo(
            # This is the description that will appear on the modules page.
            module_type="metric",
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            # This defines the format of each prediction and reference
            features=datasets.Features(
                {
                'predictions': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))),
                'references': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))),
                }
            ),
            # Additional links to the codebase or references
            codebase_urls=["https://github.com/SEA-AI/metrics/tree/main",
                           "https://github.com/cocodataset/cocoapi/tree/master"]
        )
    
    def add_batch(
            self, 
            data: dict,
            model: str = None
        ):
        """Add predictions and ground truths of a single image to update the metric.

        Args:
            data (dict): containing standard payload of data that should be evaluated
                format should be as returned by function `fo_to_payload()` in seametrics library
            model (str): should be one out of values given in data["models"]
                if not defined, defaults to data["models"][0], as only one model can be evaluated a time.
        """
        # populate two empty lists in format suitable for hugging face metric
        # nothing is computed based on them but prevents huggingface error


        self, predictions,references = _add_batch(self, data, model)    

        # prevents hugging face error, doesn't do a lot       
        super(evaluate.Metric, self).add_batch(
            predictions=predictions,
            references=references
        )


    def _compute(
            self, 
            predictions, 
            references
        ):
        """Returns the scores"""
        result = self.coco_metric.compute()["metrics"]
        return result