yeah

my_metricv2.py

@@ -29,8 +29,10 @@ year={2020}
 
 # TODO: Add description of the module here
 _DESCRIPTION = """\
-This new module is designed to solve this great ML task and is crafted with a lot of care.
-"""
+The MOT Metrics module is designed to evaluate multi-object tracking (MOT) 
+algorithms by computing various metrics based on predicted and ground truth bounding 
+boxes. It serves as a crucial tool in assessing the performance of MOT systems, 
+aiding in the iterative improvement of tracking algorithms."""
 
 
 # TODO: Add description of the arguments of the module here
@@ -42,35 +44,70 @@ Args:
         should be a string with tokens separated by spaces.
     references: list of reference for each prediction. Each
         reference should be a string with tokens separated by spaces.
+    max_iou (`float`, *optional*):
+        If specified, this is the minimum Intersection over Union (IoU) threshold to consider a detection as a true positive.
+        Default is 0.5.
 Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
+    summary: pandas.DataFrame with the following columns:
+        - idf1 (IDF1 Score): The F1 score for the identity assignment, computed as 2 * (IDP * IDR) / (IDP + IDR).
+        - idp (ID Precision): Identity Precision, representing the ratio of correctly assigned identities to the total number of predicted identities.
+        - idr (ID Recall): Identity Recall, representing the ratio of correctly assigned identities to the total number of ground truth identities.
+        - recall: Recall, computed as the ratio of the number of correctly tracked objects to the total number of ground truth objects.
+        - precision: Precision, computed as the ratio of the number of correctly tracked objects to the total number of predicted objects.
+        - num_unique_objects: Total number of unique objects in the ground truth.
+        - mostly_tracked: Number of objects that are mostly tracked throughout the sequence.
+        - partially_tracked: Number of objects that are partially tracked but not mostly tracked.
+        - mostly_lost: Number of objects that are mostly lost throughout the sequence.
+        - num_false_positives: Number of false positive detections (predicted objects not present in the ground truth).
+        - num_misses: Number of missed detections (ground truth objects not detected in the predictions).
+        - num_switches: Number of identity switches.
+        - num_fragmentations: Number of fragmented objects (objects that are broken into multiple tracks).
+        - mota (MOTA - Multiple Object Tracking Accuracy): Overall tracking accuracy, computed as 1 - ((num_false_positives + num_misses + num_switches) / num_unique_objects).
+        - motp (MOTP - Multiple Object Tracking Precision): Average precision of the object localization, computed as the mean of the localization errors of correctly detected objects.
+        - num_transfer: Number of track transfers.
+        - num_ascend: Number of ascended track IDs.
+        - num_migrate: Number of track ID migrations.
+
 Examples:
     >>> import numpy as np
-    >>> mean_iou = evaluate.load("mean_iou")
-
-    >>> # suppose one has 3 different segmentation maps predicted
-    >>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]])
-    >>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]])
-
-    >>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]])
-    >>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]])
-
-    >>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]])
-    >>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]])
-
-    >>> predicted = [predicted_1, predicted_2, predicted_3]
-    >>> ground_truth = [actual_1, actual_2, actual_3]
-
-    >>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255, reduce_labels=False)
-    >>> print(results) # doctest: +NORMALIZE_WHITESPACE
-    {'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0.   , 0.   , 0.375, 0.4  , 0.5  , 0.   , 0.5  , 1.   , 1.   , 1.   ]), 'per_category_accuracy': array([0.        , 0.        , 0.75      , 0.66666667, 1.        , 0.        , 0.5       , 1.        , 1.        , 1.        ])}
-
+    >>> module = evaluate.load("bascobasculino/my_metricv2")
+
+    >>> predicted =[
+            [1,1,10,20,30,40,0.85],
+            [1,2,50,60,70,80,0.92],
+            [1,3,80,90,100,110,0.75],
+            [2,1,15,25,35,45,0.78],
+            [2,2,55,65,75,85,0.95],
+            [3,1,20,30,40,50,0.88],
+            [3,2,60,70,80,90,0.82],
+            [4,1,25,35,45,55,0.91],
+            [4,2,65,75,85,95,0.89]
+        ]
+
+    >>> ground_truth = [
+            [1, 1, 10, 20, 30, 40],
+            [1, 2, 50, 60, 70, 80],
+            [1, 3, 85, 95, 105, 115],
+            [2, 1, 15, 25, 35, 45],
+            [2, 2, 55, 65, 75, 85],
+            [3, 1, 20, 30, 40, 50],
+            [3, 2, 60, 70, 80, 90],
+            [4, 1, 25, 35, 45, 55],
+            [5, 1, 30, 40, 50, 60],
+            [5, 2, 70, 80, 90, 100]
+        ]
+    >>> predicted = [np.array(a) for a in predicted]
+    >>> ground_truth = [np.array(a) for a in ground_truth]
+
+    >>> results = module._compute(predictions=predicted, references=ground_truth, max_iou=0.5)
+    >>> print(results)
+    {'idf1': {0: 0.8421052631578947}, 'idp': {0: 0.8888888888888888}, 'idr': {0: 0.8}, 'recall': {0: 0.8}, 
+    'precision': {0: 0.8888888888888888}, 'num_unique_objects': {0: 3}, 'mostly_tracked': {0: 2}, 
+    'partially_tracked': {0: 1}, 'mostly_lost': {0: 0}, 'num_false_positives': {0: 1}, 'num_misses': {0: 2}, 
+    'num_switches': {0: 0}, 'num_fragmentations': {0: 0}, 'mota': {0: 0.7}, 'motp': {0: 0.02981870229007634}, 
+    'num_transfer': {0: 0}, 'num_ascend': {0: 0}, 'num_migrate': {0: 0}}
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class MyMetricv2(evaluate.Metric):
@@ -145,6 +182,4 @@ def calculate(predictions, references, max_iou: float = 0.5):
 
     mh = mm.metrics.create()
     summary = mh.compute(acc)
-
-
-    return summary
+    return summary.to_dict()

tests.py

@@ -1,17 +1,37 @@
+import numpy as np
 test_cases = [
     {
-        "predictions": [0, 0],
-        "references": [1, 1],
-        "result": {"metric_score": 0}
-    },
-    {
-        "predictions": [1, 1],
-        "references": [1, 1],
-        "result": {"metric_score": 1}
-    },
-    {
-        "predictions": [1, 0],
-        "references": [1, 1],
-        "result": {"metric_score": 0.5}
-    }
+        "predictions": [np.array(a) for a in [
+                            [1,1,10,20,30,40,0.85],
+                            [1,2,50,60,70,80,0.92],
+                            [1,3,80,90,100,110,0.75],
+                            [2,1,15,25,35,45,0.78],
+                            [2,2,55,65,75,85,0.95],
+                            [3,1,20,30,40,50,0.88],
+                            [3,2,60,70,80,90,0.82],
+                            [4,1,25,35,45,55,0.91],
+                            [4,2,65,75,85,95,0.89]
+                        ]],
+        "references": [np.array(a) for a in [
+                            [1, 1, 10, 20, 30, 40],
+                            [1, 2, 50, 60, 70, 80],
+                            [1, 3, 85, 95, 105, 115],
+                            [2, 1, 15, 25, 35, 45],
+                            [2, 2, 55, 65, 75, 85],
+                            [3, 1, 20, 30, 40, 50],
+                            [3, 2, 60, 70, 80, 90],
+                            [4, 1, 25, 35, 45, 55],
+                            [5, 1, 30, 40, 50, 60],
+                            [5, 2, 70, 80, 90, 100]
+                        ]],
+        "result": {'idf1': {0: 0.8421052631578947}, 'idp': {0: 0.8888888888888888},
+                    'idr': {0: 0.8}, 'recall': {0: 0.8}, 'precision': {0: 0.8888888888888888},
+                    'num_unique_objects': {0: 3}, 'mostly_tracked': {0: 2},
+                    'partially_tracked': {0: 1}, 'mostly_lost': {0: 0},
+                    'num_false_positives': {0: 1}, 'num_misses': {0: 2},
+                    'num_switches': {0: 0}, 'num_fragmentations': {0: 0},
+                    'mota': {0: 0.7}, 'motp': {0: 0.02981870229007634},
+                    'num_transfer': {0: 0}, 'num_ascend': {0: 0},
+                    'num_migrate': {0: 0}}
+    },   
 ]