Merge branch 'main' of https://huggingface.co/spaces/SEA-AI/PanopticQuality

PanopticQuality.py

@@ -109,6 +109,7 @@ class PQMetric(evaluate.Metric):
         DEFAULT_STUFF = ["WATER", "SKY", "LAND", "CONSTRUCTION", "ICE", "OWN_BOAT"]
 
         self.label2id = label2id if label2id is not None else DEFAULT_LABEL2ID
+        self.id2label = {id: label for label, id in self.label2id.items()}
         self.stuff =  stuff if stuff is not None else DEFAULT_STUFF
         self.per_class = per_class
         self.split_sq_rq = split_sq_rq
@@ -118,6 +119,7 @@ class PQMetric(evaluate.Metric):
             return_per_class=per_class,
             return_sq_and_rq=split_sq_rq
         )
+        self.things_stuffs = sorted(self.pq_metric.things) + sorted(self.pq_metric.stuffs)
 
     def _info(self):
         return evaluate.MetricInfo(
@@ -171,9 +173,6 @@ class PQMetric(evaluate.Metric):
         fn = self.pq_metric.metric.false_negatives.clone()
         iou = self.pq_metric.metric.iou_sum.clone()
 
-        id2label = {id: label for label, id in self.label2id.items()}
-        things_stuffs = sorted(self.pq_metric.things) + sorted(self.pq_metric.stuffs)
-
         # compute scores
         result = self.pq_metric.compute() # shape : (n_classes (sorted things + sorted stuffs), scores (pq, sq, rq))
 
@@ -182,11 +181,11 @@ class PQMetric(evaluate.Metric):
         if self.per_class:
             if not self.split_sq_rq:
                 result = result.T
-            result_dict["scores"] = {id2label[numeric_label]: result[i].tolist() \
-                                     for i, numeric_label in enumerate(things_stuffs)}
+            result_dict["scores"] = {self.id2label[numeric_label]: result[i].tolist() \
+                                     for i, numeric_label in enumerate(self.things_stuffs)}
             result_dict["scores"].update({"ALL": result.mean(axis=0).tolist()})
-            result_dict["numbers"] =  {id2label[numeric_label]: [tp[i].item(), fp[i].item(), fn[i].item(), iou[i].item()] \
-                                       for i, numeric_label in enumerate(things_stuffs)}
+            result_dict["numbers"] =  {self.id2label[numeric_label]: [tp[i].item(), fp[i].item(), fn[i].item(), iou[i].item()] \
+                                       for i, numeric_label in enumerate(self.things_stuffs)}
             result_dict["numbers"].update({"ALL": [tp.sum().item(), fp.sum().item(), fn.sum().item(), iou.sum().item()]})
         else:
             result_dict["scores"] = {"ALL": result.tolist() if self.split_sq_rq else [result.tolist()]}

README.md

@@ -5,7 +5,7 @@ tags:
 - metric
 description: PanopticQuality score
 sdk: gradio
-sdk_version: 3.19.1
+sdk_version: 4.36.0
 app_file: app.py
 pinned: false
 emoji: 🖼️
@@ -36,11 +36,52 @@ To get started with PanopticQuality, make sure you have the necessary dependenci
 Added data ...
 Start computing ...
 Finished!
-tensor(0.2082, dtype=torch.float64)
+{'scores': {'MOTORBOAT': [0.18632257426639526,
+   0.698709617058436,
+   0.2666666805744171],
+  'FAR_AWAY_OBJECT': [0.0, 0.0, 0.0],
+  'SAILING_BOAT_WITH_CLOSED_SAILS': [0.0, 0.0, 0.0],
+  'SHIP': [0.3621737026917471, 0.684105846616957, 0.529411792755127],
+  'WATERCRAFT': [0.0, 0.0, 0.0],
+  'SPHERICAL_BUOY': [0.0, 0.0, 0.0],
+  'FLOTSAM': [0.0, 0.0, 0.0],
+  'SAILING_BOAT_WITH_OPEN_SAILS': [0.0, 0.0, 0.0],
+  'CONTAINER': [0.0, 0.0, 0.0],
+  'PILLAR_BUOY': [0.0, 0.0, 0.0],
+  'AERIAL_ANIMAL': [0.0, 0.0, 0.0],
+  'HUMAN_IN_WATER': [0.0, 0.0, 0.0],
+  'WOODEN_LOG': [0.0, 0.0, 0.0],
+  'MARITIME_ANIMAL': [0.0, 0.0, 0.0],
+  'WATER': [0.9397601008415222, 0.9397601008415222, 1.0],
+  'SKY': [0.9674496332804362, 0.9674496332804362, 1.0],
+  'LAND': [0.30757412078761204, 0.8304501533508301, 0.37037035822868347],
+  'CONSTRUCTION': [0.0, 0.0, 0.0],
+  'OWN_BOAT': [0.0, 0.0, 0.0],
+  'ALL': [0.14543579641409013, 0.21686712374464112, 0.16665520166095935]},
+ 'numbers': {'MOTORBOAT': [6, 15, 18, 4.1922577023506165],
+  'FAR_AWAY_OBJECT': [0, 8, 9, 0.0],
+  'SAILING_BOAT_WITH_CLOSED_SAILS': [0, 2, 0, 0.0],
+  'SHIP': [9, 1, 15, 6.156952619552612],
+  'WATERCRAFT': [0, 9, 12, 0.0],
+  'SPHERICAL_BUOY': [0, 4, 22, 0.0],
+  'FLOTSAM': [0, 0, 1, 0.0],
+  'SAILING_BOAT_WITH_OPEN_SAILS': [0, 6, 0, 0.0],
+  'CONTAINER': [0, 0, 0, 0.0],
+  'PILLAR_BUOY': [0, 0, 9, 0.0],
+  'AERIAL_ANIMAL': [0, 0, 0, 0.0],
+  'HUMAN_IN_WATER': [0, 0, 0, 0.0],
+  'WOODEN_LOG': [0, 0, 0, 0.0],
+  'MARITIME_ANIMAL': [0, 0, 0, 0.0],
+  'WATER': [15, 0, 0, 14.096401512622833],
+  'SKY': [15, 0, 0, 14.511744499206543],
+  'LAND': [5, 9, 8, 4.15225076675415],
+  'CONSTRUCTION': [0, 0, 0, 0.0],
+  'OWN_BOAT': [0, 0, 8, 0.0],
+  'ALL': [50, 54, 102, 43.109607100486755]}}
 ```
 
 ## Metric Settings
-The metric takes two optional input parameters: __label2id__ and __stuff__.
+The metric takes four optional input parameters: __label2id__, __stuff__, __per_class__ and __split_sq_rq__.
 
 * `label2id: Dict[str, int]`: this dictionary is used to map string labels to an integer representation.
     if not provided a default setting will be used:
@@ -69,8 +110,23 @@ The metric takes two optional input parameters: __label2id__ and __stuff__.
         `
         ["WATER", "SKY", "LAND", "CONSTRUCTION", "ICE", "OWN_BOAT"]`
 
+* `per_class: bool = True`: By default, the results are split up per class.
+    Setting this to False will aggregate the results (average the _scores_, sum up the _numbers_; see below for explanation of _scores_ and _numbers_)
+* `split_sq_rq: bool = True`: By default, the PQ-score is returned in three parts: the PQ score itself, and split into the segmentation quality (SQ) and recognition quality (RQ) part.
+    Setting this to False will return the PQ score only (PQ=RQ*SQ).
+
 ## Output Values
-A single float number between 0 and 1 is returned, which represents the PQ score. The bigger the number the better the PQ score, and vice versa.
+A dictionary containing the following keys:
+* __scores__: This is a dictionary, that contains a key for each label, if `per_class == True`. Otherwise it only contains the key _all_.
+              For each key, it contains a list that holds the scores in the following order: PQ, SQ and RQ. If `split_sq_rq == False`, the list consists of PQ only.
+* __numbers__: This is a dictionary, that contains a key for each label, if `per_class == True`. Otherwise it only contains the key _all_.
+               For each key, it contains a list that consists of four elements: TP, FP, FN and IOU:
+  * __TP__: number of true positive predictions
+  * __FP__: number of false positive predictions
+  * __FN__: number of false negative predictions
+  * __IOU__: sum of IOU of all TP predictions with ground truth
+
+  With all these values, it is possible to calculate the final scores.
 
 ## Further References