update

src/display/utils.py

@@ -27,7 +27,7 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-#auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
@@ -51,7 +51,7 @@ auto_eval_column_quota_dict = []
 auto_eval_column_quota_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_quota_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-#auto_eval_column_quota_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_quota_dict.append(["average_quota", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Quotas:
     auto_eval_column_quota_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, Quotas
+from src.display.utils import AutoEvalColumn, AutoEvalColumnQuota, ModelType, Tasks, Precision, WeightType, Quotas
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -99,7 +99,11 @@ class EvalResult:
 
             mean_acc = np.mean(accs) if len(accs) > 0 else 0
             print("mean_acc", task.metric, mean_acc)
-            results[task.metric] = mean_acc
+            if task.metric == "overall":
+                results[task.benchmark] = mean_acc
+            else:
+                results[task.metric] = mean_acc
+            
 
         return self(
             eval_name=result_key,
@@ -144,7 +148,7 @@ class EvalResult:
         average = 0
         nums = 0
         for k,v in self.results.items():
-            if k not in ["Visual Grounding","Counting","State & Activity Understanding","Dynamic","Relative direction","Multi-view matching","Relative distance","Depth estimation","Relative shape","Size estimation","Trajectory","Future prediction","Goal Decomposition","Navigation"]:
+            if k not in ["Perception","SpatialReasoning","Prediction","Planning","Visual Grounding","Counting","State & Activity Understanding","Dynamic","Relative direction","Multi-view matching","Relative distance","Depth estimation","Relative shape","Size estimation","Trajectory","Future prediction","Goal Decomposition","Navigation"]:
                 if v is not None and v != 0:
                     average += v
                     nums += 1
@@ -152,6 +156,17 @@ class EvalResult:
             average = 0
         else:
             average = average/nums
+        
+        nums,average_quota=0,0
+        for k,v in self.results.items():
+            if k in ["Perception","SpatialReasoning","Prediction","Planning"]:
+                f v is not None and v != 0:
+                    average_quota += v
+                    nums += 1
+        if nums ==0:
+            average_quota = 0
+        else:
+            average_quota = average_quota/nums
 
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
@@ -163,7 +178,8 @@ class EvalResult:
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
             AutoEvalColumn.revision.name: self.revision,
-            #AutoEvalColumn.average.name: average,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumnQuota.average_quota.name: average_quota,
 
             #AutoEvalColumn.license.name: self.license,
             #AutoEvalColumn.likes.name: self.likes,
@@ -186,13 +202,14 @@ class EvalResult:
 
         for task in Quotas:
             #data_dict[task.value.col_name] = self.results.get(task.value.metric, 0)
-            if task.value.col_name != "CLCC-H":
+            if task.value.metric != "overall":
                 data_dict[task.value.col_name] = self.results.get(task.value.metric, 0)
             else:
-                if self.results.get(task.value.metric, 0) == 0:
-                    data_dict[task.value.col_name] = "-"
-                else:
-                    data_dict[task.value.col_name] = "%.2f" % self.results.get(task.value.metric, 0)
+                data_dict[task.value.col_name] = self.results.get(task.value.bench, 0)
+                #if self.results.get(task.value.benchmark, 0) == 0:
+                #    data_dict[task.value.col_name] = "-"
+                #else:
+                #    data_dict[task.value.col_name] = "%.2f" % self.results.get(task.value.metric, 0)
 
         return data_dict
 

src/populate.py

@@ -27,6 +27,23 @@ def get_leaderboard_df(results_path: str, requests_path: str, dynamic_path: str,
     return raw_data, df
 
 
+def get_leaderboard_df_quota(results_path: str, requests_path: str, dynamic_path: str,cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path, dynamic_path)
+    for v in raw_data:
+        print(v.to_dict())
+    all_data_json = [v.to_dict() for v in raw_data]
+    #all_data_json.append(baseline_row)
+    filter_models_flags(all_data_json)
+    df = pd.DataFrame.from_records(all_data_json)
+    print("AutoEvalColumn.average.name",AutoEvalColumn.average.name)
+    df = df.sort_values(by=[AutoEvalColumnQuota.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return raw_data, df
+
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation queues requestes"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]