updated scores algorithm

utils_v2.py

@@ -66,17 +66,9 @@ def load_data(base_dir=SCORE_BASE_DIR):
             all_data.append(data)
     return all_data
 
-def calculate_score(raw_scores=None):
-    """This function calculates the overall average scores for all datasets as well as avg scores for each modality and sub-task based on the raw scores.
-    Algorithm summary: 
-    """
-    def get_avg(sum_score, leng):
-        avg = sum_score / leng if leng > 0 else 0.0
-        avg = round(avg, 2)  # Round to 2 decimal places
-        return avg
-    
-    avg_scores = {}
-    overall_scores_summary = {} # Stores the scores sum and length for each modality and all datasets
+def load_scores(raw_scores=None):
+    """This function loads the raw scores from the user provided scores summary and flattens them into a single dictionary."""
+    all_scores = {}
     for modality, datasets_list in DATASETS.items(): # Ex.: ('image', {'I-CLS': [...], 'I-QA': [...]})
         overall_scores_summary[modality] = (0.0, 0) # Initialize the sum and count for each modality
         for sub_task, datasets in datasets_list.items(): # Ex.: ('I-CLS', ['VOC2007', 'N24News', ...])
@@ -87,26 +79,42 @@ def calculate_score(raw_scores=None):
                 metric = SPECIAL_METRICS.get(dataset, 'hit@1')
                 if isinstance(score, dict):
                     score = score.get(metric, 0.0)
-                sub_task_sum_score += score
-
-            sub_task_overall = get_avg(sub_task_sum_score, sub_task_datasets_len)
-            avg_scores[sub_task] = sub_task_overall
-
-            # Accumulate the scores sum and length for the each modality
-            modality_sum_score, modality_datasets_len = overall_scores_summary[modality]
-            modality_sum_score += sub_task_sum_score
-            modality_datasets_len += sub_task_datasets_len
-            overall_scores_summary[modality] = (modality_sum_score, modality_datasets_len)
-
-    all_datasets_sum_score, all_datasets_len = 0.0, 0
-    for modality, (modality_sum_score, modality_datasets_len) in overall_scores_summary.items():
-        name = f"{modality.capitalize()}-Overall"
-        avg_scores[name] = get_avg(modality_sum_score, modality_datasets_len)
-        # Accumulate the scores sum and length for all datasets
-        all_datasets_sum_score += modality_sum_score
-        all_datasets_len += modality_datasets_len
-    avg_scores['Overall'] = get_avg(all_datasets_sum_score, all_datasets_len)
-    return avg_scores
+                single_dataset_score = {'dataset': dataset, 'score': score}
+                all_scores.update(single_dataset_score)
+    return all_scores
+
+def calculate_score(raw_scores=None):
+    """This function calculates the overall average scores for all datasets as well as avg scores for each modality and sub-task based on the raw scores.
+    """
+    def get_avg(sum_score, leng):
+        avg = sum_score / leng if leng > 0 else 0.0
+        avg = round(avg, 2)  # Round to 2 decimal places
+        return avg
+    
+    all_scores = load_scores(raw_scores)
+    avg_scores = {}
+
+    # Calculate overall score for all datasets
+    avg_scores['Overall'] = get_avg(sum(
+                            all_scores.values()), 
+                            len(ALL_DATASETS))
+
+    # Calculate scores for each modality
+    for modality in MODALITIES:
+        datasets_for_each_modality = ALL_DATASETS_SPLITS.get(modality, [])
+        avg_scores[f"{modality.capitalize()}-Overall"] = get_avg(
+            sum(all_scores.get(dataset, 0.0) for dataset in datasets_for_each_modality),
+            len(datasets_for_each_modality)
+        )
+
+    # Calculate scores for each sub-task
+    for modality, datasets_list in DATASETS.items():
+        for sub_task, datasets in datasets_list.items():
+            sub_task_score = sum(all_scores.get(dataset, 0.0) for dataset in datasets)
+            avg_scores[sub_task] = get_avg(sub_task_score, len(datasets))
+
+    all_scores.update(avg_scores)
+    return all_scores
 
 def generate_model_row(data):
     metadata = data['metadata']
@@ -127,7 +135,6 @@ def get_df():
     df = df.sort_values(by='Overall', ascending=False).reset_index(drop=True)
     df['Rank'] = range(1, len(df) + 1)
     df = create_hyperlinked_names(df)
-
     return df
 
 def refresh_data():