Update space

app.py

@@ -96,7 +96,7 @@ def init_leaderboard(dataframe):
         interactive=False,
     )
 
-# model_result_path = "./src/results/models_2024-10-07-14:50:12.666068.jsonl"
+model_result_path = "./src/results/models_2024-10-07-14:50:12.666068.jsonl"
 model_leaderboard_df = get_model_leaderboard_df(model_result_path)
 
 def overall_leaderboard(dataframe):
@@ -129,7 +129,7 @@ with demo:
             
 
         with gr.TabItem("🎯 Overall", elem_id="llm-benchmark-tab-table", id=1):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            leaderboard = overall_leaderboard(model_leaderboard_df)
             
         with gr.TabItem("🔢 Math", elem_id="math-tab-table", id=2):
             

src/leaderboard/read_evals.py

@@ -30,10 +30,10 @@ class ModelResult:
         config = data.get("config")
         # Get model and org
         model = config.get("model_name")
-        org = config.get("org_name")
+        org = config.get("organization")
         license = config.get("license")
         knowledge_cutoff = config.get("knowledge_cutoff")
-                
+        
         # Extract results available in this file (some results are split in several files)
         results = {}
         for domain in Domains:
@@ -75,8 +75,8 @@ class ModelResult:
             # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        # for task in Tasks:
+        #     data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         for domain in Domains:
             data_dict[domain.value.col_name] = self.results[domain.value.dimension]
@@ -277,26 +277,48 @@ def get_raw_model_results(results_path: str) -> list[EvalResult]:
     except:
         data = eval(open(results_path).read()) # a list of dicts   
 
+    # print("data", len(data))
+    # print(data[0]) 
+    # {'config': {'model_name': 'ChatGPT-4o-latest (2024-09-03)', 
+    # 'organization': 'OpenAI', 'license': 'Proprietary', 
+    # 'knowledge_cutoff': '2023/10'}, 
+    # 'results': {'math-algebra': 
+    # {'Score': 99.19484702, 'Avg Rank': 1.666666667, 'Min Rank': 1, 'Max Rank': 3}, 
+    # 'math-probability': {'Score': 100, 'Avg Rank': 1, 'Min Rank': 1, 'Max Rank': 1}, 
+    # 'reasoning-logical': {'Avg Rank': 1, 'Min Rank': 1, 'Max Rank': 1}, 
+    # 'overall': {'Avg Rank': 2, 'Min Rank': 2, 'Max Rank': 2}}}
     eval_results = {}
 
     for result in data:
         # Creation of result
         eval_result = ModelResult.init_from_json_dict(result)
+        # print(eval_result)
+        # ModelResult(eval_name='OpenAI_ChatGPT-4o-latest (2024-09-03)', 
+        # full_model='OpenAI/ChatGPT-4o-latest (2024-09-03)', 
+        # org='OpenAI', model='ChatGPT-4o-latest (2024-09-03)', 
+        # results={'overall': None}, license='Proprietary', knowledge_cutoff='2023/10')
         
-        # Store results of same eval together
         eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
+        eval_results[eval_name] = eval_result
+
+        # # Store results of same eval together
+        # if eval_name in eval_results.keys():
+        #     eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        # else:
+        #     eval_results[eval_name] = eval_result
 
     results = []
     for v in eval_results.values():
+        # print(v.to_dict())
+        # {'eval_name': 'OpenAI_ChatGPT-4o-latest (2024-09-03)', 
+        # 'Model': '<a target="_blank" href="https://huggingface.co/OpenAI/ChatGPT-4o-latest (2024-09-03)" 
+        # style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">OpenAI/ChatGPT-4o-latest (2024-09-03)</a>', 
+        # 'Hub License': 'Proprietary', 'Organization': 'OpenAI', 'Knowledge cutoff': '2023/10', 'Overall': None}
         try:
             v.to_dict() # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             continue
-
+    
     return results
 

src/populate.py

@@ -15,6 +15,9 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
 
     df = pd.DataFrame.from_records(all_data_json)
     # df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    # print(cols) # []
+    # print(df.columns) # ['eval_name', 'Model', 'Hub License', 'Organization', 'Knowledge cutoff', 'Overall']
+    # exit()
     for col in cols:
         if col not in df.columns:
             df[col] = None