Update space

app.py

@@ -97,8 +97,11 @@ def init_leaderboard(dataframe):
         interactive=False,
     )
 
-model_result_path = "./src/results/models_2024-10-07-14:50:12.666068.jsonl"
-model_leaderboard_df = get_model_leaderboard_df(model_result_path)
+# model_result_path = "./src/results/models_2024-10-07-14:50:12.666068.jsonl"
+# model_result_path = "./src/results/models_2024-10-08-03:10:26.811832.jsonl"
+model_result_path = "./src/results/models_2024-10-08-03:25:44.801310.jsonl"
+# model_leaderboard_df = get_model_leaderboard_df(model_result_path)
+
 
 def overall_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -118,6 +121,25 @@ def overall_leaderboard(dataframe):
     
     
 
+def overview_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=None,
+        search_columns=SearchColumns(primary_column=AutoEvalColumn.model.name, secondary_columns=[],
+                                     placeholder="Search by the model name",
+                                     label="Searching"),
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=None,
+        interactive=False,
+    )
+    
+
+    
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -126,33 +148,117 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
 
         with gr.TabItem("🏅 Overview", elem_id="llm-benchmark-tab-table", id=0):
-            # leaderboard = init_leaderboard(LEADERBOARD_DF)
-            leaderboard = overall_leaderboard(model_leaderboard_df)
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            # leaderboard = overview_leaderboard(model_leaderboard_df)
             
 
         with gr.TabItem("🎯 Overall", elem_id="llm-benchmark-tab-table", id=1):
-            leaderboard = overall_leaderboard(model_leaderboard_df)
+            
+            leaderboard = overall_leaderboard(
+                get_model_leaderboard_df(
+                    model_result_path,
+                    benchmark_cols=[
+                        AutoEvalColumn.rank_overall.name,
+                        AutoEvalColumn.model.name, 
+                        AutoEvalColumn.score_overall.name,
+                        AutoEvalColumn.sd_overall.name,
+                        AutoEvalColumn.license.name,
+                        AutoEvalColumn.organization.name,
+                        AutoEvalColumn.knowledge_cutoff.name,
+                        ],
+                    rank_col=[AutoEvalColumn.rank_overall.name],
+                ))
             
         with gr.TabItem("🔢 Math", elem_id="math-tab-table", id=2):
             
             # leaderboard = init_leaderboard(LEADERBOARD_DF)
             with gr.TabItem("🧮 Algebra", elem_id="algebra_subtab", id=0, elem_classes="subtab"): 
-                leaderboard = overall_leaderboard(model_leaderboard_df)
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_math_algebra.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_math_algebra.name,
+                            AutoEvalColumn.sd_math_algebra.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_math_algebra.name],
+                    )
+                )
                 
             with gr.TabItem("📐 Geometry", elem_id="geometry_subtab", id=1, elem_classes="subtab"): 
-                leaderboard = overall_leaderboard(model_leaderboard_df)
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_math_geometry.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_math_geometry.name,
+                            AutoEvalColumn.sd_math_geometry.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_math_geometry.name],
+                    )
+                )
 
             with gr.TabItem("📊 Probability", elem_id="prob_subtab", id=2, elem_classes="subtab"):
-                leaderboard = overall_leaderboard(model_leaderboard_df)
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_math_probability.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_math_probability.name,
+                            AutoEvalColumn.sd_math_probability.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_math_probability.name],
+                    )
+                )
                 
 
         with gr.TabItem("🧠 Reasoning", elem_id="reasonong-tab-table", id=3):
 
             with gr.TabItem("🧩 Logical", elem_id="logical_subtab", id=0, elem_classes="subtab"):         
-                leaderboard = overall_leaderboard(model_leaderboard_df)
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_reason_logical.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_reason_logical.name,
+                            AutoEvalColumn.sd_reason_logical.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_reason_logical.name],
+                    )
+                )
 
             with gr.TabItem("🗣️ Social", elem_id="social_subtab", id=1, elem_classes="subtab"):         
-                leaderboard = overall_leaderboard(model_leaderboard_df)
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_reason_social.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_reason_social.name,
+                            AutoEvalColumn.sd_reason_social.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_reason_social.name],
+                    )
+                )
 
 
         with gr.TabItem("</> Coding", elem_id="coding-tab-table", id=4):

src/about.py

@@ -1,6 +1,23 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
+# @dataclass
+# class Ranking:
+#     dimension: str
+#     metric: str
+#     col_name: str
+
+# class Rankings(Enum):
+#     # dimension_key in the json file, metric_key in the json file, name to display in the leaderboard 
+#     rank0 = Ranking("overall", "Avg Score", "Overall")
+#     rank1 = Ranking("math_algebra", "Avg Score", "Math (Algebra)")
+#     rank2 = Ranking("math_geometry", "Avg Score", "Math (Geometry)")
+#     rank3 = Ranking("math_prob", "Avg Score", "Math (Probability)")
+#     rank4 = Ranking("reason_logical", "Avg Score", "Logical Reasoning")
+#     rank5 = Ranking("reason_social", "Avg Score", "Social Reasoning")
+
+
 @dataclass
 class Domain:
     dimension: str

src/display/utils.py

@@ -63,6 +63,28 @@ auto_eval_column_dict.append(["score", ColumnContent, field(default_factory=lamb
 auto_eval_column_dict.append(["score_sd", ColumnContent, field(default_factory=lambda: ColumnContent("Score SD", "number", True))])
 auto_eval_column_dict.append(["rank", ColumnContent, field(default_factory=lambda: ColumnContent("Rank", "number", True))])
 
+# fine-graine dimensions
+auto_eval_column_dict.append(["score_overall", ColumnContent, field(default_factory=lambda: ColumnContent("Overall", "number", True))])
+auto_eval_column_dict.append(["score_math_algebra", ColumnContent, field(default_factory=lambda: ColumnContent("Math (Algebra)", "number", True))])
+auto_eval_column_dict.append(["score_math_geometry", ColumnContent, field(default_factory=lambda: ColumnContent("Math (Geometry)", "number", True))])
+auto_eval_column_dict.append(["score_math_probability", ColumnContent, field(default_factory=lambda: ColumnContent("Math (Probability)", "number", True))])
+auto_eval_column_dict.append(["score_reason_logical", ColumnContent, field(default_factory=lambda: ColumnContent("Logical Reasoning", "number", True))])
+auto_eval_column_dict.append(["score_reason_social", ColumnContent, field(default_factory=lambda: ColumnContent("Social Reasoning", "number", True))])
+
+auto_eval_column_dict.append(["sd_overall", ColumnContent, field(default_factory=lambda: ColumnContent("SD Overall", "number", True))])
+auto_eval_column_dict.append(["sd_math_algebra", ColumnContent, field(default_factory=lambda: ColumnContent("SD Math (Algebra)", "number", True))])
+auto_eval_column_dict.append(["sd_math_geometry", ColumnContent, field(default_factory=lambda: ColumnContent("SD Math (Geometry)", "number", True))])
+auto_eval_column_dict.append(["sd_math_probability", ColumnContent, field(default_factory=lambda: ColumnContent("SD Math (Probability)", "number", True))])
+auto_eval_column_dict.append(["sd_reason_logical", ColumnContent, field(default_factory=lambda: ColumnContent("SD Logical Reasoning", "number", True))])
+auto_eval_column_dict.append(["sd_reason_social", ColumnContent, field(default_factory=lambda: ColumnContent("SD Social Reasoning", "number", True))])
+
+auto_eval_column_dict.append(["rank_overall", ColumnContent, field(default_factory=lambda: ColumnContent("Rank Overall", "number", True))])
+auto_eval_column_dict.append(["rank_math_algebra", ColumnContent, field(default_factory=lambda: ColumnContent("Rank Math (Algebra)", "number", True))])
+auto_eval_column_dict.append(["rank_math_geometry", ColumnContent, field(default_factory=lambda: ColumnContent("Rank Math (Geometry)", "number", True))])
+auto_eval_column_dict.append(["rank_math_probability", ColumnContent, field(default_factory=lambda: ColumnContent("Rank Math (Probability)", "number", True))])
+auto_eval_column_dict.append(["rank_reason_logical", ColumnContent, field(default_factory=lambda: ColumnContent("Rank Logical Reasoning", "number", True))])
+auto_eval_column_dict.append(["rank_reason_social", ColumnContent, field(default_factory=lambda: ColumnContent("Rank Social Reasoning", "number", True))])
+                             
 
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, field(default_factory=lambda: ColumnContent(task.value.col_name, "number", True))])

src/leaderboard/read_evals.py

@@ -11,14 +11,10 @@ from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, Domains
 from src.submission.check_validity import is_model_on_hub
 
-# @dataclass
-# class RankResult:
-    
-
 
 @dataclass
-class ModelResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+class RankResult:
+    """Represents one the overall ranking table
     """
     eval_name: str
     full_model: str
@@ -74,7 +70,7 @@ class ModelResult:
 
             # AutoEvalColumn.precision.name: self.precision.value.name,
             # AutoEvalColumn.model_type.name: self.model_type.value.name,
-            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            # AutoEvalColumn.model_type_symbol.name
             # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             # AutoEvalColumn.architecture.name: self.architecture,
             # AutoEvalColumn.revision.name: self.revision,
@@ -83,6 +79,116 @@ class ModelResult:
             # AutoEvalColumn.params.name: self.num_params,
             # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
+        
+
+
+@dataclass
+class ModelResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str
+    full_model: str
+    org: str 
+    model: str
+    results: dict
+    license: str = "?"
+    knowledge_cutoff: str = ""
+    
+    @classmethod
+    def init_from_json_dict(self, data):
+        
+        config = data.get("config")
+        # Get model and org
+        model = config.get("model_name")
+        org = config.get("organization")
+        license = config.get("license")
+        knowledge_cutoff = config.get("knowledge_cutoff")
+        
+        model_results = data.get("results")
+        new_results = {}
+        for k, v in model_results.items():
+            new_v = {}
+            for kk, vv in v.items():
+                if vv == 'N/A':
+                    new_v[kk] = None
+                else:
+                    new_v[kk] = vv
+                    
+            new_results[k] = new_v
+
+        # Extract results available in this file (some results are split in several files)
+        # results = {}
+        # for domain in Domains:
+        #     domain = domain.value
+        #     results[domain.dimension] = model_results.get(domain.dimension).get(domain.metric, None)
+        
+        return self(
+            eval_name=f"{org}_{model}",
+            full_model=f"{org}/{model}",
+            org=org,
+            model=model,
+            results=new_results,
+            license=license,
+            knowledge_cutoff=knowledge_cutoff
+        )
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        
+        data_dict = {
+            # "eval_name": self.eval_name,  # not a column, just a save name,
+            # AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            # AutoEvalColumn.rank.name: None, # placeholder for the rank
+            AutoEvalColumn.model.name: self.model,
+            # AutoEvalColumn.score.name: self.results[Domains.dim0.value.dimension],
+            # AutoEvalColumn.score_sd.name: None, # placeholder for the score sd 
+
+            # AutoEvalColumn.score_overall.name: float(self.results.get("OVERALL").get("Average Score", None)),
+            # AutoEvalColumn.score_math_algebra.name: float(self.results.get("Algebra").get("Average Score", None)),
+            # AutoEvalColumn.score_math_geometry.name: float(self.results.get("Geometry").get("Average Score", None)),
+            # AutoEvalColumn.score_math_probability.name: float(self.results.get("Probability").get("Average Score", None)),
+            # AutoEvalColumn.score_reason_logical.name: float(self.results.get("Logical").get("Average Score", None)),
+            # AutoEvalColumn.score_reason_social.name: float(self.results.get("Social").get("Average Score", None)),
+            
+            # AutoEvalColumn.sd_overall.name: float(self.results.get("OVERALL").get("Standard Deviation", None)),
+            # AutoEvalColumn.sd_math_algebra.name: float(self.results.get("Algebra").get("Standard Deviation", None)),
+            # AutoEvalColumn.sd_math_geometry.name: float(self.results.get("Geometry").get("Standard Deviation", None)),
+            # AutoEvalColumn.sd_math_probability.name: float(self.results.get("Probability").get("Standard Deviation", None)),
+            # AutoEvalColumn.sd_reason_logical.name: float(self.results.get("Logical").get("Standard Deviation", None)),
+            # AutoEvalColumn.sd_reason_social.name: float(self.results.get("Social").get("Standard Deviation", None)),
+
+            # AutoEvalColumn.rank_overall.name: int(self.results.get("OVERALL").get("Rank", None)),
+            # AutoEvalColumn.rank_math_algebra.name: int(self.results.get("Algebra").get("Rank", None)),
+            # AutoEvalColumn.rank_math_geometry.name: int(self.results.get("Geometry").get("Rank", None)),
+            # AutoEvalColumn.rank_math_probability.name: int(self.results.get("Probability").get("Rank", None)),
+            # AutoEvalColumn.rank_reason_logical.name: int(self.results.get("Logical").get("Rank", None)),
+            # AutoEvalColumn.rank_reason_social.name: int(self.results.get("Social").get("Rank", None)),
+            
+            AutoEvalColumn.score_overall.name: self.results.get("OVERALL").get("Average Score", None),
+            AutoEvalColumn.score_math_algebra.name: self.results.get("Algebra").get("Average Score", None),
+            AutoEvalColumn.score_math_geometry.name: self.results.get("Geometry").get("Average Score", None),
+            AutoEvalColumn.score_math_probability.name: self.results.get("Probability").get("Average Score", None),
+            AutoEvalColumn.score_reason_logical.name: self.results.get("Logical").get("Average Score", None),
+            AutoEvalColumn.score_reason_social.name: self.results.get("Social").get("Average Score", None),
+            
+            AutoEvalColumn.sd_overall.name: self.results.get("OVERALL").get("Standard Deviation", None),
+            AutoEvalColumn.sd_math_algebra.name: self.results.get("Algebra").get("Standard Deviation", None),
+            AutoEvalColumn.sd_math_geometry.name: self.results.get("Geometry").get("Standard Deviation", None),
+            AutoEvalColumn.sd_math_probability.name: self.results.get("Probability").get("Standard Deviation", None),
+            AutoEvalColumn.sd_reason_logical.name: self.results.get("Logical").get("Standard Deviation", None),
+            AutoEvalColumn.sd_reason_social.name: self.results.get("Social").get("Standard Deviation", None),
+
+            AutoEvalColumn.rank_overall.name: self.results.get("OVERALL").get("Rank", None),
+            AutoEvalColumn.rank_math_algebra.name: self.results.get("Algebra").get("Rank", None),
+            AutoEvalColumn.rank_math_geometry.name: self.results.get("Geometry").get("Rank", None),
+            AutoEvalColumn.rank_math_probability.name: self.results.get("Probability").get("Rank", None),
+            AutoEvalColumn.rank_reason_logical.name: self.results.get("Logical").get("Rank", None),
+            AutoEvalColumn.rank_reason_social.name: self.results.get("Social").get("Rank", None),
+            
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.organization.name: self.org,
+            AutoEvalColumn.knowledge_cutoff.name: self.knowledge_cutoff,
+        }
 
         # for task in Tasks:
         #     data_dict[task.value.col_name] = self.results[task.value.benchmark]
@@ -306,7 +412,23 @@ def get_raw_model_results(results_path: str) -> list[EvalResult]:
         # full_model='OpenAI/ChatGPT-4o-latest (2024-09-03)', 
         # org='OpenAI', model='ChatGPT-4o-latest (2024-09-03)', 
         # results={'overall': None}, license='Proprietary', knowledge_cutoff='2023/10')
+
+        # all_num_results = eval_result.results
+
+        # def get_terminal_values(data):
+        #     terminal_values = []
+        #     for key, value in data.items():
+        #         if isinstance(value, dict):
+        #             terminal_values.extend(get_terminal_values(value))
+        #         else:
+        #             terminal_values.append(value)
+        #     return terminal_values
+    
+        # all_values = get_terminal_values(all_num_results)
         
+        # if 'N/A' in all_values:
+        #     continue
+            
         eval_name = eval_result.eval_name
         eval_results[eval_name] = eval_result
 
@@ -319,6 +441,7 @@ def get_raw_model_results(results_path: str) -> list[EvalResult]:
     results = []
     for v in eval_results.values():
         # print(v.to_dict())
+        # exit()
         # {'eval_name': 'OpenAI_ChatGPT-4o-latest (2024-09-03)', 
         # 'Model': '<a target="_blank" href="https://huggingface.co/OpenAI/ChatGPT-4o-latest (2024-09-03)" 
         # style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">OpenAI/ChatGPT-4o-latest (2024-09-03)</a>', 

src/populate.py

@@ -9,44 +9,53 @@ from src.leaderboard.read_evals import get_raw_eval_results, get_raw_model_resul
 
 
 
-def get_overview_leaderboard_df(results_path: str) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+# def get_overview_leaderboard_df(results_path: str) -> pd.DataFrame:
+#     """Creates a dataframe from all the individual experiment results"""
+#     raw_data = get_raw_eval_results(results_path, requests_path)
+#     all_data_json = [v.to_dict() for v in raw_data]
 
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    for col in cols:
-        if col not in df.columns:
-            df[col] = None
-        else:
-            df[col] = df[col].round(decimals=2)
+#     df = pd.DataFrame.from_records(all_data_json)
+#     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+#     for col in cols:
+#         if col not in df.columns:
+#             df[col] = None
+#         else:
+#             df[col] = df[col].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+#     # filter out if any of the benchmarks have not been produced
+#     df = df[has_no_nan_values(df, benchmark_cols)]
+#     return df
 
 
 
-def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: list=[], benchmark_cols: list=[]) -> pd.DataFrame:
+def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: list=[], benchmark_cols: list=[], rank_col: list=[]) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_model_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.score.name], ascending=True)
-    df[AutoEvalColumn.rank.name] = df[AutoEvalColumn.score.name].rank(ascending=True, method="min")
+    
+    df = df[benchmark_cols]
+    df = df.dropna(subset=benchmark_cols)
+    
+    if rank_col:
+        df = df.sort_values(by=[rank_col[0]], ascending=True)
+    
+    # df = df.sort_values(by=[AutoEvalColumn.score.name], ascending=True)
+    # df[AutoEvalColumn.rank.name] = df[AutoEvalColumn.score.name].rank(ascending=True, method="min")
     # print(cols) # []
     # print(df.columns) # ['eval_name', 'Model', 'Hub License', 'Organization', 'Knowledge cutoff', 'Overall']
     # exit()
-    for col in cols:
-        if col not in df.columns:
-            df[col] = None
-        else:
-            df = df[cols].round(decimals=2)
+    # only keep the columns that are in the cols list
+    
+    # for col in cols:
+    #     if col not in df.columns:
+    #         df[col] = None
+    #     else:
+    #         df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
-    # df = df[has_no_nan_values(df, benchmark_cols)]
+    df = df[has_no_nan_values(df, benchmark_cols)]
     return df