Add singletable metrics

app.py

@@ -19,6 +19,7 @@ from src.display.utils import (
     EVAL_COLS,
     EVAL_TYPES,
     AutoEvalColumn,
+    singletable_AutoEvalColumn,
     ModelType,
     fields,
     # WeightType,
@@ -49,7 +50,7 @@ except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+SINGLETABLE_LEADERBOARD_DF, MULTITABLE_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
 
 (
     finished_eval_queue_df,
@@ -57,7 +58,7 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-def init_leaderboard(dataframe):
+def init_multitable_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
@@ -71,7 +72,8 @@ def init_leaderboard(dataframe):
         search_columns=[AutoEvalColumn.model.name], # AutoEvalColumn.license.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.dataset.name, type="checkboxgroup", label="Datasets"),
+            ColumnFilter(AutoEvalColumn.model.name, type="checkboxgroup", label="Models"),
             # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
             # ColumnFilter(
             #     AutoEvalColumn.params.name,
@@ -88,6 +90,27 @@ def init_leaderboard(dataframe):
         interactive=False,
     )
 
+def init_singletable_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(singletable_AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(singletable_AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(singletable_AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[singletable_AutoEvalColumn.model.name], # AutoEvalColumn.license.name],
+        hide_columns=[c.name for c in fields(singletable_AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(singletable_AutoEvalColumn.dataset.name, type="checkboxgroup", label="Datasets"),
+            ColumnFilter(singletable_AutoEvalColumn.model.name, type="checkboxgroup", label="Models"),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -95,8 +118,13 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Syntherela Benchmark", elem_id="syntherela-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏅 MultiTable", elem_id="syntherela-benchmark-tab-table", id=0):
+            leaderboard = init_multitable_leaderboard(MULTITABLE_LEADERBOARD_DF)
+        
+        with gr.TabItem("🏅 SingleTable", elem_id="syntherela-benchmark-tab-table", id=1):
+            singletable_leaderboard = init_singletable_leaderboard(SINGLETABLE_LEADERBOARD_DF)
+
+
 
         with gr.TabItem("📝 About", elem_id="syntherela-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

src/about.py

@@ -14,9 +14,18 @@ class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
     # task0 = Task("anli_r1", "acc", "ANLI")
     # task1 = Task("logiqa", "acc_norm", "LogiQA")
-    task_0 = Task("multi-table", "AggregationDetection-LogisticRegression", "AggregationDetection-LogisticRegression")
-    task_1 = Task("multi-table", "AggregationDetection-XGBClassifier", "AggregationDetection-XGBClassifier")
-    task_2 = Task("multi-table", "CardinalityShapeSimilarity", "CardinalityShapeSimilarity")
+    task_0 = Task("multi-table", "AggregationDetection-LogisticRegression", "AggregationDetection-LogisticRegression ⬇️")
+    task_1 = Task("multi-table", "AggregationDetection-XGBClassifier", "AggregationDetection-XGBClassifier ⬇️")
+    task_2 = Task("multi-table", "CardinalityShapeSimilarity", "CardinalityShapeSimilarity ⬆️")
+
+class SingleTableTasks(Enum):
+    task_0 = Task("single-table", "MaximumMeanDiscrepancy", "MaximumMeanDiscrepancy ⬇️")
+    # PairwiseCorrelationDifference
+    task_1 = Task("single-table", "PairwiseCorrelationDifference", "PairwiseCorrelationDifference ⬇️")
+    # SingleTableDetection-LogisticRegression
+    task_2 = Task("single-table", "SingleTableDetection-LogisticRegression", "SingleTableDetection-LogisticRegression ⬇️")
+    # SingleTableDetection-XGBClassifier
+    task_3 = Task("single-table", "SingleTableDetection-XGBClassifier", "SingleTableDetection-XGBClassifier ⬇️")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -24,7 +33,7 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">Syntherela leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, SingleTableTasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -44,6 +44,17 @@ for task in Tasks:
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+singletable_auto_eval_column_dict = []
+# Init
+singletable_auto_eval_column_dict.append(["dataset", ColumnContent, ColumnContent("Dataset", "str", True, never_hidden=True)])
+# singletable_auto_eval_column_dict.append(["table", ColumnContent, ColumnContent("Table", "str", True, never_hidden=True)])
+singletable_auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+for task in SingleTableTasks:
+    singletable_auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
+singletable_AutoEvalColumn = make_dataclass("AutoEvalColumn", singletable_auto_eval_column_dict, frozen=True)
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column

src/populate.py

@@ -7,6 +7,7 @@ import numpy as np
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
+from src.about import Tasks, SingleTableTasks
 
 
 # def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -22,6 +23,9 @@ from src.leaderboard.read_evals import get_raw_eval_results
 #     df = df[has_no_nan_values(df, benchmark_cols)]
 #     return df
 
+def strip_emoji(text: str) -> str:
+    """Removes emojis from text"""
+    return text.encode("ascii", "ignore").decode("ascii").rstrip()
 
 def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
@@ -36,14 +40,13 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
                 data = json.load(fp)
                 all_data_json.append(data)
 
-    multi_table_metrics = [
-        "AggregationDetection-LogisticRegression",
-        "AggregationDetection-XGBClassifier",
-        "CardinalityShapeSimilarity",
-    ]
+    multi_table_metrics = [task.value.col_name for task in Tasks]
+
+    single_table_metrics = [task.value.col_name for task in SingleTableTasks]
 
     # create empty dataframe with the columns multi_table_metrics
     multitable_df = pd.DataFrame(columns=["Dataset", "Model"] + multi_table_metrics)
+    singletable_df = pd.DataFrame(columns=["Dataset", "Model"] + single_table_metrics)
 
     # iterate through all json files and add the data to the dataframe
     for data in all_data_json:
@@ -51,17 +54,33 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
         dataset = data["dataset"]
         row = {"Dataset": dataset, "Model": model}
         for metric in multi_table_metrics:
-            if metric in data["multi_table_metrics"]:
+            stripped_metric = strip_emoji(metric)
+            if stripped_metric in data["multi_table_metrics"]:
                 metric_values = []
-                for table in data["multi_table_metrics"][metric].keys():
-                    if "accuracy" in data["multi_table_metrics"][metric][table]:
-                        metric_values.append(data["multi_table_metrics"][metric][table]["accuracy"])
-                    if "statistic" in data["multi_table_metrics"][metric][table]:
-                        metric_values.append(data["multi_table_metrics"][metric][table]["statistic"])
+                for table in data["multi_table_metrics"][stripped_metric].keys():
+                    if "accuracy" in data["multi_table_metrics"][stripped_metric][table]:
+                        metric_values.append(data["multi_table_metrics"][stripped_metric][table]["accuracy"])
+                    if "statistic" in data["multi_table_metrics"][stripped_metric][table]:
+                        metric_values.append(data["multi_table_metrics"][stripped_metric][table]["statistic"])
 
-                row[metric] = np.mean(metric_values)
+                row[metric] = np.mean(metric_values).round(decimals=2)
         multitable_df = pd.concat([multitable_df, pd.DataFrame([row])], ignore_index=True)
-    return multitable_df
+        
+        singletable_row = {"Dataset": dataset, "Model": model}
+        for metric in single_table_metrics:
+            stripped_metric = strip_emoji(metric)
+            if stripped_metric in data["single_table_metrics"]:
+                metric_values = []
+                for table in data["single_table_metrics"][stripped_metric].keys():
+                    if "accuracy" in data["single_table_metrics"][stripped_metric][table]:
+                        metric_values.append(data["single_table_metrics"][stripped_metric][table]["accuracy"])
+                    if "value" in data["single_table_metrics"][stripped_metric][table]:
+                        metric_values.append(data["single_table_metrics"][stripped_metric][table]["value"])
+                
+                singletable_row[metric] = np.mean(metric_values).round(decimals=2)
+        singletable_df = pd.concat([singletable_df, pd.DataFrame([singletable_row])], ignore_index=True)
+                
+    return singletable_df, multitable_df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: