Add first version of syntherela leaderboard

.python-version

@@ -0,0 +1 @@
+syntherela_leaderboard

app.py

@@ -21,8 +21,8 @@ from src.display.utils import (
     AutoEvalColumn,
     ModelType,
     fields,
-    WeightType,
-    Precision
+    # WeightType,
+    # Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -49,7 +49,7 @@ except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
 
 (
     finished_eval_queue_df,
@@ -68,21 +68,21 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.model.name], # AutoEvalColumn.license.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(
+            #     AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            # ),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
@@ -95,13 +95,13 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 Syntherela Benchmark", elem_id="syntherela-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="syntherela-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        with gr.TabItem("🚀 Submit here! ", elem_id="syntherela-benchmark-tab-table", id=3):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
@@ -156,21 +156,21 @@ with demo:
                         interactive=True,
                     )
 
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
+                # with gr.Column():
+                    # precision = gr.Dropdown(
+                    #     choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                    #     label="Precision",
+                    #     multiselect=False,
+                    #     value="float16",
+                    #     interactive=True,
+                    # )
+                    # weight_type = gr.Dropdown(
+                    #     choices=[i.value.name for i in WeightType],
+                    #     label="Weights type",
+                    #     multiselect=False,
+                    #     value="Original",
+                    #     interactive=True,
+                    # )
                     base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
             submit_button = gr.Button("Submit Eval")
@@ -181,8 +181,8 @@ with demo:
                     model_name_textbox,
                     base_model_name_textbox,
                     revision_name_textbox,
-                    precision,
-                    weight_type,
+                    # precision,
+                    # weight_type,
                     model_type,
                 ],
                 submission_result,

src/about.py

@@ -12,8 +12,11 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    # task0 = Task("anli_r1", "acc", "ANLI")
+    # task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task_0 = Task("multi-table", "AggregationDetection-LogisticRegression", "AggregationDetection-LogisticRegression")
+    task_1 = Task("multi-table", "AggregationDetection-XGBClassifier", "AggregationDetection-XGBClassifier")
+    task_2 = Task("multi-table", "CardinalityShapeSimilarity", "CardinalityShapeSimilarity")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -23,22 +23,23 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["dataset", ColumnContent, ColumnContent("Dataset", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -47,11 +48,11 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
+    # revision = ColumnContent("revision", "str", True)
+    # private = ColumnContent("private", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
+    # status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need
 @dataclass
@@ -62,10 +63,12 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    OS = ModelDetails(name="open-source", symbol="🆓")
+    CS = ModelDetails(name="closed-source", symbol="🔒")
+    # PT = ModelDetails(name="pretrained", symbol="🟢")
+    # FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    # IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    # RL = ModelDetails(name="RL-tuned", symbol="🟦")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -73,32 +76,28 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
+        if "open-source" in type or "���" in type:
+            return ModelType.OS
+        if "closed-source" in type or "🟢" in type:
+            return ModelType.CS
         return ModelType.Unknown
 
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
+# class WeightType(Enum):
+#     Adapter = ModelDetails("Adapter")
+#     Original = ModelDetails("Original")
+#     Delta = ModelDetails("Delta")
+
+# class Precision(Enum):
+#     float16 = ModelDetails("float16")
+#     bfloat16 = ModelDetails("bfloat16")
+#     Unknown = ModelDetails("?")
+
+#     def from_str(precision):
+#         if precision in ["torch.float16", "float16"]:
+#             return Precision.float16
+#         if precision in ["torch.bfloat16", "bfloat16"]:
+#             return Precision.bfloat16
+#         return Precision.Unknown
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]

src/envs.py

@@ -6,12 +6,13 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "syntherela"
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+QUEUE_REPO = "demo-leaderboard-backend/requests"
+RESULTS_REPO = f"{OWNER}/results-demo"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks # Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -22,9 +22,9 @@ class EvalResult:
     model: str
     revision: str # commit hash, "" if main
     results: dict
-    precision: Precision = Precision.Unknown
+    # precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
+    # weight_type: WeightType = WeightType.Original # Original or Adapter
     architecture: str = "Unknown" 
     license: str = "?"
     likes: int = 0
@@ -41,7 +41,7 @@ class EvalResult:
         config = data.get("config")
 
         # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
+        # precision = Precision.from_str(config.get("model_dtype"))
 
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
@@ -50,11 +50,11 @@ class EvalResult:
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model}_" #{precision.value.name}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model}_" # {precision.value.name}"
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
@@ -85,7 +85,7 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
+            # precision=precision,  
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture
@@ -99,7 +99,7 @@ class EvalResult:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            # self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)

src/populate.py

@@ -2,24 +2,66 @@ import json
 import os
 
 import pandas as pd
+import numpy as np
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+# def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+#     """Creates a dataframe from all the individual experiment results"""
+#     raw_data = get_raw_eval_results(results_path, requests_path)
+#     all_data_json = [v.to_dict() for v in raw_data]
+
+#     df = pd.DataFrame.from_records(all_data_json)
+#     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+#     df = df[cols].round(decimals=2)
+
+#     # filter out if any of the benchmarks have not been produced
+#     df = df[has_no_nan_values(df, benchmark_cols)]
+#     return df
+
+
+def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
 
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    # iterate thorugh all files in the results path and read them into json
+    all_data_json = []
+    res_path = os.path.join(results_path, "demo-leaderboard", "syntherela-demo")
+    for entry in os.listdir(res_path):
+        if entry.endswith(".json"):
+            file_path = os.path.join(res_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+                all_data_json.append(data)
+
+    multi_table_metrics = [
+        "AggregationDetection-LogisticRegression",
+        "AggregationDetection-XGBClassifier",
+        "CardinalityShapeSimilarity",
+    ]
+
+    # create empty dataframe with the columns multi_table_metrics
+    multitable_df = pd.DataFrame(columns=["Dataset", "Model"] + multi_table_metrics)
+
+    # iterate through all json files and add the data to the dataframe
+    for data in all_data_json:
+        model = data["model"]
+        dataset = data["dataset"]
+        row = {"Dataset": dataset, "Model": model}
+        for metric in multi_table_metrics:
+            if metric in data["multi_table_metrics"]:
+                metric_values = []
+                for table in data["multi_table_metrics"][metric].keys():
+                    if "accuracy" in data["multi_table_metrics"][metric][table]:
+                        metric_values.append(data["multi_table_metrics"][metric][table]["accuracy"])
+                    if "statistic" in data["multi_table_metrics"][metric][table]:
+                        metric_values.append(data["multi_table_metrics"][metric][table]["statistic"])
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+                row[metric] = np.mean(metric_values)
+        multitable_df = pd.concat([multitable_df, pd.DataFrame([row])], ignore_index=True)
+    return multitable_df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
@@ -39,7 +81,9 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            sub_entries = [
+                e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")
+            ]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp: