[ADD] Open-ended evaluation

app.py

@@ -21,9 +21,9 @@ from src.about import (
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     DATASET_BENCHMARK_COLS,
-    TYPES_BENCHMARK_COLS,
+    OPEN_ENDED_BENCHMARK_COLS,
     DATASET_COLS,
-    Clinical_TYPES_COLS,
+    OPEN_ENDED_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
@@ -64,9 +64,10 @@ except Exception:
 _, harness_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "accuracy", "datasets")
 harness_datasets_leaderboard_df = harness_datasets_original_df.copy()
 
-# _, span_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "SpanBasedWithPartialOverlap", "clinical_types")
-# span_based_types_leaderboard_df = span_based_types_original_df.copy()
+_, open_ended_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OPEN_ENDED_COLS, OPEN_ENDED_BENCHMARK_COLS, "score", "open_ended")
+open_ended_leaderboard_df = open_ended_original_df.copy()
 
+# breakpoint()
 # # Token based results
 # _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
 # token_based_datasets_leaderboard_df = token_based_datasets_original_df.copy()
@@ -83,8 +84,12 @@ harness_datasets_leaderboard_df = harness_datasets_original_df.copy()
 
 
 def update_df(shown_columns, subset="datasets"):
-    leaderboard_table_df = harness_datasets_leaderboard_df.copy()
-    hidden_leader_board_df = harness_datasets_original_df
+    if subset == "datasets":
+        leaderboard_table_df = harness_datasets_leaderboard_df.copy()
+        hidden_leader_board_df = harness_datasets_original_df
+    elif subset == "open_ended":
+        leaderboard_table_df = open_ended_leaderboard_df.copy()
+        hidden_leader_board_df = open_ended_original_df
     # else:
     #     match evaluation_metric:
     #         case "Span Based":
@@ -98,7 +103,7 @@ def update_df(shown_columns, subset="datasets"):
     
 
     value_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns
-
+    # breakpoint()
     return leaderboard_table_df[value_cols], hidden_leader_board_df
 
 
@@ -196,60 +201,6 @@ def filter_models(
 
     return filtered_df
 
-def change_submit_request_form(model_architecture):
-    match model_architecture:
-        case "Encoder":
-            return (
-                gr.Textbox(label="Threshold for gliner models", visible=False), 
-                gr.Radio(
-                        choices=["True", "False"],
-                        label="Load GLiNER Tokenizer",
-                        visible=False
-                    ),
-                gr.Dropdown(
-                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
-                        label="Prompt for generation",
-                        multiselect=False,
-                        # value="HTML Highlighted Spans",
-                        interactive=True,
-                        visible=False
-                    )
-            )
-        case "Decoder":
-            return (
-                gr.Textbox(label="Threshold for gliner models", visible=False), 
-                gr.Radio(
-                        choices=["True", "False"],
-                        label="Load GLiNER Tokenizer",
-                        visible=False
-                    ),
-                gr.Dropdown(
-                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
-                        label="Prompt for generation",
-                        multiselect=False,
-                        # value="HTML Highlighted Spans",
-                        interactive=True,
-                        visible=True
-                    )
-            )
-        case "GLiNER Encoder":
-            return (
-                gr.Textbox(label="Threshold for gliner models", visible=True), 
-                gr.Radio(
-                        choices=["True", "False"],
-                        label="Load GLiNER Tokenizer",
-                        visible=True
-                    ),
-                gr.Dropdown(
-                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
-                        label="Prompt for generation",
-                        multiselect=False,
-                        # value="HTML Highlighted Spans",
-                        interactive=True,
-                        visible=False
-                    )
-            )
-
             
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -269,11 +220,11 @@ with demo:
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and not c.clinical_type_col],
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
                             value=[
                                 c.name
                                 for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and not c.clinical_type_col
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
                             ],
                             label="Select columns to show",
                             elem_id="column-select",
@@ -371,8 +322,117 @@ with demo:
                 )
 
         with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
-            gr.Markdown("# Coming Soon!!!", elem_classes="markdown-text")
-            pass
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain specific models",
+                        choices=["Yes", "No"],
+                        value=["Yes", "No"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="open_ended")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[OPEN_ENDED_COLS],
+                headers=OPEN_ENDED_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                # filter_columns_architecture,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                        # filter_columns_architecture,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
         with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown("# Coming Soon!!!", elem_classes="markdown-text")
             pass

src/about.py

@@ -27,19 +27,15 @@ class HarnessTasks(Enum):
     # task6 = Task("", "f1", "")
 
 @dataclass
-class ClinicalType:
+class OpenEndedColumn:
     benchmark: str
     metric: str
     col_name: str
 
-class ClinicalTypes(Enum):
+class OpenEndedColumns(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    type0 = ClinicalType("condition", "f1", "CONDITION") 
-    type1 = ClinicalType("measurement", "f1", "MEASUREMENT") 
-    type2 = ClinicalType("drug", "f1", "DRUG") 
-    type3 = ClinicalType("procedure", "f1", "PROCEDURE") 
-    type4 = ClinicalType("gene", "f1", "GENE")
-    type5 = ClinicalType("gene variant", "f1", "GENE VARIANT")
+    column0 = OpenEndedColumn("ELO", "score", "ELO") 
+    column1 = OpenEndedColumn("Score", "score", "Score") 
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/display/utils.py

@@ -3,8 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import HarnessTasks
-from src.about import ClinicalTypes
+from src.about import HarnessTasks, OpenEndedColumns
 
 
 def fields(raw_class):
@@ -20,9 +19,12 @@ class ColumnContent:
     type: str
     displayed_by_default: bool
     hidden: bool = False
+    invariant: bool = True
     never_hidden: bool = False
     dataset_task_col: bool = False
-    clinical_type_col: bool = False
+    open_ended_col: bool = False
+    med_safety_col: bool = False
+    cross_examination_col: bool = False
 
 
 ## Leaderboard columns
@@ -32,9 +34,11 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, invariant=False)])
 for task in HarnessTasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True, invariant=False)])
+for column in OpenEndedColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_col=True, invariant=False)])
 auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
 auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -181,8 +185,11 @@ class EvaluationMetrics(Enum):
 
 
 # Column selection
-DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.clinical_type_col]
-Clinical_TYPES_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col]
+DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
+OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
+MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.dataset_task_col and not c.cross_examination_col]
+CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.dataset_task_col]
+
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
@@ -191,7 +198,9 @@ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 DATASET_BENCHMARK_COLS = [t.value.col_name for t in HarnessTasks]
-TYPES_BENCHMARK_COLS = [t.value.col_name for t in ClinicalTypes]
+OPEN_ENDED_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedColumns]
+# MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyTasks]
+# CROSS_EXAMINATION_BENCHMARK_COLS = [t.value.col_name for t in CrossExaminationTasks]
 
 NUMERIC_INTERVALS = {
     "?": pd.Interval(-1, 0, closed="right"),

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, ClinicalTypes
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -22,6 +22,9 @@ class EvalResult:
     model: str
     revision: str  # commit hash, "" if main
     dataset_results: dict
+    open_ended_results: dict
+    med_safety_results: dict
+    cross_examination_results: dict
     is_domain_specific: bool
     use_chat_template: bool
     # clinical_type_results:dict
@@ -90,6 +93,19 @@ class EvalResult:
                     continue
                 mean_acc = np.mean(accs)  # * 100.0
                 harness_results[task.benchmark] = mean_acc
+        open_ended_results = {}
+        if "open-ended" in data["results"]:
+            for task in OpenEndedColumns:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = np.array([v for k, v in data["results"]["open-ended"]["overall"].items() if task.benchmark == k])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+                mean_acc = np.mean(accs)  # * 100.0
+                open_ended_results[task.benchmark] = mean_acc
+        # breakpoint()
+        med_safety_results = {}
+        cross_examination_results = {}
         # types_results = {}
         # for clinical_type in ClinicalTypes:
         #     clinical_type = clinical_type.value
@@ -109,6 +125,9 @@ class EvalResult:
             model=model,
             revision=config.get("revision", ""),
             dataset_results=harness_results,
+            open_ended_results=open_ended_results,
+            med_safety_results=med_safety_results,
+            cross_examination_results=cross_examination_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
@@ -146,60 +165,51 @@ class EvalResult:
 
     def to_dict(self, subset):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol + (" 🏥" if self.is_domain_specific else ""),
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture.value.name,
+            # AutoEvalColumn.backbone.name: self.backbone,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.is_domain_specific.name: self.is_domain_specific,
+            AutoEvalColumn.use_chat_template.name: self.use_chat_template,
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.date.name: self.date,
+            "display_result" : self.display_result,
+        }
+
         if subset == "datasets":
             average = sum([v for v in self.dataset_results.values() if v is not None]) / len(HarnessTasks)
-            data_dict = {
-                "eval_name": self.eval_name,  # not a column, just a save name,
-                AutoEvalColumn.precision.name: self.precision.value.name,
-                AutoEvalColumn.model_type.name: self.model_type.value.name,
-                AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol + (" 🏥" if self.is_domain_specific else ""),
-                AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-                # AutoEvalColumn.architecture.name: self.architecture.value.name,
-                # AutoEvalColumn.backbone.name: self.backbone,
-                AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-                AutoEvalColumn.is_domain_specific.name: self.is_domain_specific,
-                AutoEvalColumn.use_chat_template.name: self.use_chat_template,
-                AutoEvalColumn.revision.name: self.revision,
-                AutoEvalColumn.average.name: average,
-                AutoEvalColumn.license.name: self.license,
-                AutoEvalColumn.likes.name: self.likes,
-                AutoEvalColumn.params.name: self.num_params,
-                AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-                AutoEvalColumn.date.name: self.date,
-                "display_result" : self.display_result,
-            }
+            data_dict[AutoEvalColumn.average.name] = average
             if len(self.dataset_results) > 0:
                 for task in HarnessTasks:
                     data_dict[task.value.col_name] = self.dataset_results[task.value.benchmark]
-
             return data_dict
         
-        if subset == "clinical_types":
-            average = sum([v for v in self.clinical_type_results.values() if v is not None]) / len(ClinicalTypes)
-            data_dict = {
-                "eval_name": self.eval_name,  # not a column, just a save name,
-                AutoEvalColumn.precision.name: self.precision.value.name,
-                AutoEvalColumn.model_type.name: self.model_type.value.name,
-                AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-                AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-                AutoEvalColumn.architecture.name: self.architecture.value.name,
-                AutoEvalColumn.backbone.name: self.backbone,
-                AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-                AutoEvalColumn.revision.name: self.revision,
-                AutoEvalColumn.average.name: average,
-                AutoEvalColumn.license.name: self.license,
-                AutoEvalColumn.likes.name: self.likes,
-                AutoEvalColumn.params.name: self.num_params,
-                AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-                "display_result" : self.display_result,
-            }
-
-            for clinical_type in ClinicalTypes:
-                data_dict[clinical_type.value.col_name] = self.clinical_type_results[clinical_type.value.benchmark]
-
+        if subset == "open_ended":
+            if len(self.open_ended_results) > 0:
+                for task in OpenEndedColumns:
+                    data_dict[task.value.col_name] = self.open_ended_results[task.value.benchmark]
             return data_dict
         
-
+        # if subset == "med_safety":
+        #     if len(self.med_safety_results) > 0:
+        #         for task in MedSafetyTasks:
+        #             data_dict[task.value.col_name] = self.med_safety_results[task.value.benchmark]
+        #     return data_dict
+        
+        # if subset == "cross_examination":
+        #     if len(self.cross_examination_results) > 0:
+        #         for task in CrossExaminationTasks:
+        #             data_dict[task.value.col_name] = self.cross_examination_results[task.value.benchmark]
+        #     return data_dict
 
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""

src/populate.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
@@ -16,7 +16,10 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict(subset=subset) for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    if subset == "datasets":
+        df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    elif subset == "open_ended":
+        df = df.sort_values(by=["ELO"], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)