[ADD] Cross-examination framework

app.py

@@ -24,9 +24,15 @@ from src.display.utils import (
     DATASET_BENCHMARK_COLS,
     OPEN_ENDED_BENCHMARK_COLS,
     MED_SAFETY_BENCHMARK_COLS,
+    MEDICAL_SUMMARIZATION_BENCHMARK_COLS,
+    ACI_BENCHMARK_COLS,
+    SOAP_BENCHMARK_COLS,
     DATASET_COLS,
     OPEN_ENDED_COLS,
     MED_SAFETY_COLS,
+    MEDICAL_SUMMARIZATION_COLS,
+    ACI_COLS,
+    SOAP_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
@@ -75,7 +81,15 @@ open_ended_leaderboard_df = open_ended_original_df.copy()
 _, med_safety_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MED_SAFETY_COLS, MED_SAFETY_BENCHMARK_COLS, "score", "med_safety")
 med_safety_leaderboard_df = med_safety_original_df.copy()
 
-# breakpoint()
+_, medical_summarization_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDICAL_SUMMARIZATION_COLS, MEDICAL_SUMMARIZATION_BENCHMARK_COLS, "score", "medical_summarization")
+medical_summarization_leaderboard_df = medical_summarization_original_df.copy()
+
+_, aci_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ACI_COLS, ACI_BENCHMARK_COLS, "score", "aci")
+aci_leaderboard_df = aci_original_df.copy()
+
+_, soap_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, SOAP_COLS, SOAP_BENCHMARK_COLS, "score", "soap")
+soap_leaderboard_df = soap_original_df.copy()
+
 # breakpoint()
 # # Token based results
 # _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
@@ -91,7 +105,7 @@ med_safety_leaderboard_df = med_safety_original_df.copy()
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-
+# breakpoint()
 def update_df(shown_columns, subset="datasets"):
     # changes to be made here
     if subset == "datasets":
@@ -103,6 +117,15 @@ def update_df(shown_columns, subset="datasets"):
     elif subset == "med_safety":
         leaderboard_table_df = med_safety_leaderboard_df.copy()
         hidden_leader_board_df = med_safety_original_df
+    elif subset == "medical_summarization":
+        leaderboard_table_df = medical_summarization_leaderboard_df.copy()
+        hidden_leader_board_df = medical_summarization_original_df
+    elif subset == "aci":
+        leaderboard_table_df = aci_leaderboard_df.copy()
+        hidden_leader_board_df = aci_original_df
+    elif subset == "soap":
+        leaderboard_table_df = soap_leaderboard_df.copy()
+        hidden_leader_board_df = soap_original_df
     # else:
     #     match evaluation_metric:
     #         case "Span Based":
@@ -217,6 +240,7 @@ def filter_models(
             
 demo = gr.Blocks(css=custom_css)
 with demo:
+    print("hello")
     gr.HTML(TITLE)
     gr.HTML(LOGO, elem_classes="logo")
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
@@ -555,18 +579,344 @@ with demo:
                     queue=True,
                 )
 
-        with gr.TabItem("🏅 Cross Examination", elem_id="llm-benchmark-tab-table", id=3):
-            gr.Markdown("# Coming Soon!!!", elem_classes="markdown-text")
-            pass
+        with gr.TabItem("🏅 Medical Summarization", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain specific models",
+                        choices=["Yes", "No"],
+                        value=["Yes", "No"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="medical_summarization")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=4):
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[MEDICAL_SUMMARIZATION_COLS],
+                headers=MEDICAL_SUMMARIZATION_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+        with gr.TabItem("🏅 Note generation", elem_id="llm-benchmark-tab-table", id=4):
+            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
+                with gr.TabItem("ACI Bench", elem_id="llm-benchmark-tab-table2", id=0):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain specific models",
+                                choices=["Yes", "No"],
+                                value=["Yes", "No"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="aci")
+
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[ACI_COLS],
+                        headers=ACI_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+
+                with gr.TabItem("SOAP Notes", elem_id="llm-benchmark-tab-table2", id=1):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain specific models",
+                                choices=["Yes", "No"],
+                                value=["Yes", "No"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="soap")
+
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[SOAP_COLS],
+                        headers=SOAP_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=5):
             gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
             # gr.HTML(EVALUATION_EXAMPLE_IMG, elem_classes="logo")
             # gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
             # gr.HTML(ENTITY_DISTRIBUTION_IMG, elem_classes="logo")
             # gr.Markdown(LLM_BENCHMARKS_TEXT_3, elem_classes="markdown-text")
             
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=5):
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=6):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")

src/about.py

@@ -57,6 +57,42 @@ class MedSafetyColumns(Enum):
     med_safety_column7 = MedSafetyColumn("Responsibility to Patient", "score", "Responsibility to Patient") 
     med_safety_column8 = MedSafetyColumn("Law and Responsibility to Society", "score", "Law and Responsibility to Society") 
 
+@dataclass
+class MedicalSummarizationColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedicalSummarizationColumns(Enum):
+    medical_summarization_column0 = MedicalSummarizationColumn("brief", "score", "Brief Score")
+    medical_summarization_column1 = MedicalSummarizationColumn("coverage", "score", "Coverage Score")
+    medical_summarization_column2 = MedicalSummarizationColumn("conform", "score", "Conform Score")
+    medical_summarization_column3 = MedicalSummarizationColumn("fact", "score", "Fact Score")
+
+@dataclass
+class ACIColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class ACIColumns(Enum):
+    aci_column0 = ACIColumn("brief", "score", "Brief Score") 
+    aci_column1 = ACIColumn("coverage", "score", "Coverage Score") 
+    aci_column2 = ACIColumn("conform", "score", "Conform Score") 
+    aci_column3 = ACIColumn("fact", "score", "Fact Score") 
+
+@dataclass
+class SOAPColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class SOAPColumns(Enum):
+    soap_column0 = SOAPColumn("brief", "score", "Brief Score") 
+    soap_column1 = SOAPColumn("coverage", "score", "Coverage Score") 
+    soap_column2 = SOAPColumn("conform", "score", "Conform Score") 
+    soap_column3 = SOAPColumn("fact", "score", "Fact Score") 
+
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------
 

src/display/utils.py

@@ -4,7 +4,7 @@ from enum import Enum
 import pandas as pd
 
 # changes to be made here
-from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns
+from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns
 
 
 def fields(raw_class):
@@ -26,7 +26,9 @@ class ColumnContent:
     dataset_task_col: bool = False
     open_ended_col: bool = False
     med_safety_col: bool = False
-    cross_examination_col: bool = False
+    medical_summarization_col: bool = False
+    aci_col: bool = False
+    soap_col: bool = False
 
 
 ## Leaderboard columns
@@ -44,6 +46,12 @@ for column in OpenEndedColumns:
 # changes to be made here
 for column in MedSafetyColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, med_safety_col=True, invariant=False)])
+for column in MedicalSummarizationColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medical_summarization_col=True, invariant=False)])
+for column in ACIColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, aci_col=True, invariant=False)])
+for column in SOAPColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, soap_col=True, invariant=False)])
 auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
 auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -74,6 +82,8 @@ class EvalQueueColumn:  # Queue column
     closed_ended_status = ColumnContent("closed_ended_status", "str", True)
     open_ended_status = ColumnContent("open_ended_status", "str", True)
     med_safety_status = ColumnContent("med_safety_status", "str", True)
+    medical_summarization_status = ColumnContent("medical_summarization_status", "str", True)
+    note_generation_status = ColumnContent("note_generation_status", "str", True)
 
 ## All the model information that we might need
 @dataclass
@@ -196,7 +206,10 @@ class EvaluationMetrics(Enum):
 DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.dataset_task_col or c.invariant)]
 OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_col or c.invariant)]
 MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.med_safety_col or c.invariant)]
-CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.cross_examination_col or c.invariant)]
+MEDICAL_SUMMARIZATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medical_summarization_col or c.invariant)]
+ACI_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.aci_col or c.invariant)]
+SOAP_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.soap_col or c.invariant)]
+# CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.cross_examination_col or c.invariant)]
 # DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
 # OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
 # MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.dataset_task_col and not c.cross_examination_col]
@@ -213,6 +226,9 @@ EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 DATASET_BENCHMARK_COLS = [t.value.col_name for t in HarnessTasks]
 OPEN_ENDED_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedColumns]
 MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyColumns]
+MEDICAL_SUMMARIZATION_BENCHMARK_COLS = [t.value.col_name for t in MedicalSummarizationColumns]
+ACI_BENCHMARK_COLS = [t.value.col_name for t in ACIColumns]
+SOAP_BENCHMARK_COLS = [t.value.col_name for t in SOAPColumns]
 # CROSS_EXAMINATION_BENCHMARK_COLS = [t.value.col_name for t in CrossExaminationTasks]
 
 NUMERIC_INTERVALS = {

src/leaderboard/read_evals.py

@@ -9,7 +9,7 @@ import numpy as np
 
 from src.display.formatting import make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -26,7 +26,9 @@ class EvalResult:
     # changes to be made here
     open_ended_results: dict
     med_safety_results: dict
-    cross_examination_results: dict
+    medical_summarization_results: dict
+    aci_results: dict
+    soap_results: dict
     is_domain_specific: bool
     use_chat_template: bool
     # clinical_type_results:dict
@@ -123,7 +125,42 @@ class EvalResult:
                     continue
                 mean_acc = np.mean(accs)  # * 100.0
                 med_safety_results[task.benchmark] = mean_acc
-        cross_examination_results = {}
+        medical_summarization_results = {}
+        if "medical-summarization" in data["results"]:
+            for task in MedicalSummarizationColumns:
+                task = task.value
+                try:
+                    accs = np.array([v for k, v in data["results"]["medical-summarization"]["clinical_trial"].items() if task.benchmark == k])
+                except:
+                    accs = np.array([])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+                mean_acc = np.mean(accs)  # * 100.0
+                medical_summarization_results[task.benchmark] = mean_acc
+        aci_results = {}
+        if "note-generation" in data["results"] and "aci" in data["results"]["note-generation"]:
+            for task in ACIColumns:
+                task = task.value
+                try:
+                    accs = np.array([v for k, v in data["results"]["note-generation"]["aci"].items() if task.benchmark == k])
+                except:
+                    accs = np.array([])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+                mean_acc = np.mean(accs)  # * 100.0
+                aci_results[task.benchmark] = mean_acc
+        soap_results = {}
+        if "note-generation" in data["results"] and "soap" in data["results"]["note-generation"]:
+            for task in SOAPColumns:
+                task = task.value
+                try:
+                    accs = np.array([v for k, v in data["results"]["note-generation"]["soap"].items() if task.benchmark == k])
+                except:
+                    accs = np.array([])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+                mean_acc = np.mean(accs)  # * 100.0
+                soap_results[task.benchmark] = mean_acc
         # types_results = {}
         # for clinical_type in ClinicalTypes:
         #     clinical_type = clinical_type.value
@@ -145,7 +182,9 @@ class EvalResult:
             dataset_results=harness_results,
             open_ended_results=open_ended_results,
             med_safety_results=med_safety_results,
-            cross_examination_results=cross_examination_results,
+            medical_summarization_results=medical_summarization_results,
+            aci_results=aci_results,
+            soap_results=soap_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
@@ -224,12 +263,22 @@ class EvalResult:
                 for task in MedSafetyColumns:
                     data_dict[task.value.col_name] = self.med_safety_results[task.value.benchmark]
             return data_dict
+        if subset == "medical_summarization":
+            if len(self.medical_summarization_results) > 0:
+                for task in MedicalSummarizationColumns:
+                    data_dict[task.value.col_name] = self.medical_summarization_results[task.value.benchmark]
+            return data_dict
+        if subset == "aci":
+            if len(self.aci_results) > 0:
+                for task in ACIColumns:
+                    data_dict[task.value.col_name] = self.aci_results[task.value.benchmark]
+            return data_dict
+        if subset == "soap":
+            if len(self.soap_results) > 0:
+                for task in SOAPColumns:
+                    data_dict[task.value.col_name] = self.soap_results[task.value.benchmark]
+            return data_dict
         
-        # if subset == "cross_examination":
-        #     if len(self.cross_examination_results) > 0:
-        #         for task in CrossExaminationTasks:
-        #             data_dict[task.value.col_name] = self.cross_examination_results[task.value.benchmark]
-        #     return data_dict
 
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""

src/populate.py

@@ -5,7 +5,7 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
@@ -24,6 +24,12 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=True)
     elif subset == "open_ended":
         df = df.sort_values(by=["ELO"], ascending=False)
+    elif subset == "medical_summarization":
+        df = df.sort_values(by=["Coverage Score"], ascending=False)
+    elif subset == "aci":
+        df = df.sort_values(by=["Coverage Score"], ascending=False)
+    elif subset == "soap":
+        df = df.sort_values(by=["Coverage Score"], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
     # filter out if any of the benchmarks have not been produced
@@ -46,6 +52,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
             data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
             data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
+            data[EvalQueueColumn.medical_summarization_status.name] = data["status"]["medical-summarization"]
+            data[EvalQueueColumn.note_generation_status.name] = data["status"]["note-generation"]
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
@@ -60,6 +68,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
                 data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
                 data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
+                data[EvalQueueColumn.medical_summarization_status.name] = data["status"]["medical-summarization"]
+                data[EvalQueueColumn.note_generation_status.name] = data["status"]["note-generation"]
                 all_evals.append(data)
     # breakpoint()
     pending_list = []
@@ -67,8 +77,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     finished_list = []
     for run in all_evals:
         # changes to be made here
-        status_list = [run["status"]["closed-ended"], run["status"]["open-ended"], run["status"]["med-safety"], run["status"]["cross-examination"]]
-        status_list = status_list[:3]
+        status_list = [run["status"]["closed-ended"], run["status"]["open-ended"], run["status"]["med-safety"], run["status"]["medical-summarization"], run["status"]["note-generation"]]
+        # status_list = status_list
         if "RUNNING" in status_list:
             running_list.append(run)
         elif "PENDING" in status_list or "RERUN" in status_list: