[ADD] Closed ended arabic

app.py

@@ -31,12 +31,14 @@ from src.display.utils import (
     MEDICAL_SUMMARIZATION_BENCHMARK_COLS,
     ACI_BENCHMARK_COLS,
     SOAP_BENCHMARK_COLS,
+    CLOSED_ENDED_ARABIC_BENCHMARK_COLS,
     DATASET_COLS,
     OPEN_ENDED_COLS,
     MED_SAFETY_COLS,
     MEDICAL_SUMMARIZATION_COLS,
     ACI_COLS,
     SOAP_COLS,
+    CLOSED_ENDED_ARABIC_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
@@ -94,6 +96,10 @@ aci_leaderboard_df = aci_original_df.copy()
 _, soap_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, SOAP_COLS, SOAP_BENCHMARK_COLS, "score", "soap")
 soap_leaderboard_df = soap_original_df.copy()
 
+if PRIVATE_REPO:
+    _, closed_ended_arabic_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, CLOSED_ENDED_ARABIC_COLS, CLOSED_ENDED_ARABIC_BENCHMARK_COLS, "score", "closed_ended_arabic")
+    closed_ended_arabic_leaderboard_df = closed_ended_arabic_original_df.copy()
+
 # breakpoint()
 # # Token based results
 # _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
@@ -130,6 +136,9 @@ def update_df(shown_columns, subset="datasets"):
     elif subset == "soap":
         leaderboard_table_df = soap_leaderboard_df.copy()
         hidden_leader_board_df = soap_original_df
+    elif PRIVATE_REPO and subset == "closed-ended-arabic":
+        leaderboard_table_df = closed_ended_arabic_leaderboard_df.copy()
+        hidden_leader_board_df = closed_ended_arabic_original_df
     # else:
     #     match evaluation_metric:
     #         case "Span Based":
@@ -941,7 +950,121 @@ with demo:
                 with gr.Accordion("Question generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
                 with gr.Accordion("Cross Examination", open=False):
-                        system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")  
+                        system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
+        if PRIVATE_REPO:
+            with gr.TabItem("Dev Evals", elem_id="llm-benchmark-tab-table", id=100):
+                with gr.Tabs(elem_classes="tab-buttons2") as tabs:
+                    with gr.TabItem("🏅 Arabic Closed Ended Evaluation", elem_id="llm-benchmark-tab-table100", id=0):
+                        with gr.Row():
+                            with gr.Column():
+                                with gr.Row():
+                                    search_bar = gr.Textbox(
+                                        placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                        show_label=False,
+                                        elem_id="search-bar",
+                                    )
+                                with gr.Row():
+                                    shown_columns = gr.CheckboxGroup(
+                                        choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_arabic_col)],
+                                        value=[
+                                            c.name
+                                            for c in fields(AutoEvalColumn)
+                                            if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_arabic_col)
+                                        ],
+                                        label="Select columns to show",
+                                        elem_id="column-select",
+                                        interactive=True,
+                                    )
+                                # with gr.Row():
+                                #     deleted_models_visibility = gr.Checkbox(
+                                #         value=False, label="Show gated/private/deleted models", interactive=True
+                                #     )
+                            with gr.Column(min_width=320):
+                                # with gr.Box(elem_id="box-filter"):
+                                filter_columns_type = gr.CheckboxGroup(
+                                    label="Model Types",
+                                    choices=[t.to_str() for t in ModelType],
+                                    value=[t.to_str() for t in ModelType],
+                                    interactive=True,
+                                    elem_id="filter-columns-type",
+                                )
+                                # filter_columns_architecture = gr.CheckboxGroup(
+                                #     label="Architecture Types",
+                                #     choices=[i.value.name for i in ModelArch],
+                                #     value=[i.value.name for i in ModelArch],
+                                #     interactive=True,
+                                #     elem_id="filter-columns-architecture",
+                                # )
+                                filter_domain_specific = gr.CheckboxGroup(
+                                    label="Domain Specificity",
+                                    choices=["🏥  Clinical models", "Generic models"],
+                                    value=["🏥  Clinical models", "Generic models"],
+                                    interactive=True,
+                                    elem_id="filter-columns-type",
+                                )
+                                filter_columns_size = gr.CheckboxGroup(
+                                    label="Model sizes (in billions of parameters)",
+                                    choices=list(NUMERIC_INTERVALS.keys()),
+                                    value=list(NUMERIC_INTERVALS.keys()),
+                                    interactive=True,
+                                    elem_id="filter-columns-size",
+                                )
+
+                        closed_ended_arabic_leaderboard_df, closed_ended_arabic_original_df = update_df(shown_columns.value, subset="closed-ended-arabic")
+
+                        leaderboard_table = gr.components.Dataframe(
+                            value=closed_ended_arabic_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                            headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                            datatype=TYPES,
+                            elem_id="leaderboard-table",
+                            interactive=False,
+                            visible=True,
+                        )
+
+                        # Dummy leaderboard for handling the case when the user uses backspace key
+                        hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                            value=closed_ended_arabic_original_df[CLOSED_ENDED_ARABIC_COLS],
+                            headers=CLOSED_ENDED_ARABIC_COLS,
+                            datatype=TYPES,
+                            visible=False,
+                        )
+
+                                    
+                        search_bar.submit(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                                # filter_columns_architecture
+                            ],
+                            leaderboard_table,
+                        )
+                        for selector in [
+                            shown_columns,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            # filter_columns_architecture,
+                            filter_columns_size,
+                            # deleted_models_visibility,
+                        ]:
+                            selector.change(
+                                update_table,
+                                [
+                                    hidden_leaderboard_table_for_search,
+                                    shown_columns,
+                                    search_bar,
+                                    filter_columns_type,
+                                    filter_domain_specific,
+                                    filter_columns_size
+                                    # filter_columns_architecture,
+                                ],
+                                leaderboard_table,
+                                queue=True,
+                            )
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=5):
             gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
             gr.HTML(FIVE_PILLAR_DIAGRAM)

src/about.py

@@ -96,6 +96,19 @@ class SOAPColumns(Enum):
     soap_column2 = SOAPColumn("fact", "score", "Consistency") 
     # soap_column3 = SOAPColumn("brief", "score", "Conciseness") 
 
+@dataclass
+class ClosedEndedArabicColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class ClosedEndedArabicColumns(Enum):
+    arabictask0 = ClosedEndedArabicColumn("MMLU-Arabic", "accuracy", "MMLU-Arabic")
+    arabictask2 = ClosedEndedArabicColumn("MedMCQA-Arabic", "accuracy", "MedMCQA-Arabic")
+    arabictask3 = ClosedEndedArabicColumn("MedQA-Arabic", "accuracy", "MedQA-Arabic")
+    arabictask5 = ClosedEndedArabicColumn("PubMedQA-Arabic", "accuracy", "PubMedQA-Arabic")
+
+
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------
 

src/display/utils.py

@@ -4,7 +4,8 @@ from enum import Enum
 import pandas as pd
 
 # changes to be made here
-from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns
+from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.envs import PRIVATE_REPO
 import json
 import gradio as gr
 
@@ -30,6 +31,7 @@ class ColumnContent:
     medical_summarization_col: bool = False
     aci_col: bool = False
     soap_col: bool = False
+    closed_ended_arabic_col: bool = False
 
 
 ## Leaderboard columns
@@ -39,7 +41,7 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, invariant=False)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, closed_ended_arabic_col=True, invariant=False)])
 auto_eval_column_dict.append(["overall", ColumnContent, ColumnContent("Overall Score", "number", True, False, medical_summarization_col=True, aci_col=True, soap_col=True, invariant=False)])
 for task in HarnessTasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True, invariant=False)])
@@ -57,6 +59,9 @@ for column in ACIColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, aci_col=True, invariant=False)])
 for column in SOAPColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, soap_col=True, invariant=False)])
+# if PRIVATE_REPO:
+for column in ClosedEndedArabicColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, closed_ended_arabic_col=True, invariant=False)])
 auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
 auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -89,6 +94,8 @@ class EvalQueueColumn:  # Queue column
     med_safety_status = ColumnContent("med_safety_status", "str", True)
     medical_summarization_status = ColumnContent("medical_summarization_status", "str", True)
     note_generation_status = ColumnContent("note_generation_status", "str", True)
+    if PRIVATE_REPO:
+        closed_ended_arabic_status = ColumnContent("closed_ended_arabic_status", "str", True)
 
 ## All the model information that we might need
 @dataclass
@@ -214,6 +221,8 @@ MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c
 MEDICAL_SUMMARIZATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medical_summarization_col or c.invariant)]
 ACI_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.aci_col or c.invariant)]
 SOAP_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.soap_col or c.invariant)]
+# if PRIVATE_REPO:
+CLOSED_ENDED_ARABIC_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.closed_ended_arabic_col or c.invariant)]
 # CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.cross_examination_col or c.invariant)]
 # DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
 # OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
@@ -234,6 +243,8 @@ MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyColumns]
 MEDICAL_SUMMARIZATION_BENCHMARK_COLS = [t.value.col_name for t in MedicalSummarizationColumns]
 ACI_BENCHMARK_COLS = [t.value.col_name for t in ACIColumns]
 SOAP_BENCHMARK_COLS = [t.value.col_name for t in SOAPColumns]
+# if PRIVATE_REPO:
+CLOSED_ENDED_ARABIC_BENCHMARK_COLS = [t.value.col_name for t in ClosedEndedArabicColumns]
 # CROSS_EXAMINATION_BENCHMARK_COLS = [t.value.col_name for t in CrossExaminationTasks]
 
 NUMERIC_INTERVALS = {

src/leaderboard/read_evals.py

@@ -9,8 +9,9 @@ import numpy as np
 
 from src.display.formatting import make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
 from src.submission.check_validity import is_model_on_hub
+from src.envs import PRIVATE_REPO
 
 
 @dataclass
@@ -29,6 +30,7 @@ class EvalResult:
     medical_summarization_results: dict
     aci_results: dict
     soap_results: dict
+    closed_ended_arabic_results: dict
     is_domain_specific: bool
     use_chat_template: bool
     # clinical_type_results:dict
@@ -162,6 +164,20 @@ class EvalResult:
                     continue
                 mean_acc = np.mean(accs)  # * 100.0
                 soap_results[task.benchmark] = mean_acc
+        closed_ended_arabic_results = {}
+        if PRIVATE_REPO and "closed-ended-arabic" in data["results"]:
+            for task in ClosedEndedArabicColumns:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                try:
+                    accs = np.array([v.get(task.metric, None) for k, v in data["results"]["closed-ended-arabic"].items() if task.benchmark == k])
+                except:
+                    # breakpoint()
+                    accs = np.array([])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+                mean_acc = np.mean(accs)  # * 100.0
+                closed_ended_arabic_results[task.benchmark] = mean_acc
         if open_ended_results == {} or med_safety_results == {} or medical_summarization_results == {} or aci_results == {} or soap_results == {}:
             open_ended_results = {}
             med_safety_results = {}
@@ -192,6 +208,7 @@ class EvalResult:
             medical_summarization_results=medical_summarization_results,
             aci_results=aci_results,
             soap_results=soap_results,
+            closed_ended_arabic_results=closed_ended_arabic_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
@@ -294,7 +311,13 @@ class EvalResult:
                 for task in SOAPColumns:
                     data_dict[task.value.col_name] = self.soap_results[task.value.benchmark]
             return data_dict
-        
+        if PRIVATE_REPO and subset == "closed_ended_arabic":
+            average = sum([v for v in self.closed_ended_arabic_results.values() if v is not None]) / len(ClosedEndedArabicColumns)
+            data_dict[AutoEvalColumn.average.name] = average
+            if len(self.closed_ended_arabic_results) > 0:
+                for task in ClosedEndedArabicColumns:
+                    data_dict[task.value.col_name] = self.closed_ended_arabic_results[task.value.benchmark]
+            return data_dict
 
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""

src/populate.py

@@ -5,8 +5,9 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
 from src.leaderboard.read_evals import get_raw_eval_results
+from src.envs import PRIVATE_REPO
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, evaluation_metric:str, subset:str) -> pd.DataFrame:
@@ -30,6 +31,8 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         df = df.sort_values(by=[AutoEvalColumn.overall.name], ascending=False)
     elif subset == "soap":
         df = df.sort_values(by=[AutoEvalColumn.overall.name], ascending=False)
+    elif subset == "closed_ended_arabic":
+        df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
     # filter out if any of the benchmarks have not been produced
@@ -54,6 +57,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
             data[EvalQueueColumn.medical_summarization_status.name] = data["status"]["medical-summarization"]
             data[EvalQueueColumn.note_generation_status.name] = data["status"]["note-generation"]
+            if PRIVATE_REPO:
+                data[EvalQueueColumn.closed_ended_arabic_status.name] = data["status"]["closed-ended-arabic"]
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
@@ -70,6 +75,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
                 data[EvalQueueColumn.medical_summarization_status.name] = data["status"]["medical-summarization"]
                 data[EvalQueueColumn.note_generation_status.name] = data["status"]["note-generation"]
+                if PRIVATE_REPO:
+                    data[EvalQueueColumn.closed_ended_arabic_status.name] = data["status"]["closed-ended-arabic"]
                 all_evals.append(data)
     # breakpoint()
     pending_list = []
@@ -78,6 +85,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     for run in all_evals:
         # changes to be made here
         status_list = [run["status"]["closed-ended"], run["status"]["open-ended"], run["status"]["med-safety"], run["status"]["medical-summarization"], run["status"]["note-generation"]]
+        if PRIVATE_REPO:
+            status_list.append(run["status"]["closed-ended-arabic"])
         # status_list = status_list
         if "RUNNING" in status_list:
             running_list.append(run)