[MODIFY] Metrics for medical summarization, aci bench and soap notes

app.py

@@ -704,11 +704,11 @@ with demo:
                                 )
                             with gr.Row():
                                 shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)],
                                     value=[
                                         c.name
                                         for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)
                                     ],
                                     label="Select columns to show",
                                     elem_id="column-select",
@@ -814,11 +814,11 @@ with demo:
                                 )
                             with gr.Row():
                                 shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)],
                                     value=[
                                         c.name
                                         for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)
                                     ],
                                     label="Select columns to show",
                                     elem_id="column-select",

src/about.py

@@ -79,7 +79,7 @@ class ACIColumns(Enum):
     aci_column0 = ACIColumn("coverage", "score", "Coverage") 
     aci_column1 = ACIColumn("conform", "score", "Conformity") 
     aci_column2 = ACIColumn("fact", "score", "Consistency") 
-    aci_column3 = ACIColumn("brief", "score", "Conciseness") 
+    # aci_column3 = ACIColumn("brief", "score", "Conciseness") 
 
 @dataclass
 class SOAPColumn:
@@ -91,7 +91,7 @@ class SOAPColumns(Enum):
     soap_column0 = SOAPColumn("coverage", "score", "Coverage") 
     soap_column1 = SOAPColumn("conform", "score", "Conformity") 
     soap_column2 = SOAPColumn("fact", "score", "Consistency") 
-    soap_column3 = SOAPColumn("brief", "score", "Conciseness") 
+    # soap_column3 = SOAPColumn("brief", "score", "Conciseness") 
 
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -39,6 +39,7 @@ auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent(
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, med_safety_col=True, invariant=False)])
+auto_eval_column_dict.append(["overall", ColumnContent, ColumnContent("Overall Score", "number", True, False, medical_summarization_col=True, aci_col=True, soap_col=True, invariant=False)])
 for task in HarnessTasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True, invariant=False)])
 for column in OpenEndedColumns:

src/leaderboard/read_evals.py

@@ -272,15 +272,26 @@ class EvalResult:
             return data_dict
         if subset == "medical_summarization":
             if len(self.medical_summarization_results) > 0:
+                adjusted_conciseness = max(0, self.medical_summarization_results["brief"])
+                coverage = self.medical_summarization_results["coverage"]
+                hm = 2 / (1/coverage + 1/adjusted_conciseness) if not (adjusted_conciseness == 0 or coverage == 0) else 0
+                conformity = self.medical_summarization_results["conform"]
+                consistency = self.medical_summarization_results["fact"]
+                overall = sum([hm, conformity, consistency]) / 3
+                data_dict[AutoEvalColumn.overall.name] = overall
                 for task in MedicalSummarizationColumns:
                     data_dict[task.value.col_name] = self.medical_summarization_results[task.value.benchmark]
             return data_dict
         if subset == "aci":
+            overall = sum([v for v in self.aci_results.values() if v is not None]) / len(ACIColumns)
+            data_dict[AutoEvalColumn.overall.name] = overall
             if len(self.aci_results) > 0:
                 for task in ACIColumns:
                     data_dict[task.value.col_name] = self.aci_results[task.value.benchmark]
             return data_dict
         if subset == "soap":
+            overall = sum([v for v in self.soap_results.values() if v is not None]) / len(SOAPColumns)
+            data_dict[AutoEvalColumn.overall.name] = overall
             if len(self.soap_results) > 0:
                 for task in SOAPColumns:
                     data_dict[task.value.col_name] = self.soap_results[task.value.benchmark]

src/populate.py

@@ -25,11 +25,11 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     elif subset == "open_ended":
         df = df.sort_values(by=["ELO"], ascending=False)
     elif subset == "medical_summarization":
-        df = df.sort_values(by=["Coverage"], ascending=False)
+        df = df.sort_values(by=["Overall Score"], ascending=False)
     elif subset == "aci":
-        df = df.sort_values(by=["Coverage"], ascending=False)
+        df = df.sort_values(by=["Overall Score"], ascending=False)
     elif subset == "soap":
-        df = df.sort_values(by=["Coverage"], ascending=False)
+        df = df.sort_values(by=["Overall Score"], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
     # filter out if any of the benchmarks have not been produced