[ADD] CI for open-ended

src/about.py

@@ -34,9 +34,10 @@ class OpenEndedColumn:
 
 class OpenEndedColumns(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    column0 = OpenEndedColumn("ELO", "score", "ELO") 
-    column1 = OpenEndedColumn("Score", "score", "Score") 
-
+    column0 = OpenEndedColumn("ELO", "score", "ELO")
+    column1 = OpenEndedColumn("ELO_intervals", "score", "ELO 95% CI")
+    column2 = OpenEndedColumn("Score", "score", "Score") 
+    column3 = OpenEndedColumn("Score_intervals", "score", "Score 95% CI")
 # changes to be made here
 
 @dataclass

src/leaderboard/read_evals.py

@@ -106,25 +106,19 @@ class EvalResult:
             for task in OpenEndedColumns:
                 task = task.value
                 # We average all scores of a given metric (not all metrics are present in all files)
-                accs = np.array([v for k, v in data["results"]["open-ended"]["overall"].items() if task.benchmark == k])
-                if accs.size == 0 or any([acc is None for acc in accs]):
-                    continue
-                mean_acc = np.mean(accs)  # * 100.0
-                open_ended_results[task.benchmark] = mean_acc
+                accs = data["results"]["open-ended"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended"]["overall"] else None
+                open_ended_results[task.benchmark] = accs
+            if open_ended_results["ELO_intervals"] is not None and open_ended_results["Score_intervals"] is not None:
+                open_ended_results["ELO_intervals"] = "+" + str(open_ended_results["ELO_intervals"][1]) + "/-" + str(abs(open_ended_results["ELO_intervals"][0]))
+                open_ended_results["Score_intervals"] = "+" + str(open_ended_results["Score_intervals"][1]) + "/-" + str(abs(open_ended_results["Score_intervals"][0]))
         # breakpoint()
         # changes to be made here
         med_safety_results = {}
         if "med-safety" in data["results"]:
             for task in MedSafetyColumns:
                 task = task.value
-                try:
-                    accs = np.array([v.get(task.metric, None) for k, v in data["results"]["med-safety"].items() if task.benchmark == k])
-                except:
-                    accs = np.array([])
-                if accs.size == 0 or any([acc is None for acc in accs]):
-                    continue
-                mean_acc = np.mean(accs)  # * 100.0
-                med_safety_results[task.benchmark] = mean_acc
+                accs = data["results"]["med-safety"][task.benchmark]["score"]
+                med_safety_results[task.benchmark] = accs
         medical_summarization_results = {}
         if "medical-summarization" in data["results"]:
             for task in MedicalSummarizationColumns: