[MODIFY] Added support for other frameworks in submit, evaluation queue and harness results displau

medic-harness-requests/.gitattributes

@@ -1,58 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.lz4 filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-# Audio files - uncompressed
-*.pcm filter=lfs diff=lfs merge=lfs -text
-*.sam filter=lfs diff=lfs merge=lfs -text
-*.raw filter=lfs diff=lfs merge=lfs -text
-# Audio files - compressed
-*.aac filter=lfs diff=lfs merge=lfs -text
-*.flac filter=lfs diff=lfs merge=lfs -text
-*.mp3 filter=lfs diff=lfs merge=lfs -text
-*.ogg filter=lfs diff=lfs merge=lfs -text
-*.wav filter=lfs diff=lfs merge=lfs -text
-# Image files - uncompressed
-*.bmp filter=lfs diff=lfs merge=lfs -text
-*.gif filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text
-*.tiff filter=lfs diff=lfs merge=lfs -text
-# Image files - compressed
-*.jpg filter=lfs diff=lfs merge=lfs -text
-*.jpeg filter=lfs diff=lfs merge=lfs -text
-*.webp filter=lfs diff=lfs merge=lfs -text
-# Video files - compressed
-*.mp4 filter=lfs diff=lfs merge=lfs -text
-*.webm filter=lfs diff=lfs merge=lfs -text

medic-harness-results/.gitattributes

@@ -1,58 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.lz4 filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-# Audio files - uncompressed
-*.pcm filter=lfs diff=lfs merge=lfs -text
-*.sam filter=lfs diff=lfs merge=lfs -text
-*.raw filter=lfs diff=lfs merge=lfs -text
-# Audio files - compressed
-*.aac filter=lfs diff=lfs merge=lfs -text
-*.flac filter=lfs diff=lfs merge=lfs -text
-*.mp3 filter=lfs diff=lfs merge=lfs -text
-*.ogg filter=lfs diff=lfs merge=lfs -text
-*.wav filter=lfs diff=lfs merge=lfs -text
-# Image files - uncompressed
-*.bmp filter=lfs diff=lfs merge=lfs -text
-*.gif filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text
-*.tiff filter=lfs diff=lfs merge=lfs -text
-# Image files - compressed
-*.jpg filter=lfs diff=lfs merge=lfs -text
-*.jpeg filter=lfs diff=lfs merge=lfs -text
-*.webp filter=lfs diff=lfs merge=lfs -text
-# Video files - compressed
-*.mp4 filter=lfs diff=lfs merge=lfs -text
-*.webm filter=lfs diff=lfs merge=lfs -text

medic-harness-results/aaditya/Llama3-OpenBioLLM-70B/results_2024-07-24T15:26:36Z.json

@@ -1,37 +0,0 @@
-{
-  "config": {
-    "model_name": "aaditya/Llama3-OpenBioLLM-70B",
-    "revision": "main",
-    "submitted_time": "2024-07-24 14:33:56+00:00",
-    "model_type": "domain-specific",
-    "num_params": 70000000000,
-    "private": false,
-    "evaluated_time": "2024-07-24T15:26:36Z"
-  },
-  "results": {
-    "MMLU": {
-      "accuracy": 90.4
-    },
-    "MMLU-Pro": {
-      "accuracy": 64.2
-    },
-    "MedMCQA": {
-      "accuracy": 73.2
-    },
-    "MedQA": {
-      "accuracy": 76.9
-    },
-    "USMLE": {
-      "accuracy": 79.0
-    },
-    "PubMedQA": {
-      "accuracy": 73.2
-    },
-    "ToxiGen": {
-      "accuracy": 91.3
-    },
-    "Average": {
-      "accuracy": 78.3
-    }
-  }
-}

medic-harness-results/meta-llama/Llama-3.1-8B-Instruct/results_2024-07-24T15:26:36Z.json

@@ -1,39 +0,0 @@
-{
-  "config": {
-    "model_name": "meta-llama/Llama-3.1-8B-Instruct",
-    "revision": "main",
-    "submitted_time": "2024-07-24 14:33:56+00:00",
-    "model_type": "instruction-tuned",
-    "num_params": 8000000000,
-    "private": false,
-    "evaluated_time": "2024-07-24T15:26:36Z"
-  },
-  "results": {
-    "MMLU": {
-      "accuracy": 73.4
-    },
-    "MMLU-Pro": {
-      "accuracy": 49.9
-    },
-    "MedMCQA": {
-      "accuracy": 58.4
-    },
-    "MedQA": {
-      "accuracy": 62.0
-    },
-    "USMLE": {
-      "accuracy": 68.2
-    },
-    "PubMedQA": {
-      "accuracy": 76.2
-    },
-    "ToxiGen": {
-      "accuracy": 82.3
-    },
-    "Average": {
-      "accuracy": 67.2
-    }
-  }
-}
-
-

src/display/utils.py

@@ -61,8 +61,8 @@ class EvalQueueColumn:  # Queue column
     model_type = ColumnContent("model_type", "str", True)
     precision = ColumnContent("precision", "str", True)
     weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
+    closed_ended_status = ColumnContent("closed_ended_status", "str", True)
+    open_ended_status = ColumnContent("open_ended_status", "str", True)
 
 ## All the model information that we might need
 @dataclass

src/leaderboard/read_evals.py

@@ -76,18 +76,20 @@ class EvalResult:
                 backbone = ";".join(backbones)
 
         # Extract results available in this file (some results are split in several files)
-        dataset_results = {}
+        harness_results = {}
         for task in HarnessTasks:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            try:
+                accs = np.array([v.get(task.metric, None) for k, v in data["results"]["closed-ended"].items() if task.benchmark == k])
+            except:
+                breakpoint()
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
             mean_acc = np.mean(accs)  # * 100.0
-            dataset_results[task.benchmark] = mean_acc
-        print(dataset_results)
+            harness_results[task.benchmark] = mean_acc
         # types_results = {}
         # for clinical_type in ClinicalTypes:
         #     clinical_type = clinical_type.value
@@ -106,7 +108,7 @@ class EvalResult:
             org=org,
             model=model,
             revision=config.get("revision", ""),
-            dataset_results=dataset_results,
+            dataset_results=harness_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,

src/populate.py

@@ -36,6 +36,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data = json.load(fp)
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
+            data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
@@ -47,11 +49,23 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 # print(data)
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
+                data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
                 all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    # breakpoint()
+    pending_list = []
+    running_list = []
+    finished_list = []
+    for run in all_evals:
+        status_list = [run["status"]["closed-ended"], run["status"]["open-ended"], run["status"]["med-safety"], run["status"]["cross-examination"]]
+        status_list = status_list[:2]
+        if "RUNNING" in status_list:
+            running_list.append(run)
+        elif "PENDING" in status_list or "RERUN" in status_list:
+            pending_list.append(run)
+        else:
+            finished_list.append(run)
+        # breakpoint()
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)

src/submission/submit.py

@@ -135,7 +135,12 @@ def add_new_eval(
         "weight_type": weight_type,
         "is_domain_specific": domain_specific,
         "use_chat_template": chat_template,
-        "status": "PENDING",
+        "status": {
+            "closed-ended": "PENDING",
+            "open-ended": "PENDING",
+            "med-safety": "PENDING",
+            "cross-examination": "PENDING",
+        },
         "submitted_time": current_time,
         "model_type": model_type,
         "likes": model_info.likes,