Add data

app.py

@@ -1,7 +1,6 @@
 import gradio as gr
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 import pandas as pd
+import json
 
 from src.about import (
     REPRODUCIBILITY_TEXT,
@@ -10,64 +9,65 @@ from src.about import (
     TITLE,
 )
 from src.display.css_html_js import custom_css, custom_js
-from src.display.utils import (
-    COLS,
-    ST_BENCHMARK_COLS,
-    AGENTIC_BENCHMARK_COLS,
-    EVAL_COLS,
-    AutoEvalColumn,
-    fields,
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df, TASK_NAME_INVERSE_MAP
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-ST_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, ST_BENCHMARK_COLS)
-AGENTIC_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, AGENTIC_BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def bold_max(s):
-    is_max = s == s.max()  # Boolean Series: True for the max value(s)
-    return ['font-weight: bold' if v else '' for v in is_max]
-
-def init_leaderboard(df, benchmark_type):
-    if df is None or df.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
+# from src.display.utils import (
+#     COLS,
+#     ST_BENCHMARK_COLS,
+#     AGENTIC_BENCHMARK_COLS,
+#     EVAL_COLS,
+#     AutoEvalColumn,
+#     fields,
+# )
+# from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+# from src.populate import get_evaluation_queue_df, get_leaderboard_df, TASK_NAME_INVERSE_MAP
+# from src.submission.submit import add_new_eval
+from src.display.formatting import make_clickable_field
+
+
+# def restart_space():
+#     API.restart_space(repo_id=REPO_ID)
+
+# ### Space initialisation
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print(EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+
+
+# ST_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, ST_BENCHMARK_COLS)
+# AGENTIC_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, AGENTIC_BENCHMARK_COLS)
+
+# (
+#     finished_eval_queue_df,
+#     running_eval_queue_df,
+#     pending_eval_queue_df,
+# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+# def bold_max(s):
+#     is_max = s == s.max()  # Boolean Series: True for the max value(s)
+#     return ['font-weight: bold' if v else '' for v in is_max]
+
+# def init_leaderboard(df, benchmark_type):
+#     if df is None or df.empty:
+#         raise ValueError("Leaderboard DataFrame is empty or None.")
     
-    non_task_cols = ["Model"]
-    if benchmark_type == "agentic":
-        # Include agent column
-        non_task_cols.append("Agent")
-    elif benchmark_type == "base":
-        # Drop agent column
-        dataframe = dataframe.drop(columns=["Agent"])
-    AutoEvalColumnSubset = [c for c in fields(AutoEvalColumn) if ((c.name in non_task_cols) or (TASK_NAME_INVERSE_MAP.get(c.name, dict()).get("type", "")==benchmark_type))]
+#     non_task_cols = ["Model"]
+#     if benchmark_type == "agentic":
+#         # Include agent column
+#         non_task_cols.append("Agent")
+#     elif benchmark_type == "base":
+#         # Drop agent column
+#         dataframe = dataframe.drop(columns=["Agent"])
+#     AutoEvalColumnSubset = [c for c in fields(AutoEvalColumn) if ((c.name in non_task_cols) or (TASK_NAME_INVERSE_MAP.get(c.name, dict()).get("type", "")==benchmark_type))]
 
     # styler = dataframe.style.apply(bold_max, subset=pd.IndexSlice[:, dataframe.columns[1:]])
     # df.style.set_table_styles([
@@ -85,13 +85,78 @@ def init_leaderboard(df, benchmark_type):
     # styled_df = df.style.set_tooltips(tooltips)
 
 
+    # return gr.components.Dataframe(
+    #     value=df,
+    #     datatype=[c.type for c in AutoEvalColumnSubset],
+    #     column_widths=["150px" if c.name != "Model" else "250px" for c in AutoEvalColumnSubset],
+    #     wrap=False,
+    # )
+
+
+
+def build_leaderboard(type):
+    with open('data/results.json', 'r') as f:
+        results = json.load(f)
+
+    with open('data/tasks.json', 'r') as f:
+        tasks = json.load(f)
+
+    # Filter tasks based on type
+    filtered_tasks = {k: v for k, v in tasks.items() if v['type'] == type}
+
+    data = []
+    for model_name, model_data in results.items():
+        # For agentic type, skip models that have all null values for agentic tasks
+        if type == "agentic":
+            has_agentic_results = any(
+                model_data['results'].get(task, {}).get(tasks[task]['metric']) is not None 
+                for task in filtered_tasks
+            )
+            if not has_agentic_results:
+                continue
+
+        model_sha = model_data["config"]["model_sha"]
+        model_name = model_data["config"]["model_name"]
+        row = {
+            'Model': make_clickable_field(model_name, model_sha)
+        }
+        
+        for dataset, metrics in model_data['results'].items():
+            # Only include metrics for tasks of the specified type
+            if dataset in filtered_tasks:
+                value = next(iter(metrics.values()))
+                log_url = metrics.get('log_url')
+                # Use display name from tasks.json instead of raw dataset name
+                display_name = filtered_tasks[dataset]['display_name']
+                # Round non-null values to 2 decimal places and make clickable if log_url exists
+                if value is not None:
+                    value = round(value*100, 2)
+                    if log_url:
+                        value = make_clickable_field(value, log_url)
+                row[display_name] = value
+        data.append(row)
+
+    results_df = pd.DataFrame(data)
+    
+    # Round all numeric columns to 2 decimal places
+    numeric_cols = results_df.select_dtypes(include=['float64', 'float32']).columns
+    results_df[numeric_cols] = results_df[numeric_cols].round(2)
+
+    # Fill null values with "-"
+    results_df = results_df.fillna("--")
+
+    if type == "agentic":
+        # Include agent column as second column after Model
+        results_df.insert(1, 'Agent', '[Basic Agent](https://inspect.ai-safety-institute.org.uk/agents.html#sec-basic-agent)')
+    
     return gr.components.Dataframe(
-        value=df,
-        datatype=[c.type for c in AutoEvalColumnSubset],
-        column_widths=["150px" if c.name != "Model" else "250px" for c in AutoEvalColumnSubset],
+        value=results_df,
+        datatype=["html" for _ in results_df.columns],
+        column_widths=["250px" if c == "Model" else "150px" for c in results_df.columns],
         wrap=False,
     )
 
+
 black_logo_path = "src/assets/logo-icon-black.png"
 white_logo_path = "src/assets/logo-icon-white.png"
 
@@ -123,10 +188,10 @@ with demo:
 
     with gr.Tabs(elem_classes=["leaderboard-table", "tab-buttons"]) as tabs:
         with gr.TabItem("Base Benchmarks", elem_classes="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(ST_LEADERBOARD_DF, "base")
+            build_leaderboard("base")
 
         with gr.TabItem("Agentic Benchmarks", elem_classes="llm-benchmark-tab-table", id=1):
-            leaderboard = init_leaderboard(AGENTIC_LEADERBOARD_DF, "agentic")
+            build_leaderboard("agentic")
 
         with gr.TabItem("About", elem_classes="llm-benchmark-tab-table", id=2):
             gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text", sanitize_html=False)
@@ -135,8 +200,5 @@ with demo:
             gr.Markdown(REPRODUCIBILITY_TEXT, elem_classes="markdown-text", sanitize_html=False)
 
 assets = [black_logo_path, white_logo_path]
+demo.launch()
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch(allowed_paths=assets)

data/populate_results.py

@@ -0,0 +1,41 @@
+import json
+
+def get_log_url(model_name: str, log_file_name: str) -> str:
+    """Returns the URL to the log file for a given model and benchmark"""
+    if log_file_name is None:
+        return None
+    else:
+        # replace .json with .eval
+        log_file_name = log_file_name.replace(".json", ".eval")
+        return f"https://storage.googleapis.com/inspect-evals/eval/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
+
+def main():
+    # Load the results and log file names
+    with open("data/results.json", "r") as f:
+        results = json.load(f)
+    
+    with open("data/inspect_log_file_names.json", "r") as f:
+        log_files = json.load(f)
+
+    # For each model in results
+    for model_name, model_data in results.items():
+        # Get the log files for this model
+        model_logs = log_files.get(model_name, {})
+        
+        # For each task in the model's results
+        for task_name, task_data in model_data["results"].items():
+            # Get the log file name for this task
+            log_file_name = model_logs.get(task_name)
+            
+            # Add the log URL to the task data
+            if log_file_name:
+                task_data["log_url"] = get_log_url(model_name, log_file_name)
+            else:
+                task_data["log_url"] = None
+
+    # Save the updated results
+    with open("data/results_with_logs.json", "w") as f:
+        json.dump(results, f, indent=4)
+
+if __name__ == "__main__":
+    main()

data/results.json

@@ -0,0 +1,948 @@
+{
+    "DeepSeek-R1": {
+        "config": {
+            "model_name": "DeepSeek-R1",
+            "model_sha": "https://api-docs.deepseek.com/news/news250120",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "mmlu_pro": {
+                "accuracy": 0.8382646276595744,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-12T11-02-35-05-00_mmlu-pro_BhD89DYN9KM3k4weSDfaQK.eval"
+            },
+            "humaneval": {
+                "mean": 0.9567901234567902,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-03T11-45-22-05-00_humaneval_hnkHWYqrb5HxiBt2CWzCnq.eval"
+            },
+            "math": {
+                "accuracy": 0.9272,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-11T11-38-10-05-00_math_ZYFSqsWsmP5kLRLHEMWULU.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.954510993176649,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-02T16-28-05-05-00_gsm8k_YMw6WiZkgTBQ54z5UHtDDX.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9667235494880546,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-01-30T15-42-39-05-00_arc-challenge_CviW9ro6rKBbctkwJzQstp.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.9179163378058406,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-04T00-25-12-05-00_winogrande_NPgTbtqom2QSPKxeThWrdZ.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9873737373737373,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.7045454545454546,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-11T11-37-45-05-00_gpqa-diamond_MwnVeLwyuiEAALr3M5q3dn.eval"
+            },
+            "drop": {
+                "mean": null,
+                "log_url": null
+            },
+            "hellaswag": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "ifeval": {
+                "final_acc": null,
+                "log_url": null
+            },
+            "mmlu": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "mmmu_open": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gaia": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "agentharm": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "agentharm_benign": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "swe_bench": {
+                "mean": null,
+                "log_url": null
+            }
+        }
+    },
+    "Meta-Llama-3.1-70B-Instruct": {
+        "config": {
+            "model_name": "Meta-Llama-3.1-70B-Instruct",
+            "model_sha": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "hellaswag": {
+                "accuracy": 0.869946225851424,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T00-45-54-04-00_hellaswag_BKfQG9yGAr383MGnooMLBH.eval"
+            },
+            "drop": {
+                "mean": 0.8811263765076035,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T21-01-02-04-00_drop_LzAWvLWkNrNKu5qf56wXRo.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.4318181818181818,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T23-41-39-04-00_gpqa-diamond_TdLdYmVM6GCVMAECcXkuhj.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.8666140489344909,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T09-20-56-04-00_winogrande_WnUgkSRhSMvh3zUjnuJWQZ.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9469294920394238,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T00-03-31-04-00_gsm8k_bKsUfCAfcmBCeryboNaLoX.eval"
+            },
+            "math": {
+                "accuracy": 0.6004,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T02-34-50-04-00_math_2xiNcrGih26uzJdG4q88bM.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.8604907201780166,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T02-29-32-04-00_ifeval_Dwh3CF2ZYFrvw7UcTwrsvK.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9445392491467577,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T20-58-56-04-00_arc-challenge_oFL5wFjT7KwNFhMFfe72JN.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9823232323232324,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T20-53-12-04-00_arc-easy_UXzR7cDeNteP39NoXUYnhm.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.6688829787234043,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T06-11-16-04-00_mmlu-pro_oQiEBJdeKtEEt4cm9KL7uy.eval"
+            },
+            "humaneval": {
+                "mean": 0.7865853658536586,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T02-28-25-04-00_humaneval_KcJV2rHuHJ2JLxijihEkcW.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.8033755875231449,
+                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T03-51-50-04-00_mmlu_6SNjs2QmPRvqGnvbnNtaqb.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "mmmu_open": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gaia": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "agentharm": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "agentharm_benign": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "swe_bench": {
+                "mean": null,
+                "log_url": null
+            }
+        }
+    },
+    "Mistral-Large-Instruct-2407": {
+        "config": {
+            "model_name": "Mistral-Large-Instruct-2407",
+            "model_sha": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "drop": {
+                "mean": 0.7424257996853698,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T01-56-12-04-00_drop_NtvuCoU2LoMbH8DztcCTen.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.8285172231900246,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T06-30-16-04-00_ifeval_TLkvCSFEWo4PLv6hAha7YB.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.8035892323030908,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T07-21-48-04-00_mmlu_YnUhmHoStr3WuJdchWmNPt.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.4734848484848485,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T04-22-52-04-00_gpqa-diamond_SuZUZxGdqS2ZecbLRNkKd4.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9378316906747536,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T04-28-49-04-00_gsm8k_5tQp9tbwUMj6NpjNKCAfVm.eval"
+            },
+            "math": {
+                "accuracy": 0.6574,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T06-33-09-04-00_math_2CmjBedAfUxqvmcHRdBgyB.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9852693602693603,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T01-48-39-04-00_arc-easy_YbfuBT3usZXt2xgZkkR5dq.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.6942320478723404,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T09-41-25-04-00_mmlu-pro_fyYT4aabPesfY5TpzFMPnd.eval"
+            },
+            "humaneval": {
+                "mean": 0.8658536585365854,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T06-29-24-04-00_humaneval_nu8SUSGekKJWB8HLKDigYK.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.9047998406691894,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T04-50-00-04-00_hellaswag_ZzQoZ6gkRQsTzMhQr7GYNn.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9436860068259386,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T01-54-13-04-00_arc-challenge_WfQRhMkFcywefpU46isBVP.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.8547750591949487,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T11-57-58-04-00_winogrande_TP3UGwpp37Dyv6ks9Ty5Hk.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "mmmu_open": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gaia": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "agentharm": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "agentharm_benign": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "swe_bench": {
+                "mean": null,
+                "log_url": null
+            }
+        }
+    },
+    "c4ai-command-r-plus": {
+        "config": {
+            "model_name": "c4ai-command-r-plus",
+            "model_sha": "https://huggingface.co/CohereForAI/c4ai-command-r-plus"
+        },
+        "results": {
+            "ifeval": {
+                "final_acc": 0.7779591483929307,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T17-23-04-04-00_ifeval_RGucUMwdGmUnRpqyMTZTzW.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.7490134175217048,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T14-42-18-04-00_winogrande_bY8yg7aRR5dCCK7NDCZEcc.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.8506825938566553,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-29T17-30-03-04-00_arc-challenge_XB7LURXEGaxskWuLtYwdnW.eval"
+            },
+            "drop": {
+                "mean": 0.743557420031463,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T12-06-30-04-00_drop_itY9cLiYAW2BF7NTeDceNd.eval"
+            },
+            "math": {
+                "accuracy": 0.2626,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T17-26-34-04-00_math_kohBUMpMFuMsR4jz4vUNWM.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.3194444444444444,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-29T22-47-45-04-00_gpqa-diamond_JKpb6ya4pec9hh7uovPPCZ.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.441156914893617,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-31T01-11-38-04-00_mmlu-pro_gZVAuy3zMKR23BieM5PqAX.eval"
+            },
+            "humaneval": {
+                "mean": 0.6219512195121951,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T17-22-23-04-00_humaneval_5ByPqUhoofSbKgvsUQNFCX.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.7816527672479151,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T15-03-35-04-00_gsm8k_QxbfbriJsKGQAg96JyjkoT.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.7954590718980283,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T15-18-17-04-00_hellaswag_UYyBTR6N8VJnKRmnbCrB8N.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.695128899017234,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T21-55-26-04-00_mmlu_JUPPLTzfe3Kme6UuorPTqg.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9377104377104377,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.eval"
+            }
+        }
+    },
+    "claude-3-5-sonnet-20241022": {
+        "config": {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "model_sha": "https://www.anthropic.com/claude/sonnet",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "mmmu_multiple_choice": {
+                "accuracy": 0.6481700118063755,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-21T11-20-03-05-00_mmmu-multiple-choice_CWhKvGdoFo6pdHhDyi9GNm.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7762632978723404,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T19-01-05-05-00_mmlu-pro_3vi84or97gQupuj5sT6vgZ.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.9228241386178052,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T15-09-33-05-00_hellaswag_QXqFxojvSToMu8ckHEMLkB.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.6098484848484849,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T13-56-36-05-00_gpqa-diamond_eg4gFaMRENjnnYvQNtSB59.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9620924943138741,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T14-23-25-05-00_gsm8k_nHB8Z4uZAwRAZFYpKmTptA.eval"
+            },
+            "mmmu_open": {
+                "accuracy": 0.41509433962264153,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-21T11-24-21-05-00_mmmu-open_SSjv3Dq9gZkEEUnvJUd5xf.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9915824915824916,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T10-06-24-05-00_arc-easy_oBReQZQM5SAwMMD2jFshPb.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9692832764505119,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T10-12-11-05-00_arc-challenge_X8i6caCzkcQo5AT5zXkXso.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.8665432274604757,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T15-16-51-05-00_mmlu_NFDs2kxmh3kQEbpbd8sz3w.eval"
+            },
+            "math": {
+                "accuracy": 0.7942,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T12-29-54-05-00_math_NvNQU58M8r3fpiwPGnvq8h.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.8958114469607309,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T11-28-44-05-00_ifeval_fmWxch4ZjbmYCST6yUZsdV.eval"
+            },
+            "humaneval": {
+                "mean": 0.9451219512195121,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T11-26-12-05-00_humaneval_kUASiaNd9uZfWvCwYHhdF5.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.9021310181531176,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T22-09-41-05-00_winogrande_mSWGAKg75E5RP79KWizvb9.eval"
+            },
+            "drop": {
+                "mean": 0.8977608809648663,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T10-15-15-05-00_drop_Z9A2Y84HYponNxnzNT9TNq.eval"
+            },
+            "gaia": {
+                "accuracy": 0.3381818181818182,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-12T23-57-37-05-00_claude-3-5-sonnet_gaia_merged.eval"
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.8556962025316455,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-11T02-47-45-05-00_claude-3-5-sonnet_gdm-intercode-ctf_merged.eval"
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.6153846153846154,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-11T07-41-14+00-00_claude-3-5-sonnet_gdm-in-house-ctf.eval"
+            },
+            "agentharm": {
+                "avg_score": 0.14767992424242424,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T08-05-14-08-00_agentharm_VJGhWKLrVLdQczBZVgCXHc.eval"
+            },
+            "agentharm_benign": {
+                "avg_score": 0.800704570051161,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-21T15-09-48-08-00_agentharm-benign_A3uBBWNvv88P5BsgqwFCfg.eval"
+            },
+            "swe_bench": {
+                "mean": 0.0672,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T18-56-55+00-00_anthropic-claude-3-5-sonnet.eval"
+            }
+        }
+    },
+    "gemini-1.5-flash": {
+        "config": {
+            "model_name": "gemini-1.5-flash",
+            "model_sha": "https://deepmind.google/technologies/gemini/flash",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "gpqa_diamond": {
+                "accuracy": 0.40404040404040403,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9308873720136519,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T17-34-51-04-00_arc-challenge_FbGgLswBZbRE4EhWiMyRt6.eval"
+            },
+            "math": {
+                "accuracy": 0.452,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.eval"
+            },
+            "mmmu_open": {
+                "accuracy": 0.16981132075471697,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2025-01-20T23-13-27-05-00_mmmu-open_GWi6XNYUSLq99BdabtScGm.eval"
+            },
+            "drop": {
+                "mean": 0.751044572627163,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T17-36-25-04-00_drop_6TzJGqqEkpFUCxGD4QejV6.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.5993184840425532,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.7681296737102001,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T21-15-06-04-00_ifeval_nYs9KujQMQjcpbpbLtVx8G.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.8557060346544513,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T19-35-34-04-00_hellaswag_2SAz3cvMpDxFaApdHDR3s4.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.7884767166535123,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-31T00-59-07-04-00_winogrande_Ci55vHvbGGW38zVpMCwtWa.eval"
+            },
+            "humaneval": {
+                "mean": 0.7439024390243902,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.984006734006734,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T17-29-56-04-00_arc-easy_XcEzqqPqJsRV29NqYDfnNo.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.8582259287338894,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T19-32-39-04-00_gsm8k_nLSssETKDDWNktAFWnVwfv.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.7714713003845606,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-31T10-49-43-04-00_mmlu_oGb9mspeGbYS2gfbkknskN.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.5702479338842975,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2025-01-20T23-10-01-05-00_mmmu-multiple-choice_c5rLkrXkV83udX6DVJui5F.eval"
+            },
+            "gaia": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "agentharm": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "agentharm_benign": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "swe_bench": {
+                "mean": null,
+                "log_url": null
+            }
+        }
+    },
+    "gemini-1.5-pro": {
+        "config": {
+            "model_name": "gemini-1.5-pro",
+            "model_sha": "https://deepmind.google/technologies/gemini/pro",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "mmlu": {
+                "accuracy": 0.8467454778521578,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T16-56-26-05-00_mmlu_Z9KrcK7x4ZLAR5nJ9JaVUe.eval"
+            },
+            "humaneval": {
+                "mean": 0.8719512195121951,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-43-07-05-00_humaneval_5JBjtymGtK23qwVKxqidhV.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.6304604486422668,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2025-01-20T23-16-04-05-00_mmmu-multiple-choice_NLmxmHYt6CJymRVVa5UsbD.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7563996010638298,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T20-13-09-05-00_mmlu-pro_Hv2ujvKLV6H7ZwQu2q8LNw.eval"
+            },
+            "math": {
+                "accuracy": 0.852,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T15-48-46-05-00_math_9DAZmGEfhpa3nUcmMAwqZe.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9877946127946128,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-31-43-05-00_arc-easy_eGxYWywpLuREcaCKvHa8Uk.eval"
+            },
+            "mmmu_open": {
+                "accuracy": 0.3584905660377358,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2025-01-20T23-19-25-05-00_mmmu-open_CDbtEQ7tjs5zkj4ScBbzod.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9613343442001516,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T15-15-26-05-00_gsm8k_cTebw3ugfrVz3dyPwxtdUZ.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.5782828282828283,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-05T09-56-31-05-00_gpqa-diamond_FBq2bnoyGYQ3NF96xQw8iy.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.8982344623377084,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-43-32-05-00_ifeval_mSwZ7AwA7akj5PjZbQMjgC.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.8768745067087609,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-40-46-05-00_winogrande_5SmD6rx47zmZvHHkQSSfHK.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9633105802047781,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-37-36-05-00_arc-challenge_5VVApyQD22QpJoMm53EMdU.eval"
+            },
+            "drop": {
+                "mean": 0.8800912427897221,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-44-32-05-00_drop_9dzPKVJojSVsxmiBFnej2m.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.9123680541724756,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-05T13-14-31-05-00_hellaswag_N98eeftuY2pucRtgpUYk5m.eval"
+            },
+            "gaia": {
+                "accuracy": 0.13818181818181818,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T15-33-29-05-00_gemini-1.5-pro_gaia_merged.eval"
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.5291139240506328,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T23-59-58+00-00_gemini-1.5-pro_gdm-intercode-ctf_merged.eval"
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.23076923076923078,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-22T03-42-16+00-00_gemini-1.5-pro_gdm-in-house-ctf.eval"
+            },
+            "agentharm": {
+                "avg_score": 0.2898649645808737,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T12-45-43-08-00_agentharm_VmD26soLwmRgWPo3hpRHBr.eval"
+            },
+            "agentharm_benign": {
+                "avg_score": 0.5961489079102715,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T13-18-51-08-00_agentharm-benign_gP3pQPxAuCtFLiHzt2Egt7.eval"
+            },
+            "swe_bench": {
+                "mean": 0.004,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-22T03-00-08+00-00_google-gemini-1.5-pro_swe.eval"
+            }
+        }
+    },
+    "gpt-4o": {
+        "config": {
+            "model_name": "gpt-4o",
+            "model_sha": "https://openai.com/index/hello-gpt-4o",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "gpqa_diamond": {
+                "accuracy": 0.51010101010101,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T03-29-33-04-00_gpqa-diamond_nFmRv5MJiYjHjezmq4V6Va.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9633105802047781,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T01-45-55-04-00_arc-challenge_nrsPPxh4DpzgLPQDFdcfVp.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9446550416982562,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T03-31-16-04-00_gsm8k_jVXeSvHowbietZCFsFYCwB.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.8435408061529697,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T10-49-43-04-00_mmlu_GarLpfQFSpM3C22nbbGp54.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.8780386042367585,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T05-00-11-04-00_ifeval_jxreUu8JqRdkrcHP4E3hLR.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7450964095744681,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T06-59-42-04-00_mmlu-pro_EuAKDwAWSfNVpqyyqrf2Ba.eval"
+            },
+            "mmmu_open": {
+                "accuracy": 0.3584905660377358,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2025-01-20T23-07-46-05-00_mmmu-open_d3Q2HvuPZzEX6FAM4NBhnp.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.9013417521704814,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T09-02-03-04-00_winogrande_44kKF7M9mKoqVC7ixZVXuq.eval"
+            },
+            "drop": {
+                "mean": 0.7511693759832198,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T01-47-20-04-00_drop_3gxDcn6vUoR3nvHX9BcSq4.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9915824915824916,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T01-41-34-04-00_arc-easy_nUavRHdiRVfrxo6dmCPadh.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.5903187721369539,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2025-01-20T23-03-21-05-00_mmmu-multiple-choice_eoycAFLMirSqiURdXmBP2e.eval"
+            },
+            "humaneval": {
+                "mean": 0.9085365853658537,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T04-59-42-04-00_humaneval_nmJcd84CcNKjWS8fBfMbZM.eval"
+            },
+            "math": {
+                "accuracy": 0.7054,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T05-01-22-04-00_math_cDSpKPp3nLrFy8uYfYKEbM.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.924317864967138,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T03-33-47-04-00_hellaswag_JNnnPuz3dhZRpyXzizMUBF.eval"
+            },
+            "gaia": {
+                "accuracy": 0.16606060606060608,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-13T15-53-22+00-00_gpt-4o_gaia_merged.eval"
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.6379746835443038,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-08T10-06-29-05-00_gpt-4o_gdm-intercode-ctf_merged.eval"
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.23076923076923078,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-11T07-02-14+00-00_gpt-4o_gdm-in-house-ctf.eval"
+            },
+            "agentharm": {
+                "avg_score": 0.49953844451003543,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-07T16-34-15-08-00_agentharm_UfSoyHEAH2E5RVdrPVUemy.eval"
+            },
+            "agentharm_benign": {
+                "avg_score": 0.8249433048012594,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-21T13-45-18-08-00_agentharm-benign_8DhGJqEAvw6o8uCv4a4dVz.eval"
+            },
+            "swe_bench": {
+                "mean": 0.012,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-14T23-09-10+00-00_openai-gpt-4o_swe.eval"
+            }
+        }
+    },
+    "gpt-4o-mini": {
+        "config": {
+            "model_name": "gpt-4o-mini",
+            "model_sha": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "drop": {
+                "mean": 0.8065915049816466,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T17-36-25-04-00_drop_6TzJGqqEkpFUCxGD4QejV6.eval"
+            },
+            "humaneval": {
+                "mean": 0.8597560975609756,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.3838383838383838,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.eval"
+            },
+            "mmmu_open": {
+                "accuracy": 0.18867924528301888,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2025-01-20T23-13-27-05-00_mmmu-open_GWi6XNYUSLq99BdabtScGm.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9249146757679181,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T17-34-51-04-00_arc-challenge_FbGgLswBZbRE4EhWiMyRt6.eval"
+            },
+            "mmlu": {
+                "accuracy": 0.7698333570716422,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-31T10-49-43-04-00_mmlu_oGb9mspeGbYS2gfbkknskN.eval"
+            },
+            "hellaswag": {
+                "accuracy": 0.8750248954391555,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T19-35-34-04-00_hellaswag_2SAz3cvMpDxFaApdHDR3s4.eval"
+            },
+            "ifeval": {
+                "final_acc": 0.8419061423689144,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T21-15-06-04-00_ifeval_nYs9KujQMQjcpbpbLtVx8G.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.5395513577331759,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2025-01-20T23-10-01-05-00_mmmu-multiple-choice_c5rLkrXkV83udX6DVJui5F.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9793771043771043,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T17-29-56-04-00_arc-easy_XcEzqqPqJsRV29NqYDfnNo.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.7529597474348856,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-31T00-59-07-04-00_winogrande_Ci55vHvbGGW38zVpMCwtWa.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.6396276595744681,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.eval"
+            },
+            "math": {
+                "accuracy": 0.633,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9181197877179682,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T19-32-39-04-00_gsm8k_nLSssETKDDWNktAFWnVwfv.eval"
+            },
+            "gaia": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "agentharm": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "agentharm_benign": {
+                "avg_score": null,
+                "log_url": null
+            },
+            "swe_bench": {
+                "mean": null,
+                "log_url": null
+            }
+        }
+    },
+    "o1": {
+        "config": {
+            "model_name": "o1",
+            "model_sha": "https://openai.com/o1",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "winogrande": {
+                "accuracy": 0.9392265193370166,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T16-46-06-05-00_winogrande_YUtAdEsForRffqe4Sm3wtR.eval"
+            },
+            "humaneval": {
+                "mean": 0.9695121951219512,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T14-59-12-05-00_humaneval_RRL8GMy9NakTxUHsDVWNng.eval"
+            },
+            "mmmu_open": {
+                "accuracy": 0.6981132075471698,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T22-48-09-05-00_mmmu-open_oBzxJBYbvnktbbAwhoCrYK.eval"
+            },
+            "math": {
+                "accuracy": 0.959,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T15-03-22-05-00_math_6BbvHFF8hLMsVYozyNLbyQ.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9911616161616161,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T11-29-26-05-00_arc-easy_DFbir4BdgQDbKd52r7tRKR.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9786689419795221,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T11-44-42-05-00_arc-challenge_PsWXaBqrgv3EcTZC55gRzJ.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9416224412433661,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T12-56-38-05-00_gsm8k_iD8275qeyNTgX523pn45bF.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.7550505050505051,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T11-53-53-05-00_gpqa-diamond_EJV7ULFSQLRoFTEqsv3t6q.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.8447473404255319,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T14-02-37-05-00_mmlu-pro_EvDzvqaahQwhv6fJovN4BT.eval"
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.8063754427390791,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T21-04-57-05-00_mmmu-multiple-choice_MctxjookaeTLCL8KpUeazT.eval"
+            },
+            "drop": {
+                "mean": null,
+                "log_url": null
+            },
+            "hellaswag": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "ifeval": {
+                "final_acc": null,
+                "log_url": null
+            },
+            "mmlu": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gaia": {
+                "accuracy": 0.41090909090909084,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-22T13-42-00-05-00_o1_gaia_merged.eval"
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.8481012658227849,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-22T20-46-35+00-00_o1_gdm-intercode-ctf_merged.eval"
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.46153846153846156,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-22T05-52-25+00-00_o1_gdm-in-house-ctf.eval"
+            },
+            "agentharm": {
+                "avg_score": 0.08782061688311688,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-21T09-05-42-08-00_agentharm_UGDq2yJeLAnPH6p7FgDgD8.eval"
+            },
+            "agentharm_benign": {
+                "avg_score": 0.7235176849665487,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-21T18-20-15-08-00_agentharm-benign_bkW2Bf5xLyDQdNtfLdjCpJ.eval"
+            },
+            "swe_bench": {
+                "mean": 0.0036,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-21T17-42-11+00-00_openai-o1_swe.eval "
+            }
+        }
+    },
+    "o3-mini": {
+        "config": {
+            "model_name": "o3-mini",
+            "model_sha": "https://openai.com/index/openai-o3-mini",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "math": {
+                "accuracy": 0.9691320905993185,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T18-33-30-05-00_math_86Gx8n4BxhpyfaSHmRcCUm.eval"
+            },
+            "humaneval": {
+                "mean": 0.9817073170731707,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T20-58-48-05-00_humaneval_Dkod7CS9RmbbogYx9aEXtx.eval"
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7924606807023383,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T19-49-27-05-00_mmlu-pro_jz9woKfdKt8VMzqNFsy7kY.eval"
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.7365319865319865,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T17-57-54-05-00_gpqa-diamond_2znyMtdc7X4LJufxXeXA8Z.eval"
+            },
+            "winogrande": {
+                "accuracy": 0.8492501973164956,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T22-50-40-05-00_winogrande_VsTW2uU2Kj66YoNoFfRfUj.eval"
+            },
+            "gsm8k": {
+                "accuracy": 0.9454131918119788,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T18-23-05-05-00_gsm8k_d523pJzkcvobxamhhobCRb.eval"
+            },
+            "arc_challenge": {
+                "accuracy": 0.9641638225255973,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T17-53-30-05-00_arc-challenge_AYFHec7wmd4jELF2Rgzfya.eval"
+            },
+            "arc_easy": {
+                "accuracy": 0.9755892255892256,
+                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T17-45-57-05-00_arc-easy_Nd8NP3K48tvwLVZb8kXDwg.eval"
+            },
+            "drop": {
+                "mean": null,
+                "log_url": null
+            },
+            "hellaswag": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "ifeval": {
+                "final_acc": null,
+                "log_url": null
+            },
+            "mmlu": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "mmmu_open": {
+                "accuracy": null,
+                "log_url": null
+            },
+            "gaia": {
+                "accuracy": 0.27030303030303043,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-02-05T23-21-20+00-00_gaia_hyMq8MzMm6NgAeq3dNqZSU.eval"
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.8278481012658225,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-05T21-43-18+00-00_gdm-intercode-ctf_gdm29C6DuTEsX9qm9ymmrC.eval"
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.38461538461538464,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-05T23-59-08+00-00_gdm-in-house-ctf_2zkAX5nkJoxDnVKpJL9VgW.eval"
+            },
+            "agentharm": {
+                "avg_score": 0.1241931080283353,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-03T18-17-03-08-00_agentharm_DmN6i5HrgXHNARjsuSewjg.eval"
+            },
+            "agentharm_benign": {
+                "avg_score": 0.5429306867375049,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-03T18-49-08-08-00_agentharm-benign_Gv94YFpAXaaCJqe3Fc6yr3.eval"
+            },
+            "swe_bench": {
+                "mean": 0.0024,
+                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-03T06-49-09+00-00_openai-o3-mini_swe.eval"
+            }
+        }
+    }
+}

data/results.json.bak

@@ -0,0 +1,760 @@
+{
+    "DeepSeek-R1": {
+        "config": {
+            "model_name": "DeepSeek-R1",
+            "model_sha": "https://api-docs.deepseek.com/news/news250120",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "mmlu_pro": {
+                "accuracy": 0.8382646276595744
+            },
+            "humaneval": {
+                "mean": 0.9567901234567902
+            },
+            "math": {
+                "accuracy": 0.9272
+            },
+            "gsm8k": {
+                "accuracy": 0.954510993176649
+            },
+            "arc_challenge": {
+                "accuracy": 0.9667235494880546
+            },
+            "winogrande": {
+                "accuracy": 0.9179163378058406
+            },
+            "arc_easy": {
+                "accuracy": 0.9873737373737373
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.7045454545454546
+            },
+            "drop": {
+                "mean": null
+            },
+            "hellaswag": {
+                "accuracy": null
+            },
+            "ifeval": {
+                "final_acc": null
+            },
+            "mmlu": {
+                "accuracy": null
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null
+            },
+            "mmmu_open": {
+                "accuracy": null
+            },
+            "gaia": {
+                "accuracy": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null
+            },
+            "agentharm": {
+                "avg_score": null
+            },
+            "agentharm_benign": {
+                "avg_score": null
+            },
+            "swe_bench": {
+                "mean": null
+            }
+        }
+    },
+    "Meta-Llama-3.1-70B-Instruct": {
+        "config": {
+            "model_name": "Meta-Llama-3.1-70B-Instruct",
+            "model_sha": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "hellaswag": {
+                "accuracy": 0.869946225851424
+            },
+            "drop": {
+                "mean": 0.8811263765076035
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.4318181818181818
+            },
+            "winogrande": {
+                "accuracy": 0.8666140489344909
+            },
+            "gsm8k": {
+                "accuracy": 0.9469294920394238
+            },
+            "math": {
+                "accuracy": 0.6004
+            },
+            "ifeval": {
+                "final_acc": 0.8604907201780166
+            },
+            "arc_challenge": {
+                "accuracy": 0.9445392491467577
+            },
+            "arc_easy": {
+                "accuracy": 0.9823232323232324
+            },
+            "mmlu_pro": {
+                "accuracy": 0.6688829787234043
+            },
+            "humaneval": {
+                "mean": 0.7865853658536586
+            },
+            "mmlu": {
+                "accuracy": 0.8033755875231449
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null
+            },
+            "mmmu_open": {
+                "accuracy": null
+            },
+            "gaia": {
+                "accuracy": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null
+            },
+            "agentharm": {
+                "avg_score": null
+            },
+            "agentharm_benign": {
+                "avg_score": null
+            },
+            "swe_bench": {
+                "mean": null
+            }
+        }
+    },
+    "Mistral-Large-Instruct-2407": {
+        "config": {
+            "model_name": "Mistral-Large-Instruct-2407",
+            "model_sha": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "drop": {
+                "mean": 0.7424257996853698
+            },
+            "ifeval": {
+                "final_acc": 0.8285172231900246
+            },
+            "mmlu": {
+                "accuracy": 0.8035892323030908
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.4734848484848485
+            },
+            "gsm8k": {
+                "accuracy": 0.9378316906747536
+            },
+            "math": {
+                "accuracy": 0.6574
+            },
+            "arc_easy": {
+                "accuracy": 0.9852693602693603
+            },
+            "mmlu_pro": {
+                "accuracy": 0.6942320478723404
+            },
+            "humaneval": {
+                "mean": 0.8658536585365854
+            },
+            "hellaswag": {
+                "accuracy": 0.9047998406691894
+            },
+            "arc_challenge": {
+                "accuracy": 0.9436860068259386
+            },
+            "winogrande": {
+                "accuracy": 0.8547750591949487
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null
+            },
+            "mmmu_open": {
+                "accuracy": null
+            },
+            "gaia": {
+                "accuracy": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null
+            },
+            "agentharm": {
+                "avg_score": null
+            },
+            "agentharm_benign": {
+                "avg_score": null
+            },
+            "swe_bench": {
+                "mean": null
+            }
+        }
+    },
+    "c4ai-command-r-plus": {
+        "config": {
+            "model_name": "c4ai-command-r-plus",
+            "model_sha": "https://huggingface.co/CohereForAI/c4ai-command-r-plus"
+        },
+        "results": {
+            "ifeval": {
+                "final_acc": 0.7779591483929307
+            },
+            "winogrande": {
+                "accuracy": 0.7490134175217048
+            },
+            "arc_challenge": {
+                "accuracy": 0.8506825938566553
+            },
+            "drop": {
+                "mean": 0.743557420031463
+            },
+            "math": {
+                "accuracy": 0.2626
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.3194444444444444
+            },
+            "mmlu_pro": {
+                "accuracy": 0.441156914893617
+            },
+            "humaneval": {
+                "mean": 0.6219512195121951
+            },
+            "gsm8k": {
+                "accuracy": 0.7816527672479151
+            },
+            "hellaswag": {
+                "accuracy": 0.7954590718980283
+            },
+            "mmlu": {
+                "accuracy": 0.695128899017234
+            },
+            "arc_easy": {
+                "accuracy": 0.9377104377104377
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null
+            },
+            "mmmu_open": {
+                "accuracy": null
+            },
+            "gaia": {
+                "accuracy": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null
+            },
+            "agentharm": {
+                "avg_score": null
+            },
+            "agentharm_benign": {
+                "avg_score": null
+            },
+            "swe_bench": {
+                "mean": null
+            }
+        }
+    },
+    "claude-3-5-sonnet-20241022": {
+        "config": {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "model_sha": "https://www.anthropic.com/claude/sonnet",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "mmmu_multiple_choice": {
+                "accuracy": 0.6481700118063755
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7762632978723404
+            },
+            "hellaswag": {
+                "accuracy": 0.9228241386178052
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.6098484848484849
+            },
+            "gsm8k": {
+                "accuracy": 0.9620924943138741
+            },
+            "mmmu_open": {
+                "accuracy": 0.41509433962264153
+            },
+            "arc_easy": {
+                "accuracy": 0.9915824915824916
+            },
+            "arc_challenge": {
+                "accuracy": 0.9692832764505119
+            },
+            "mmlu": {
+                "accuracy": 0.8665432274604757
+            },
+            "math": {
+                "accuracy": 0.7942
+            },
+            "ifeval": {
+                "final_acc": 0.8958114469607309
+            },
+            "humaneval": {
+                "mean": 0.9451219512195121
+            },
+            "winogrande": {
+                "accuracy": 0.9021310181531176
+            },
+            "drop": {
+                "mean": 0.8977608809648663
+            },
+            "gaia": {
+                "accuracy": 0.3381818181818182
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.8556962025316455
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.6153846153846154
+            },
+            "agentharm": {
+                "avg_score": 0.14767992424242424
+            },
+            "agentharm_benign": {
+                "avg_score": 0.800704570051161
+            },
+            "swe_bench": {
+                "mean": 0.0672
+            }
+        }
+    },
+    "gemini-1.5-flash": {
+        "config": {
+            "model_name": "gemini-1.5-flash",
+            "model_sha": "https://deepmind.google/technologies/gemini/flash",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "gpqa_diamond": {
+                "accuracy": 0.40404040404040403
+            },
+            "arc_challenge": {
+                "accuracy": 0.9308873720136519
+            },
+            "math": {
+                "accuracy": 0.452
+            },
+            "mmmu_open": {
+                "accuracy": 0.16981132075471697
+            },
+            "drop": {
+                "mean": 0.751044572627163
+            },
+            "mmlu_pro": {
+                "accuracy": 0.5993184840425532
+            },
+            "ifeval": {
+                "final_acc": 0.7681296737102001
+            },
+            "hellaswag": {
+                "accuracy": 0.8557060346544513
+            },
+            "winogrande": {
+                "accuracy": 0.7884767166535123
+            },
+            "humaneval": {
+                "mean": 0.7439024390243902
+            },
+            "arc_easy": {
+                "accuracy": 0.984006734006734
+            },
+            "gsm8k": {
+                "accuracy": 0.8582259287338894
+            },
+            "mmlu": {
+                "accuracy": 0.7714713003845606
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.5702479338842975
+            },
+            "gaia": {
+                "accuracy": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null
+            },
+            "agentharm": {
+                "avg_score": null
+            },
+            "agentharm_benign": {
+                "avg_score": null
+            },
+            "swe_bench": {
+                "mean": null
+            }
+        }
+    },
+    "gemini-1.5-pro": {
+        "config": {
+            "model_name": "gemini-1.5-pro",
+            "model_sha": "https://deepmind.google/technologies/gemini/pro",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "mmlu": {
+                "accuracy": 0.8467454778521578
+            },
+            "humaneval": {
+                "mean": 0.8719512195121951
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.6304604486422668
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7563996010638298
+            },
+            "math": {
+                "accuracy": 0.852
+            },
+            "arc_easy": {
+                "accuracy": 0.9877946127946128
+            },
+            "mmmu_open": {
+                "accuracy": 0.3584905660377358
+            },
+            "gsm8k": {
+                "accuracy": 0.9613343442001516
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.5782828282828283
+            },
+            "ifeval": {
+                "final_acc": 0.8982344623377084
+            },
+            "winogrande": {
+                "accuracy": 0.8768745067087609
+            },
+            "arc_challenge": {
+                "accuracy": 0.9633105802047781
+            },
+            "drop": {
+                "mean": 0.8800912427897221
+            },
+            "hellaswag": {
+                "accuracy": 0.9123680541724756
+            },
+            "gaia": {
+                "accuracy": 0.13818181818181818
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.5291139240506328
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.23076923076923078
+            },
+            "agentharm": {
+                "avg_score": 0.2898649645808737
+            },
+            "agentharm_benign": {
+                "avg_score": 0.5961489079102715
+            },
+            "swe_bench": {
+                "mean": 0.004
+            }
+        }
+    },
+    "gpt-4o": {
+        "config": {
+            "model_name": "gpt-4o",
+            "model_sha": "https://openai.com/index/hello-gpt-4o",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "gpqa_diamond": {
+                "accuracy": 0.51010101010101
+            },
+            "arc_challenge": {
+                "accuracy": 0.9633105802047781
+            },
+            "gsm8k": {
+                "accuracy": 0.9446550416982562
+            },
+            "mmlu": {
+                "accuracy": 0.8435408061529697
+            },
+            "ifeval": {
+                "final_acc": 0.8780386042367585
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7450964095744681
+            },
+            "mmmu_open": {
+                "accuracy": 0.3584905660377358
+            },
+            "winogrande": {
+                "accuracy": 0.9013417521704814
+            },
+            "drop": {
+                "mean": 0.7511693759832198
+            },
+            "arc_easy": {
+                "accuracy": 0.9915824915824916
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.5903187721369539
+            },
+            "humaneval": {
+                "mean": 0.9085365853658537
+            },
+            "math": {
+                "accuracy": 0.7054
+            },
+            "hellaswag": {
+                "accuracy": 0.924317864967138
+            },
+            "gaia": {
+                "accuracy": 0.16606060606060608
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.6379746835443038
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.23076923076923078
+            },
+            "agentharm": {
+                "avg_score": 0.49953844451003543
+            },
+            "agentharm_benign": {
+                "avg_score": 0.8249433048012594
+            },
+            "swe_bench": {
+                "mean": 0.012
+            }
+        }
+    },
+    "gpt-4o-mini": {
+        "config": {
+            "model_name": "gpt-4o-mini",
+            "model_sha": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "drop": {
+                "mean": 0.8065915049816466
+            },
+            "humaneval": {
+                "mean": 0.8597560975609756
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.3838383838383838
+            },
+            "mmmu_open": {
+                "accuracy": 0.18867924528301888
+            },
+            "arc_challenge": {
+                "accuracy": 0.9249146757679181
+            },
+            "mmlu": {
+                "accuracy": 0.7698333570716422
+            },
+            "hellaswag": {
+                "accuracy": 0.8750248954391555
+            },
+            "ifeval": {
+                "final_acc": 0.8419061423689144
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.5395513577331759
+            },
+            "arc_easy": {
+                "accuracy": 0.9793771043771043
+            },
+            "winogrande": {
+                "accuracy": 0.7529597474348856
+            },
+            "mmlu_pro": {
+                "accuracy": 0.6396276595744681
+            },
+            "math": {
+                "accuracy": 0.633
+            },
+            "gsm8k": {
+                "accuracy": 0.9181197877179682
+            },
+            "gaia": {
+                "accuracy": null
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": null
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": null
+            },
+            "agentharm": {
+                "avg_score": null
+            },
+            "agentharm_benign": {
+                "avg_score": null
+            },
+            "swe_bench": {
+                "mean": null
+            }
+        }
+    },
+    "o1": {
+        "config": {
+            "model_name": "o1",
+            "model_sha": "https://openai.com/o1",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "winogrande": {
+                "accuracy": 0.9392265193370166
+            },
+            "humaneval": {
+                "mean": 0.9695121951219512
+            },
+            "mmmu_open": {
+                "accuracy": 0.6981132075471698
+            },
+            "math": {
+                "accuracy": 0.959
+            },
+            "arc_easy": {
+                "accuracy": 0.9911616161616161
+            },
+            "arc_challenge": {
+                "accuracy": 0.9786689419795221
+            },
+            "gsm8k": {
+                "accuracy": 0.9416224412433661
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.7550505050505051
+            },
+            "mmlu_pro": {
+                "accuracy": 0.8447473404255319
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": 0.8063754427390791
+            },
+            "drop": {
+                "mean": null
+            },
+            "hellaswag": {
+                "accuracy": null
+            },
+            "ifeval": {
+                "final_acc": null
+            },
+            "mmlu": {
+                "accuracy": null
+            },
+            "gaia": {
+                "accuracy": 0.41090909090909084
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.8481012658227849
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.46153846153846156
+            },
+            "agentharm": {
+                "avg_score": 0.08782061688311688
+            },
+            "agentharm_benign": {
+                "avg_score": 0.7235176849665487
+            },
+            "swe_bench": {
+                "mean": 0.0036
+            }
+        }
+    },
+    "o3-mini": {
+        "config": {
+            "model_name": "o3-mini",
+            "model_sha": "https://openai.com/index/openai-o3-mini",
+            "model_dtype": "torch.float16"
+        },
+        "results": {
+            "math": {
+                "accuracy": 0.9691320905993185
+            },
+            "humaneval": {
+                "mean": 0.9817073170731707
+            },
+            "mmlu_pro": {
+                "accuracy": 0.7924606807023383
+            },
+            "gpqa_diamond": {
+                "accuracy": 0.7365319865319865
+            },
+            "winogrande": {
+                "accuracy": 0.8492501973164956
+            },
+            "gsm8k": {
+                "accuracy": 0.9454131918119788
+            },
+            "arc_challenge": {
+                "accuracy": 0.9641638225255973
+            },
+            "arc_easy": {
+                "accuracy": 0.9755892255892256
+            },
+            "drop": {
+                "mean": null
+            },
+            "hellaswag": {
+                "accuracy": null
+            },
+            "ifeval": {
+                "final_acc": null
+            },
+            "mmlu": {
+                "accuracy": null
+            },
+            "mmmu_multiple_choice": {
+                "accuracy": null
+            },
+            "mmmu_open": {
+                "accuracy": null
+            },
+            "gaia": {
+                "accuracy": 0.27030303030303043
+            },
+            "gdm_intercode_ctf": {
+                "accuracy": 0.8278481012658225
+            },
+            "gdm_in_house_ctf": {
+                "accuracy": 0.38461538461538464
+            },
+            "agentharm": {
+                "avg_score": 0.1241931080283353
+            },
+            "agentharm_benign": {
+                "avg_score": 0.5429306867375049
+            },
+            "swe_bench": {
+                "mean": 0.0024
+            }
+        }
+    }
+}

data/tasks.json

@@ -0,0 +1,142 @@
+{
+    "arc_easy": {
+        "benchmark": "arc_easy",
+        "metric": "accuracy",
+        "display_name": "ARC-Easy",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc"
+    },
+    "arc_challenge": {
+        "benchmark": "arc_challenge",
+        "metric": "accuracy",
+        "display_name": "ARC-Challenge",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc"
+    },
+    "drop": {
+        "benchmark": "drop",
+        "metric": "mean",
+        "display_name": "DROP",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/drop"
+    },
+    "winogrande": {
+        "benchmark": "winogrande",
+        "metric": "accuracy",
+        "display_name": "WinoGrande",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/winogrande"
+    },
+    "gsm8k": {
+        "benchmark": "gsm8k",
+        "metric": "accuracy",
+        "display_name": "GSM8K",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gsm8k"
+    },
+    "hellaswag": {
+        "benchmark": "hellaswag",
+        "metric": "accuracy",
+        "display_name": "HellaSwag",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/hellaswag"
+    },
+    "humaneval": {
+        "benchmark": "humaneval",
+        "metric": "mean",
+        "display_name": "HumanEval",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/humaneval"
+    },
+    "ifeval": {
+        "benchmark": "ifeval",
+        "metric": "final_acc",
+        "display_name": "IFEval",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/ifeval"
+    },
+    "math": {
+        "benchmark": "math",
+        "metric": "accuracy",
+        "display_name": "MATH",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mathematics"
+    },
+    "mmlu": {
+        "benchmark": "mmlu",
+        "metric": "accuracy",
+        "display_name": "MMLU",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu"
+    },
+    "mmlu_pro": {
+        "benchmark": "mmlu_pro",
+        "metric": "accuracy",
+        "display_name": "MMLU-Pro",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu_pro"
+    },
+    "gpqa_diamond": {
+        "benchmark": "gpqa_diamond",
+        "metric": "accuracy",
+        "display_name": "GPQA-Diamond",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gpqa"
+    },
+    "mmmu_multiple_choice": {
+        "benchmark": "mmmu_multiple_choice",
+        "metric": "accuracy",
+        "display_name": "MMMU-Multiple-Choice",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu"
+    },
+    "mmmu_open": {
+        "benchmark": "mmmu_open",
+        "metric": "accuracy",
+        "display_name": "MMMU-Open-Ended",
+        "type": "base",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu"
+    },
+    "gaia": {
+        "benchmark": "gaia",
+        "metric": "accuracy",
+        "display_name": "GAIA",
+        "type": "agentic",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gaia"
+    },
+    "gdm_intercode_ctf": {
+        "benchmark": "gdm_intercode_ctf",
+        "metric": "accuracy",
+        "display_name": "InterCode-CTF",
+        "type": "agentic",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf"
+    },
+    "gdm_in_house_ctf": {
+        "benchmark": "gdm_in_house_ctf",
+        "metric": "accuracy",
+        "display_name": "In-House-CTF",
+        "type": "agentic",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/in_house_ctf"
+    },
+    "agentharm": {
+        "benchmark": "agentharm",
+        "metric": "avg_score",
+        "display_name": "AgentHarm",
+        "type": "agentic",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm"
+    },
+    "agentharm_benign": {
+        "benchmark": "agentharm_benign",
+        "metric": "avg_score",
+        "display_name": "AgentHarm-Benign",
+        "type": "agentic",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm"
+    },
+    "swe_bench": {
+        "benchmark": "swe_bench",
+        "metric": "mean",
+        "display_name": "SWE-Bench",
+        "type": "agentic",
+        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/swe_bench"
+    }
+}

src/about.py

@@ -33,18 +33,13 @@ class Tasks(Enum):
 
     # agentic
     task14 = Task("gaia", "accuracy", "GAIA", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gaia")
-    task15 = Task("gdm_intercode_ctf", "accuracy", "GDM-InterCode-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf")
-    task16 = Task("gdm_in_house_ctf", "accuracy", "GDM-In-House-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/in_house_ctf")
+    task15 = Task("gdm_intercode_ctf", "accuracy", "InterCode-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf")
+    task16 = Task("gdm_in_house_ctf", "accuracy", "In-House-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/in_house_ctf")
     task17 = Task("agentharm", "avg_score", "AgentHarm", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm")
     task18 = Task("agentharm_benign", "avg_score", "AgentHarm-Benign", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm")
     task19 = Task("swe_bench", "mean", "SWE-Bench", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/swe_bench")
 
 
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">State of Evaluation Leaderboard</h1>"""
 

src/display/formatting.py

@@ -5,6 +5,8 @@ def model_hyperlink(link, model_name):
 def make_clickable_model(model_name, model_sha):
     return model_hyperlink(model_sha, model_name)
 
+def make_clickable_field(name, url):
+    return model_hyperlink(url, name)
 
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"

src/submission/submit.py

@@ -1,119 +1,119 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
+# import json
+# import os
+# from datetime import datetime, timezone
+
+# from src.display.formatting import styled_error, styled_message, styled_warning
+# from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+# from src.submission.check_validity import (
+#     already_submitted_models,
+#     check_model_card,
+#     get_model_size,
+#     is_model_on_hub,
+# )
+
+# REQUESTED_MODELS = None
+# USERS_TO_SUBMISSION_DATES = None
+
+# def add_new_eval(
+#     model: str,
+#     base_model: str,
+#     revision: str,
+#     precision: str,
+#     weight_type: str,
+#     model_type: str,
+# ):
+#     global REQUESTED_MODELS
+#     global USERS_TO_SUBMISSION_DATES
+#     if not REQUESTED_MODELS:
+#         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+#     user_name = ""
+#     model_path = model
+#     if "/" in model:
+#         user_name = model.split("/")[0]
+#         model_path = model.split("/")[1]
+
+#     precision = precision.split(" ")[0]
+#     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+#     if model_type is None or model_type == "":
+#         return styled_error("Please select a model type.")
+
+#     # Does the model actually exist?
+#     if revision == "":
+#         revision = "main"
+
+#     # Is the model on the hub?
+#     if weight_type in ["Delta", "Adapter"]:
+#         base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+#         if not base_model_on_hub:
+#             return styled_error(f'Base model "{base_model}" {error}')
+
+#     if not weight_type == "Adapter":
+#         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+#         if not model_on_hub:
+#             return styled_error(f'Model "{model}" {error}')
+
+#     # Is the model info correctly filled?
+#     try:
+#         model_info = API.model_info(repo_id=model, revision=revision)
+#     except Exception:
+#         return styled_error("Could not get your model information. Please fill it up properly.")
+
+#     model_size = get_model_size(model_info=model_info, precision=precision)
+
+#     # Were the model card and license filled?
+#     try:
+#         license = model_info.cardData["license"]
+#     except Exception:
+#         return styled_error("Please select a license for your model")
+
+#     modelcard_OK, error_msg = check_model_card(model)
+#     if not modelcard_OK:
+#         return styled_error(error_msg)
+
+#     # Seems good, creating the eval
+#     print("Adding new eval")
+
+#     eval_entry = {
+#         "model": model,
+#         "base_model": base_model,
+#         "revision": revision,
+#         "precision": precision,
+#         "weight_type": weight_type,
+#         "status": "PENDING",
+#         "submitted_time": current_time,
+#         "model_type": model_type,
+#         "likes": model_info.likes,
+#         "params": model_size,
+#         "license": license,
+#         "private": False,
+#     }
+
+#     # Check for duplicate submission
+#     if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+#         return styled_warning("This model has been already submitted.")
+
+#     print("Creating eval file")
+#     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+#     os.makedirs(OUT_DIR, exist_ok=True)
+#     out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+#     with open(out_path, "w") as f:
+#         f.write(json.dumps(eval_entry))
+
+#     print("Uploading eval file")
+#     API.upload_file(
+#         path_or_fileobj=out_path,
+#         path_in_repo=out_path.split("eval-queue/")[1],
+#         repo_id=QUEUE_REPO,
+#         repo_type="dataset",
+#         commit_message=f"Add {model} to eval queue",
+#     )
+
+#     # Remove the local file
+#     os.remove(out_path)
+
+#     return styled_message(
+#         "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+#     )