File size: 4,231 Bytes
4a78d34 b1f9063 4a78d34 bbde2b0 4a78d34 bbde2b0 a2189ab 36244aa bbde2b0 0796d85 a319d81 bbde2b0 7c6bd6c bbde2b0 323e17d bbde2b0 7c6bd6c 8ad1a09 4a78d34 dea22be a2f2df3 8ad1a09 4a78d34 8555000 a319d81 d7d56ae bbde2b0 a319d81 d7d56ae cdca101 36244aa 4a78d34 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
import json
import os
import numpy as np
import pandas as pd
from src.about import Tasks
from src.display.formatting import has_no_nan_values, make_clickable_model
from src.display.utils import AutoEvalColumn, EvalQueueColumn
from src.leaderboard.read_evals import get_raw_eval_results
TASK_NAME_INVERSE_MAP = dict()
for task in Tasks:
TASK_NAME_INVERSE_MAP[task.value.col_name] = {
"name": task.value.benchmark,
"type": task.value.type,
"source": task.value.source,
}
EMPTY_SYMBOL = "--"
def get_inspect_log_url(model_name: str, benchmark_name: str) -> str:
"""Returns the URL to the log file for a given model and benchmark"""
with open("./inspect_log_file_names.json", "r") as f:
inspect_log_files = json.load(f)
log_file_name = inspect_log_files[model_name].get(benchmark_name, None)
if log_file_name is None:
return ""
else:
# replace .json with .eval
log_file_name = log_file_name.replace(".json", ".eval")
return f"https://storage.googleapis.com/inspect-evals/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
"""Creates a dataframe from all the individual experiment results"""
raw_data = get_raw_eval_results(results_path, requests_path)
all_data_json = [v.to_dict() for v in raw_data]
df = pd.DataFrame.from_records(all_data_json)
# df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
df = df[cols].round(decimals=2)
# subset for model and benchmark cols
df = df[[AutoEvalColumn.model.name] + benchmark_cols]
# # filter out if any of the benchmarks have not been produced
# df = df[has_no_nan_values(df, benchmark_cols)]
df = df.fillna(EMPTY_SYMBOL)
# make values clickable and link to log files
for col in benchmark_cols:
df[col] = df[[AutoEvalColumn.model.name, col]].apply(lambda x: f"[{x[col]}]({get_inspect_log_url(model_name=x[AutoEvalColumn.model.name].split('>')[1].split('<')[0], benchmark_name=TASK_NAME_INVERSE_MAP[col]['name'])})" if x[col] != EMPTY_SYMBOL else x[col], axis=1)
# # make task names clickable and link to inspect-evals repository - this creates issues later
# df = df.rename(columns={col: f"[{col}]({TASK_NAME_INVERSE_MAP[col]['source']})" for col in benchmark_cols})
return df
def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
"""Creates the different dataframes for the evaluation queues requestes"""
entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
all_evals = []
for entry in entries:
if ".json" in entry:
file_path = os.path.join(save_path, entry)
with open(file_path) as fp:
data = json.load(fp)
data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
data[EvalQueueColumn.revision.name] = data.get("revision", "main")
all_evals.append(data)
elif ".md" not in entry:
# this is a folder
sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
for sub_entry in sub_entries:
file_path = os.path.join(save_path, entry, sub_entry)
with open(file_path) as fp:
data = json.load(fp)
data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
data[EvalQueueColumn.revision.name] = data.get("revision", "main")
all_evals.append(data)
pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
running_list = [e for e in all_evals if e["status"] == "RUNNING"]
finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
df_running = pd.DataFrame.from_records(running_list, columns=cols)
df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
return df_finished[cols], df_running[cols], df_pending[cols]
|