Spaces:
Running
Running
File size: 15,757 Bytes
2d9c70c 2ce0cc2 2d9c70c 90e7c81 2d9c70c 97e6937 da65ce9 90e7c81 da65ce9 90e7c81 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c e2f5761 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 90e7c81 2ce0cc2 2d9c70c 97e6937 2ce0cc2 90e7c81 97e6937 2d9c70c 97e6937 2ce0cc2 97e6937 90e7c81 2d9c70c 90e7c81 2d9c70c d719fb7 2d9c70c 97e6937 2d9c70c 97e6937 90e7c81 2d9c70c 97e6937 2d9c70c 97e6937 90e7c81 2d9c70c 90e7c81 2d9c70c 90e7c81 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 90e7c81 2d9c70c 97e6937 2d9c70c 97e6937 2d9c70c 90e7c81 97e6937 2d9c70c 90e7c81 2d9c70c e2f5761 90e7c81 2d9c70c 97e6937 2d9c70c 90e7c81 97e6937 90e7c81 97e6937 90e7c81 2d9c70c 90e7c81 97e6937 2d9c70c 6a95b89 2d9c70c 2ce0cc2 2d9c70c 90e7c81 2d9c70c 90e7c81 2d9c70c 90e7c81 2d9c70c 90e7c81 2d9c70c 97e6937 90e7c81 2ce0cc2 90e7c81 2d9c70c 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c 6a95b89 90e7c81 6a95b89 2ce0cc2 2d9c70c 6a95b89 2d9c70c 90e7c81 6a95b89 2ce0cc2 2d9c70c 6a95b89 97e6937 2ce0cc2 2d9c70c 2ce0cc2 2d9c70c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 |
import ast
from collections import defaultdict
from functools import partial
import itertools
import os
import re
from concurrent.futures import ThreadPoolExecutor
import numpy as np
from datetime import datetime
import gradio as gr
import pandas as pd
from datatrove.io import DataFolder
FALLBACK_TOKEN_NAME = "HF_TOKEN"
def is_arary_like(x):
return isinstance(x, list) or isinstance(x, tuple) or isinstance(x, np.ndarray)
def get_task_type(df):
# Compatibility with old lighteval
if all(isinstance(pred, str) or (is_arary_like(pred) and all(isinstance(item, str) for item in pred)) for pred in df['predictions'].iloc[0]):
return "generative"
if all(is_arary_like(pred) and all(isinstance(item, float) for item in pred) for pred in df['predictions'].iloc[0]):
return "multiple_choice"
return "mixed"
def fix_df(df):
# For some reason some metrics and predictions are stored as strings
for col in ["predictions", "metrics", "choices", "gold", "gold_index"]:
if col in df.columns:
df[col] = [ast.literal_eval(x) if isinstance(x, str) else x for x in df[col].values]
if col == "predictions":
# For multiple choice
df[col] = df[col].apply(lambda x: [[z[0] for z in x]] if is_arary_like(x) and len(x[0]) == 2 else x)
# For unwraping of generative
df[col] = df[col].apply(lambda x: x[0] if is_arary_like(x) and len(x) == 1 else x)
return df
def get_run_name_seed(run_name):
if "-seed-" not in run_name:
return run_name, 5
run_name, seed = run_name.split("-seed-")
return run_name, int(seed)
def fetch_repo_structure(results_uri, oauth_token: gr.OAuthToken | None = None):
token = os.environ.get(FALLBACK_TOKEN_NAME)
if oauth_token:
token = oauth_token.token
data_folder = DataFolder(results_uri, token=token)
runs = [f.removeprefix("details/") for f in data_folder.list_files("details", recursive=False, include_directories=True) if f != "details"]
if not runs:
return {}, gr.update(choices=[], value=None)
def process_run(run):
run_files = [f.removeprefix(f"details/{run}/") for f in data_folder.list_files(f"details/{run}", recursive=False, include_directories=True) if f != f"details/{run}"]
return run, run_files
with ThreadPoolExecutor() as executor:
results = list(executor.map(process_run, runs))
checkpoints_dict = dict(results)
return checkpoints_dict, gr.update(choices=list(checkpoints_dict), value=None)
def update_checkpoints(selected_runs, checkpoints):
if not selected_runs:
return gr.update(choices=[], value=None)
common_checkpoints = set(checkpoints[selected_runs[0]])
for run in selected_runs[1:]:
common_checkpoints.intersection_update(set(checkpoints[run]))
common_checkpoints = sorted(list(common_checkpoints))
return gr.update(choices=common_checkpoints, value=common_checkpoints[0] if common_checkpoints else None)
def select_runs_by_regex(runs, current_selected, regex_to_select):
comp_re = re.compile(regex_to_select)
return list(sorted(set((current_selected if current_selected else []) +
[run for run in runs if comp_re.fullmatch(run)])))
def select_runs_by_language(runs, current_selected, language):
if language:
return select_runs_by_regex(runs, current_selected, f".*-{language}-.*")
return current_selected
def fetch_available_tasks(results_uri, runs_to_fetch, checkpoint) -> dict[str, dict[str, str]]:
token = os.environ.get(FALLBACK_TOKEN_NAME)
data_folder = DataFolder(results_uri, token=token)
all_tasks = defaultdict(lambda: defaultdict(dict))
for run in runs_to_fetch:
try:
details_folder = f"details/{run}/{checkpoint}"
files = data_folder.list_files(details_folder, recursive=True)
parquet_files = [f.removeprefix(details_folder + "/") for f in files if f.endswith('.parquet')]
for full_filename in parquet_files:
task_name, date_str = full_filename.replace('.parquet', '').rsplit('_', 1)
date = datetime.strptime(date_str, '%Y-%m-%dT%H-%M-%S.%f')
if run not in all_tasks[task_name] or date > all_tasks[task_name][run]['date']:
all_tasks[task_name][run] = {'filename': full_filename, 'date': date}
except FileNotFoundError:
print(f"Checkpoint not found for run: {run}")
available_tasks = {
task: {run: info['filename'] for run, info in runs.items()}
for task, runs in all_tasks.items()
if set(runs.keys()) == set(runs_to_fetch)
}
return available_tasks
def fetch_run_results(results_uri, runs_to_fetch, checkpoint,
oauth_token: gr.OAuthToken | None = None, progress=gr.Progress()):
task_runs_dict = fetch_available_tasks(results_uri, runs_to_fetch, checkpoint)
task_names = list(task_runs_dict.keys())
return gr.update(choices=task_names, value=task_names[0] if task_names else None), task_runs_dict
def render_table(df, selected_runs, metric_names):
if df is None or not selected_runs or not metric_names:
return None, "0"
kept_metrics = [f"metric_{metric_name}_{run_name}" for run_name in selected_runs for metric_name in metric_names]
other_metrics = [col for col in df.columns if col.startswith(f"metric_") and col not in kept_metrics]
df = df.drop(columns=other_metrics)
df = shorten_column_names(df, selected_runs, metric_names)
# Sample 100
n_samples = len(df)
df = df.sample(n=min(100, len(df)), random_state=42)
return df, str(n_samples)
def get_column_widths(df):
column_widths = []
for col in df.columns:
if col == "prompt":
column_widths.append("300px")
elif col in ["choices", "gold"]:
column_widths.append("250px")
elif col.startswith("metric_"):
column_widths.append("50px")
else:
column_widths.append("200px") # Default width for other columns
return column_widths
def shorten_column_names(df, run_names: list[str], metric_names: list[str]):
"""
Turns metric columns (metric_{metric}_{run_name}) into {metric}_i
Turns generation_{run_name} into generation_i
Also truncates full_prompt column to 200 chars with expandable view
"""
# Handle metric columns
columns_to_rename = {}
for idx, run_name in enumerate(run_names):
for metric_name in metric_names:
original_metric_column = f"metric_{metric_name}_{run_name}"
if original_metric_column in df.columns:
columns_to_rename[original_metric_column] = f"{metric_name}_{idx}"
original_generation_column = f"generation_{run_name}"
if original_generation_column in df.columns:
columns_to_rename[original_generation_column] = f"generation_{idx}"
# Rename columns in a single operation
df = df.rename(columns=columns_to_rename)
# Add markdown formatting to full_prompt column for truncation with expansion
if 'prompt' in df.columns:
df['prompt'] = df['prompt'].apply(
lambda x: f"<details><summary>{x[:100]}...</summary>\n\n{x}</details>" if len(x) > 100 else x
)
return df
def load_task_data(results_uri, runs_to_fetch, checkpoint, task_name, tasks_files, prompt_column, progress=gr.Progress()):
token = os.environ.get(FALLBACK_TOKEN_NAME)
if not runs_to_fetch or not task_name:
return None, None
data_folder = DataFolder(f"filecache::{results_uri}", token=token, cache_storage="./results-cache")
def fetch_run_file(run_to_fetch):
file_path = f"details/{run_to_fetch}/{checkpoint}/{tasks_files[task_name][run_to_fetch]}"
try:
with data_folder.open(file_path, "rb") as f:
df = pd.read_parquet(f)
return df, run_to_fetch
except FileNotFoundError:
print(f"File not found: {tasks_files[task_name][run_to_fetch]}")
return None, run_to_fetch
with ThreadPoolExecutor() as pool:
results = list(progress.tqdm(pool.map(fetch_run_file, runs_to_fetch), total=len(runs_to_fetch),
desc="Fetching run data..."))
dfs = [fix_df(df) for df, _ in results if df is not None]
run_names = [run for _, run in results if run is not None]
if not dfs:
return None, None, gr.update(choices=[], value=None)
task_type = get_task_type(dfs[0])
def prepare_df(df, run_name, task_type, prompt_column):
def get_choice_predictions(df, task_type):
predictions = df['predictions']
if task_type == "generative":
return predictions
if task_type == "multiple_choice":
n_choices = len(df['choices'])
return [pred[0] for pred in predictions[:n_choices]]
if task_type == "mixed":
return predictions[0]
return predictions
generative_columns = {
f"generation_{run_name}": df.apply(partial(get_choice_predictions, task_type=task_type), axis=1)
} if task_type == "generative" or task_type == "mixed" else {}
prepared_df = pd.DataFrame({
'prompt': df[prompt_column],
'choices': df['choices'].apply(tuple), # Convert lists to tuples
'gold': df['gold'].apply(lambda x: tuple(x) if isinstance(x, list) else x), # Convert lists to tuples
'gold_index': df['gold_index'],
**generative_columns,
})
# For some reason some metrics are stored as strings
metrics = df['metrics']
available_metrics = set(metric for row_metrics in metrics for metric in row_metrics)
for metric_key in available_metrics:
prepared_df[f'metric_{metric_key}_{run_name}'] = [metric.get(metric_key, None) for metric in metrics]
# Merge rows with the same full_prompt
prepared_df = prepared_df.groupby('prompt').agg(lambda x: next((item for item in x if item is not None), None)).reset_index()
prepared_df["prompt"] = prepared_df["prompt"].astype(str)
return prepared_df
def get_gold_label(df, task_type):
if task_type == "generative":
return df['gold']
return df['gold_index']
# Prepare the first DataFrame with choices and gold
# Join all prepared DataFrames
prepared_dfs = [
prepare_df(df, run_name, task_type, prompt_column)
for df, run_name in zip(dfs, run_names)
]
combined_df = prepared_dfs[0]
for idx, prepared_df in enumerate(prepared_dfs[1:]):
combined_df = combined_df.merge(prepared_df, how='outer', on=("prompt", "gold"), suffixes=(None, f"_{idx}"))
to_keep = ["prompt", "gold"]
if task_type in ["multiple_choice", "mixed"]:
to_keep.append("choices")
elif task_type == "generative":
to_keep.extend([col for col in combined_df.columns if col.startswith("generation_")])
combined_df['gold'] = combined_df.apply(lambda row: get_gold_label(row, task_type), axis=1).values
metric_cols = [col for col in combined_df.columns if col.startswith("metric_")]
combined_df = combined_df[to_keep + metric_cols]
available_metrics = list(set("_".join(col.split('_')[1:-1]) for col in metric_cols))
chosen_metrics = available_metrics[:1]
return combined_df, gr.update(choices=available_metrics, value=chosen_metrics)
with gr.Blocks() as demo:
runs_checkpoints = gr.State({})
results_df_full = gr.State(None)
tasks_files = gr.State({})
login_button = gr.LoginButton(visible=False)
results_uri = gr.Textbox(label="Results URI", value="s3://fineweb-multilingual-v1/evals/test/", visible=True)
with gr.Column():
gr.Markdown("# FineWeb experiments results explorer")
split_checkpoints = gr.Checkbox(label="Split checkpoints from models", value=True)
with gr.Row():
with gr.Column():
select_by_regex_text = gr.Textbox(label="Regex to select runs",
value="ind_minhash(-CC-MAIN-|_)\\d{4}-\\d{2}-seed.*")
select_by_regex_button = gr.Button("Select matching runs")
with gr.Column():
select_by_language = gr.Dropdown(choices=["ar", "fr", "ru", "hi", "th", "tr", "zh", "sw", "te"],
interactive=True, label="Select by language",
info="Choose a language to prefill the regex")
selected_runs = gr.Dropdown(choices=[], interactive=True, multiselect=True, label="Selected runs")
checkpoint = gr.Dropdown(choices=[], interactive=True, label="Checkpoint", visible=True)
fetch_res = gr.Button("Fetch results")
task_name = gr.Dropdown(choices=[], interactive=True, label="Task name")
metric_names = gr.Dropdown(choices=[], interactive=True, multiselect=True, label="Metric")
results_df = gr.Dataframe(
interactive=False,
wrap=True,
line_breaks=True,
datatype="markdown"
)
with gr.Row():
with gr.Column():
num_samples = gr.Text(interactive=False, label="# Samples")
prompt_column = gr.Radio(choices=["full_prompt", "example"], label="Prompt display", value="example")
# Run selection
gr.on(
triggers=[results_uri.change],
fn=fetch_repo_structure, inputs=[results_uri], outputs=[runs_checkpoints, selected_runs],
)
gr.on(
triggers=[select_by_regex_button.click],
fn=select_runs_by_regex,
inputs=[runs_checkpoints, selected_runs, select_by_regex_text], outputs=[selected_runs]
)
gr.on(
triggers=[select_by_language.change],
fn=select_runs_by_language,
inputs=[runs_checkpoints, selected_runs, select_by_language], outputs=[selected_runs]
)
# Update checkpoints based on selected runs
gr.on(
triggers=[selected_runs.change],
fn=update_checkpoints,
inputs=[selected_runs, runs_checkpoints],
outputs=[checkpoint]
)
# Fetch available tasks
gr.on(
triggers=[fetch_res.click],
fn=fetch_run_results,
inputs=[results_uri, selected_runs, checkpoint],
outputs=[task_name, tasks_files]
).then(
fn=load_task_data,
inputs=[results_uri, selected_runs, checkpoint, task_name, tasks_files, prompt_column],
outputs=[results_df_full, metric_names]
).then(
fn=render_table,
inputs=[results_df_full, selected_runs, metric_names],
outputs=[results_df, num_samples]
)
# Update results when task name or metric changes
gr.on(
triggers=[task_name.input],
fn=load_task_data,
inputs=[results_uri, selected_runs, checkpoint, task_name, tasks_files, prompt_column],
outputs=[results_df_full, metric_names]
).then(
fn=render_table,
inputs=[results_df_full, selected_runs, metric_names],
outputs=[results_df, num_samples]
)
gr.on(
triggers=[metric_names.input],
fn=render_table,
inputs=[results_df_full, selected_runs, metric_names],
outputs=[results_df, num_samples]
)
demo.load(fn=fetch_repo_structure, inputs=[results_uri], outputs=[runs_checkpoints, selected_runs])
demo.launch() |