import gradio as gr from gradio_leaderboard import Leaderboard import pandas as pd from huggingface_hub import snapshot_download, create_repo from huggingface_hub.utils import RepositoryNotFoundError import os from src.about import ( INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.display.utils import ( BENCHMARK_COLS, COLS, AutoEvalColumn, fields, ) from src.envs import API, EVAL_RESULTS_PATH, RESULTS_REPO, TOKEN, OWNER from src.populate import get_leaderboard_df from src.evaluation.dynamic_eval import run_dynamic_perplexity_eval def init_leaderboard(dataframe): if dataframe is None: raise ValueError("Leaderboard DataFrame is None.") print("\n=== Initializing Leaderboard ===", flush=True) print(f"DataFrame shape: {dataframe.shape}", flush=True) print(f"DataFrame columns: {dataframe.columns.tolist()}", flush=True) return Leaderboard( value=dataframe, select_columns=[c.name for c in fields(AutoEvalColumn) if not c.hidden], search_columns=[AutoEvalColumn.model.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ AutoEvalColumn.model_type.name, AutoEvalColumn.precision.name, ], ) def refresh_leaderboard(): """Refresh leaderboard data from disk""" print("\n=== Refreshing Leaderboard ===", flush=True) try: # Download latest results print("Downloading latest results...", flush=True) snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) print("Download complete", flush=True) except Exception as e: print(f"Error refreshing results: {e}", flush=True) # Get fresh leaderboard data print("Getting fresh leaderboard data...", flush=True) df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS) print(f"Got DataFrame with shape: {df.shape}", flush=True) return init_leaderboard(df) def run_perplexity_test(model_name, revision, precision): """Run perplexity evaluation on demand.""" print(f"\n=== Running Perplexity Test ===", flush=True) print(f"Model: {model_name}", flush=True) print(f"Revision: {revision}", flush=True) print(f"Precision: {precision}", flush=True) if not model_name: return "Please enter a model name.", None success, result = run_dynamic_perplexity_eval(model_name, revision, precision) print(f"Evaluation result - Success: {success}, Result: {result}", flush=True) if success: # Get updated leaderboard print("Refreshing leaderboard...", flush=True) new_leaderboard = refresh_leaderboard() print("Leaderboard refresh complete", flush=True) return f"โœ… Perplexity evaluation completed!\nPerplexity: {result:.4f}", new_leaderboard else: return f"โŒ Evaluation failed: {result}", None # Initialize results repository and directory try: # Try to download existing repository try: snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) except RepositoryNotFoundError: # Create the repository if it doesn't exist print(f"Creating new results repository: {RESULTS_REPO}") create_repo( repo_id=RESULTS_REPO, repo_type="dataset", private=False, token=TOKEN ) # Create local directory os.makedirs(EVAL_RESULTS_PATH, exist_ok=True) except Exception as e: print(f"Error initializing results: {e}") # Ensure local directory exists even if repo operations fail os.makedirs(EVAL_RESULTS_PATH, exist_ok=True) # Get initial leaderboard data LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS) # Create the Gradio interface demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("๐Ÿ… Leaderboard", elem_id="leaderboard-tab", id=0): leaderboard = init_leaderboard(LEADERBOARD_DF) with gr.TabItem("๐Ÿ“ About", elem_id="about-tab", id=1): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") with gr.TabItem("๐Ÿงช Test Model", elem_id="test-model-tab", id=2): with gr.Row(): with gr.Column(): model_name = gr.Textbox(label="Model name", placeholder="org/model-name") revision = gr.Textbox(label="Revision", placeholder="main", value="main") precision = gr.Dropdown( choices=["float16", "bfloat16"], label="Precision", value="float16" ) with gr.Column(): test_button = gr.Button("๐Ÿš€ Run Perplexity Test", variant="primary") result = gr.Markdown() test_button.click( run_perplexity_test, [model_name, revision, precision], [result, leaderboard] ) demo.queue(default_concurrency_limit=5).launch()