Update app.py

app.py

@@ -1,346 +1,925 @@
-import subprocess
-import gradio as gr
+import os
+import json
+import numpy as np
 import pandas as pd
+import gradio as gr
+from datetime import datetime, timedelta
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig, AutoTokenizer
+from huggingface_hub import HfApi, hf_hub_download, ModelCard
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-    BOTTOM_LOGO,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    NUMERIC_INTERVALS,
-    TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
 
+################################################################################
+# GLOBALS & CONSTANTS
+################################################################################
 
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
+OWNER = "OALL"
+REPO_ID = f"{OWNER}/Open-Arabic-LLM-Leaderboard-v2-exp"
+RESULTS_REPO_ID = f"{OWNER}/v2_results"
+REQUESTS_REPO_ID = f"{OWNER}/requests_v2"
 
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
-    return df
+# Global HF API instance (set once)
+hf_api_token = os.environ.get('HF_API_TOKEN', None)
+API = HfApi(token=hf_api_token)
+
+TASKS = [
+    ("community|alghafa:_average|0",        "acc_norm",     "AlGhafa"),
+    ("community|arabic_mmlu:_average|0",    "acc_norm",     "ArabicMMLU"),
+    ("community|arabic_exams|0",            "acc_norm",     "EXAMS"),
+    ("community|madinah_qa:_average|0",     "acc_norm",     "MadinahQA"),
+    ("community|aratrust:_average|0",       "acc_norm",     "AraTrust"),
+    ("community|alrage_qa|0",               "llm_as_judge", "ALRAGE"),
+    ("community|arabic_mmlu_ht:_average|0", "acc_norm",     "ArbMMLU-HT"),
+]
+
+MODEL_TYPE_TO_EMOJI = {
+    "🟢 : pretrained": "🟢",
+    "🟩 : continuously pretrained": "🟩",
+    "💬 : chat models (RLHF, DPO, IFT, ...)": "💬",
+    "🔶 : fine-tuned on domain-specific datasets": "🔶",
+    "🤝 : base merges and merges": "🤝",
+    "Missing": "?",
+}
+
+HEADER = """
+<img src="https://raw.githubusercontent.com/alielfilali01/OALL-assets/main/TITLE.png" style="width:30%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">
+"""
+
+BOTTOM_LOGO = """<img src="https://raw.githubusercontent.com/alielfilali01/OALL-assets/main/BOTTOM.png" style="width:50%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">"""
+
+SUBMISSION_TEXT = """
+# Submit Your Model for Evaluation 🌴 
+
+**The Open Arabic LLM Leaderboard** aims to help you evaluate and compare the performance of Arabic Large Language Models.
+
+When you submit a model on this page, it is automatically evaluated on a set of arabic native benchmarks ([find here](https://github.com/huggingface/lighteval/blob/main/examples/tasks/OALL_v2_tasks.txt)) with one additional human-translated version of [MMLU](https://arxiv.org/abs/2009.03300).
+
+The GPU used for evaluation is operated with the support of  __[Technology Innovation Institute (TII)](https://www.tii.ae/)__.
+
+More details about the benchmarks and the evaluation process is provided on the “About” section below.
+"""
+
+ABOUT_SECTION = """
+## About
+
+While outstanding LLM models are being released competitively, most of them are centered on English and are familiar with the English cultural sphere. We operate the Open Arabic LLM Leaderboard (OALL), to evaluate models that reflect the characteristics of the Arabic language, culture and heritage. Through this, we hope that users can conveniently use the leaderboard, participate, and contribute to the advancement of research in the Arab region 🔥.
+
+### Icons & Model types
+🟢 : `pretrained`
+
+🟩 : `continuously pretrained`
 
+💬 : `chat models (RLHF, DPO, IFT, ...)`
 
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
+🔶 : `fine-tuned on domain-specific datasets`
 
+🤝 : `base merges and moerges`
+
+### Notes:
+- We reserve the right to correct any incorrect tags or icons after manual verification to ensure the accuracy and reliability of the leaderboard. This helps maintain the integrity and trustworthiness of the platform.
+- Some models may be flagged as “Subjects of Caution” by the community. These models might have used the evaluation set for training, attempted to manipulate rankings, or raised ethical concerns. Models deemed as such may face restricted visibility or removal from the leaderboard. Users are advised to exercise discretion when interpreting rankings.
+- The leaderboard automatically hides models that were submitted, evaluated, and subsequently made private or gated post-evaluation. This platform is designed for **“open”** models that benefit the wider community. If you intend to restrict your model’s accessibility after using the leaderboard’s resources or exploit the platform solely for personal gains, please refrain from submitting. Violators may face bans on their usernames and/or organization IDs from future submissions.
+- The leaderboard no longer accepts models in **float32** precision except under special circumstances. If you are the developer of a float32 model and believe it deserves inclusion, please reach out to us.
+- To ensure fair and equitable access to leaderboard resources, all usernames and organization IDs are limited to **5 submissions per week**. This policy minimizes spamming, encourages thoughtful participation, and allows everyone in the community to benefit from the platform.
+
+By adhering to these guidelines, we aim to foster a fair, collaborative, and transparent environment for evaluating and advancing open models for the arabic/arabic-interested communities.
+
+### How it works
+
+📈 We evaluate models using [LightEval](https://github.com/huggingface/lighteval), a unified and straightforward framework from the HuggingFace Eval Team to test and assess causal language models on a large number of different evaluation tasks.
+
+To ensure a fair and unbiased assessment of the models' true capabilities, all evaluations are conducted in zero-shot settings `0-shots`. This approach eliminates any potential advantage from task-specific fine-tuning, providing a clear indication of how well the models can generalize to new tasks.
+
+Also, given the nature of the tasks, which include multiple-choice questions, the leaderboard primarily uses normalized log likelihood accuracy `loglikelihood_acc_norm` for all tasks.
+
+Please, consider reaching out to us through the discussions tab if you are working on benchmarks for Arabic LLMs and willing to see them on this leaderboard as well. Your benchmark might change the whole game for Arabic models !
+
+### Details and Logs
+- Detailed numerical results in the `results` OALL dataset: https://huggingface.co/datasets/OALL/v2_results
+- Community queries and running status in the `requests` OALL dataset: https://huggingface.co/datasets/OALL/requests_v2
+
+### More resources
+If you still have questions, you can check our FAQ [here](https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard/discussions/15)!
+
+"""
+
+CITATION_BUTTON_LABEL = """
+Copy the following snippet to cite these results
+"""
+
+CITATION_BUTTON_TEXT = """
+@misc{OALL-2,
+  author = {El Filali, Ali and ALOUI, Manel and Husaain, Tarique and Alzubaidi, Ahmed and Boussaha, Basma El Amel and Cojocaru, Ruxandra and Fourrier, Clémentine and Habib, Nathan and Hacid, Hakim},
+  title = {Open Arabic LLM Leaderboard 2},
+  year = {2025},
+  publisher = {OALL},
+  howpublished = {https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard}
+}
+"""
+
+################################################################################
+# UTILITY & HELPER FUNCTIONS
+################################################################################
+
+def model_hyperlink(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    # return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+    # return f'[{model_name}]({link})' # WHYYYYYYYY It is not working !!!???
+    return f"{model_name}"
+
+def restart_space():
+    """Restart the Gradio space periodically."""
+    API.restart_space(repo_id=REPO_ID)
 
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
+def unify_precision(raw_precision: str) -> str:
+    """
+    Map raw precision strings (e.g. 'torch.float16', 'fp16', 'float16')
+    to canonical forms: 'float16', 'float32', 'bfloat16', '8bit', '4bit', 'Missing'.
+    """
+    if not raw_precision or raw_precision.lower() in ["missing", "unk", "none"]:
+        return "Missing"
+    p = raw_precision.lower()
+    if p in ["torch.float16", "float16", "fp16"]:
+        return "float16"
+    if p in ["torch.float32", "float32", "fp32"]:
+        return "float32"
+    if p in ["torch.bfloat16", "bfloat16", "bf16"]:
+        return "bfloat16"
+    if p == "8bit":
+        return "8bit"
+    if p == "4bit":
+        return "4bit"
+    return "Missing"
+
+def load_requests(status_folder: str) -> pd.DataFrame:
+    """
+    Load all .json requests from REQUESTS_REPO_ID, filtering by 'status' == status_folder.
+    """
+    df_out = []
+    try:
+        files_info = API.list_repo_files(
+            repo_id=REQUESTS_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+        json_files = [f for f in files_info if f.endswith(".json")]
+    except Exception as e:
+        print(f"Error listing files in {REQUESTS_REPO_ID}: {e}")
+        return pd.DataFrame()
+
+    for path in json_files:
+        try:
+            local_path = hf_hub_download(
+                repo_id=REQUESTS_REPO_ID,
+                filename=path,
+                repo_type="dataset",
+                token=hf_api_token
+            )
+            with open(local_path, "r", encoding="utf-8") as f:
+                req = json.load(f)
+        except Exception as e:
+            print(f"Error loading {path}: {e}")
+            continue
+
+        if str(req.get("status", "")).strip().lower() == status_folder.lower():
+            df_out.append(req)
+
+    return pd.DataFrame(df_out)
+
+def load_all_requests() -> pd.DataFrame:
+    """
+    Load *all* requests from the dataset (pending, finished, failed, etc.).
+    Returns a single DataFrame with columns from all requests.
+    """
+    df_out = []
+    try:
+        files_info = API.list_repo_files(
+            repo_id=REQUESTS_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+        json_files = [f for f in files_info if f.endswith(".json")]
+    except Exception as e:
+        print(f"Error listing files in {REQUESTS_REPO_ID}: {e}")
+        return pd.DataFrame()
+
+    for path in json_files:
+        try:
+            local_path = hf_hub_download(
+                repo_id=REQUESTS_REPO_ID,
+                filename=path,
+                repo_type="dataset",
+                token=hf_api_token
+            )
+            with open(local_path, "r", encoding="utf-8") as f:
+                req = json.load(f)
+            df_out.append(req)
+        except Exception as e:
+            print(f"Error loading {path}: {e}")
+            continue
+
+    return pd.DataFrame(df_out)
+
+def already_in_queue(df_pending: pd.DataFrame, model_name: str, revision: str, precision: str) -> bool:
+    """
+    Check if (model, revision, precision) is already in the 'pending' queue.
+    """
+    if df_pending.empty:
+        return False
+    matched = df_pending[
+        (df_pending["model"] == model_name)
+        & (df_pending["revision"] == revision)
+        & (df_pending["precision"] == unify_precision(precision))
     ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
+    return not matched.empty
+
+def get_model_size(model_info: ModelInfo, precision: str) -> float:
+    """
+    Return approximate model parameter size in billions, if safetensors info is available.
+    Return 0 if unknown.
+    For GPTQ, we do a small multiplier to reflect the extra bits, etc.
+    """
+    try:
+        param_bytes = model_info.safetensors.get("total", 0)
+        model_size = round(param_bytes / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0.0
+
+    if "gptq" in model_info.modelId.lower():
+        return model_size * 8
+    return model_size
+
+def parse_datetime(dt_str: str) -> datetime:
+    """
+    Safely parse an ISO datetime string into a Python datetime object.
+    """
+    try:
+        return datetime.fromisoformat(dt_str.replace("Z", ""))
+    except Exception:
+        return datetime.min
+
+################################################################################
+# SCOREBOARD LOADING & DISPLAY
+################################################################################
+
+def load_scoreboard() -> pd.DataFrame:
+    """
+    1) Reads JSON "results_*.json" from RESULTS_REPO_ID to collect scores.
+    2) Combines with "finished" requests data for license, revision, etc.
+    3) Removes any model that is no longer public or accessible.
+    4) Returns a DataFrame ready for display & filtering.
+    
+    Only models with finished evaluations are kept.
+    """
+    # Step A: Get scoreboard files from the results dataset
+    try:
+        files_info = API.list_repo_files(
+            repo_id=RESULTS_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+    except Exception as e:
+        print(f"Error listing scoreboard files in {RESULTS_REPO_ID}: {e}")
+        return pd.DataFrame()
+
+    candidate_json_paths = [
+        path for path in files_info
+        if path.endswith(".json") and len(path.split("/")) == 3 and path.split("/")[2].startswith("results_")
     ]
-    return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-            )
 
-    return filtered_df
-
-
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
+    rows = []
+    # Step B: Read each scoreboard file
+    for file_path in candidate_json_paths:
+        try:
+            local_file = hf_hub_download(
+                repo_id=RESULTS_REPO_ID,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_api_token
+            )
+            with open(local_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+        except Exception as e:
+            print(f"Error loading scoreboard file {file_path}: {e}")
+            continue
+
+        config_general = data.get("config_general", {})
+        results_block = data.get("results", {})
+
+        model_name  = config_general.get("model_name", "UNK")
+        scoreboard_precision = unify_precision(config_general.get("model_dtype", "Missing"))
+        # To be consistent with submission logic:
+        if scoreboard_precision == "Missing":
+            scoreboard_precision = "UNK"
+        scoreboard_model_type  = config_general.get("model_type", "Missing")
+
+        row_dict = {
+            "Model Name":   model_name,
+            "Revision":     "Missing",  # We'll fill from requests
+            "License":      "Missing",  # We'll fill from requests
+            "Precision":    scoreboard_precision,
+            "Full Type":    scoreboard_model_type,
+            "Model Size":   0.0,        # We'll fill from requests
+            "Hub ❤️":       0,          # We'll fill from requests
+        }
+
+        # Fill tasks
+        for (task_key, metric_field, display_name) in TASKS:
+            val = np.nan
+            if task_key in results_block:
+                subd = results_block[task_key]
+                if isinstance(subd, dict) and metric_field in subd:
+                    val = subd[metric_field]
+            row_dict[display_name] = val
+
+        rows.append(row_dict)
+
+    df = pd.DataFrame(rows)
+    if df.empty:
+        base_cols = [
+            "Model Name","Revision","License",
+            "Precision","Full Type","Model Size","Hub ❤️"
+        ]
+        task_names = [t[2] for t in TASKS]
+        return pd.DataFrame(columns=base_cols + task_names)
+
+    # Step C: Convert tasks to numeric & multiply by 100
+    task_cols = [t[2] for t in TASKS if t[2] in df.columns]
+    df[task_cols] = df[task_cols].apply(pd.to_numeric, errors="coerce")
+    for c in task_cols:
+        df[c] = (df[c] * 100).round(2)
+
+    # Step D: Compute average
+    if task_cols:
+        df["Average ⬆️"] = df[task_cols].mean(axis=1).round(2)
+    else:
+        df["Average ⬆️"] = np.nan
+
+    # Step E: Overwrite scoreboard data with "finished" requests (except for Precision)
+    df_finished = load_requests("finished")
+    if not df_finished.empty:
+        df_finished["precision"] = df_finished["precision"].apply(unify_precision)
+        df_finished["license"]   = df_finished["license"].apply(
+            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
+        )
+        df_finished["model_type"] = df_finished["model_type"].apply(
+            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
+        )
+
+        # Group by model name and precision to correctly distinguish multiple submissions
+        dff_grouped = df_finished.groupby(["model", "precision"], as_index=False).last()
+
+        request_map = {}
+        for _, row_ in dff_grouped.iterrows():
+            key = f"{row_['model']}__{row_['precision']}"
+            request_map[key] = {
+                "license":   row_["license"],
+                "revision":  row_["revision"],
+                "precision": row_["precision"],
+                "model_type":row_["model_type"],
+                "params":    row_["params"],
+                "likes":     row_["likes"]
+            }
+
+        def apply_request_info(row_):
+            key = f"{row_['Model Name']}__{row_['Precision']}"
+            if key in request_map:
+                row_["License"]    = request_map[key]["license"]
+                row_["Revision"]   = request_map[key]["revision"]
+                # Do NOT update "Precision": keep the value from the results file.
+                row_["Full Type"]  = request_map[key]["model_type"]
+                row_["Model Size"] = request_map[key]["params"]
+                row_["Hub ❤️"]     = request_map[key]["likes"]
+            return row_
+
+        df = df.apply(apply_request_info, axis=1)
+
+    # Step E2: Remove rows that do not have finished request info (i.e. Revision is still "Missing")
+    df = df[df["Revision"] != "Missing"]
+
+    # Step F: Remove any model not public
+    remove_idx = []
+    for idx, row in df.iterrows():
+        model_name = row["Model Name"]
+        if model_name == "UNK":
+            remove_idx.append(idx)
+            continue
+        try:
+            API.model_info(model_name)
+        except Exception:
+            remove_idx.append(idx)
+    df.drop(remove_idx, inplace=True)
+    df.reset_index(drop=True, inplace=True)
+
+    # Step G: Sort scoreboard
+    df = df.sort_values(by="Average ⬆️", ascending=False).reset_index(drop=True)
+
+    # Step H: Insert ranking & create a "Model Size Filter" for slider usage
+    df.insert(0, "Rank", range(1, len(df) + 1))
+    df["Model Size Filter"] = df["Model Size"]
+
+    # Step I: Short label for the model type
+    def map_type_to_emoji(full_str):
+        if not isinstance(full_str, str):
+            return "Missing"
+        return MODEL_TYPE_TO_EMOJI.get(full_str.strip(), full_str.strip())
+    df["T"] = df["Full Type"].apply(map_type_to_emoji)
+
+    # At this point, convert "Model Name" to a clickable link
+    df["Model Name"] = df["Model Name"].apply(model_hyperlink)
+
+    # Reorder columns
+    final_cols = ["Rank", "T", "Model Name", "Average ⬆️"] + task_cols
+    remainder = ["Model Size", "Hub ❤️", "License", "Precision", "Revision", "Model Size Filter", "Full Type"]
+    for rc in remainder:
+        if rc not in final_cols and rc in df.columns:
+            final_cols.append(rc)
+
+    return df[final_cols]
+
+################################################################################
+# SUBMISSION LOGIC
+################################################################################
+
+def check_model_card(repo_id: str) -> (bool, str):
+    """Check if model card is present, has a license, and is not too short."""
+    try:
+        card = ModelCard.load(repo_id)
+    except Exception:
+        return (False, "No model card found. Please add a README.md describing your model and license.")
+    if card.data.license is None and not ("license_name" in card.data and "license_link" in card.data):
+        return (False, "No license metadata found in the model card.")
+    if len(card.text) < 200:
+        return (False, "Model card is too short (<200 chars). Please add more details.")
+    return (True, "")
+
+def is_model_on_hub(model_name, revision, token=None, trust_remote_code=False, test_tokenizer=True):
+    """Check if the model & tokenizer can be loaded from the Hub."""
+    try:
+        config = AutoConfig.from_pretrained(
+            model_name,
+            revision=revision,
+            trust_remote_code=trust_remote_code,
+            token=token
+        )
+    except ValueError:
+        return (False, "requires `trust_remote_code=True`. Not automatically allowed.", None)
+    except Exception as e:
+        return (False, f"not loadable from hub: {str(e)}", None)
+
+    if test_tokenizer:
+        try:
+            _ = AutoTokenizer.from_pretrained(
+                model_name,
+                revision=revision,
+                trust_remote_code=trust_remote_code,
+                token=token
+            )
+        except Exception as e:
+            return (False, f"tokenizer not loadable: {str(e)}", None)
+
+    return (True, "", config)
+
+def check_org_threshold(org_name: str) -> (bool, str):
+    """
+    Each org can only submit 5 models in the last 7 days.
+    Return (True, "") if allowed. Otherwise, (False, "error message").
+    """
+    df_all = load_all_requests()
+    if df_all.empty:
+        return (True, "")
+
+    def get_org(m):
+        try:
+            return m.split("/")[0]
+        except:
+            return m
+
+    df_all["org_name"] = df_all["model"].apply(get_org)
+    df_org = df_all[df_all["org_name"] == org_name].copy()
+    if df_org.empty:
+        return (True, "")
+
+    df_org["datetime"] = df_org["submitted_time"].apply(parse_datetime)
+    df_org.dropna(subset=["datetime"], inplace=True)
+
+    now = datetime.utcnow()
+    week_ago = now - timedelta(days=7)
+    df_recent = df_org[df_org["datetime"] >= week_ago]
+
+    if len(df_recent) >= 5:
+        df_recent_sorted = df_recent.sort_values(by="datetime")
+        earliest = df_recent_sorted.iloc[0]["datetime"]
+        next_ok = earliest + timedelta(days=7)
+        msg_next = next_ok.isoformat(timespec="seconds") + "Z"
+        return (
+            False,
+            f"Your org '{org_name}' has reached the 5-submissions-per-week limit. You can submit again after {msg_next}."
+        )
+    return (True, "")
+
+def submit_model(
+    model_name: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+    chat_template: str
+):
+    # -------------------------------------------------------------------------
+    # 0) Strip inputs to avoid trailing or leading spaces
+    # -------------------------------------------------------------------------
+    model_name  = model_name.strip()
+    base_model  = base_model.strip()
+    revision    = revision.strip()
+    precision   = precision.strip()
+    
+    if not model_name:
+        return "**Error**: Model name cannot be empty (use 'org/model')."
+    if not revision:
+        revision = "main"
+
+    # 1) Check model card
+    card_ok, card_msg = check_model_card(model_name)
+    if not card_ok:
+        return f"**Error**: {card_msg}"
+
+    # 2) If adapter/delta, check base_model
+    if weight_type.lower() in ["adapter", "delta"]:
+        if not base_model:
+            return "**Error**: For adapter/delta, you must provide a valid `base_model`."
+        ok_base, base_err, _ = is_model_on_hub(base_model, revision, hf_api_token, test_tokenizer=True)
+        if not ok_base:
+            return f"**Error**: Base model '{base_model}' {base_err}"
+    else:
+        ok_model, model_err, _ = is_model_on_hub(model_name, revision, hf_api_token, test_tokenizer=True)
+        if not ok_model:
+            return f"**Error**: Model '{model_name}' {model_err}"
+
+    # 3) Retrieve ModelInfo
+    try:
+        info = API.model_info(model_name, revision=revision, token=hf_api_token)
+    except Exception as e:
+        return f"**Error**: Could not fetch model info. {str(e)}"
+
+    model_license = info.card_data.license
+    model_likes   = info.likes or 0
+    model_private = bool(getattr(info, "private", False))
+
+    # 4) Check duplicates
+    df_pending = load_requests("pending")
+    df_finished = load_requests("finished")
+    
+    if already_in_queue(df_finished, model_name, revision, precision):
+        return f"**Warning**: '{model_name}' with (rev='{revision}', prec='{precision}') has already been evaluated (status FINISHED)."
+    elif already_in_queue(df_pending, model_name, revision, precision):
+        return f"**Warning**: '{model_name}' with (rev='{revision}', prec='{precision}') is already in PENDING."
+
+    # 5) Check threshold
+    try:
+        org = model_name.split("/")[0]
+    except:
+        org = model_name
+    under_threshold, message = check_org_threshold(org)
+    if not under_threshold:
+        return f"**Error**: {message}"
+
+    precision_final = unify_precision(precision)
+    if precision_final == "Missing":
+        precision_final = "UNK"
+
+    model_params = get_model_size(model_info=info, precision=precision)
+    current_time = datetime.utcnow().isoformat() + "Z"
+    # Convert chat_template input to boolean: True if "Yes", False if "No"
+    chat_template_bool = True if chat_template.strip().lower() == "yes" else False
+
+    submission = {
+        "model":        model_name,
+        "base_model":   base_model.strip(),
+        "revision":     revision,
+        "precision":    precision_final,
+        "weight_type":  weight_type,
+        "status":       "PENDING",
+        "submitted_time": current_time,
+        "model_type":   model_type,
+        "likes":        model_likes,
+        "params":       model_params,
+        "license":      model_license,
+        "private":      model_private,
+        "job_id":       None,
+        "job_start_time": None,
+        "chat_template": chat_template_bool
+    }
+
+    # Must be "org/repo"
+    try:
+        org_, repo_id = model_name.split("/")
+    except ValueError:
+        return "**Error**: Please specify model as 'org/model'. Note that `org` can be `username` as well."
+
+    private_str = "True" if model_private else "False"
+    file_path_in_repo = f"{org_}/{repo_id}_eval_request_{private_str}_{precision_final}_{weight_type}.json"
+
+    # 6) Upload submission
+    try:
+        API.upload_file(
+            path_or_fileobj=json.dumps(submission, indent=2).encode("utf-8"),
+            path_in_repo=file_path_in_repo,
+            repo_id=REQUESTS_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token,
+            commit_message=f"Add {model_name} to eval queue"
+        )
+    except Exception as e:
+        return f"**Error**: Could not upload to '{REQUESTS_REPO_ID}': {str(e)}"
+
+    return f"**Success**: Model '{model_name}' submitted for evaluation!"
+
+################################################################################
+# MAIN GRADIO APP
+################################################################################
+
+def main():
+    # Periodically restart the Space (e.g., every 30 minutes)
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(restart_space, "interval", hours=1)
+    scheduler.start()
+
+    df_tasks = load_scoreboard()
+
+    # Prepare filter choices from 'finished' requests
+    df_finished = load_requests("finished")
+    if not df_finished.empty:
+        df_finished["precision"]   = df_finished["precision"].apply(unify_precision)
+        df_finished["license"]     = df_finished["license"].apply(
+            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
+        )
+        df_finished["model_type"]  = df_finished["model_type"].apply(
+            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
+        )
+
+        precision_options = sorted(df_finished["precision"].dropna().unique().tolist())
+        license_options   = sorted(df_finished["license"].dropna().unique().tolist())
+        model_type_opts   = sorted(df_finished["model_type"].dropna().unique().tolist())
+
+        for lst in [precision_options, license_options, model_type_opts]:
+            if "Missing" not in lst:
+                lst.append("Missing")
+    else:
+        precision_options = ["float16", "bfloat16", "8bit", "4bit", "Missing"]
+        license_options   = ["Missing"]
+        model_type_opts   = ["Missing"]
+
+    if not df_tasks.empty:
+        min_model_size = int(df_tasks["Model Size Filter"].min())
+        max_model_size = int(df_tasks["Model Size Filter"].max())
+    else:
+        min_model_size, max_model_size = 0, 1000
+
+    all_columns = df_tasks.columns.tolist() if not df_tasks.empty else []
+    # We don't want to show "Model Size Filter" or "Full Type" directly
+    hidden_cols = {"Model Size Filter", "Full Type"}
+    for h in hidden_cols:
+        if h in all_columns:
+            all_columns.remove(h)
+
+    task_cols   = [t[2] for t in TASKS if t[2] in df_tasks.columns]
+    default_cols = ["Rank", "T", "Model Name", "Average ⬆️"] + task_cols
+    default_cols = [c for c in default_cols if c in all_columns]
+
+    with gr.Blocks() as demo:
+        gr.HTML(HEADER)
+        with gr.Tabs():
+            ####################################################################
+            # TAB 1: LLM Leaderboard
+            ####################################################################
+            with gr.Tab("🏅 LLM Leaderboard"):
+                with gr.Row():
+                    search_box = gr.Textbox(
+                        label="Search",
+                        placeholder="Search for models...",
+                        interactive=True
                     )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
+                with gr.Row():
+                    col_selector = gr.CheckboxGroup(
+                        choices=all_columns,
+                        value=default_cols,
+                        label="Select columns to display"
                     )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
+                    t_filter = gr.CheckboxGroup(
+                        choices=model_type_opts,
+                        value=model_type_opts.copy(),
+                        label="Filter by Model Type"
+                    )
+                with gr.Row():
+                    license_filter = gr.CheckboxGroup(
+                        choices=license_options,
+                        value=license_options.copy(),
+                        label="Filter by License"
+                    )
+                    precision_filter = gr.CheckboxGroup(
+                        choices=precision_options,
+                        value=precision_options.copy(),
+                        label="Filter by Precision"
+                    )
+                with gr.Row():
+                    size_min_slider = gr.Slider(
+                        minimum=min_model_size,
+                        maximum=max_model_size,
+                        value=min_model_size,
+                        step=1,
+                        label="Minimum Model Size (params)",
+                        interactive=True
+                    )
+                    size_max_slider = gr.Slider(
+                        minimum=min_model_size,
+                        maximum=max_model_size,
+                        value=max_model_size,
+                        step=1,
+                        label="Maximum Model Size (params)",
+                        interactive=True
                     )
 
-            leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-                ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
+                leaderboard = gr.Dataframe(
+                    value=df_tasks[default_cols] if not df_tasks.empty else pd.DataFrame(columns=default_cols),
+                    interactive=False
                 )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+                def filter_by_full_type(dff, selected_full_types):
+                    incl_missing = "Missing" in selected_full_types
+                    if incl_missing:
+                        return dff[
+                            (dff["Full Type"].isin(selected_full_types))
+                            | (dff["Full Type"].isna())
+                            | (dff["Full Type"] == "")
+                            | (dff["Full Type"] == "Missing")
+                        ]
+                    else:
+                        return dff[dff["Full Type"].isin(selected_full_types)]
+
+                def filter_leaderboard(
+                    search_query,
+                    selected_cols,
+                    t_filter_values,
+                    lic_filter_values,
+                    prec_filter_values,
+                    min_sz,
+                    max_sz
+                ):
+                    dff = df_tasks.copy()
+                    # 1) Filter by size
+                    if min_sz > max_sz:
+                        min_sz, max_sz = max_sz, min_sz
+                    dff = dff[(dff["Model Size Filter"] >= min_sz+1) & (dff["Model Size Filter"] <= max_sz+1)]
+                    # 2) Search by name
+                    dff["plain_name"] = dff["Model Name"].str.replace(r'<.*?>', '', regex=True)
+                    if search_query:
+                        dff = dff[dff["plain_name"].str.contains(search_query, case=False, na=False)]
+                    # 3) Filter by model type
+                    if t_filter_values:
+                        dff = filter_by_full_type(dff, t_filter_values)
+                    # 4) Filter by license
+                    if lic_filter_values:
+                        incl_missing = "Missing" in lic_filter_values
+                        chosen = [l for l in lic_filter_values if l != "Missing"]
+                        if incl_missing:
+                            dff = dff[
+                                dff["License"].isin(chosen)
+                                | (dff["License"] == "Missing")
+                                | (dff["License"].isna())
+                            ]
+                        else:
+                            dff = dff[dff["License"].isin(chosen)]
+                    # 5) Filter by precision
+                    if prec_filter_values:
+                        incl_missing = "Missing" in prec_filter_values
+                        chosen = [p for p in prec_filter_values if p != "Missing"]
+                        if incl_missing:
+                            dff = dff[
+                                dff["Precision"].isin(chosen)
+                                | (dff["Precision"].isna())
+                                | (dff["Precision"] == "Missing")
+                                | (dff["Precision"] == "UNK")
+                            ]
+                        else:
+                            dff = dff[dff["Precision"].isin(chosen)]
+
+                    dff.drop(columns=["plain_name"], inplace=True, errors="ignore")
+                    final_cols = [col for col in dff.columns if col in selected_cols]
+                    return dff[final_cols]
+
+                filter_inputs = [
+                    search_box, col_selector, t_filter,
+                    license_filter, precision_filter,
+                    size_min_slider, size_max_slider
+                ]
+                search_box.submit(filter_leaderboard, inputs=filter_inputs, outputs=leaderboard)
+                for comp in filter_inputs:
+                    comp.change(filter_leaderboard, inputs=filter_inputs, outputs=leaderboard)
+
+            ####################################################################
+            # TAB 2: Submit here
+            ####################################################################
+            with gr.Tab("🚀 Submit here!"):
+                gr.Markdown(SUBMISSION_TEXT)
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
                 with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    # with gr.Accordion(
-                    #     f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                    #     open=False,
-                    # ):
-                    #     with gr.Row():
-                    #         running_eval_table = gr.components.Dataframe(
-                    #             value=running_eval_queue_df,
-                    #             headers=EVAL_COLS,
-                    #             datatype=EVAL_TYPES,
-                    #             row_count=5,
-                    #         )
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
+                    model_name_box = gr.Textbox(
+                        label="Model Name",
+                        placeholder="myorg/mymodel",
+                        interactive=True
                     )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
+                    revision_box = gr.Textbox(
+                        label="Revision Commit",
+                        placeholder="main",
+                        value="main",
+                        interactive=True
+                    )
+                with gr.Row():
+                    model_type_box = gr.Dropdown(
+                        label="Model Type",
+                        choices=list(MODEL_TYPE_TO_EMOJI.keys()),
+                        value="💬 : chat models (RLHF, DPO, IFT, ...)",
+                        interactive=True
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
+                    weight_type_box = gr.Dropdown(
+                        label="Weight Type",
+                        choices=["Original", "Adapter", "Delta"],
                         value="Original",
-                        interactive=True,
+                        interactive=True
+                    )
+                with gr.Row():
+                    precision_box = gr.Dropdown(
+                        label="Precision",
+                        choices=["float16", "bfloat16", "8bit", "4bit"],
+                        value="bfloat16",
+                        interactive=True
+                    )
+                    base_model_box = gr.Textbox(
+                        label="Base Model (if adapter or delta weights)",
+                        placeholder="(Optional) e.g. myorg/base-model",
+                        interactive=True
+                    )
+                with gr.Row():
+                    chat_template_box = gr.Radio(
+                        label="Evaluate using chat-template?",
+                        choices=["Yes", "No"],
+                        value="No",
+                        interactive=True
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-    gr.HTML(BOTTOM_LOGO)
+                submit_btn = gr.Button("Submit Model")
+                submit_out = gr.Markdown()
+
+                submit_btn.click(
+                    fn=submit_model,
+                    inputs=[model_name_box, base_model_box, revision_box, precision_box, weight_type_box, model_type_box, chat_template_box],
+                    outputs=submit_out
+                )
+
+                df_pending  = load_requests("pending")
+                df_running  = load_requests("running")
+                df_finished2= load_requests("finished")
+                df_failed   = load_requests("failed")
+
+                gr.Markdown("## Evaluation Status")
+                with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
+                    if not df_pending.empty:
+                        gr.Dataframe(df_pending)
+                    else:
+                        gr.Markdown("No pending evaluations.")
+                with gr.Accordion(f"Running Evaluations ({len(df_running)})", open=False):
+                    if not df_running.empty:
+                        gr.Dataframe(df_running)
+                    else:
+                        gr.Markdown("No running evaluations.")
+                with gr.Accordion(f"Finished Evaluations ({len(df_finished2)})", open=False):
+                    if not df_finished2.empty:
+                        gr.Dataframe(df_finished2)
+                    else:
+                        gr.Markdown("No finished evaluations.")
+                with gr.Accordion(f"Failed Evaluations ({len(df_failed)})", open=False):
+                    if not df_failed.empty:
+                        gr.Dataframe(df_failed)
+                    else:
+                        gr.Markdown("No failed evaluations.")
+
+                gr.Markdown(ABOUT_SECTION)
+
+        with gr.Row():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_box = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    lines=20,
+                    elem_id="citation-button",
+                    show_copy_button=True
+                )
+                
+        gr.HTML(BOTTOM_LOGO)
+        
+    demo.queue(default_concurrency_limit=40).launch()
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    main()