app.py

import os
import json
import numpy as np
import pandas as pd
import gradio as gr
from datetime import datetime, timedelta
from huggingface_hub.hf_api import ModelInfo
from transformers import AutoConfig, AutoTokenizer
from huggingface_hub import HfApi, hf_hub_download, ModelCard
from apscheduler.schedulers.background import BackgroundScheduler

################################################################################
# GLOBALS & CONSTANTS
################################################################################

OWNER = "OALL"
REPO_ID = f"{OWNER}/Open-Arabic-LLM-Leaderboard-v2-exp"
RESULTS_REPO_ID = f"{OWNER}/v2_results"
REQUESTS_REPO_ID = f"{OWNER}/requests_v2"

# Global HF API instance (set once)
hf_api_token = os.environ.get('HF_API_TOKEN', None)
API = HfApi(token=hf_api_token)

TASKS = [
    ("community|alghafa:_average|0",        "acc_norm",     "AlGhafa"),
    ("community|arabic_mmlu:_average|0",    "acc_norm",     "ArabicMMLU"),
    ("community|arabic_exams|0",            "acc_norm",     "EXAMS"),
    ("community|madinah_qa:_average|0",     "acc_norm",     "MadinahQA"),
    ("community|aratrust:_average|0",       "acc_norm",     "AraTrust"),
    ("community|alrage_qa|0",               "llm_as_judge", "ALRAGE"),
    ("community|arabic_mmlu_ht:_average|0", "acc_norm",     "ArbMMLU-HT"),
]

MODEL_TYPE_TO_EMOJI = {
    "🟢 : pretrained": "🟢",
    "🟩 : continuously pretrained": "🟩",
    "💬 : chat models (RLHF, DPO, IFT, ...)": "💬",
    "🔶 : fine-tuned on domain-specific datasets": "🔶",
    "🤝 : base merges and merges": "🤝",
    "Missing": "?",
}

HEADER = """
<img src="https://raw.githubusercontent.com/alielfilali01/OALL-assets/main/TITLE.png" style="width:30%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">
"""

BOTTOM_LOGO = """<img src="https://raw.githubusercontent.com/alielfilali01/OALL-assets/main/BOTTOM.png" style="width:50%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">"""

SUBMISSION_TEXT = """
# Submit Your Model for Evaluation 🌴 

**The Open Arabic LLM Leaderboard** aims to help you evaluate and compare the performance of Arabic Large Language Models.

When you submit a model on this page, it is automatically evaluated on a set of arabic native benchmarks ([find here](https://github.com/huggingface/lighteval/blob/main/examples/tasks/OALL_v2_tasks.txt)) with one additional human-translated version of [MMLU](https://arxiv.org/abs/2009.03300).

The GPU used for evaluation is operated with the support of  __[Technology Innovation Institute (TII)](https://www.tii.ae/)__.

More details about the benchmarks and the evaluation process is provided on the “About” section below.

Find the first version of the leaderboard hosted as Legacy in this [Space](https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard-v1).
"""

ABOUT_SECTION = """
## About

While outstanding LLM models are being released competitively, most of them are centered on English and are familiar with the English cultural sphere. We operate the Open Arabic LLM Leaderboard (OALL), to evaluate models that reflect the characteristics of the Arabic language, culture and heritage. Through this, we hope that users can conveniently use the leaderboard, participate, and contribute to the advancement of research in the Arab region 🔥.

### Icons & Model types
🟢 : `pretrained`

🟩 : `continuously pretrained`

💬 : `chat models (RLHF, DPO, IFT, ...)`

🔶 : `fine-tuned on domain-specific datasets`

🤝 : `base merges and moerges`

### Notes:
- We reserve the right to correct any incorrect tags or icons after manual verification to ensure the accuracy and reliability of the leaderboard. This helps maintain the integrity and trustworthiness of the platform.
- Some models may be flagged as “Subjects of Caution” by the community. These models might have used the evaluation set for training, attempted to manipulate rankings, or raised ethical concerns. Models deemed as such may face restricted visibility or removal from the leaderboard. Users are advised to exercise discretion when interpreting rankings.
- The leaderboard automatically hides models that were submitted, evaluated, and subsequently made private or gated post-evaluation. This platform is designed for **“open”** models that benefit the wider community. If you intend to restrict your model’s accessibility after using the leaderboard’s resources or exploit the platform solely for personal gains, please refrain from submitting. Violators may face bans on their usernames and/or organization IDs from future submissions.
- The leaderboard no longer accepts models in **float32** precision except under special circumstances. If you are the developer of a float32 model and believe it deserves inclusion, please reach out to us.
- To ensure fair and equitable access to leaderboard resources, all usernames and organization IDs are limited to **5 submissions per week**. This policy minimizes spamming, encourages thoughtful participation, and allows everyone in the community to benefit from the platform.

By adhering to these guidelines, we aim to foster a fair, collaborative, and transparent environment for evaluating and advancing open models for the arabic/arabic-interested communities.

### How it works

📈 We evaluate models using [LightEval](https://github.com/huggingface/lighteval), a unified and straightforward framework from the HuggingFace Eval Team to test and assess causal language models on a large number of different evaluation tasks.

To ensure a fair and unbiased assessment of the models' true capabilities, all evaluations are conducted in zero-shot settings `0-shots`. This approach eliminates any potential advantage from task-specific fine-tuning, providing a clear indication of how well the models can generalize to new tasks.

Also, given the nature of the tasks, which include multiple-choice questions, the leaderboard primarily uses normalized log likelihood accuracy `loglikelihood_acc_norm` for all tasks.

Please, consider reaching out to us through the discussions tab if you are working on benchmarks for Arabic LLMs and willing to see them on this leaderboard as well. Your benchmark might change the whole game for Arabic models !

### Details and Logs
- Detailed numerical results in the `results` OALL dataset: https://huggingface.co/datasets/OALL/v2_results
- Community queries and running status in the `requests` OALL dataset: https://huggingface.co/datasets/OALL/requests_v2

### More resources
For evaluations of chat models using 3C3H on generative tasks benchmarks, please refer to the [AraGen-Leaderboard](https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard).

If you still have questions, you can check our FAQ [here](https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard/discussions/15)!

"""

CITATION_BUTTON_LABEL = """
Copy the following snippet to cite these results
"""

CITATION_BUTTON_TEXT = """
@misc{OALL-2,
  author = {El Filali, Ali and ALOUI, Manel and Husaain, Tarique and Alzubaidi, Ahmed and Boussaha, Basma El Amel and Cojocaru, Ruxandra and Fourrier, Clémentine and Habib, Nathan and Hacid, Hakim},
  title = {Open Arabic LLM Leaderboard 2},
  year = {2025},
  publisher = {OALL},
  howpublished = {https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard}
}
"""

################################################################################
# UTILITY & HELPER FUNCTIONS
################################################################################

def model_hyperlink(model_name):
    link = f"https://huggingface.co/{model_name}"
    # return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
    # return f'[{model_name}]({link})' # WHYYYYYYYY It is not working !!!???
    return f"{model_name}"

def restart_space():
    """Restart the Gradio space periodically."""
    API.restart_space(repo_id=REPO_ID)

def unify_precision(raw_precision: str) -> str:
    """
    Map raw precision strings (e.g. 'torch.float16', 'fp16', 'float16')
    to canonical forms: 'float16', 'float32', 'bfloat16', '8bit', '4bit', 'Missing'.
    """
    if not raw_precision or raw_precision.lower() in ["missing", "unk", "none"]:
        return "Missing"
    p = raw_precision.lower()
    if p in ["torch.float16", "float16", "fp16"]:
        return "float16"
    if p in ["torch.float32", "float32", "fp32"]:
        return "float32"
    if p in ["torch.bfloat16", "bfloat16", "bf16"]:
        return "bfloat16"
    if p == "8bit":
        return "8bit"
    if p == "4bit":
        return "4bit"
    return "Missing"

def load_requests(status_folder: str) -> pd.DataFrame:
    """
    Load all .json requests from REQUESTS_REPO_ID, filtering by 'status' == status_folder.
    """
    df_out = []
    try:
        files_info = API.list_repo_files(
            repo_id=REQUESTS_REPO_ID,
            repo_type="dataset",
            token=hf_api_token
        )
        json_files = [f for f in files_info if f.endswith(".json")]
    except Exception as e:
        print(f"Error listing files in {REQUESTS_REPO_ID}: {e}")
        return pd.DataFrame()

    for path in json_files:
        try:
            local_path = hf_hub_download(
                repo_id=REQUESTS_REPO_ID,
                filename=path,
                repo_type="dataset",
                token=hf_api_token
            )
            with open(local_path, "r", encoding="utf-8") as f:
                req = json.load(f)
        except Exception as e:
            print(f"Error loading {path}: {e}")
            continue

        if str(req.get("status", "")).strip().lower() == status_folder.lower():
            df_out.append(req)

    return pd.DataFrame(df_out)

def load_all_requests() -> pd.DataFrame:
    """
    Load *all* requests from the dataset (pending, finished, failed, etc.).
    Returns a single DataFrame with columns from all requests.
    """
    df_out = []
    try:
        files_info = API.list_repo_files(
            repo_id=REQUESTS_REPO_ID,
            repo_type="dataset",
            token=hf_api_token
        )
        json_files = [f for f in files_info if f.endswith(".json")]
    except Exception as e:
        print(f"Error listing files in {REQUESTS_REPO_ID}: {e}")
        return pd.DataFrame()

    for path in json_files:
        try:
            local_path = hf_hub_download(
                repo_id=REQUESTS_REPO_ID,
                filename=path,
                repo_type="dataset",
                token=hf_api_token
            )
            with open(local_path, "r", encoding="utf-8") as f:
                req = json.load(f)
            df_out.append(req)
        except Exception as e:
            print(f"Error loading {path}: {e}")
            continue

    return pd.DataFrame(df_out)

def already_in_queue(df_pending: pd.DataFrame, model_name: str, revision: str, precision: str) -> bool:
    """
    Check if (model, revision, precision) is already in the 'pending' queue.
    """
    if df_pending.empty:
        return False
    matched = df_pending[
        (df_pending["model"] == model_name)
        & (df_pending["revision"] == revision)
        & (df_pending["precision"] == unify_precision(precision))
    ]
    return not matched.empty

def get_model_size(model_info: ModelInfo, precision: str) -> float:
    """
    Return approximate model parameter size in billions, if safetensors info is available.
    Return 0 if unknown.
    For GPTQ, we do a small multiplier to reflect the extra bits, etc.
    """
    try:
        param_bytes = model_info.safetensors.get("total", 0)
        model_size = round(param_bytes / 1e9, 3)
    except (AttributeError, TypeError):
        return 0.0

    if "gptq" in model_info.modelId.lower():
        return model_size * 8
    return model_size

def parse_datetime(dt_str: str) -> datetime:
    """
    Safely parse an ISO datetime string into a Python datetime object.
    """
    try:
        return datetime.fromisoformat(dt_str.replace("Z", ""))
    except Exception:
        return datetime.min

################################################################################
# SCOREBOARD LOADING & DISPLAY
################################################################################

def load_scoreboard() -> pd.DataFrame:
    """
    1) Reads JSON "results_*.json" from RESULTS_REPO_ID to collect scores.
    2) Combines with "finished" requests data for license, revision, etc.
    3) Removes any model that is no longer public or accessible.
    4) Returns a DataFrame ready for display & filtering.
    
    Only models with finished evaluations are kept.
    """
    # Step A: Get scoreboard files from the results dataset
    try:
        files_info = API.list_repo_files(
            repo_id=RESULTS_REPO_ID,
            repo_type="dataset",
            token=hf_api_token
        )
    except Exception as e:
        print(f"Error listing scoreboard files in {RESULTS_REPO_ID}: {e}")
        return pd.DataFrame()

    candidate_json_paths = [
        path for path in files_info
        if path.endswith(".json") and len(path.split("/")) == 3 and path.split("/")[2].startswith("results_")
    ]

    rows = []
    # Step B: Read each scoreboard file
    for file_path in candidate_json_paths:
        try:
            local_file = hf_hub_download(
                repo_id=RESULTS_REPO_ID,
                filename=file_path,
                repo_type="dataset",
                token=hf_api_token
            )
            with open(local_file, "r", encoding="utf-8") as f:
                data = json.load(f)
        except Exception as e:
            print(f"Error loading scoreboard file {file_path}: {e}")
            continue

        config_general = data.get("config_general", {})
        results_block = data.get("results", {})

        model_name  = config_general.get("model_name", "UNK")
        scoreboard_precision = unify_precision(config_general.get("model_dtype", "Missing"))
        # To be consistent with submission logic:
        if scoreboard_precision == "Missing":
            scoreboard_precision = "UNK"
        scoreboard_model_type  = config_general.get("model_type", "Missing")

        row_dict = {
            "Model Name":   model_name,
            "Revision":     "Missing",  # We'll fill from requests
            "License":      "Missing",  # We'll fill from requests
            "Precision":    scoreboard_precision,
            "Full Type":    scoreboard_model_type,
            "Model Size":   0.0,        # We'll fill from requests
            "Hub ❤️":       0,          # We'll fill from requests
        }

        # Fill tasks
        for (task_key, metric_field, display_name) in TASKS:
            val = np.nan
            if task_key in results_block:
                subd = results_block[task_key]
                if isinstance(subd, dict) and metric_field in subd:
                    val = subd[metric_field]
            row_dict[display_name] = val

        rows.append(row_dict)

    df = pd.DataFrame(rows)
    if df.empty:
        base_cols = [
            "Model Name","Revision","License",
            "Precision","Full Type","Model Size","Hub ❤️"
        ]
        task_names = [t[2] for t in TASKS]
        return pd.DataFrame(columns=base_cols + task_names)

    # Step C: Convert tasks to numeric & multiply by 100
    task_cols = [t[2] for t in TASKS if t[2] in df.columns]
    df[task_cols] = df[task_cols].apply(pd.to_numeric, errors="coerce")
    for c in task_cols:
        df[c] = (df[c] * 100).round(2)

    # Step D: Compute average
    if task_cols:
        df["Average ⬆️"] = df[task_cols].mean(axis=1).round(2)
    else:
        df["Average ⬆️"] = np.nan

    # Step E: Overwrite scoreboard data with "finished" requests (except for Precision)
    df_finished = load_requests("finished")
    if not df_finished.empty:
        df_finished["precision"] = df_finished["precision"].apply(unify_precision)
        df_finished["license"]   = df_finished["license"].apply(
            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
        )
        df_finished["model_type"] = df_finished["model_type"].apply(
            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
        )

        # Group by model name and precision to correctly distinguish multiple submissions
        dff_grouped = df_finished.groupby(["model", "precision"], as_index=False).last()

        request_map = {}
        for _, row_ in dff_grouped.iterrows():
            key = f"{row_['model']}__{row_['precision']}"
            request_map[key] = {
                "license":   row_["license"],
                "revision":  row_["revision"],
                "precision": row_["precision"],
                "model_type":row_["model_type"],
                "params":    row_["params"],
                "likes":     row_["likes"]
            }

        def apply_request_info(row_):
            key = f"{row_['Model Name']}__{row_['Precision']}"
            if key in request_map:
                row_["License"]    = request_map[key]["license"]
                row_["Revision"]   = request_map[key]["revision"]
                # Do NOT update "Precision": keep the value from the results file.
                row_["Full Type"]  = request_map[key]["model_type"]
                row_["Model Size"] = request_map[key]["params"]
                row_["Hub ❤️"]     = request_map[key]["likes"]
            return row_

        df = df.apply(apply_request_info, axis=1)

    # Step E2: Remove rows that do not have finished request info (i.e. Revision is still "Missing")
    df = df[df["Revision"] != "Missing"]

    # Step F: Remove any model not public
    remove_idx = []
    for idx, row in df.iterrows():
        model_name = row["Model Name"]
        if model_name == "UNK":
            remove_idx.append(idx)
            continue
        try:
            API.model_info(model_name)
        except Exception:
            remove_idx.append(idx)
    df.drop(remove_idx, inplace=True)
    df.reset_index(drop=True, inplace=True)

    # Step G: Sort scoreboard
    df = df.sort_values(by="Average ⬆️", ascending=False).reset_index(drop=True)

    # Step H: Insert ranking & create a "Model Size Filter" for slider usage
    df.insert(0, "Rank", range(1, len(df) + 1))
    df["Model Size Filter"] = df["Model Size"]

    # Step I: Short label for the model type
    def map_type_to_emoji(full_str):
        if not isinstance(full_str, str):
            return "Missing"
        return MODEL_TYPE_TO_EMOJI.get(full_str.strip(), full_str.strip())
    df["T"] = df["Full Type"].apply(map_type_to_emoji)

    # At this point, convert "Model Name" to a clickable link
    df["Model Name"] = df["Model Name"].apply(model_hyperlink)

    # Reorder columns
    final_cols = ["Rank", "T", "Model Name", "Average ⬆️"] + task_cols
    remainder = ["Model Size", "Hub ❤️", "License", "Precision", "Revision", "Model Size Filter", "Full Type"]
    for rc in remainder:
        if rc not in final_cols and rc in df.columns:
            final_cols.append(rc)

    return df[final_cols]

################################################################################
# SUBMISSION LOGIC
################################################################################

def check_model_card(repo_id: str) -> (bool, str):
    """Check if model card is present, has a license, and is not too short."""
    try:
        card = ModelCard.load(repo_id)
    except Exception:
        return (False, "No model card found. Please add a README.md describing your model and license.")
    if card.data.license is None and not ("license_name" in card.data and "license_link" in card.data):
        return (False, "No license metadata found in the model card.")
    if len(card.text) < 200:
        return (False, "Model card is too short (<200 chars). Please add more details.")
    return (True, "")

def is_model_on_hub(model_name, revision, token=None, trust_remote_code=False, test_tokenizer=True):
    """Check if the model & tokenizer can be loaded from the Hub."""
    try:
        config = AutoConfig.from_pretrained(
            model_name,
            revision=revision,
            trust_remote_code=trust_remote_code,
            token=token
        )
    except ValueError:
        return (False, "requires `trust_remote_code=True`. Not automatically allowed.", None)
    except Exception as e:
        return (False, f"not loadable from hub: {str(e)}", None)

    if test_tokenizer:
        try:
            _ = AutoTokenizer.from_pretrained(
                model_name,
                revision=revision,
                trust_remote_code=trust_remote_code,
                token=token
            )
        except Exception as e:
            return (False, f"tokenizer not loadable: {str(e)}", None)

    return (True, "", config)

def check_org_threshold(org_name: str) -> (bool, str):
    """
    Each org can only submit 5 models in the last 7 days.
    Return (True, "") if allowed. Otherwise, (False, "error message").
    """
    df_all = load_all_requests()
    if df_all.empty:
        return (True, "")

    def get_org(m):
        try:
            return m.split("/")[0]
        except:
            return m

    df_all["org_name"] = df_all["model"].apply(get_org)
    df_org = df_all[df_all["org_name"] == org_name].copy()
    if df_org.empty:
        return (True, "")

    df_org["datetime"] = df_org["submitted_time"].apply(parse_datetime)
    df_org.dropna(subset=["datetime"], inplace=True)

    now = datetime.utcnow()
    week_ago = now - timedelta(days=7)
    df_recent = df_org[df_org["datetime"] >= week_ago]

    if len(df_recent) >= 5:
        df_recent_sorted = df_recent.sort_values(by="datetime")
        earliest = df_recent_sorted.iloc[0]["datetime"]
        next_ok = earliest + timedelta(days=7)
        msg_next = next_ok.isoformat(timespec="seconds") + "Z"
        return (
            False,
            f"Your org '{org_name}' has reached the 5-submissions-per-week limit. You can submit again after {msg_next}."
        )
    return (True, "")

def submit_model(
    model_name: str,
    base_model: str,
    revision: str,
    precision: str,
    weight_type: str,
    model_type: str,
    chat_template: str
):
    # -------------------------------------------------------------------------
    # 0) Strip inputs to avoid trailing or leading spaces
    # -------------------------------------------------------------------------
    model_name  = model_name.strip()
    base_model  = base_model.strip()
    revision    = revision.strip()
    precision   = precision.strip()
    
    if not model_name:
        return "**Error**: Model name cannot be empty (use 'org/model')."
    if not revision:
        revision = "main"

    # 1) Check model card
    card_ok, card_msg = check_model_card(model_name)
    if not card_ok:
        return f"**Error**: {card_msg}"

    # 2) If adapter/delta, check base_model
    if weight_type.lower() in ["adapter", "delta"]:
        if not base_model:
            return "**Error**: For adapter/delta, you must provide a valid `base_model`."
        ok_base, base_err, _ = is_model_on_hub(base_model, revision, hf_api_token, trust_remote_code=True, test_tokenizer=True)
        if not ok_base:
            return f"**Error**: Base model '{base_model}' {base_err}"
    else:
        ok_model, model_err, _ = is_model_on_hub(model_name, revision, hf_api_token, trust_remote_code=True, test_tokenizer=True)
        if not ok_model:
            return f"**Error**: Model '{model_name}' {model_err}"

    # 3) Retrieve ModelInfo
    try:
        info = API.model_info(model_name, revision=revision, token=hf_api_token)
    except Exception as e:
        return f"**Error**: Could not fetch model info. {str(e)}"

    model_license = info.card_data.license
    model_likes   = info.likes or 0
    model_private = bool(getattr(info, "private", False))

    # 4) Check duplicates
    df_pending = load_requests("pending")
    df_finished = load_requests("finished")
    
    if already_in_queue(df_finished, model_name, revision, precision):
        return f"**Warning**: '{model_name}' with (rev='{revision}', prec='{precision}') has already been evaluated (status FINISHED)."
    elif already_in_queue(df_pending, model_name, revision, precision):
        return f"**Warning**: '{model_name}' with (rev='{revision}', prec='{precision}') is already in PENDING."

    # 5) Check threshold
    try:
        org = model_name.split("/")[0]
    except:
        org = model_name
    under_threshold, message = check_org_threshold(org)
    if not under_threshold:
        return f"**Error**: {message}"

    precision_final = unify_precision(precision)
    if precision_final == "Missing":
        precision_final = "UNK"

    model_params = get_model_size(model_info=info, precision=precision)
    current_time = datetime.utcnow().isoformat() + "Z"
    # Convert chat_template input to boolean: True if "Yes", False if "No"
    chat_template_bool = True if chat_template.strip().lower() == "yes" else False

    submission = {
        "model":        model_name,
        "base_model":   base_model.strip(),
        "revision":     revision,
        "precision":    precision_final,
        "weight_type":  weight_type,
        "status":       "PENDING",
        "submitted_time": current_time,
        "model_type":   model_type,
        "likes":        model_likes,
        "params":       model_params,
        "license":      model_license,
        "private":      model_private,
        "job_id":       None,
        "job_start_time": None,
        "chat_template": chat_template_bool
    }

    # Must be "org/repo"
    try:
        org_, repo_id = model_name.split("/")
    except ValueError:
        return "**Error**: Please specify model as 'org/model'. Note that `org` can be `username` as well."

    private_str = "True" if model_private else "False"
    file_path_in_repo = f"{org_}/{repo_id}_eval_request_{private_str}_{precision_final}_{weight_type}.json"

    # 6) Upload submission
    try:
        API.upload_file(
            path_or_fileobj=json.dumps(submission, indent=2).encode("utf-8"),
            path_in_repo=file_path_in_repo,
            repo_id=REQUESTS_REPO_ID,
            repo_type="dataset",
            token=hf_api_token,
            commit_message=f"Add {model_name} to eval queue"
        )
    except Exception as e:
        return f"**Error**: Could not upload to '{REQUESTS_REPO_ID}': {str(e)}"

    return f"**Success**: Model '{model_name}' submitted for evaluation!"

################################################################################
# MAIN GRADIO APP
################################################################################

def main():
    # Periodically restart the Space (e.g., every 30 minutes)
    scheduler = BackgroundScheduler()
    scheduler.add_job(restart_space, "interval", hours=1)
    scheduler.start()

    df_tasks = load_scoreboard()

    # Prepare filter choices from 'finished' requests
    df_finished = load_requests("finished")
    if not df_finished.empty:
        df_finished["precision"]   = df_finished["precision"].apply(unify_precision)
        df_finished["license"]     = df_finished["license"].apply(
            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
        )
        df_finished["model_type"]  = df_finished["model_type"].apply(
            lambda x: ", ".join(x) if isinstance(x, list) else str(x)
        )

        precision_options = sorted(df_finished["precision"].dropna().unique().tolist())
        license_options   = sorted(df_finished["license"].dropna().unique().tolist())
        model_type_opts   = sorted(df_finished["model_type"].dropna().unique().tolist())

        for lst in [precision_options, license_options, model_type_opts]:
            if "Missing" not in lst:
                lst.append("Missing")
    else:
        precision_options = ["float16", "bfloat16", "8bit", "4bit", "Missing"]
        license_options   = ["Missing"]
        model_type_opts   = ["Missing"]

    if not df_tasks.empty:
        min_model_size = int(df_tasks["Model Size Filter"].min())
        max_model_size = int(df_tasks["Model Size Filter"].max())
    else:
        min_model_size, max_model_size = 0, 1000

    all_columns = df_tasks.columns.tolist() if not df_tasks.empty else []
    # We don't want to show "Model Size Filter" or "Full Type" directly
    hidden_cols = {"Model Size Filter", "Full Type"}
    for h in hidden_cols:
        if h in all_columns:
            all_columns.remove(h)

    task_cols   = [t[2] for t in TASKS if t[2] in df_tasks.columns]
    default_cols = ["Rank", "T", "Model Name", "Average ⬆️"] + task_cols
    default_cols = [c for c in default_cols if c in all_columns]

    with gr.Blocks() as demo:
        gr.HTML(HEADER)
        with gr.Tabs():
            ####################################################################
            # TAB 1: LLM Leaderboard
            ####################################################################
            with gr.Tab("🏅 LLM Leaderboard"):
                with gr.Row():
                    search_box = gr.Textbox(
                        label="Search",
                        placeholder="Search for models...",
                        interactive=True
                    )
                with gr.Row():
                    col_selector = gr.CheckboxGroup(
                        choices=all_columns,
                        value=default_cols,
                        label="Select columns to display"
                    )
                    t_filter = gr.CheckboxGroup(
                        choices=model_type_opts,
                        value=model_type_opts.copy(),
                        label="Filter by Model Type"
                    )
                with gr.Row():
                    license_filter = gr.CheckboxGroup(
                        choices=license_options,
                        value=license_options.copy(),
                        label="Filter by License"
                    )
                    precision_filter = gr.CheckboxGroup(
                        choices=precision_options,
                        value=precision_options.copy(),
                        label="Filter by Precision"
                    )
                with gr.Row():
                    size_min_slider = gr.Slider(
                        minimum=min_model_size,
                        maximum=max_model_size,
                        value=min_model_size,
                        step=1,
                        label="Minimum Model Size (params)",
                        interactive=True
                    )
                    size_max_slider = gr.Slider(
                        minimum=min_model_size,
                        maximum=max_model_size,
                        value=max_model_size,
                        step=1,
                        label="Maximum Model Size (params)",
                        interactive=True
                    )

                leaderboard = gr.Dataframe(
                    value=df_tasks[default_cols] if not df_tasks.empty else pd.DataFrame(columns=default_cols),
                    interactive=False
                )

                def filter_by_full_type(dff, selected_full_types):
                    incl_missing = "Missing" in selected_full_types
                    if incl_missing:
                        return dff[
                            (dff["Full Type"].isin(selected_full_types))
                            | (dff["Full Type"].isna())
                            | (dff["Full Type"] == "")
                            | (dff["Full Type"] == "Missing")
                        ]
                    else:
                        return dff[dff["Full Type"].isin(selected_full_types)]

                def filter_leaderboard(
                    search_query,
                    selected_cols,
                    t_filter_values,
                    lic_filter_values,
                    prec_filter_values,
                    min_sz,
                    max_sz
                ):
                    dff = df_tasks.copy()
                    # 1) Filter by size
                    if min_sz > max_sz:
                        min_sz, max_sz = max_sz, min_sz
                    dff = dff[(dff["Model Size Filter"] >= min_sz+1) & (dff["Model Size Filter"] <= max_sz+1)]
                    # 2) Search by name
                    dff["plain_name"] = dff["Model Name"].str.replace(r'<.*?>', '', regex=True)
                    if search_query:
                        dff = dff[dff["plain_name"].str.contains(search_query, case=False, na=False)]
                    # 3) Filter by model type
                    if t_filter_values:
                        dff = filter_by_full_type(dff, t_filter_values)
                    # 4) Filter by license
                    if lic_filter_values:
                        incl_missing = "Missing" in lic_filter_values
                        chosen = [l for l in lic_filter_values if l != "Missing"]
                        if incl_missing:
                            dff = dff[
                                dff["License"].isin(chosen)
                                | (dff["License"] == "Missing")
                                | (dff["License"].isna())
                            ]
                        else:
                            dff = dff[dff["License"].isin(chosen)]
                    # 5) Filter by precision
                    if prec_filter_values:
                        incl_missing = "Missing" in prec_filter_values
                        chosen = [p for p in prec_filter_values if p != "Missing"]
                        if incl_missing:
                            dff = dff[
                                dff["Precision"].isin(chosen)
                                | (dff["Precision"].isna())
                                | (dff["Precision"] == "Missing")
                                | (dff["Precision"] == "UNK")
                            ]
                        else:
                            dff = dff[dff["Precision"].isin(chosen)]

                    dff.drop(columns=["plain_name"], inplace=True, errors="ignore")
                    final_cols = [col for col in dff.columns if col in selected_cols]
                    return dff[final_cols]

                filter_inputs = [
                    search_box, col_selector, t_filter,
                    license_filter, precision_filter,
                    size_min_slider, size_max_slider
                ]
                search_box.submit(filter_leaderboard, inputs=filter_inputs, outputs=leaderboard)
                for comp in filter_inputs:
                    comp.change(filter_leaderboard, inputs=filter_inputs, outputs=leaderboard)

            ####################################################################
            # TAB 2: Submit here
            ####################################################################
            with gr.Tab("🚀 Submit here!"):
                gr.Markdown(SUBMISSION_TEXT)

                with gr.Row():
                    model_name_box = gr.Textbox(
                        label="Model Name",
                        placeholder="myorg/mymodel",
                        interactive=True
                    )
                    revision_box = gr.Textbox(
                        label="Revision Commit",
                        placeholder="main",
                        value="main",
                        interactive=True
                    )
                with gr.Row():
                    model_type_box = gr.Dropdown(
                        label="Model Type",
                        choices=list(MODEL_TYPE_TO_EMOJI.keys()),
                        value="💬 : chat models (RLHF, DPO, IFT, ...)",
                        interactive=True
                    )
                    weight_type_box = gr.Dropdown(
                        label="Weight Type",
                        choices=["Original", "Adapter", "Delta"],
                        value="Original",
                        interactive=True
                    )
                with gr.Row():
                    precision_box = gr.Dropdown(
                        label="Precision",
                        choices=["float16", "bfloat16", "8bit", "4bit"],
                        value="bfloat16",
                        interactive=True
                    )
                    base_model_box = gr.Textbox(
                        label="Base Model (if adapter or delta weights)",
                        placeholder="(Optional) e.g. myorg/base-model",
                        interactive=True
                    )
                with gr.Row():
                    chat_template_box = gr.Radio(
                        label="Evaluate using chat-template?",
                        choices=["Yes", "No"],
                        value="No",
                        interactive=True
                    )

                submit_btn = gr.Button("Submit Model")
                submit_out = gr.Markdown()

                submit_btn.click(
                    fn=submit_model,
                    inputs=[model_name_box, base_model_box, revision_box, precision_box, weight_type_box, model_type_box, chat_template_box],
                    outputs=submit_out
                )

                df_pending  = load_requests("pending")
                df_running  = load_requests("running")
                df_finished2= load_requests("finished")
                df_failed   = load_requests("failed")

                gr.Markdown("## Evaluation Status")
                with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
                    if not df_pending.empty:
                        gr.Dataframe(df_pending)
                    else:
                        gr.Markdown("No pending evaluations.")
                with gr.Accordion(f"Running Evaluations ({len(df_running)})", open=False):
                    if not df_running.empty:
                        gr.Dataframe(df_running)
                    else:
                        gr.Markdown("No running evaluations.")
                with gr.Accordion(f"Finished Evaluations ({len(df_finished2)})", open=False):
                    if not df_finished2.empty:
                        gr.Dataframe(df_finished2)
                    else:
                        gr.Markdown("No finished evaluations.")
                with gr.Accordion(f"Failed Evaluations ({len(df_failed)})", open=False):
                    if not df_failed.empty:
                        gr.Dataframe(df_failed)
                    else:
                        gr.Markdown("No failed evaluations.")

                gr.Markdown(ABOUT_SECTION)

        with gr.Row():
            with gr.Accordion("📙 Citation", open=False):
                citation_box = gr.Textbox(
                    value=CITATION_BUTTON_TEXT,
                    label=CITATION_BUTTON_LABEL,
                    lines=9,
                    elem_id="citation-button",
                    show_copy_button=True
                )
                
        gr.HTML(BOTTOM_LOGO)
        
    demo.queue(default_concurrency_limit=40).launch()

if __name__ == "__main__":
    main()