src/dataloading.py

from huggingface_hub import HfApi
from functools import lru_cache


def get_leaderboard_models():
    api = HfApi()
    
    # List all datasets in the open-llm-leaderboard organization
    datasets = api.list_datasets(author="open-llm-leaderboard")
    
    models = []
    for dataset in datasets:
        if dataset.id.endswith("-details"):
            # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
            model_part = dataset.id.split("/")[-1].replace("-details", "")
            provider, model = model_part.split("__", 1)
            models.append(f"{provider}/{model}")
    
    # Example models
    models = ["meta_llama/Llama-3.1-70B-Instruct"]

    return sorted(models)



@lru_cache(maxsize=1)
def get_leaderboard_models_cached():
    return get_leaderboard_models()


def get_leaderboard_datasets():
    return [
        "ai2_arc",
        "hellaswag",
        "mmlu",
        "truthful_qa",
        "winogrande",
        "gsm8k"
    ]