[Add] filter gated models

app.py

@@ -87,7 +87,7 @@ def update_datasets_based_on_models(selected_models, current_dataset):
         return gr.update(choices=[], value=None)
 
 links_markdown = """
-[📄 Paper](https://arxiv.org/pdf/%3CARXIV%20PAPER%20ID%3E.pdf)   |  
+[📄 Paper](https://arxiv.org/abs/6181841)   |  
 [☯ Homepage](https://model-similarity.github.io/)   |  
 [🐱 Code](https://github.com/model-similarity/lm-similarity)   |  
 [🐍 pip install lm-sim](https://pypi.org/project/lm-sim/)   |  
@@ -103,6 +103,7 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
         dataset_dropdown = gr.Dropdown(
             choices=get_leaderboard_datasets(None),
             label="Select Dataset",
+            value="mmlu_pro",
             filterable=True,
             interactive=True,
             allow_custom_value=False,
@@ -117,6 +118,7 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
     model_dropdown = gr.Dropdown(
         choices=get_leaderboard_models_cached(),
         label="Select Models",
+        value=["Qwen/Qwen2.5-"],
         multiselect=True,
         filterable=True,
         allow_custom_value=False,
@@ -154,8 +156,8 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
 - **Datasets**: [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard#/) benchmark datasets \n
     - Some datasets are not multiple-choice - for these, the metrics are not applicable. \n
 - **Models**: Open LLM Leaderboard models \n
-    - Every model is gated on Hugging Face and access has to be requested. \n
-    - We requested access to the most popular models, but some may be missing. \n
+    - Every model evaluation is gated on Hugging Face and access has to be requested. \n
+    - We requested access for the most popular models, but some may be missing. \n
 - **Metrics**: Kappa_p (probabilistic), Kappa_p (deterministic), Error Consistency""")
 
 if __name__ == "__main__":

src/dataloading.py

@@ -14,13 +14,19 @@ def get_leaderboard_models():
     models = []
     for dataset in datasets:
         if dataset.id.endswith("-details"):
-            # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
-            model_part = dataset.id.split("/")[-1].replace("-details", "")
-            if "__" in model_part:
-                provider, model = model_part.split("__", 1)
-                models.append(f"{provider}/{model}")
-            else:
-                models.append(model_part)
+            dataset_id = dataset.id
+            try:
+                # Check if the dataset can be loaded
+                check_gated = datasets.get_dataset_config_names(dataset_id)
+                # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
+                model_part = dataset.id.split("/")[-1].replace("-details", "")
+                if "__" in model_part:
+                    provider, model = model_part.split("__", 1)
+                    models.append(f"{provider}/{model}")
+                else:
+                    models.append(model_part)
+            except Exception as e:
+                pass
     
     return sorted(models)