[Add] create heatmaps for multiselection

app.py

@@ -1,68 +1,77 @@
 import gradio as gr
-
+import plotly.graph_objects as go
+import numpy as np
 
 
 from src.dataloading import get_leaderboard_models_cached, get_leaderboard_datasets
 
 
-def create_demo():
-    # Fetch data once on startup (cache this in production)
-    models = get_leaderboard_models_cached()
-    datasets = get_leaderboard_datasets()
+def create_heatmap(selected_models, benchmark):
+    if not selected_models:
+        return gr.update(visible=False)
+    
+    # Generate random similarity matrix
+    size = len(selected_models)
+    similarities = np.random.rand(size, size)
+    
+    # Create symmetric matrix (for demo purposes)
+    similarities = (similarities + similarities.T) / 2
+    
+    # Create heatmap with Plotly
+    fig = go.Figure(data=go.Heatmap(
+        z=similarities,
+        x=selected_models,
+        y=selected_models,
+        colorscale='Viridis',
+        hoverongaps=False
+    ))
+    
+    fig.update_layout(
+        title=f"Model Similarity for {benchmark}",
+        xaxis_title="Models",
+        yaxis_title="Models",
+        height=600,
+        width=800
+    )
+    
+    return fig
 
-    with gr.Blocks(title="LLM Similarity Analyzer") as demo:
-        gr.Markdown("## Compare Models/Datasets from Open LLM Leaderboard")
-        
-        with gr.Row():
-            model_dropdown = gr.Dropdown(
-                choices=models,
-                label="Select Model",
-                filterable=True,
-                interactive=True,
-                allow_custom_value=False,
-                info="Search models from Open LLM Leaderboard"
-            )
-            
-            dataset_dropdown = gr.Dropdown(
-                choices=datasets,
+with gr.Blocks(title="LLM Similarity Analyzer") as demo:
+    gr.Markdown("## Model Similarity Comparison Tool")
+    
+    # Model selection section
+    with gr.Row():
+        dataset_dropdown = gr.Dropdown(
+                choices=get_leaderboard_datasets(),
                 label="Select Dataset",
                 filterable=True,
                 interactive=True,
                 info="Leaderboard benchmark datasets"
             )
 
-        # Add your similarity computation and visualization components here
-        # Example placeholder:
-        similarity_output = gr.Textbox(label="Similarity Score")
-        compute_btn = gr.Button("Compute Similarity")
-        
-        def compute_similarity(model, dataset):
-            # Replace with your actual similarity metric
-            return f"Similarity between {model} and {dataset}: {0.85:.2f}"
-        
-        compute_btn.click(
-            fn=compute_similarity,
-            inputs=[model_dropdown, dataset_dropdown],
-            outputs=similarity_output
+        model_dropdown = gr.Dropdown(
+            choices=get_leaderboard_models_cached(),
+            label="Select Models",
+            multiselect=True,
+            filterable=True,
+            allow_custom_value=False,
+            info="Search and select multiple models (click selected models to remove)"
         )
-        
-    return demo
-
-
-def create_demo_with_refresh():
-    demo = create_demo()
     
-    with demo:
-        refresh_btn = gr.Button("Refresh Model List")
-        def refresh_models():
-            return gr.Dropdown(choices=get_leaderboard_models_cached())
-        
-        refresh_btn.click(
-            fn=refresh_models,
-            outputs=model_dropdown
-        )
+    # Heatmap display
+    heatmap = gr.Plot(
+        label="Similarity Heatmap",
+        visible=False,
+        container=False
+    )
     
-    return demo
+    # Interactive updates
+    model_dropdown.input(
+        fn=create_heatmap,
+        inputs=(model_dropdown, dataset_dropdown),
+        outputs=heatmap
+    )
+
 
-demo = create_demo_with_refresh()
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()

src/dataloading.py

@@ -5,16 +5,16 @@ from functools import lru_cache
 def get_leaderboard_models():
     api = HfApi()
     
-    # List all files in the "open_llm_leaderboard" directory of the Space
+    # List all files in the repository
     files = api.list_repo_files(
         repo_id="open-llm-leaderboard/open_llm_leaderboard",
-        repo_type="space",
-        path="open_llm_leaderboard"
+        repo_type="space"
     )
     
     models = []
     for file in files:
-        if "-details" in file and "__" in file:
+        # Filter files in the "open_llm_leaderboard" directory
+        if file.startswith("open_llm_leaderboard/") and "-details" in file and "__" in file:
             # Extract provider and model name from filename
             filename = file.split("/")[-1].replace("-details", "")
             provider, model = filename.split("__", 1)
@@ -23,6 +23,7 @@ def get_leaderboard_models():
     return sorted(list(set(models)))  # Remove duplicates
 
 
+
 @lru_cache(maxsize=1)
 def get_leaderboard_models_cached():
     return get_leaderboard_models()