Update SDK version and refactor leaderboard tabs for improved structure

README.md

@@ -4,7 +4,7 @@ emoji: 📊
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 5.24.0
+sdk_version: 5.31.0
 app_file: app.py
 pinned: true
 short_description: The only leaderboard you will require for your RAG needs 🏆

app.py

@@ -1,8 +1,10 @@
 import gradio as gr
-from retrieval_leaderboard import create_retrieval_tab
-from reranking_leaderboard import create_reranking_tab
-from llm_in_context_leaderboard import create_llm_in_context_tab
 from dotenv import load_dotenv
+
+from llm_in_context_leaderboard import create_llm_in_context_tab
+from reranking_leaderboard import create_reranking_tab
+from retrieval_leaderboard import create_retrieval_tab
+
 load_dotenv()
 
 HEADER = """<div style="text-align: center; margin-bottom: 20px;">
@@ -30,18 +32,19 @@ CITATION_BUTTON_TEXT = """
 }
 """
 
-def main():
-    with gr.Blocks() as demo:
+
+def create_app():
+    with gr.Blocks() as app:
         gr.HTML(HEADER)
-        
+
         with gr.Tabs():
             with gr.Tab("🕵️‍♂️ Retrieval"):
                 create_retrieval_tab()
 
             with gr.Tab("📊 Reranking"):
                 create_reranking_tab()
-                    
-            # with gr.Tab("📊 LLM in Context"):
+
+            # with gr.Tab("🧠 LLM in Context"):
             #     create_llm_in_context_tab()
 
             with gr.Row():
@@ -53,8 +56,9 @@ def main():
                         elem_id="citation-button",
                         show_copy_button=True,
                     )
+    return app
 
-    demo.launch()
 
 if __name__ == "__main__":
-    main()
+    app = create_app()
+    app.queue().launch()

leaderboard_tab.py

@@ -1,40 +1,46 @@
 import gradio as gr
-import pandas as pd
 from fuzzywuzzy import fuzz
+
 from utils import submit_gradio_module
 
+
 def search_leaderboard(df, model_name, columns_to_show, threshold=95):
     """
     Search the leaderboard for models matching the search term using fuzzy matching.
-    
+
     Args:
         df: The dataframe containing all leaderboard data
         model_name: The search term to find models
         columns_to_show: List of columns to include in the result
         threshold: Minimum similarity threshold (default: 95)
-    
+
     Returns:
         Filtered dataframe with only matching models and selected columns
     """
     if not model_name.strip():
         return df.loc[:, columns_to_show]
     search_name = model_name.lower()  # compute once for efficiency
+
     def calculate_similarity(row):
         return fuzz.partial_ratio(search_name, row["Model"].lower())
+
     filtered_df = df.copy()
     filtered_df["similarity"] = filtered_df.apply(calculate_similarity, axis=1)
-    filtered_df = filtered_df[filtered_df["similarity"] >= threshold].sort_values('similarity', ascending=False)
-    filtered_df = filtered_df.drop('similarity', axis=1).loc[:, columns_to_show]
+    filtered_df = filtered_df[filtered_df["similarity"] >= threshold].sort_values(
+        "similarity", ascending=False
+    )
+    filtered_df = filtered_df.drop("similarity", axis=1).loc[:, columns_to_show]
     return filtered_df
 
+
 def update_columns_to_show(df, columns_to_show):
     """
     Update the displayed columns in the dataframe.
-    
+
     Args:
         df: The dataframe to update
         columns_to_show: List of columns to include
-    
+
     Returns:
         gradio.update object with the updated dataframe
     """
@@ -49,10 +55,19 @@ def update_columns_to_show(df, columns_to_show):
             columns_widths.append(150)
     return gr.update(value=dummy_df, column_widths=columns_widths)
 
-def create_leaderboard_tab(df, initial_columns_to_show, search_function, update_function, about_section, task_type):
+
+def create_leaderboard_tab(
+    df,
+    initial_columns_to_show,
+    search_function,
+    update_function,
+    about_section,
+    task_type,
+    model_param_limit=2000,
+):
     """
     Create a complete leaderboard tab with search, column selection, and data display.
-    
+
     Args:
         df: The dataframe containing the leaderboard data
         initial_columns_to_show: Initial list of columns to display
@@ -60,31 +75,32 @@ def create_leaderboard_tab(df, initial_columns_to_show, search_function, update_
         update_function: Function to handle column updates
         about_section: Markdown text for the About tab
         task_type: Type of the task ("Retriever" or "Reranker")
-    
+
     Returns:
         A gradio Tabs component with the complete leaderboard interface
     """
-    columns_widths = [80 if col == "Rank" else 400 if col == "Model" else 150 for col in initial_columns_to_show]
-    
+    columns_widths = [
+        80 if col == "Rank" else 400 if col == "Model" else 150
+        for col in initial_columns_to_show
+    ]
+
     with gr.Tabs() as tabs:
         with gr.Tab("👑 Leaderboard"):
             with gr.Column():
                 with gr.Row(equal_height=True):
                     search_box = gr.Textbox(
-                        placeholder="Search for models...", 
-                        label="Search (You can also press Enter to search)", 
-                        scale=5
+                        placeholder="Search for models...",
+                        label="Search (You can also press Enter to search)",
+                        scale=5,
                     )
                     search_button = gr.Button(
-                        value="Search",
-                        variant="primary",
-                        scale=1
+                        value="Search", variant="primary", scale=1
                     )
                 columns_to_show_input = gr.CheckboxGroup(
                     label="Columns to Show",
                     choices=df.columns.tolist(),
                     value=initial_columns_to_show,
-                    scale=4
+                    scale=4,
                 )
 
             leaderboard = gr.Dataframe(
@@ -93,30 +109,28 @@ def create_leaderboard_tab(df, initial_columns_to_show, search_function, update_
                 wrap=True,
                 show_fullscreen_button=True,
                 interactive=False,
-                column_widths=columns_widths
+                column_widths=columns_widths,
             )
-            
+
             # Connect events
             search_box.submit(
                 search_function,
                 inputs=[search_box, columns_to_show_input],
-                outputs=leaderboard
+                outputs=leaderboard,
             )
             columns_to_show_input.select(
-                update_function,
-                inputs=columns_to_show_input,
-                outputs=leaderboard
+                update_function, inputs=columns_to_show_input, outputs=leaderboard
             )
             search_button.click(
                 search_function,
                 inputs=[search_box, columns_to_show_input],
-                outputs=leaderboard
+                outputs=leaderboard,
             )
-            
+
         with gr.Tab("🏵️ Submit"):
-            submit_gradio_module(task_type)
+            submit_gradio_module(task_type, model_param_limit=model_param_limit)
 
         with gr.Tab("ℹ️ About"):
             gr.Markdown(about_section)
-    
+
     return tabs

reranking_leaderboard.py

@@ -1,6 +1,11 @@
 from pathlib import Path
+
+from leaderboard_tab import (
+    create_leaderboard_tab,
+    search_leaderboard,
+    update_columns_to_show,
+)
 from utils import load_json_results
-from leaderboard_tab import search_leaderboard, update_columns_to_show, create_leaderboard_tab
 
 # Constants
 RERANKER_ABOUT_SECTION = """
@@ -37,45 +42,54 @@ All metrics are averaged across multiple evaluation datasets to provide a compre
 # Global variables
 reranking_df = None
 
-def load_reranking_results(prepare_for_display=False, sort_col=None, drop_cols=None):
-    dataframe_path = Path(__file__).parent / "results" / "reranking_results.json"
-    return load_json_results(
-        dataframe_path,
-        prepare_for_display=prepare_for_display,
-        sort_col=sort_col,
-        drop_cols=drop_cols
-    )
 
 def load_reranking_leaderboard():
     """Load and prepare the reranking leaderboard data"""
     global reranking_df
-    
+
     # Prepare reranking dataframe
-    reranking_df = load_reranking_results(True, sort_col="Average Score", drop_cols=["Revision", "Precision", "Task"])
+    dataframe_path = Path(__file__).parent / "results" / "reranking_results.json"
+    reranking_df = load_json_results(
+        dataframe_path,
+        prepare_for_display=True,
+        sort_col="Average Score",
+        drop_cols=["Revision", "Task"],
+    )
     reranking_df.insert(0, "Rank", range(1, 1 + len(reranking_df)))
-    
+
     return reranking_df
 
+
 def reranking_search_leaderboard(model_name, columns_to_show):
     """Search function for reranking leaderboard"""
     return search_leaderboard(reranking_df, model_name, columns_to_show)
 
+
 def update_reranker_columns_to_show(columns_to_show):
     """Update displayed columns for reranking leaderboard"""
     return update_columns_to_show(reranking_df, columns_to_show)
 
+
 def create_reranking_tab():
     """Create the complete reranking leaderboard tab"""
     global reranking_df
-    
+
     # Load data if not already loaded
-    if (reranking_df is None):
+    if reranking_df is None:
         reranking_df = load_reranking_leaderboard()
-    
+
     # Define default columns to show
-    default_columns = ["Rank", "Model", "Average Score", "Model Size (MB)", "Context Length",
-                      "Embedding Dimension", "Namaa Global Knowledge", "Navid General Knowledge"]
-    
+    default_columns = [
+        "Rank",
+        "Model",
+        "Average Score",
+        "Model Size (MB)",
+        "Context Length",
+        "Embedding Dimension",
+        "Namaa Global Knowledge",
+        "Navid General Knowledge",
+    ]
+
     # Create and return the tab
     return create_leaderboard_tab(
         df=reranking_df,
@@ -83,5 +97,5 @@ def create_reranking_tab():
         search_function=reranking_search_leaderboard,
         update_function=update_reranker_columns_to_show,
         about_section=RERANKER_ABOUT_SECTION,
-        task_type="Reranker"
+        task_type="Reranker",
     )

retrieval_leaderboard.py

@@ -1,6 +1,11 @@
 from pathlib import Path
+
+from leaderboard_tab import (
+    create_leaderboard_tab,
+    search_leaderboard,
+    update_columns_to_show,
+)
 from utils import load_json_results
-from leaderboard_tab import search_leaderboard, update_columns_to_show, create_leaderboard_tab
 
 # Constants
 RETRIEVAL_ABOUT_SECTION = """
@@ -37,45 +42,51 @@ The retrieval evaluation assesses a model's ability to find and retrieve relevan
 # Global variables
 retrieval_df = None
 
-def load_retrieval_results(prepare_for_display=False, sort_col=None, drop_cols=None):
-    dataframe_path = Path(__file__).parent / "results" / "retrieval_results.json"
-    return load_json_results(
-        dataframe_path,
-        prepare_for_display=prepare_for_display,
-        sort_col=sort_col,
-        drop_cols=drop_cols
-    )
 
 def load_retrieval_leaderboard():
     """Load and prepare the retrieval leaderboard data"""
     global retrieval_df
-    
+
     # Prepare retrieval dataframe
-    retrieval_df = load_retrieval_results(True, "Average Score", drop_cols=["Revision", "Precision", "Task"])
+    dataframe_path = Path(__file__).parent / "results" / "retrieval_results.json"
+    retrieval_df = load_json_results(
+        dataframe_path, True, "Average Score", drop_cols=["Revision", "Task"]
+    )
     retrieval_df.insert(0, "Rank", range(1, 1 + len(retrieval_df)))
-    
+
     return retrieval_df
 
+
 def retrieval_search_leaderboard(model_name, columns_to_show):
     """Search function for retrieval leaderboard"""
     return search_leaderboard(retrieval_df, model_name, columns_to_show)
 
+
 def update_retrieval_columns_to_show(columns_to_show):
     """Update displayed columns for retrieval leaderboard"""
     return update_columns_to_show(retrieval_df, columns_to_show)
 
+
 def create_retrieval_tab():
     """Create the complete retrieval leaderboard tab"""
     global retrieval_df
-    
+
     # Load data if not already loaded
     if retrieval_df is None:
         retrieval_df = load_retrieval_leaderboard()
-    
+
     # Define default columns to show
-    default_columns = ["Rank", "Model", "Average Score", "Model Size (MB)", "Context Length",
-                       "Embedding Dimension", "Web Search Dataset", "Islamic Knowledge Dataset"]
-    
+    default_columns = [
+        "Rank",
+        "Model",
+        "Average Score",
+        "Model Size (MB)",
+        "Context Length",
+        "Embedding Dimension",
+        "Web Search Dataset",
+        "Islamic Knowledge Dataset",
+    ]
+
     # Create and return the tab
     return create_leaderboard_tab(
         df=retrieval_df,
@@ -83,5 +94,5 @@ def create_retrieval_tab():
         search_function=retrieval_search_leaderboard,
         update_function=update_retrieval_columns_to_show,
         about_section=RETRIEVAL_ABOUT_SECTION,
-        task_type="Retriever"
+        task_type="Retriever",
     )