Upload 16 files

app.py

@@ -0,0 +1,284 @@
+"""
+Main application for Dynamic Highscores system.
+
+This file integrates all components into a unified application.
+"""
+
+import os
+import gradio as gr
+import threading
+import time
+from database_schema import DynamicHighscoresDB
+from auth import HuggingFaceAuth
+from benchmark_selection import BenchmarkSelector, create_benchmark_selection_ui
+from evaluation_queue import EvaluationQueue, create_model_submission_ui
+from leaderboard import Leaderboard, create_leaderboard_ui
+from sample_benchmarks import add_sample_benchmarks
+
+# Initialize components in main thread
+db = DynamicHighscoresDB()
+auth_manager = HuggingFaceAuth(db)
+benchmark_selector = BenchmarkSelector(db, auth_manager)
+evaluation_queue = EvaluationQueue(db, auth_manager)
+leaderboard = Leaderboard(db)
+
+# Initialize sample benchmarks if none exist
+print("Checking for existing benchmarks...")
+benchmarks = db.get_benchmarks()
+if not benchmarks or len(benchmarks) == 0:
+    print("No benchmarks found. Adding sample benchmarks...")
+    try:
+        # Make sure the database path is clear
+        print(f"Database path: {db.db_path}")
+        
+        # Import and call the function directly 
+        num_added = add_sample_benchmarks()
+        print(f"Added {num_added} sample benchmarks.")
+    except Exception as e:
+        print(f"Error adding sample benchmarks: {str(e)}")
+        # Try direct DB insertion as fallback
+        try:
+            print("Attempting direct benchmark insertion...")
+            db.add_benchmark(
+                name="MMLU (Massive Multitask Language Understanding)",
+                dataset_id="cais/mmlu",
+                description="Tests knowledge across 57 subjects"
+            )
+            print("Added fallback benchmark.")
+        except Exception as inner_e:
+            print(f"Fallback insertion failed: {str(inner_e)}")
+else:
+    print(f"Found {len(benchmarks)} existing benchmarks.")
+
+# Custom CSS with theme awareness
+css = """
+/* Theme-adaptive colored info box */
+.info-text {
+    background-color: rgba(53, 130, 220, 0.1);
+    padding: 12px;
+    border-radius: 8px;
+    border-left: 4px solid #3498db;
+    margin: 12px 0;
+}
+
+/* High-contrast text for elements - works in light and dark themes */
+.info-text, .header, .footer, .tab-content, 
+button, input, textarea, select, option, 
+.gradio-container *, .markdown-text {
+    color: var(--text-color, inherit) !important;
+}
+
+/* Container styling */
+.container {
+    max-width: 1200px;
+    margin: 0 auto;
+}
+
+/* Header styling */
+.header {
+    text-align: center;
+    margin-bottom: 20px;
+    font-weight: bold;
+    font-size: 24px;
+}
+
+/* Footer styling */
+.footer {
+    text-align: center;
+    margin-top: 40px;
+    padding: 20px;
+    border-top: 1px solid var(--border-color-primary, #eee);
+}
+
+/* Login section styling */
+.login-section {
+    padding: 10px;
+    margin-bottom: 15px;
+    border-radius: 8px;
+    background-color: rgba(250, 250, 250, 0.1);
+    text-align: center;
+}
+
+/* Login button styling */
+.login-button {
+    background-color: #4CAF50 !important;
+    color: white !important;
+    font-weight: bold;
+}
+
+/* Force high contrast on specific input areas */
+input[type="text"], input[type="password"], textarea {
+    background-color: var(--background-fill-primary) !important;
+    color: var(--body-text-color) !important;
+}
+
+/* Force text visibility in multiple contexts */
+.gradio-markdown p, .gradio-markdown h1, .gradio-markdown h2, 
+.gradio-markdown h3, .gradio-markdown h4, .gradio-markdown li {
+    color: var(--body-text-color) !important;
+}
+
+/* Fix dark mode text visibility */
+@media (prefers-color-scheme: dark) {
+    input, textarea, select {
+        color: #ffffff !important;
+    }
+    
+    ::placeholder {
+        color: rgba(255, 255, 255, 0.5) !important;
+    }
+}
+"""
+
+# JavaScript login implementation
+def js_login_script():
+    space_host = os.environ.get("SPACE_HOST", "localhost:7860")
+    redirect_uri = f"https://{space_host}"
+    
+    return f"""
+    <script src="https://unpkg.com/@huggingface/[email protected]/dist/index.umd.min.js"></script>
+    <script>
+    (async function() {{
+        const HfHub = window.HfHub;
+        try {{
+            // Check if we're returning from OAuth redirect
+            const oauthResult = await HfHub.oauthHandleRedirectIfPresent();
+            
+            if (oauthResult) {{
+                console.log("User logged in:", oauthResult);
+                
+                // Store the user info in localStorage
+                localStorage.setItem("hf_user", JSON.stringify(oauthResult.userInfo));
+                localStorage.setItem("hf_token", oauthResult.accessToken);
+                
+                // Update the UI to show logged in state
+                document.getElementById("login-status").textContent = "Logged in as: " + oauthResult.userInfo.name;
+                document.getElementById("login-button").style.display = "none";
+                
+                // Refresh the page to update server-side state
+                setTimeout(() => window.location.reload(), 1000);
+            }}
+        }} catch (error) {{
+            console.error("OAuth error:", error);
+        }}
+        
+        // Check if user is already logged in from localStorage
+        const storedUser = localStorage.getItem("hf_user");
+        if (storedUser) {{
+            try {{
+                const userInfo = JSON.parse(storedUser);
+                document.getElementById("login-status").textContent = "Logged in as: " + userInfo.name;
+                document.getElementById("login-button").style.display = "none";
+            }} catch (e) {{
+                console.error("Error parsing stored user:", e);
+            }}
+        }}
+        
+        // Setup login button
+        document.getElementById("login-button").addEventListener("click", async function() {{
+            try {{
+                const loginUrl = await HfHub.oauthLoginUrl({{
+                    redirectUrl: "{redirect_uri}",
+                    scopes: ["openid", "profile"]
+                }});
+                window.location.href = loginUrl;
+            }} catch (error) {{
+                console.error("Error generating login URL:", error);
+                alert("Error starting login process. Please try again.");
+            }}
+        }});
+    }})();
+    </script>
+    """
+
+# Simple manual authentication check
+def check_user(request: gr.Request):
+    if request:
+        username = request.headers.get("HF-User")
+        if username:
+            # User is logged in via HF Spaces
+            print(f"User logged in: {username}")
+            user = db.get_user_by_username(username)
+            if not user:
+                # Create user if they don't exist
+                print(f"Creating new user: {username}")
+                is_admin = (username == "Quazim0t0")
+                db.add_user(username, username, is_admin)
+                user = db.get_user_by_username(username)
+            return username
+    return None
+
+# Start evaluation queue worker
+def start_queue_worker():
+    # Wait a moment to ensure app is initialized
+    time.sleep(2)
+    try:
+        print("Starting evaluation queue worker...")
+        evaluation_queue.start_worker()
+    except Exception as e:
+        print(f"Error starting queue worker: {e}")
+
+# Create Gradio app
+with gr.Blocks(css=css, title="Dynamic Highscores") as app:
+    # State to track user
+    user_state = gr.State(None)
+    
+    # Login section
+    with gr.Row(elem_classes=["login-section"]):
+        with gr.Column():
+            gr.HTML("""
+            <div style="display: flex; justify-content: space-between; align-items: center;">
+                <div id="login-status">Not logged in</div>
+                <button id="login-button" style="padding: 8px 16px; background-color: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer;">Login with HuggingFace</button>
+            </div>
+            """)
+    
+    # Add the JS login script
+    gr.HTML(js_login_script())
+    
+    gr.Markdown("# 🏆 Dynamic Highscores", elem_classes=["header"])
+    gr.Markdown("""
+    *Not Active yet, Check back soon!* Welcome to Dynamic Highscores - a community benchmark platform for evaluating and comparing language models.
+    
+    - **Add your own benchmarks** from HuggingFace datasets
+    - **Submit your models** for CPU-only evaluation
+    - **Compare performance** across different models and benchmarks
+    - **Filter results** by model type (Merge, Agent, Reasoning, Coding, etc.)
+    """, elem_classes=["info-text"])
+    
+    # Main tabs
+    with gr.Tabs() as tabs:
+        with gr.TabItem("📊 Leaderboard", id=0):
+            leaderboard_ui = create_leaderboard_ui(leaderboard, db)
+        
+        with gr.TabItem("🚀 Submit Model", id=1):
+            submission_ui = create_model_submission_ui(evaluation_queue, auth_manager, db)
+        
+        with gr.TabItem("🔍 Benchmarks", id=2):
+            benchmark_ui = create_benchmark_selection_ui(benchmark_selector, auth_manager)
+    
+    gr.Markdown("""
+    ### About Dynamic Highscores
+    
+    This platform allows users to select benchmarks from HuggingFace datasets and evaluate models against them.
+    Each user can submit one benchmark per day (admin users are exempt from this limit).
+    All evaluations run on CPU only to ensure fair comparisons.
+    
+    Created by Quazim0t0
+    """, elem_classes=["footer"])
+    
+    # Check login on page load
+    app.load(
+        fn=check_user,
+        inputs=[],
+        outputs=[user_state]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    # Start queue worker in a separate thread
+    queue_thread = threading.Thread(target=start_queue_worker)
+    queue_thread.daemon = True
+    queue_thread.start()
+    
+    app.launch()

auth.py

@@ -0,0 +1,326 @@
+"""
+Authentication module for Dynamic Highscores system.
+
+This module handles user authentication with HuggingFace,
+user session management, and access control.
+"""
+
+import os
+import json
+import time
+import requests
+import gradio as gr
+from huggingface_hub import HfApi, login
+from functools import wraps
+
+class HuggingFaceAuth:
+    """Authentication manager for HuggingFace integration."""
+    
+    def __init__(self, db_manager):
+        """Initialize the authentication manager.
+        
+        Args:
+            db_manager: Database manager instance for user storage
+        """
+        self.db_manager = db_manager
+        self.hf_api = HfApi()
+        self.admin_username = os.environ.get("ADMIN_USERNAME", "Quazim0t0")
+        self.running_in_space = 'SPACE_ID' in os.environ
+    
+    def login_user(self, token):
+        """Log in a user with their HuggingFace token.
+        
+        Args:
+            token: HuggingFace API token
+            
+        Returns:
+            dict: User information if login successful, None otherwise
+        """
+        try:
+            # Validate token with HuggingFace
+            login(token=token, add_to_git_credential=False)
+            
+            # Get user info from HuggingFace
+            user_info = self.hf_api.whoami(token=token)
+            
+            if not user_info:
+                return None
+            
+            # Check if user exists in our database, create if not
+            username = user_info.get("name", user_info.get("fullname", ""))
+            hf_user_id = user_info.get("id", "")
+            
+            if not hf_user_id:
+                return None
+            
+            # Check if this is the admin account
+            is_admin = (username == self.admin_username)
+            
+            # Add or get user from database
+            user_id = self.db_manager.add_user(username, hf_user_id, is_admin)
+            
+            # Get complete user info from database
+            user = self.db_manager.get_user(hf_user_id)
+            
+            if user:
+                # Add token to user info for session only (not stored in database)
+                user['token'] = token
+                return user
+                
+            return None
+        except Exception as e:
+            print(f"Login error: {e}")
+            return None
+    
+    def check_login(self, request: gr.Request):
+        """Check if a user is logged in from a Gradio request.
+        
+        Args:
+            request: Gradio request object
+            
+        Returns:
+            dict: User information if logged in, None otherwise
+        """
+        if not request:
+            return None
+        
+        # First, check if we're in a HuggingFace Space with OAuth
+        if self.running_in_space:
+            # Check for HF-User header from Space OAuth
+            username = request.headers.get("HF-User")
+            if username:
+                # Check if user exists in our database, create if not
+                user = self.db_manager.get_user_by_username(username)
+                if not user:
+                    # Create a new user 
+                    is_admin = (username == self.admin_username)
+                    user_id = self.db_manager.add_user(username, username, is_admin)
+                    user = self.db_manager.get_user_by_username(username)
+                return user
+        
+        # Fallback to token-based auth for local development
+        token = request.cookies.get("hf_token")
+        
+        if not token:
+            return None
+        
+        try:
+            # Validate token with HuggingFace
+            user_info = self.hf_api.whoami(token=token)
+            
+            if not user_info:
+                return None
+            
+            # Get user from database
+            hf_user_id = user_info.get("id", "")
+            user = self.db_manager.get_user(hf_user_id)
+            
+            if user:
+                # Add token to user info for session only (not stored in database)
+                user['token'] = token
+                return user
+                
+            return None
+        except Exception as e:
+            print(f"Check login error: {e}")
+            return None
+    
+    def require_login(self, func):
+        """Decorator to require login for a function.
+        
+        Args:
+            func: Function to decorate
+            
+        Returns:
+            Function: Decorated function that requires login
+        """
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            # Find the request argument
+            request = None
+            for arg in args:
+                if isinstance(arg, gr.Request):
+                    request = arg
+                    break
+            
+            if not request and 'request' in kwargs:
+                request = kwargs['request']
+            
+            if not request:
+                return "Please log in to access this feature."
+            
+            # Check if user is logged in
+            user = self.check_login(request)
+            
+            if not user:
+                return "Please log in to access this feature."
+            
+            # Add user to kwargs
+            kwargs['user'] = user
+            
+            # Call the original function
+            return func(*args, **kwargs)
+        
+        return wrapper
+    
+    def require_admin(self, func):
+        """Decorator to require admin privileges for a function.
+        
+        Args:
+            func: Function to decorate
+            
+        Returns:
+            Function: Decorated function that requires admin privileges
+        """
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            # Find the request argument
+            request = None
+            for arg in args:
+                if isinstance(arg, gr.Request):
+                    request = arg
+                    break
+            
+            if not request and 'request' in kwargs:
+                request = kwargs['request']
+            
+            if not request:
+                return "Admin access required."
+            
+            # Check if user is logged in
+            user = self.check_login(request)
+            
+            if not user:
+                return "Admin access required."
+            
+            # Check if user is admin
+            if not user.get('is_admin', False):
+                return "Admin access required."
+            
+            # Add user to kwargs
+            kwargs['user'] = user
+            
+            # Call the original function
+            return func(*args, **kwargs)
+        
+        return wrapper
+    
+    def can_submit_benchmark(self, user_id):
+        """Check if a user can submit a benchmark today.
+        
+        Args:
+            user_id: User ID to check
+            
+        Returns:
+            bool: True if user can submit, False otherwise
+        """
+        return self.db_manager.can_submit_today(user_id)
+    
+    def update_submission_date(self, user_id):
+        """Update the last submission date for a user.
+        
+        Args:
+            user_id: User ID to update
+        """
+        self.db_manager.update_submission_date(user_id)
+
+# Authentication UI components
+def create_login_ui():
+    """Create the login UI components.
+    
+    Returns:
+        tuple: (login_button, logout_button, user_info)
+    """
+    with gr.Row():
+        with gr.Column(scale=3):
+            # If running in a HuggingFace Space, use their OAuth
+            if 'SPACE_ID' in os.environ:
+                login_button = gr.Button("Login with HuggingFace", visible=False)
+                logout_button = gr.Button("Logout", visible=False)
+            else:
+                # For local development, use token-based login
+                login_button = gr.Button("Login with HuggingFace Token")
+                logout_button = gr.Button("Logout", visible=False)
+        
+        with gr.Column(scale=2):
+            user_info = gr.Markdown("Checking login status...")
+    
+    return login_button, logout_button, user_info
+
+def login_handler(auth_manager):
+    """Handle login button click.
+    
+    Args:
+        auth_manager: Authentication manager instance
+        
+    Returns:
+        tuple: JS to redirect to login and updated UI visibility
+    """
+    # This is only used for local development
+    # For HuggingFace Spaces, the built-in OAuth is used
+    return (
+        gr.update(visible=False),  # Hide login button
+        gr.update(visible=True),   # Show logout button
+        "Redirecting to login...",
+        """
+        <script>
+        // Open a popup window for token entry
+        function promptForToken() {
+            const token = prompt("Enter your HuggingFace token:");
+            if (token) {
+                // Set the token as a cookie
+                document.cookie = "hf_token=" + token + "; path=/; SameSite=Strict";
+                // Reload the page to apply the token
+                window.location.reload();
+            }
+        }
+        
+        // Call the function
+        promptForToken();
+        </script>
+        """
+    )
+
+def logout_handler():
+    """Handle logout button click.
+    
+    Returns:
+        tuple: Updated UI components visibility and user info
+    """
+    # Clear token cookie in JavaScript
+    return (
+        gr.update(visible=True),   # Show login button
+        gr.update(visible=False),  # Hide logout button
+        "Logged out",
+        """
+        <script>
+        // Clear the token cookie
+        document.cookie = "hf_token=; path=/; max-age=0; SameSite=Strict";
+        // Reload the page
+        window.location.reload();
+        </script>
+        """
+    )
+
+def setup_auth_handlers(login_button, logout_button, user_info, auth_manager):
+    """Set up event handlers for authentication UI components.
+    
+    Args:
+        login_button: Login button component
+        logout_button: Logout button component
+        user_info: User info component
+        auth_manager: Authentication manager instance
+    """
+    # Only add event handlers if not running in a HuggingFace Space
+    if 'SPACE_ID' not in os.environ:
+        login_button.click(
+            fn=lambda: login_handler(auth_manager),
+            inputs=[],
+            outputs=[login_button, logout_button, user_info, gr.HTML()]
+        )
+        
+        logout_button.click(
+            fn=logout_handler,
+            inputs=[],
+            outputs=[login_button, logout_button, user_info, gr.HTML()]
+        )

benchmark_selection.py

@@ -0,0 +1,573 @@
+"""
+Benchmark selection module for Dynamic Highscores system.
+
+This module handles browsing, selection, and loading of HuggingFace datasets
+to be used as benchmarks for model evaluation.
+"""
+
+import os
+import json
+import gradio as gr
+from huggingface_hub import HfApi, list_datasets
+from datasets import load_dataset, get_dataset_config_names
+from functools import partial
+
+class BenchmarkSelector:
+    """Benchmark selection manager for HuggingFace datasets."""
+    
+    def __init__(self, db_manager, auth_manager):
+        """Initialize the benchmark selector.
+        
+        Args:
+            db_manager: Database manager instance for benchmark storage
+            auth_manager: Authentication manager instance for access control
+        """
+        self.db_manager = db_manager
+        self.auth_manager = auth_manager
+        self.hf_api = HfApi()
+        
+        # Common benchmark categories for filtering
+        self.categories = [
+            "All",
+            "Text Generation",
+            "Question Answering",
+            "Summarization",
+            "Translation",
+            "Classification",
+            "Code Generation",
+            "Reasoning",
+            "Math"
+        ]
+        
+        # Common metrics for different benchmark types
+        self.metric_templates = {
+            "Text Generation": ["bleu", "rouge", "meteor"],
+            "Question Answering": ["exact_match", "f1"],
+            "Summarization": ["rouge1", "rouge2", "rougeL"],
+            "Translation": ["bleu", "ter"],
+            "Classification": ["accuracy", "f1", "precision", "recall"],
+            "Code Generation": ["exact_match", "pass@k", "functional_correctness"],
+            "Reasoning": ["accuracy", "consistency"],
+            "Math": ["accuracy", "correct_steps"]
+        }
+    
+    def search_datasets(self, query, category="All", limit=50):
+        """Search for datasets on HuggingFace.
+        
+        Args:
+            query: Search query string
+            category: Dataset category to filter by
+            limit: Maximum number of results to return
+            
+        Returns:
+            list: List of dataset information dictionaries
+        """
+        try:
+            # Apply category filter if not "All"
+            filter_str = None
+            if category != "All":
+                filter_str = f"task_categories:{category}"
+            
+            # Search for datasets
+            datasets = list_datasets(
+                search=query,
+                filter=filter_str,
+                limit=limit
+            )
+            
+            # Format results
+            results = []
+            for dataset in datasets:
+                # Handle cases where description might be missing
+                dataset_description = ""
+                if hasattr(dataset, 'description') and dataset.description:
+                    dataset_description = dataset.description[:200] + "..." if len(dataset.description) > 200 else dataset.description
+                
+                # Handle cases where tags might be missing
+                dataset_tags = []
+                if hasattr(dataset, 'tags'):
+                    dataset_tags = dataset.tags
+                
+                # Handle cases where downloads might be missing
+                dataset_downloads = 0
+                if hasattr(dataset, 'downloads'):
+                    dataset_downloads = dataset.downloads
+                
+                # Handle cases where author might be missing
+                dataset_author = ""
+                if hasattr(dataset, 'author'):
+                    dataset_author = dataset.author
+                
+                results.append({
+                    "id": dataset.id,
+                    "name": dataset.id.split("/")[-1],
+                    "author": dataset_author,
+                    "description": dataset_description,
+                    "tags": dataset_tags,
+                    "downloads": dataset_downloads
+                })
+            
+            return results
+        except Exception as e:
+            print(f"Dataset search error: {e}")
+            return []
+    
+    def get_dataset_info(self, dataset_id):
+        """Get detailed information about a dataset.
+        
+        Args:
+            dataset_id: HuggingFace dataset ID
+            
+        Returns:
+            dict: Dataset information
+        """
+        try:
+            # Get dataset info from HuggingFace
+            dataset_info = self.hf_api.dataset_info(dataset_id)
+            
+            # Get available configurations
+            configs = []
+            try:
+                configs = get_dataset_config_names(dataset_id)
+            except Exception as e:
+                print(f"Error getting dataset configs: {e}")
+            
+            # Handle missing attributes safely
+            dataset_description = ""
+            if hasattr(dataset_info, 'description'):
+                dataset_description = dataset_info.description
+            
+            dataset_citation = ""
+            if hasattr(dataset_info, 'citation'):
+                dataset_citation = dataset_info.citation
+            
+            dataset_tags = []
+            if hasattr(dataset_info, 'tags'):
+                dataset_tags = dataset_info.tags
+            
+            dataset_downloads = 0
+            if hasattr(dataset_info, 'downloads'):
+                dataset_downloads = dataset_info.downloads
+            
+            dataset_author = ""
+            if hasattr(dataset_info, 'author'):
+                dataset_author = dataset_info.author
+            
+            # Format result
+            result = {
+                "id": dataset_info.id,
+                "name": dataset_info.id.split("/")[-1],
+                "author": dataset_author,
+                "description": dataset_description,
+                "citation": dataset_citation,
+                "configs": configs,
+                "tags": dataset_tags,
+                "downloads": dataset_downloads
+            }
+            
+            return result
+        except Exception as e:
+            print(f"Dataset info error: {e}")
+            return None
+    
+    def load_dataset_sample(self, dataset_id, config=None, split="train", sample_size=5):
+        """Load a sample from a dataset.
+        
+        Args:
+            dataset_id: HuggingFace dataset ID
+            config: Dataset configuration name
+            split: Dataset split to sample from
+            sample_size: Number of samples to load
+            
+        Returns:
+            dict: Dataset sample information
+        """
+        try:
+            # Load dataset
+            if config:
+                dataset = load_dataset(dataset_id, config, split=split)
+            else:
+                dataset = load_dataset(dataset_id, split=split)
+            
+            # Get sample
+            if len(dataset) > sample_size:
+                sample = dataset.select(range(sample_size))
+            else:
+                sample = dataset
+            
+            # Get features
+            features = list(sample.features.keys())
+            
+            # Convert sample to list of dictionaries
+            sample_data = []
+            for item in sample:
+                sample_item = {}
+                for key in features:
+                    # Convert non-serializable values to strings
+                    if isinstance(item[key], (list, dict)):
+                        sample_item[key] = str(item[key])
+                    else:
+                        sample_item[key] = item[key]
+                sample_data.append(sample_item)
+            
+            # Format result
+            result = {
+                "id": dataset_id,
+                "config": config,
+                "split": split,
+                "features": features,
+                "sample": sample_data,
+                "total_size": len(dataset)
+            }
+            
+            return result
+        except Exception as e:
+            print(f"Dataset sample error: {e}")
+            return None
+    
+    def add_benchmark(self, dataset_id, name=None, description=None, metrics=None, config=None):
+        """Add a dataset as a benchmark.
+        
+        Args:
+            dataset_id: HuggingFace dataset ID
+            name: Benchmark name (defaults to dataset name)
+            description: Benchmark description (defaults to dataset description)
+            metrics: Metrics to use for evaluation
+            config: Dataset configuration to use
+            
+        Returns:
+            int: Benchmark ID if successful, None otherwise
+        """
+        try:
+            # Get dataset info if name or description not provided
+            if not name or not description:
+                dataset_info = self.get_dataset_info(dataset_id)
+                if not dataset_info:
+                    return None
+                
+                if not name:
+                    name = dataset_info["name"]
+                
+                if not description:
+                    description = dataset_info["description"]
+            
+            # Format dataset ID with config if provided
+            full_dataset_id = dataset_id
+            if config:
+                full_dataset_id = f"{dataset_id}:{config}"
+            
+            # Add benchmark to database
+            benchmark_id = self.db_manager.add_benchmark(
+                name=name,
+                dataset_id=full_dataset_id,
+                description=description,
+                metrics=metrics
+            )
+            
+            return benchmark_id
+        except Exception as e:
+            print(f"Add benchmark error: {e}")
+            return None
+    
+    def get_benchmarks(self):
+        """Get all available benchmarks.
+        
+        Returns:
+            list: List of benchmark information dictionaries
+        """
+        return self.db_manager.get_benchmarks()
+
+# Benchmark selection UI components
+def create_benchmark_selection_ui(benchmark_selector, auth_manager):
+    """Create the benchmark selection UI components.
+    
+    Args:
+        benchmark_selector: Benchmark selector instance
+        auth_manager: Authentication manager instance
+        
+    Returns:
+        gr.Blocks: Gradio Blocks component with benchmark selection UI
+    """
+    with gr.Blocks() as benchmark_ui:
+        gr.Markdown("## 📊 Dynamic Highscores Benchmark Selection")
+        gr.Markdown("""
+        ### Add your own datasets from HuggingFace as benchmarks!
+        
+        You can add any dataset from HuggingFace to use as a benchmark for evaluating models.
+        Simply enter the dataset ID (e.g., 'squad', 'glue', 'hellaswag') and add it as a benchmark.
+        
+        Other users will be able to select your added benchmarks for their model evaluations.
+        """, elem_classes=["info-text"])
+        
+        with gr.Tabs() as tabs:
+            with gr.TabItem("➕ Add New Benchmark", id=0):
+                with gr.Row():
+                    with gr.Column(scale=3):
+                        search_input = gr.Textbox(
+                            placeholder="Search for datasets on HuggingFace...",
+                            label="Search",
+                            show_label=False
+                        )
+                    
+                    with gr.Column(scale=1):
+                        category_dropdown = gr.Dropdown(
+                            choices=benchmark_selector.categories,
+                            value="All",
+                            label="Category"
+                        )
+                    
+                    with gr.Column(scale=1):
+                        search_button = gr.Button("Search")
+                
+                dataset_results = gr.Dataframe(
+                    headers=["Name", "Author", "Description", "Downloads"],
+                    datatype=["str", "str", "str", "number"],
+                    label="Search Results",
+                    interactive=True
+                )
+                
+                with gr.Row():
+                    with gr.Column(scale=2):
+                        dataset_id_input = gr.Textbox(
+                            placeholder="Enter HuggingFace dataset ID (e.g., 'squad', 'glue', 'hellaswag')",
+                            label="Dataset ID",
+                            info="You can enter any dataset ID from HuggingFace"
+                        )
+                    
+                    with gr.Column(scale=1):
+                        view_button = gr.Button("View Dataset Details")
+                
+                with gr.Accordion("Dataset Details", open=False):
+                    dataset_info = gr.JSON(label="Dataset Information")
+                    
+                    with gr.Row():
+                        config_dropdown = gr.Dropdown(
+                            label="Configuration",
+                            choices=[],
+                            interactive=True
+                        )
+                        
+                        split_dropdown = gr.Dropdown(
+                            label="Split",
+                            choices=["train", "validation", "test"],
+                            value="train",
+                            interactive=True
+                        )
+                        
+                        sample_button = gr.Button("Load Sample")
+                    
+                    sample_data = gr.Dataframe(
+                        label="Sample Data",
+                        interactive=False
+                    )
+                
+                gr.Markdown("### Add this dataset as a benchmark")
+                with gr.Row():
+                    with gr.Column(scale=2):
+                        benchmark_name = gr.Textbox(
+                            placeholder="Enter a name for this benchmark",
+                            label="Benchmark Name",
+                            info="A descriptive name for this benchmark"
+                        )
+                        
+                        benchmark_description = gr.Textbox(
+                            placeholder="Enter a description for this benchmark",
+                            label="Description",
+                            info="Explain what this benchmark evaluates",
+                            lines=3
+                        )
+                    
+                    with gr.Column(scale=1):
+                        metrics_input = gr.CheckboxGroup(
+                            label="Evaluation Metrics",
+                            choices=[],
+                            interactive=True,
+                            info="Select metrics to use for evaluation"
+                        )
+                
+                with gr.Row():
+                    add_benchmark_button = gr.Button("Add as Benchmark", size="lg", variant="primary")
+                
+                benchmark_status = gr.Markdown("")
+            
+            with gr.TabItem("📋 Available Benchmarks", id=1):
+                gr.Markdown("### Benchmarks available for model evaluation")
+                gr.Markdown("These benchmarks can be selected when submitting models for evaluation.")
+                
+                with gr.Row():
+                    refresh_benchmarks_button = gr.Button("Refresh Benchmarks")
+                    reload_sample_benchmarks_button = gr.Button("Reload Sample Benchmarks", variant="secondary")
+                
+                reload_status = gr.Markdown("")
+                
+                benchmarks_container = gr.Column()
+                with benchmarks_container:
+                    no_benchmarks_message = gr.Markdown(
+                        "### No Datasets Added Yet\n\nBe the first to add a benchmark dataset! Go to the 'Add New Benchmark' tab to add a dataset from HuggingFace.",
+                        visible=True
+                    )
+                    
+                    my_benchmarks = gr.Dataframe(
+                        headers=["ID", "Name", "Dataset", "Description"],
+                        label="Available Benchmarks",
+                        interactive=True,
+                        visible=False
+                    )
+        
+        # Event handlers
+        def search_datasets_handler(query, category):
+            if not query:
+                return None
+            
+            results = benchmark_selector.search_datasets(query, category)
+            
+            # Format for dataframe
+            formatted_results = []
+            for result in results:
+                formatted_results.append([
+                    result["name"],
+                    result["author"],
+                    result["description"],
+                    result["downloads"]
+                ])
+            
+            return formatted_results
+        
+        def view_dataset_handler(dataset_id):
+            if not dataset_id:
+                return None, [], None
+            
+            dataset_info = benchmark_selector.get_dataset_info(dataset_id)
+            
+            if not dataset_info:
+                return None, [], None
+            
+            # Update metrics based on dataset tags
+            metrics = []
+            for category, category_metrics in benchmark_selector.metric_templates.items():
+                if any(tag.lower() in [t.lower() for t in dataset_info["tags"]] for tag in category.lower().split()):
+                    metrics.extend(category_metrics)
+            
+            # Remove duplicates
+            metrics = list(set(metrics))
+            
+            return dataset_info, dataset_info["configs"], gr.update(choices=metrics)
+        
+        def load_sample_handler(dataset_id, config, split):
+            if not dataset_id:
+                return None
+            
+            sample_info = benchmark_selector.load_dataset_sample(
+                dataset_id,
+                config=config if config else None,
+                split=split
+            )
+            
+            if not sample_info:
+                return None
+            
+            return sample_info["sample"]
+        
+        def add_benchmark_handler(dataset_id, config, name, description, metrics, request: gr.Request):
+            if not dataset_id:
+                return "Please enter a dataset ID from HuggingFace."
+            
+            # Check if user is logged in
+            user = auth_manager.check_login(request)
+            
+            if not user:
+                return "Please log in to add benchmarks."
+            
+            # Add benchmark
+            benchmark_id = benchmark_selector.add_benchmark(
+                dataset_id=dataset_id,
+                name=name if name else None,
+                description=description if description else None,
+                metrics=metrics if metrics else None,
+                config=config if config else None
+            )
+            
+            if benchmark_id:
+                return f"✅ Benchmark added successfully with ID: {benchmark_id}\n\nThis dataset is now available for model evaluation. You can view it in the 'Available Benchmarks' tab."
+            else:
+                return "❌ Failed to add benchmark. Please check the dataset ID and try again."
+        
+        def get_benchmarks_handler(request: gr.Request):
+            # Check if user is logged in
+            user = auth_manager.check_login(request)
+            
+            if not user:
+                return gr.update(visible=True), gr.update(visible=False), None
+            
+            # Get benchmarks
+            benchmarks = benchmark_selector.get_benchmarks()
+            
+            # If no benchmarks, show message
+            if not benchmarks or len(benchmarks) == 0:
+                return gr.update(visible=True), gr.update(visible=False), None
+            
+            # Format for dataframe
+            formatted_benchmarks = []
+            for benchmark in benchmarks:
+                formatted_benchmarks.append([
+                    benchmark["id"],
+                    benchmark["name"],
+                    benchmark["dataset_id"],
+                    benchmark["description"]
+                ])
+            
+            return gr.update(visible=False), gr.update(visible=True), formatted_benchmarks
+        
+        def reload_sample_benchmarks_handler():
+            try:
+                from sample_benchmarks import add_sample_benchmarks
+                num_added = add_sample_benchmarks()
+                return f"✅ Successfully reloaded {num_added} sample benchmarks."
+            except Exception as e:
+                return f"❌ Error reloading benchmarks: {str(e)}"
+        
+        # Connect event handlers
+        search_button.click(
+            fn=search_datasets_handler,
+            inputs=[search_input, category_dropdown],
+            outputs=[dataset_results]
+        )
+        
+        view_button.click(
+            fn=view_dataset_handler,
+            inputs=[dataset_id_input],
+            outputs=[dataset_info, config_dropdown, metrics_input]
+        )
+        
+        sample_button.click(
+            fn=load_sample_handler,
+            inputs=[dataset_id_input, config_dropdown, split_dropdown],
+            outputs=[sample_data]
+        )
+        
+        add_benchmark_button.click(
+            fn=add_benchmark_handler,
+            inputs=[dataset_id_input, config_dropdown, benchmark_name, benchmark_description, metrics_input],
+            outputs=[benchmark_status]
+        )
+        
+        refresh_benchmarks_button.click(
+            fn=get_benchmarks_handler,
+            inputs=[],
+            outputs=[no_benchmarks_message, my_benchmarks, my_benchmarks]
+        )
+        
+        reload_sample_benchmarks_button.click(
+            fn=reload_sample_benchmarks_handler,
+            inputs=[],
+            outputs=[reload_status]
+        )
+        
+        # Initialize benchmarks on load
+        benchmark_ui.load(
+            fn=get_benchmarks_handler,
+            inputs=[],
+            outputs=[no_benchmarks_message, my_benchmarks, my_benchmarks]
+        )
+    
+    return benchmark_ui

database_schema.py

@@ -0,0 +1,483 @@
+"""
+Database schema for Dynamic Highscores system.
+
+This module defines the SQLite database schema for the Dynamic Highscores system,
+which integrates benchmark selection, model evaluation, and leaderboard functionality.
+"""
+
+import sqlite3
+import os
+import json
+import threading
+from datetime import datetime, timedelta
+import pandas as pd
+
+class ThreadLocalDB:
+    """Thread-local database connection manager."""
+    
+    _thread_local = threading.local()
+    
+    def __init__(self, db_path):
+        """Initialize with database path."""
+        self.db_path = db_path
+    
+    def get_connection(self):
+        """Get a thread-local database connection."""
+        if not hasattr(self._thread_local, 'conn') or self._thread_local.conn is None:
+            self._thread_local.conn = sqlite3.connect(self.db_path)
+            self._thread_local.conn.row_factory = sqlite3.Row
+        return self._thread_local.conn
+    
+    def get_cursor(self):
+        """Get a cursor from the thread-local connection."""
+        conn = self.get_connection()
+        if not hasattr(self._thread_local, 'cursor') or self._thread_local.cursor is None:
+            self._thread_local.cursor = conn.cursor()
+        return self._thread_local.cursor
+    
+    def close(self):
+        """Close the thread-local connection if it exists."""
+        if hasattr(self._thread_local, 'conn') and self._thread_local.conn is not None:
+            if hasattr(self._thread_local, 'cursor') and self._thread_local.cursor is not None:
+                self._thread_local.cursor.close()
+                self._thread_local.cursor = None
+            self._thread_local.conn.close()
+            self._thread_local.conn = None
+
+class DynamicHighscoresDB:
+    """Database manager for the Dynamic Highscores system."""
+    
+    def __init__(self, db_path="dynamic_highscores.db"):
+        """Initialize the database connection and create tables if they don't exist."""
+        self.db_path = db_path
+        self.thread_local_db = ThreadLocalDB(db_path)
+        self.create_tables()
+    
+    def get_conn(self):
+        """Get the thread-local database connection."""
+        return self.thread_local_db.get_connection()
+    
+    def get_cursor(self):
+        """Get the thread-local database cursor."""
+        return self.thread_local_db.get_cursor()
+    
+    def close(self):
+        """Close the thread-local database connection."""
+        self.thread_local_db.close()
+    
+    def create_tables(self):
+        """Create all necessary tables if they don't exist."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        # Users table - stores user information
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS users (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            username TEXT UNIQUE NOT NULL,
+            hf_user_id TEXT UNIQUE NOT NULL,
+            is_admin BOOLEAN DEFAULT 0,
+            last_submission_date TEXT,
+            created_at TEXT DEFAULT CURRENT_TIMESTAMP
+        )
+        ''')
+        
+        # Benchmarks table - stores information about available benchmarks
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS benchmarks (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            dataset_id TEXT NOT NULL,
+            description TEXT,
+            metrics TEXT,  -- JSON string of metrics
+            created_at TEXT DEFAULT CURRENT_TIMESTAMP
+        )
+        ''')
+        
+        # Models table - stores information about submitted models
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS models (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            hf_model_id TEXT NOT NULL,
+            user_id INTEGER NOT NULL,
+            tag TEXT NOT NULL,  -- One of: Merge, Agent, Reasoning, Coding, etc.
+            parameters TEXT,  -- Number of parameters (can be NULL)
+            description TEXT,
+            created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (user_id) REFERENCES users (id),
+            UNIQUE (hf_model_id, user_id)
+        )
+        ''')
+        
+        # Evaluations table - stores evaluation results
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS evaluations (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            model_id INTEGER NOT NULL,
+            benchmark_id INTEGER NOT NULL,
+            status TEXT NOT NULL,  -- pending, running, completed, failed
+            results TEXT,  -- JSON string of results
+            score REAL,  -- Overall score (can be NULL)
+            submitted_at TEXT DEFAULT CURRENT_TIMESTAMP,
+            started_at TEXT,
+            completed_at TEXT,
+            FOREIGN KEY (model_id) REFERENCES models (id),
+            FOREIGN KEY (benchmark_id) REFERENCES benchmarks (id)
+        )
+        ''')
+        
+        # Queue table - stores evaluation queue
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS queue (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            evaluation_id INTEGER NOT NULL,
+            priority INTEGER DEFAULT 0,  -- Higher number = higher priority
+            added_at TEXT DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (evaluation_id) REFERENCES evaluations (id)
+        )
+        ''')
+        
+        conn.commit()
+    
+    # User management methods
+    def add_user(self, username, hf_user_id, is_admin=False):
+        """Add a new user to the database."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        try:
+            cursor.execute(
+                "INSERT INTO users (username, hf_user_id, is_admin) VALUES (?, ?, ?)",
+                (username, hf_user_id, is_admin)
+            )
+            conn.commit()
+            return cursor.lastrowid
+        except sqlite3.IntegrityError:
+            # User already exists
+            cursor.execute(
+                "SELECT id FROM users WHERE hf_user_id = ?",
+                (hf_user_id,)
+            )
+            row = cursor.fetchone()
+            return row[0] if row else None
+    
+    def get_user(self, hf_user_id):
+        """Get user information by HuggingFace user ID."""
+        cursor = self.get_cursor()
+        
+        cursor.execute(
+            "SELECT * FROM users WHERE hf_user_id = ?",
+            (hf_user_id,)
+        )
+        row = cursor.fetchone()
+        return dict(row) if row else None
+    
+    def get_user_by_username(self, username):
+        """Get user information by username."""
+        cursor = self.get_cursor()
+        
+        cursor.execute(
+            "SELECT * FROM users WHERE username = ?",
+            (username,)
+        )
+        row = cursor.fetchone()
+        return dict(row) if row else None
+    
+    def can_submit_today(self, user_id):
+        """Check if a user can submit a benchmark evaluation today."""
+        cursor = self.get_cursor()
+        
+        cursor.execute(
+            "SELECT is_admin, last_submission_date FROM users WHERE id = ?",
+            (user_id,)
+        )
+        result = cursor.fetchone()
+        
+        if not result:
+            return False
+        
+        user_data = dict(result)
+        
+        # Admin can always submit
+        if user_data['is_admin']:
+            return True
+        
+        # If no previous submission, user can submit
+        if not user_data['last_submission_date']:
+            return True
+        
+        # Check if last submission was before today
+        last_date = datetime.fromisoformat(user_data['last_submission_date'])
+        today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
+        
+        return last_date < today
+    
+    def update_submission_date(self, user_id):
+        """Update the last submission date for a user."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        current_time = datetime.now().isoformat()
+        cursor.execute(
+            "UPDATE users SET last_submission_date = ? WHERE id = ?",
+            (current_time, user_id)
+        )
+        conn.commit()
+    
+    # Benchmark management methods
+    def add_benchmark(self, name, dataset_id, description="", metrics=None):
+        """Add a new benchmark to the database."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        if metrics is None:
+            metrics = {}
+        
+        metrics_json = json.dumps(metrics)
+        
+        try:
+            cursor.execute(
+                "INSERT INTO benchmarks (name, dataset_id, description, metrics) VALUES (?, ?, ?, ?)",
+                (name, dataset_id, description, metrics_json)
+            )
+            conn.commit()
+            return cursor.lastrowid
+        except sqlite3.IntegrityError:
+            # Benchmark already exists with this dataset_id
+            cursor.execute(
+                "SELECT id FROM benchmarks WHERE dataset_id = ?",
+                (dataset_id,)
+            )
+            row = cursor.fetchone()
+            return row[0] if row else None
+    
+    def get_benchmarks(self):
+        """Get all available benchmarks."""
+        cursor = self.get_cursor()
+        
+        cursor.execute("SELECT * FROM benchmarks")
+        benchmarks = [dict(row) for row in cursor.fetchall()]
+        
+        # Parse metrics JSON
+        for benchmark in benchmarks:
+            if benchmark['metrics']:
+                benchmark['metrics'] = json.loads(benchmark['metrics'])
+            else:
+                benchmark['metrics'] = {}
+        
+        return benchmarks
+    
+    def get_benchmark(self, benchmark_id):
+        """Get benchmark information by ID."""
+        cursor = self.get_cursor()
+        
+        cursor.execute(
+            "SELECT * FROM benchmarks WHERE id = ?",
+            (benchmark_id,)
+        )
+        row = cursor.fetchone()
+        benchmark = dict(row) if row else None
+        
+        if benchmark and benchmark['metrics']:
+            benchmark['metrics'] = json.loads(benchmark['metrics'])
+        
+        return benchmark
+    
+    # Model management methods
+    def add_model(self, name, hf_model_id, user_id, tag, parameters=None, description=""):
+        """Add a new model to the database."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        try:
+            cursor.execute(
+                "INSERT INTO models (name, hf_model_id, user_id, tag, parameters, description) VALUES (?, ?, ?, ?, ?, ?)",
+                (name, hf_model_id, user_id, tag, parameters, description)
+            )
+            conn.commit()
+            return cursor.lastrowid
+        except sqlite3.IntegrityError:
+            # Model already exists for this user
+            cursor.execute(
+                "SELECT id FROM models WHERE hf_model_id = ? AND user_id = ?",
+                (hf_model_id, user_id)
+            )
+            row = cursor.fetchone()
+            return row[0] if row else None
+    
+    def get_models(self, tag=None):
+        """Get all models, optionally filtered by tag."""
+        cursor = self.get_cursor()
+        
+        if tag and tag.lower() != "all":
+            cursor.execute(
+                "SELECT * FROM models WHERE tag = ?",
+                (tag,)
+            )
+        else:
+            cursor.execute("SELECT * FROM models")
+        
+        return [dict(row) for row in cursor.fetchall()]
+    
+    def get_model(self, model_id):
+        """Get model information by ID."""
+        cursor = self.get_cursor()
+        
+        cursor.execute(
+            "SELECT * FROM models WHERE id = ?",
+            (model_id,)
+        )
+        row = cursor.fetchone()
+        return dict(row) if row else None
+    
+# Evaluation management methods
+    def add_evaluation(self, model_id, benchmark_id, priority=0):
+        """Add a new evaluation to the database and queue."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        # First, add the evaluation
+        cursor.execute(
+            "INSERT INTO evaluations (model_id, benchmark_id, status) VALUES (?, ?, 'pending')",
+            (model_id, benchmark_id)
+        )
+        evaluation_id = cursor.lastrowid
+        
+        # Then, add it to the queue
+        cursor.execute(
+            "INSERT INTO queue (evaluation_id, priority) VALUES (?, ?)",
+            (evaluation_id, priority)
+        )
+        
+        conn.commit()
+        return evaluation_id
+    
+    def update_evaluation_status(self, evaluation_id, status, results=None, score=None):
+        """Update the status of an evaluation."""
+        cursor = self.get_cursor()
+        conn = self.get_conn()
+        
+        params = [status, evaluation_id]
+        sql = "UPDATE evaluations SET status = ?"
+        
+        if results is not None:
+            sql += ", results = ?"
+            params.insert(1, json.dumps(results))
+        
+        if score is not None:
+            sql += ", score = ?"
+            params.insert(1 if results is None else 2, score)
+        
+        if status in ['completed', 'failed']:
+            sql += ", completed_at = datetime('now')"
+        elif status == 'running':
+            sql += ", started_at = datetime('now')"
+        
+        sql += " WHERE id = ?"
+        
+        cursor.execute(sql, params)
+        conn.commit()
+    
+    def get_next_in_queue(self):
+        """Get the next evaluation in the queue."""
+        cursor = self.get_cursor()
+        
+        cursor.execute("""
+            SELECT q.*, e.id as evaluation_id, e.model_id, e.benchmark_id, e.status
+            FROM queue q
+            JOIN evaluations e ON q.evaluation_id = e.id
+            WHERE e.status = 'pending'
+            ORDER BY q.priority DESC, q.added_at ASC
+            LIMIT 1
+        """)
+        
+        row = cursor.fetchone()
+        return dict(row) if row else None
+    
+    def get_evaluation_results(self, model_id=None, benchmark_id=None, tag=None, status=None, limit=None):
+        """Get evaluation results, optionally filtered by model, benchmark, tag, or status."""
+        cursor = self.get_cursor()
+        
+        sql = """
+            SELECT e.id, e.model_id, e.benchmark_id, e.status, e.results, e.score, 
+                   e.submitted_at, e.started_at, e.completed_at, m.name as model_name, m.tag, 
+                   b.name as benchmark_name
+            FROM evaluations e
+            JOIN models m ON e.model_id = m.id
+            JOIN benchmarks b ON e.benchmark_id = b.id
+            WHERE 1=1
+        """
+        
+        params = []
+        
+        if status:
+            sql += " AND e.status = ?"
+            params.append(status)
+        
+        if model_id:
+            sql += " AND e.model_id = ?"
+            params.append(model_id)
+        
+        if benchmark_id and benchmark_id != "all" and benchmark_id.lower() != "all":
+            sql += " AND e.benchmark_id = ?"
+            params.append(benchmark_id)
+        
+        if tag and tag.lower() != "all":
+            sql += " AND m.tag = ?"
+            params.append(tag)
+        
+        sql += " ORDER BY e.submitted_at DESC"
+        
+        if limit:
+            sql += " LIMIT ?"
+            params.append(limit)
+        
+        cursor.execute(sql, params)
+        results = [dict(row) for row in cursor.fetchall()]
+        
+        # Parse results JSON
+        for result in results:
+            if result['results']:
+                try:
+                    result['results'] = json.loads(result['results'])
+                except:
+                    result['results'] = {}
+        
+        return results
+    
+    def get_leaderboard_df(self, tag=None, benchmark_id=None):
+        """Get a pandas DataFrame of the leaderboard, optionally filtered by tag and benchmark."""
+        results = self.get_evaluation_results(tag=tag, benchmark_id=benchmark_id, status="completed")
+        
+        if not results:
+            return pd.DataFrame()
+        
+        # Create a list of dictionaries for the DataFrame
+        leaderboard_data = []
+        
+        for result in results:
+            entry = {
+                'model_name': result['model_name'],
+                'tag': result['tag'],
+                'benchmark_name': result['benchmark_name'],
+                'score': result['score'],
+                'completed_at': result['completed_at']
+            }
+            
+            # Add any additional metrics from results
+            if result['results'] and isinstance(result['results'], dict):
+                for key, value in result['results'].items():
+                    if isinstance(value, (int, float)) and key not in entry:
+                        entry[key] = value
+            
+            leaderboard_data.append(entry)
+        
+        # Convert to DataFrame
+        df = pd.DataFrame(leaderboard_data)
+        
+        # Sort by score (descending)
+        if not df.empty and 'score' in df.columns:
+            df = df.sort_values('score', ascending=False)
+        
+        return df

evaluation_queue.py

@@ -0,0 +1,1122 @@
+"""
+Model evaluation queue system for Dynamic Highscores.
+
+This module handles the evaluation queue, CPU-only processing,
+and enforces daily submission limits for users.
+"""
+
+import os
+import json
+import time
+import threading
+import queue as queue_module
+from datetime import datetime, timedelta
+import gradio as gr
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+from datasets import load_dataset
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+import sqlite3
+
+class EvaluationQueue:
+    """Manages the evaluation queue for model benchmarking."""
+    
+    def __init__(self, db_manager, auth_manager):
+        """Initialize the evaluation queue manager.
+        
+        Args:
+            db_manager: Database manager instance
+            auth_manager: Authentication manager instance
+        """
+        self.db_manager = db_manager
+        self.auth_manager = auth_manager
+        self.hf_api = HfApi()
+        self.queue = queue_module.Queue()
+        self.is_processing = False
+        self.worker_thread = None
+        self.model_tags = ["Merge", "Agent", "Reasoning", "Coding", "General", "Specialized", "Instruction", "Chat"]
+        self.current_evaluation = None
+        self.progress = 0
+        self.progress_lock = threading.Lock()
+        # Memory limit for models in GB (leave 2GB for system)
+        self.memory_limit_gb = 14.0
+    
+    def start_worker(self):
+        """Start the worker thread for processing the evaluation queue."""
+        if self.worker_thread is None or not self.worker_thread.is_alive():
+            self.is_processing = True
+            self.worker_thread = threading.Thread(target=self._process_queue)
+            self.worker_thread.daemon = True
+            self.worker_thread.start()
+    
+    def stop_worker(self):
+        """Stop the worker thread."""
+        self.is_processing = False
+        if self.worker_thread and self.worker_thread.is_alive():
+            self.worker_thread.join(timeout=1.0)
+    
+    def check_model_size(self, model_id):
+        """Check if a model will fit within RAM limitations.
+        
+        Args:
+            model_id: HuggingFace model ID
+            
+        Returns:
+            tuple: (will_fit, message)
+        """
+        try:
+            # Query model info from the HuggingFace API
+            model_info_obj = self.hf_api.model_info(model_id)
+            
+            # Initialize total size
+            total_size_gb = 0
+            
+            # Try different approaches to get model size based on API response structure
+            if hasattr(model_info_obj, 'safetensors') and model_info_obj.safetensors:
+                # New API format with safetensors dict
+                for file_info in model_info_obj.safetensors.values():
+                    if hasattr(file_info, 'size'):
+                        total_size_gb += file_info.size / (1024 * 1024 * 1024)
+                    elif isinstance(file_info, dict) and 'size' in file_info:
+                        total_size_gb += file_info['size'] / (1024 * 1024 * 1024)
+            
+            # Fallback to siblings method
+            if total_size_gb == 0 and hasattr(model_info_obj, 'siblings'):
+                for sibling in model_info_obj.siblings:
+                    if hasattr(sibling, 'size'):
+                        if sibling.rfilename.endswith(('.bin', '.safetensors', '.pt')):
+                            total_size_gb += sibling.size / (1024 * 1024 * 1024)
+                    elif isinstance(sibling, dict) and 'size' in sibling:
+                        if sibling.get('rfilename', '').endswith(('.bin', '.safetensors', '.pt')):
+                            total_size_gb += sibling['size'] / (1024 * 1024 * 1024)
+            
+            # If we still couldn't determine size, try a reasonable guess based on model name
+            if total_size_gb == 0:
+                # Try to guess from model name (e.g., if it has "7b" in the name)
+                model_name = model_id.lower()
+                size_indicators = {
+                    "1b": 1, "2b": 2, "3b": 3, "5b": 5, "7b": 7, "8b": 8,
+                    "10b": 10, "13b": 13, "20b": 20, "30b": 30, "65b": 65, "70b": 70
+                }
+                
+                for indicator, size in size_indicators.items():
+                    if indicator in model_name.replace("-", "").replace("_", ""):
+                        total_size_gb = size * 2  # Rough estimate: param count × 2 for size in GB
+                        break
+            
+            # If we still couldn't determine size, use a default
+            if total_size_gb == 0:
+                # Try direct API method
+                try:
+                    print(f"Checking model size with direct method for {model_id}")
+                    # Print out the entire structure for debugging
+                    print(f"Model info: {model_info_obj.__dict__}")
+                    
+                    # Default to a conservative estimate
+                    total_size_gb = 5  # Assume a 5GB model as default
+                except Exception as e:
+                    print(f"Direct size check failed: {e}")
+                    return True, "Unable to determine model size accurately, but allowing submission with caution"
+            
+            # Account for memory overhead
+            estimated_ram_needed = total_size_gb * 1.3  # 30% overhead
+            
+            # Check against limit
+            if estimated_ram_needed > self.memory_limit_gb:
+                return False, f"Model is too large (approximately {total_size_gb:.1f}GB, needs {estimated_ram_needed:.1f}GB RAM). Maximum allowed is {self.memory_limit_gb}GB."
+            
+            return True, f"Model size check passed ({total_size_gb:.1f}GB, estimated {estimated_ram_needed:.1f}GB RAM usage)"
+        
+        except Exception as e:
+            print(f"Model size check error: {e}")
+            # Log more details for debugging
+            import traceback
+            traceback.print_exc()
+            
+            # Allow submission with warning
+            return True, f"Warning: Could not verify model size ({str(e)}). Please ensure your model is under {self.memory_limit_gb}GB."
+    
+    def _process_queue(self):
+        """Process the evaluation queue in a separate thread."""
+        while self.is_processing:
+            try:
+                # Get the next evaluation from the database
+                pending_evals = self.db_manager.get_evaluation_results(status="pending")
+                
+                if pending_evals:
+                    # Sort by priority and added_at
+                    next_eval = pending_evals[0]
+                    
+                    # Update status to running
+                    self.db_manager.update_evaluation_status(next_eval['id'], 'running')
+                    
+                    # Set current evaluation and reset progress
+                    with self.progress_lock:
+                        self.current_evaluation = next_eval
+                        self.progress = 0
+                    
+                    try:
+                        # Get model and benchmark details
+                        model_info = self.db_manager.get_model(next_eval['model_id'])
+                        benchmark_info = self.db_manager.get_benchmark(next_eval['benchmark_id'])
+                        
+                        if model_info and benchmark_info:
+                            # Check if model will fit in memory
+                            will_fit, message = self.check_model_size(model_info['hf_model_id'])
+                            
+                            if not will_fit:
+                                raise Exception(f"Model too large for evaluation: {message}")
+                                
+                            # Run the evaluation
+                            results = self._run_evaluation(
+                                model_info['hf_model_id'],
+                                benchmark_info['dataset_id']
+                            )
+                            
+                            # Calculate overall score
+                            score = self._calculate_overall_score(results)
+                            
+                            # Update status to completed with results
+                            self.db_manager.update_evaluation_status(
+                                next_eval['id'],
+                                'completed',
+                                results=results,
+                                score=score
+                            )
+                        else:
+                            raise Exception("Model or benchmark not found")
+                    except Exception as e:
+                        print(f"Evaluation error: {e}")
+                        # Update status to failed with error message
+                        error_results = {"error": str(e)}
+                        self.db_manager.update_evaluation_status(
+                            next_eval['id'], 
+                            'failed',
+                            results=error_results
+                        )
+                    
+                    # Clear current evaluation
+                    with self.progress_lock:
+                        self.current_evaluation = None
+                        self.progress = 0
+                else:
+                    # No evaluations in queue, sleep for a bit
+                    time.sleep(5)
+            except Exception as e:
+                print(f"Queue processing error: {e}")
+                time.sleep(5)
+    
+    def _run_evaluation(self, model_id, dataset_id):
+        """Run an evaluation for a model on a benchmark.
+        
+        Args:
+            model_id: HuggingFace model ID
+            dataset_id: HuggingFace dataset ID (with optional config)
+            
+        Returns:
+            dict: Evaluation results
+        """
+        # Update progress
+        with self.progress_lock:
+            self.progress = 5  # Starting evaluation
+        
+        # Parse dataset ID and config
+        if ":" in dataset_id:
+            dataset_id, config = dataset_id.split(":", 1)
+        else:
+            config = None
+        
+        # Update progress
+        with self.progress_lock:
+            self.progress = 10  # Loading dataset
+        
+        # Load the dataset
+        try:
+            if config:
+                dataset = load_dataset(dataset_id, config, split="test")
+            else:
+                dataset = load_dataset(dataset_id, split="test")
+        except Exception as e:
+            return {"error": f"Failed to load dataset: {str(e)}"}
+        
+        # Update progress
+        with self.progress_lock:
+            self.progress = 20  # Loading model
+        
+        try:
+            # Load the model with memory optimization settings
+            device = "cpu"
+            model = AutoModelForCausalLM.from_pretrained(
+                model_id,
+                device_map=device,
+                torch_dtype=torch.float32,  # Use float32 for CPU
+                low_cpu_mem_usage=True,     # Enable memory optimization
+                offload_folder="offload",   # Enable offloading if needed
+                offload_state_dict=True,    # Offload state dict for memory saving
+                max_memory={0: f"{self.memory_limit_gb}GB"}  # Limit memory usage
+            )
+            tokenizer = AutoTokenizer.from_pretrained(model_id)
+        except Exception as e:
+            print(f"Model loading error: {e}")
+            return {"error": f"Failed to load model: {str(e)}"}
+        
+        # Update progress
+        with self.progress_lock:
+            self.progress = 30  # Determining task type
+        
+        # Determine task type based on dataset features
+        task_type = self._determine_task_type(dataset)
+        
+        # Update progress
+        with self.progress_lock:
+            self.progress = 40  # Starting evaluation
+        
+        try:
+            # Run appropriate evaluation based on task type
+            if task_type == "text-generation":
+                results = self._evaluate_text_generation(model, tokenizer, dataset)
+            elif task_type == "question-answering":
+                results = self._evaluate_question_answering(model, tokenizer, dataset)
+            elif task_type == "classification":
+                results = self._evaluate_classification(model, tokenizer, dataset)
+            elif task_type == "code-generation":
+                results = self._evaluate_code_generation(model, tokenizer, dataset)
+            else:
+                # Default to general evaluation
+                results = self._evaluate_general(model, tokenizer, dataset)
+        except Exception as e:
+            print(f"Evaluation task error: {e}")
+            return {"error": f"Evaluation failed: {str(e)}"}
+        
+        # Update progress
+        with self.progress_lock:
+            self.progress = 95  # Cleaning up
+        
+        # Clean up to free memory
+        del model
+        del tokenizer
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        
+        # Update progress
+        with self.progress_lock:
+            self.progress = 100  # Completed
+        
+        return results
+    
+    def get_current_progress(self):
+        """Get the current evaluation progress.
+        
+        Returns:
+            tuple: (current_evaluation, progress_percentage)
+        """
+        with self.progress_lock:
+            return self.current_evaluation, self.progress
+    
+    def _determine_task_type(self, dataset):
+        """Determine the task type based on dataset features.
+        
+        Args:
+            dataset: HuggingFace dataset
+            
+        Returns:
+            str: Task type
+        """
+        features = dataset.features
+        
+        # Check for common feature patterns
+        if "question" in features and "answer" in features:
+            return "question-answering"
+        elif "code" in features or "solution" in features:
+            return "code-generation"
+        elif "label" in features or "class" in features:
+            return "classification"
+        elif "input" in features and "output" in features:
+            return "text-generation"
+        else:
+            return "general"
+    
+    def _evaluate_text_generation(self, model, tokenizer, dataset):
+        """Evaluate a model on text generation tasks.
+        
+        Args:
+            model: HuggingFace model
+            tokenizer: HuggingFace tokenizer
+            dataset: HuggingFace dataset
+            
+        Returns:
+            dict: Evaluation results
+        """
+        # Set up generation pipeline
+        generator = pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            device="cpu"
+        )
+        
+        # Sample a subset for evaluation (to keep runtime reasonable)
+        if len(dataset) > 100:
+            dataset = dataset.select(range(100))
+        
+        # Track metrics
+        correct = 0
+        total = 0
+        generated_texts = []
+        
+        # Process each example
+        for i, example in enumerate(dataset):
+            # Update progress based on completion percentage
+            with self.progress_lock:
+                self.progress = 40 + int((i / len(dataset)) * 50)
+            
+            input_text = example.get("input", example.get("prompt", ""))
+            expected_output = example.get("output", example.get("target", ""))
+            
+            if not input_text or not expected_output:
+                continue
+            
+            # Generate text
+            generated = generator(
+                input_text,
+                max_length=100,
+                num_return_sequences=1
+            )
+            
+            generated_text = generated[0]["generated_text"]
+            generated_texts.append(generated_text)
+            
+            # Simple exact match check
+            if expected_output.strip() in generated_text:
+                correct += 1
+            
+            total += 1
+        
+        # Calculate metrics
+        accuracy = correct / total if total > 0 else 0
+        
+        return {
+            "accuracy": accuracy,
+            "samples_evaluated": total,
+            "generated_samples": generated_texts[:5]  # Include a few samples
+        }
+    
+    def _evaluate_question_answering(self, model, tokenizer, dataset):
+        """Evaluate a model on question answering tasks.
+        
+        Args:
+            model: HuggingFace model
+            tokenizer: HuggingFace tokenizer
+            dataset: HuggingFace dataset
+            
+        Returns:
+            dict: Evaluation results
+        """
+        # Set up QA pipeline
+        qa_pipeline = pipeline(
+            "question-answering",
+            model=model,
+            tokenizer=tokenizer,
+            device="cpu"
+        )
+        
+        # Sample a subset for evaluation
+        if len(dataset) > 100:
+            dataset = dataset.select(range(100))
+        
+        # Track metrics
+        exact_matches = 0
+        f1_scores = []
+        total = 0
+        
+        # Process each example
+        for i, example in enumerate(dataset):
+            # Update progress based on completion percentage
+            with self.progress_lock:
+                self.progress = 40 + int((i / len(dataset)) * 50)
+            
+            question = example.get("question", "")
+            context = example.get("context", "")
+            answer = example.get("answer", "")
+            
+            if not question or not answer:
+                continue
+            
+            # Get model prediction
+            if context:
+                result = qa_pipeline(question=question, context=context)
+            else:
+                # If no context provided, use the question as context
+                result = qa_pipeline(question=question, context=question)
+            
+            predicted_answer = result["answer"]
+            
+            # Calculate exact match
+            if predicted_answer.strip() == answer.strip():
+                exact_matches += 1
+            
+            # Calculate F1 score
+            f1 = self._calculate_f1(answer, predicted_answer)
+            f1_scores.append(f1)
+            
+            total += 1
+        
+        # Calculate metrics
+        exact_match_accuracy = exact_matches / total if total > 0 else 0
+        avg_f1 = sum(f1_scores) / len(f1_scores) if f1_scores else 0
+        
+        return {
+            "exact_match": exact_match_accuracy,
+            "f1": avg_f1,
+            "samples_evaluated": total
+        }
+    
+    def _evaluate_classification(self, model, tokenizer, dataset):
+        """Evaluate a model on classification tasks.
+        
+        Args:
+            model: HuggingFace model
+            tokenizer: HuggingFace tokenizer
+            dataset: HuggingFace dataset
+            
+        Returns:
+            dict: Evaluation results
+        """
+        # Set up classification pipeline
+        classifier = pipeline(
+            "text-classification",
+            model=model,
+            tokenizer=tokenizer,
+            device="cpu"
+        )
+        
+        # Sample a subset for evaluation
+        if len(dataset) > 100:
+            dataset = dataset.select(range(100))
+        
+        # Track metrics
+        correct = 0
+        total = 0
+        
+        # Process each example
+        for i, example in enumerate(dataset):
+            # Update progress based on completion percentage
+            with self.progress_lock:
+                self.progress = 40 + int((i / len(dataset)) * 50)
+            
+            text = example.get("text", example.get("sentence", ""))
+            label = str(example.get("label", example.get("class", "")))
+            
+            if not text or not label:
+                continue
+            
+            # Get model prediction
+            result = classifier(text)
+            predicted_label = result[0]["label"]
+            
+            # Check if correct
+            if str(predicted_label) == label:
+                correct += 1
+            
+            total += 1
+        
+        # Calculate metrics
+        accuracy = correct / total if total > 0 else 0
+        
+        return {
+            "accuracy": accuracy,
+            "samples_evaluated": total
+        }
+    
+    def _evaluate_code_generation(self, model, tokenizer, dataset):
+        """Evaluate a model on code generation tasks.
+        
+        Args:
+            model: HuggingFace model
+            tokenizer: HuggingFace tokenizer
+            dataset: HuggingFace dataset
+            
+        Returns:
+            dict: Evaluation results
+        """
+        # Set up generation pipeline
+        generator = pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            device="cpu"
+        )
+        
+        # Sample a subset for evaluation
+        if len(dataset) > 50:  # Smaller sample for code tasks
+            dataset = dataset.select(range(50))
+        
+        # Track metrics
+        exact_matches = 0
+        functional_matches = 0
+        total = 0
+        
+        # Process each example
+        for i, example in enumerate(dataset):
+            # Update progress based on completion percentage
+            with self.progress_lock:
+                self.progress = 40 + int((i / len(dataset)) * 50)
+            
+            prompt = example.get("prompt", example.get("input", ""))
+            solution = example.get("solution", example.get("output", ""))
+            
+            if not prompt or not solution:
+                continue
+            
+            # Generate code
+            generated = generator(
+                prompt,
+                max_length=200,
+                num_return_sequences=1
+            )
+            
+            generated_code = generated[0]["generated_text"]
+            
+            # Extract code from generated text (remove prompt)
+            if prompt in generated_code:
+                generated_code = generated_code[len(prompt):].strip()
+            
+            # Check exact match
+            if generated_code.strip() == solution.strip():
+                exact_matches += 1
+                functional_matches += 1
+            else:
+                # We would ideally check functional correctness here
+                # but that requires executing code which is complex and potentially unsafe
+                # For now, we'll use a simple heuristic
+                if len(generated_code) > 0 and any(keyword in generated_code for keyword in ["def ", "function", "return", "class"]):
+                    functional_matches += 0.5  # Partial credit
+            
+            total += 1
+        
+        # Calculate metrics
+        exact_match_rate = exact_matches / total if total > 0 else 0
+        functional_correctness = functional_matches / total if total > 0 else 0
+        
+        return {
+            "exact_match": exact_match_rate,
+            "functional_correctness": functional_correctness,
+            "samples_evaluated": total
+        }
+    
+    def _evaluate_general(self, model, tokenizer, dataset):
+        """General evaluation for any dataset type.
+        
+        Args:
+            model: HuggingFace model
+            tokenizer: HuggingFace tokenizer
+            dataset: HuggingFace dataset
+            
+        Returns:
+            dict: Evaluation results
+        """
+        # Set up generation pipeline
+        generator = pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            device="cpu"
+        )
+        
+        # Sample a subset for evaluation
+        if len(dataset) > 50:
+            dataset = dataset.select(range(50))
+        
+        # Find input and output fields
+        features = dataset.features
+        input_field = None
+        output_field = None
+        
+        for field in features:
+            if field.lower() in ["input", "prompt", "question", "text"]:
+                input_field = field
+            elif field.lower() in ["output", "target", "answer", "response"]:
+                output_field = field
+        
+        if not input_field:
+            # Just use the first string field as input
+            for field in features:
+                if isinstance(features[field], (str, list)):
+                    input_field = field
+                    break
+        
+        # Track metrics
+        total = 0
+        generated_texts = []
+        
+        # Process each example
+        for i, example in enumerate(dataset):
+            # Update progress based on completion percentage
+            with self.progress_lock:
+                self.progress = 40 + int((i / len(dataset)) * 50)
+            
+            if input_field and input_field in example:
+                input_text = str(example[input_field])
+                
+                # Generate text
+                generated = generator(
+                    input_text,
+                    max_length=100,
+                    num_return_sequences=1
+                )
+                
+                generated_text = generated[0]["generated_text"]
+                generated_texts.append({
+                    "input": input_text,
+                    "output": generated_text,
+                    "expected": str(example[output_field]) if output_field and output_field in example else "N/A"
+                })
+                
+                total += 1
+        
+        return {
+            "samples_evaluated": total,
+            "generated_samples": generated_texts[:5]  # Include a few samples
+        }
+    
+    def _calculate_f1(self, answer, prediction):
+        """Calculate F1 score between answer and prediction.
+        
+        Args:
+            answer: Ground truth answer
+            prediction: Model prediction
+            
+        Returns:
+            float: F1 score
+        """
+        # Tokenize
+        answer_tokens = answer.lower().split()
+        prediction_tokens = prediction.lower().split()
+        
+        # Calculate precision and recall
+        common_tokens = set(answer_tokens) & set(prediction_tokens)
+        
+        if not common_tokens:
+            return 0.0
+        
+        precision = len(common_tokens) / len(prediction_tokens)
+        recall = len(common_tokens) / len(answer_tokens)
+        
+        # Calculate F1
+        if precision + recall == 0:
+            return 0.0
+        
+        f1 = 2 * precision * recall / (precision + recall)
+        return f1
+    
+    def _calculate_overall_score(self, results):
+        """Calculate an overall score from evaluation results.
+        
+        Args:
+            results: Evaluation results dictionary
+            
+        Returns:
+            float: Overall score between 0 and 100
+        """
+        # If there was an error, return a low score
+        if "error" in results:
+            return 0.0
+            
+        score = 0.0
+        
+        # Check for common metrics and weight them
+        if "accuracy" in results:
+            score += results["accuracy"] * 100
+        
+        if "exact_match" in results:
+            score += results["exact_match"] * 100
+        
+        if "f1" in results:
+            score += results["f1"] * 100
+        
+        if "functional_correctness" in results:
+            score += results["functional_correctness"] * 100
+        
+        # If multiple metrics were found, average them
+        num_metrics = sum(1 for metric in ["accuracy", "exact_match", "f1", "functional_correctness"] if metric in results)
+        
+        if num_metrics > 0:
+            score /= num_metrics
+        else:
+            # Default score if no metrics available
+            score = 50.0
+        
+        return score
+    
+    def submit_evaluation(self, model_id, benchmark_id, user_id, priority=0):
+        """Submit a model for evaluation on a benchmark.
+        
+        Args:
+            model_id: Model ID in the database
+            benchmark_id: Benchmark ID in the database
+            user_id: User ID submitting the evaluation
+            priority: Queue priority (higher = higher priority)
+            
+        Returns:
+            tuple: (evaluation_id, message)
+        """
+        # Check if user can submit today
+        if not self.auth_manager.can_submit_benchmark(user_id):
+            return None, "Daily submission limit reached. Try again tomorrow."
+        
+        try:
+            # Get model HuggingFace ID to check size
+            model_info = self.db_manager.get_model(model_id)
+            if not model_info:
+                return None, "Model not found in database."
+            
+            # Check if model will fit in memory
+            will_fit, message = self.check_model_size(model_info['hf_model_id'])
+            
+            if not will_fit:
+                return None, message
+                
+            # Add evaluation to database and queue
+            evaluation_id = self.db_manager.add_evaluation(
+                model_id=model_id,
+                benchmark_id=benchmark_id,
+                priority=priority
+            )
+            
+            # Update user's last submission date
+            self.auth_manager.update_submission_date(user_id)
+            
+            # Make sure worker is running
+            self.start_worker()
+            
+            return evaluation_id, f"Evaluation submitted successfully. {message}"
+        except Exception as e:
+            print(f"Submit evaluation error: {e}")
+            return None, f"Failed to submit evaluation: {str(e)}"
+    
+    def get_queue_status(self):
+        """Get the current status of the evaluation queue.
+        
+        Returns:
+            dict: Queue status information
+        """
+        try:
+            # Get evaluations from database
+            pending_evals = self.db_manager.get_evaluation_results(status="pending")
+            running_evals = self.db_manager.get_evaluation_results(status="running")
+            completed_evals = self.db_manager.get_evaluation_results(status="completed")
+            failed_evals = self.db_manager.get_evaluation_results(status="failed")
+            
+            # Get current evaluation progress
+            current_eval, progress = self.get_current_progress()
+            
+            return {
+                "pending": len(pending_evals),
+                "running": len(running_evals),
+                "completed": len(completed_evals),
+                "failed": len(failed_evals),
+                "is_processing": self.is_processing,
+                "current_evaluation": current_eval,
+                "progress": progress,
+                "memory_limit_gb": self.memory_limit_gb
+            }
+        except Exception as e:
+            print(f"Queue status error: {e}")
+            return {
+                "pending": 0,
+                "running": 0,
+                "completed": 0,
+                "failed": 0,
+                "is_processing": self.is_processing,
+                "current_evaluation": None,
+                "progress": 0,
+                "memory_limit_gb": self.memory_limit_gb,
+                "error": str(e)
+            }
+
+# Model submission UI components
+def create_model_submission_ui(evaluation_queue, auth_manager, db_manager):
+    """Create the model submission UI components.
+    
+    Args:
+        evaluation_queue: Evaluation queue instance
+        auth_manager: Authentication manager instance
+        db_manager: Database manager instance
+        
+    Returns:
+        gr.Blocks: Gradio Blocks component with model submission UI
+    """
+    with gr.Blocks() as submission_ui:
+        # Store user authentication state
+        user_state = gr.State(None)
+        
+        # Check authentication on load
+        def check_auth_on_load(request: gr.Request):
+            if request:
+                # Special handling for HF Spaces OAuth
+                if 'SPACE_ID' in os.environ:
+                    username = request.headers.get("HF-User")
+                    if username:
+                        user = db_manager.get_user_by_username(username)
+                        if user:
+                            print(f"User authenticated via HF Spaces OAuth: {username}")
+                            return user
+                else:
+                    # Standard token-based auth
+                    user = auth_manager.check_login(request)
+                    if user:
+                        return user
+            return None
+        
+        with gr.Tab("Submit Model"):
+            gr.Markdown(f"""
+            ### Model Size Restrictions
+            
+            Models must fit within {evaluation_queue.memory_limit_gb}GB of RAM for evaluation.
+            Large models will be rejected to ensure all evaluations can complete successfully.
+            """, elem_classes=["info-text"])
+            
+            with gr.Row():
+                with gr.Column(scale=2):
+                    model_id_input = gr.Textbox(
+                        placeholder="HuggingFace model ID (e.g., 'gpt2', 'facebook/opt-350m')",
+                        label="Model ID"
+                    )
+                    
+                    check_size_button = gr.Button("Check Model Size")
+                    size_check_result = gr.Markdown("")
+                    model_name_input = gr.Textbox(
+                        placeholder="Display name for your model",
+                        label="Model Name"
+                    )
+                    
+                    model_description_input = gr.Textbox(
+                        placeholder="Brief description of your model",
+                        label="Description",
+                        lines=3
+                    )
+                    
+                    model_parameters_input = gr.Number(
+                        label="Number of Parameters (billions)",
+                        precision=2
+                    )
+                
+                with gr.Column(scale=1):
+                    model_tag_input = gr.Dropdown(
+                        choices=evaluation_queue.model_tags,
+                        label="Model Tag",
+                        info="Select one category that best describes your model"
+                    )
+                    
+                    # Fixed benchmark dropdown to properly show names
+                    benchmark_dropdown = gr.Dropdown(
+                        label="Benchmark",
+                        info="Select a benchmark to evaluate your model on",
+                        choices=[("none", "Loading benchmarks...")],
+                        value=None
+                    )
+                    
+                    refresh_benchmarks_button = gr.Button("Refresh Benchmarks")
+            
+            submit_model_button = gr.Button("Submit for Evaluation")
+            submission_status = gr.Markdown("")
+            auth_message = gr.Markdown("")
+        
+        with gr.Tab("Evaluation Queue"):
+            refresh_queue_button = gr.Button("Refresh Queue")
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    queue_stats = gr.JSON(
+                        label="Queue Statistics"
+                    )
+                
+                with gr.Column(scale=2):
+                    queue_status = gr.Dataframe(
+                        headers=["ID", "Model", "Benchmark", "Status", "Submitted"],
+                        label="Recent Evaluations"
+                    )
+            
+            with gr.Row(visible=True) as progress_container:
+                with gr.Column():
+                    current_eval_info = gr.Markdown("No evaluation currently running")
+                    # Use a simple text display for progress instead of Progress component
+                    progress_display = gr.Markdown("Progress: 0%")
+        
+        # Event handlers
+        def check_model_size_handler(model_id):
+            if not model_id:
+                return "Please enter a HuggingFace model ID."
+            
+            try:
+                will_fit, message = evaluation_queue.check_model_size(model_id)
+                
+                if will_fit:
+                    return f"✅ {message}"
+                else:
+                    return f"❌ {message}"
+            except Exception as e:
+                print(f"Model size check error: {e}")
+                import traceback
+                traceback.print_exc()
+                return f"Error checking model size: {str(e)}"
+            
+        def refresh_benchmarks_handler():
+            benchmarks = db_manager.get_benchmarks()
+            
+            # Format for dropdown - properly formatted to display names
+            choices = []
+            for b in benchmarks:
+                # Add as tuple of (id, name) to ensure proper display
+                choices.append((str(b["id"]), b["name"]))
+            
+            if not choices:
+                choices = [("none", "No benchmarks available - add some first")]
+            
+            return gr.update(choices=choices)
+        
+        def submit_model_handler(model_id, model_name, model_description, model_parameters, model_tag, benchmark_id, user):
+            # Check if user is logged in
+            if not user:
+                return "Please log in to submit a model."
+            
+            if not model_id or not model_name or not model_tag or not benchmark_id:
+                return "Please fill in all required fields."
+            
+            if benchmark_id == "none":
+                return "Please select a valid benchmark."
+            
+            try:
+                # Check if model will fit in RAM
+                will_fit, size_message = evaluation_queue.check_model_size(model_id)
+                
+                if not will_fit:
+                    return f"❌ {size_message}"
+                
+                # Add model to database
+                model_db_id = db_manager.add_model(
+                    name=model_name,
+                    hf_model_id=model_id,
+                    user_id=user["id"],
+                    tag=model_tag,
+                    parameters=str(model_parameters) if model_parameters else None,
+                    description=model_description
+                )
+                
+                if not model_db_id:
+                    return "Failed to add model to database."
+                
+                # Submit for evaluation
+                eval_id, message = evaluation_queue.submit_evaluation(
+                    model_id=model_db_id,
+                    benchmark_id=benchmark_id,
+                    user_id=user["id"]
+                )
+                
+                if eval_id:
+                    return f"✅ Model submitted successfully. {size_message}\nEvaluation ID: {eval_id}"
+                else:
+                    return message
+            except Exception as e:
+                print(f"Error submitting model: {str(e)}")
+                import traceback
+                traceback.print_exc()
+                return f"Error submitting model: {str(e)}"
+        
+        def refresh_queue_handler():
+            # Get queue statistics
+            stats = evaluation_queue.get_queue_status()
+            
+            # Get recent evaluations (all statuses, limited to 20)
+            evals = db_manager.get_evaluation_results(limit=20)
+            
+            # Format for dataframe
+            eval_data = []
+            for eval in evals:
+                eval_data.append([
+                    eval["id"],
+                    eval["model_name"],
+                    eval["benchmark_name"],
+                    eval["status"],
+                    eval["submitted_at"]
+                ])
+            
+            # Also update progress display
+            current_eval, progress = evaluation_queue.get_current_progress()
+            if current_eval:
+                model_info = db_manager.get_model(current_eval['model_id'])
+                benchmark_info = db_manager.get_benchmark(current_eval['benchmark_id'])
+                
+                if model_info and benchmark_info:
+                    eval_info = f"**Currently Evaluating:** {model_info['name']} on {benchmark_info['name']}"
+                    progress_text = f"Progress: {progress}%"
+                    return stats, eval_data, eval_info, progress_text
+            
+            return stats, eval_data, "No evaluation currently running", "Progress: 0%"
+        
+        # Update authentication status
+        def update_auth_message(user):
+            if user:
+                return f"Logged in as {user['username']}"
+            else:
+                return "Please log in to submit a model."
+        
+        # Connect event handlers
+        check_size_button.click(
+            fn=check_model_size_handler,
+            inputs=[model_id_input],
+            outputs=[size_check_result]
+        )
+        
+        refresh_benchmarks_button.click(
+            fn=refresh_benchmarks_handler,
+            inputs=[],
+            outputs=[benchmark_dropdown]
+        )
+        
+        submit_model_button.click(
+            fn=submit_model_handler,
+            inputs=[
+                model_id_input,
+                model_name_input,
+                model_description_input,
+                model_parameters_input,
+                model_tag_input,
+                benchmark_dropdown,
+                user_state
+            ],
+            outputs=[submission_status]
+        )
+        
+        refresh_queue_button.click(
+            fn=refresh_queue_handler,
+            inputs=[],
+            outputs=[queue_stats, queue_status, current_eval_info, progress_display]
+        )
+        
+        # Initialize on load
+        submission_ui.load(
+            fn=check_auth_on_load,
+            inputs=[],
+            outputs=[user_state]
+        )
+        
+        submission_ui.load(
+            fn=lambda user: update_auth_message(user),
+            inputs=[user_state],
+            outputs=[auth_message]
+        )
+        
+        submission_ui.load(
+            fn=refresh_benchmarks_handler,
+            inputs=[],
+            outputs=[benchmark_dropdown]
+        )
+        
+        submission_ui.load(
+            fn=refresh_queue_handler,
+            inputs=[],
+            outputs=[queue_stats, queue_status, current_eval_info, progress_display]
+        )
+    
+    return submission_ui

leaderboard.py

@@ -0,0 +1,396 @@
+"""
+Leaderboard module for Dynamic Highscores system.
+
+This module implements the unified leaderboard with tag-based filtering
+for displaying all evaluated models.
+"""
+
+import os
+import json
+import pandas as pd
+import gradio as gr
+import plotly.express as px
+import plotly.graph_objects as go
+
+class Leaderboard:
+    """Manages the unified leaderboard with filtering capabilities."""
+    
+    def __init__(self, db_manager):
+        """Initialize the leaderboard manager.
+        
+        Args:
+            db_manager: Database manager instance
+        """
+        self.db_manager = db_manager
+        self.model_tags = ["All", "Merge", "Agent", "Reasoning", "Coding", "General", "Specialized", "Instruction", "Chat"]
+        
+        # Define color scheme for tags
+        self.tag_colors = {
+            "Merge": "#FF6B6B",
+            "Agent": "#4ECDC4",
+            "Reasoning": "#FFD166",
+            "Coding": "#6B5B95",
+            "General": "#88D8B0",
+            "Specialized": "#FF8C42",
+            "Instruction": "#5D9CEC",
+            "Chat": "#AC92EB"
+        }
+    
+    def get_leaderboard_data(self, tag=None, benchmark_id=None):
+        """Get leaderboard data, optionally filtered by tag or benchmark.
+        
+        Args:
+            tag: Model tag to filter by (None for all)
+            benchmark_id: Benchmark ID to filter by (None for all)
+            
+        Returns:
+            pd.DataFrame: Leaderboard data
+        """
+        # Get evaluation results from database
+        if tag and tag != "All":
+            df = self.db_manager.get_leaderboard_df(tag=tag, benchmark_id=benchmark_id)
+        else:
+            df = self.db_manager.get_leaderboard_df(benchmark_id=benchmark_id)
+        
+        return df
+    
+    def format_leaderboard_for_display(self, df):
+        """Format leaderboard data for display.
+        
+        Args:
+            df: Leaderboard DataFrame
+            
+        Returns:
+            pd.DataFrame: Formatted leaderboard for display
+        """
+        if df.empty:
+            return pd.DataFrame(columns=['Model', 'Benchmark', 'Tag', 'Score', 'Completed'])
+        
+        # Select and rename columns for display
+        display_df = df[['model_name', 'benchmark_name', 'tag', 'score', 'completed_at']].copy()
+        display_df.columns = ['Model', 'Benchmark', 'Tag', 'Score', 'Completed']
+        
+        # Round score to 2 decimal places
+        display_df['Score'] = display_df['Score'].round(2)
+        
+        # Sort by score (descending)
+        display_df = display_df.sort_values('Score', ascending=False)
+        
+        return display_df
+    
+    def create_performance_chart(self, df, chart_type="bar"):
+        """Create a performance chart from leaderboard data.
+        
+        Args:
+            df: Leaderboard DataFrame
+            chart_type: Type of chart to create ("bar" or "scatter")
+            
+        Returns:
+            plotly.graph_objects.Figure: Performance chart
+        """
+        if df.empty:
+            # Return empty figure
+            fig = go.Figure()
+            fig.update_layout(
+                title="No data available",
+                xaxis_title="Model",
+                yaxis_title="Score"
+            )
+            return fig
+        
+        # Prepare data for visualization
+        plot_df = df[['model_name', 'benchmark_name', 'tag', 'score']].copy()
+        plot_df.columns = ['Model', 'Benchmark', 'Tag', 'Score']
+        
+        # Create chart based on type
+        if chart_type == "scatter":
+            fig = px.scatter(
+                plot_df,
+                x="Model",
+                y="Score",
+                color="Tag",
+                symbol="Benchmark",
+                size="Score",
+                hover_data=["Model", "Benchmark", "Score"],
+                color_discrete_map=self.tag_colors
+            )
+        else:  # Default to bar chart
+            fig = px.bar(
+                plot_df,
+                x="Model",
+                y="Score",
+                color="Tag",
+                barmode="group",
+                hover_data=["Model", "Benchmark", "Score"],
+                color_discrete_map=self.tag_colors
+            )
+        
+        # Customize layout
+        fig.update_layout(
+            title="Model Performance Comparison",
+            xaxis_title="Model",
+            yaxis_title="Score",
+            legend_title="Tag",
+            font=dict(size=12)
+        )
+        
+        return fig
+    
+    def create_tag_distribution_chart(self, df):
+        """Create a chart showing distribution of models by tag.
+        
+        Args:
+            df: Leaderboard DataFrame
+            
+        Returns:
+            plotly.graph_objects.Figure: Tag distribution chart
+        """
+        if df.empty:
+            # Return empty figure
+            fig = go.Figure()
+            fig.update_layout(
+                title="No data available",
+                xaxis_title="Tag",
+                yaxis_title="Count"
+            )
+            return fig
+        
+        # Count models by tag
+        tag_counts = df['tag'].value_counts().reset_index()
+        tag_counts.columns = ['Tag', 'Count']
+        
+        # Create pie chart
+        fig = px.pie(
+            tag_counts,
+            names='Tag',
+            values='Count',
+            title='Model Distribution by Tag',
+            color='Tag',
+            color_discrete_map=self.tag_colors
+        )
+        
+        # Customize layout
+        fig.update_layout(
+            font=dict(size=12)
+        )
+        
+        return fig
+    
+    def create_benchmark_comparison_chart(self, df):
+        """Create a chart comparing performance across benchmarks.
+        
+        Args:
+            df: Leaderboard DataFrame
+            
+        Returns:
+            plotly.graph_objects.Figure: Benchmark comparison chart
+        """
+        if df.empty:
+            # Return empty figure
+            fig = go.Figure()
+            fig.update_layout(
+                title="No data available",
+                xaxis_title="Benchmark",
+                yaxis_title="Average Score"
+            )
+            return fig
+        
+        # Calculate average score by benchmark
+        benchmark_avg = df.groupby('benchmark_name')['score'].mean().reset_index()
+        benchmark_avg.columns = ['Benchmark', 'Average Score']
+        
+        # Create bar chart
+        fig = px.bar(
+            benchmark_avg,
+            x='Benchmark',
+            y='Average Score',
+            title='Average Performance by Benchmark',
+            color='Benchmark'
+        )
+        
+        # Customize layout
+        fig.update_layout(
+            xaxis_title="Benchmark",
+            yaxis_title="Average Score",
+            font=dict(size=12)
+        )
+        
+        return fig
+
+# Leaderboard UI components
+def create_leaderboard_ui(leaderboard, db_manager):
+    """Create the leaderboard UI components.
+    
+    Args:
+        leaderboard: Leaderboard instance
+        db_manager: Database manager instance
+        
+    Returns:
+        gr.Blocks: Gradio Blocks component with leaderboard UI
+    """
+    with gr.Blocks() as leaderboard_ui:
+        gr.Markdown("# Dynamic Highscores Leaderboard")
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                tag_filter = gr.Dropdown(
+                    choices=leaderboard.model_tags,
+                    value="All",
+                    label="Filter by Tag"
+                )
+                
+                benchmark_filter = gr.Dropdown(
+                    choices=[("all", "All Benchmarks")],
+                    value="all",
+                    label="Filter by Benchmark"
+                )
+                
+                refresh_button = gr.Button("Refresh Leaderboard")
+            
+            with gr.Column(scale=2):
+                chart_type = gr.Radio(
+                    choices=["bar", "scatter"],
+                    value="bar",
+                    label="Chart Type"
+                )
+                
+                view_type = gr.Radio(
+                    choices=["Table", "Chart", "Dashboard"],
+                    value="Table",
+                    label="View Type"
+                )
+        
+        # Table view
+        leaderboard_table = gr.Dataframe(
+            headers=["Model", "Benchmark", "Tag", "Score", "Completed"],
+            label="Leaderboard",
+            visible=True
+        )
+        
+        # Chart view
+        with gr.Row(visible=False) as chart_view:
+            performance_chart = gr.Plot(label="Performance Chart")
+        
+        # Dashboard view
+        with gr.Row(visible=False) as dashboard_view:
+            with gr.Column(scale=2):
+                dashboard_performance_chart = gr.Plot(label="Performance Comparison")
+            
+            with gr.Column(scale=1):
+                with gr.Row():
+                    tag_distribution_chart = gr.Plot(label="Model Distribution")
+                
+                with gr.Row():
+                    benchmark_comparison_chart = gr.Plot(label="Benchmark Comparison")
+        
+        # Event handlers
+        def refresh_benchmarks():
+            try:
+                benchmarks = db_manager.get_benchmarks()
+                
+                # Format for dropdown
+                choices = [("all", "All Benchmarks")]
+                choices.extend([(str(b["id"]), b["name"]) for b in benchmarks])
+                
+                return gr.update(choices=choices)
+            except Exception as e:
+                print(f"Error refreshing benchmarks: {e}")
+                return gr.update(choices=[("all", "All Benchmarks")])
+        
+        def update_leaderboard(tag, benchmark_id, chart_type_val, view_type_val):
+            try:
+                # Get leaderboard data
+                if benchmark_id == "all":
+                    benchmark_id = None
+                
+                df = leaderboard.get_leaderboard_data(tag=tag, benchmark_id=benchmark_id)
+                
+                # Format for display
+                display_df = leaderboard.format_leaderboard_for_display(df)
+                
+                # Create charts
+                perf_chart = leaderboard.create_performance_chart(df, chart_type=chart_type_val)
+                tag_chart = leaderboard.create_tag_distribution_chart(df)
+                benchmark_chart = leaderboard.create_benchmark_comparison_chart(df)
+                
+                # Update visibility based on view type
+                table_visible = view_type_val == "Table"
+                chart_visible = view_type_val == "Chart"
+                dashboard_visible = view_type_val == "Dashboard"
+                
+                return (
+                    display_df,
+                    perf_chart,
+                    perf_chart,  # Same chart for both views
+                    tag_chart,
+                    benchmark_chart,
+                    gr.update(visible=table_visible),
+                    gr.update(visible=chart_visible),
+                    gr.update(visible=dashboard_visible)
+                )
+            except Exception as e:
+                print(f"Error updating leaderboard: {e}")
+                empty_df = pd.DataFrame(columns=['Model', 'Benchmark', 'Tag', 'Score', 'Completed'])
+                empty_chart = go.Figure()
+                empty_chart.update_layout(title="Error loading data")
+                
+                return (
+                    empty_df,
+                    empty_chart,
+                    empty_chart,
+                    empty_chart,
+                    empty_chart,
+                    gr.update(visible=True),
+                    gr.update(visible=False),
+                    gr.update(visible=False)
+                )
+        
+        # Connect event handlers
+        refresh_button.click(
+            fn=lambda tag, benchmark, chart_t, view_t: update_leaderboard(tag, benchmark, chart_t, view_t),
+            inputs=[tag_filter, benchmark_filter, chart_type, view_type],
+            outputs=[
+                leaderboard_table,
+                performance_chart,
+                dashboard_performance_chart,
+                tag_distribution_chart,
+                benchmark_comparison_chart,
+                leaderboard_table,
+                chart_view,
+                dashboard_view
+            ]
+        )
+        
+        view_type.change(
+            fn=lambda view_t: (
+                gr.update(visible=view_t == "Table"),
+                gr.update(visible=view_t == "Chart"),
+                gr.update(visible=view_t == "Dashboard")
+            ),
+            inputs=[view_type],
+            outputs=[leaderboard_table, chart_view, dashboard_view]
+        )
+        
+        # Initialize on load
+        leaderboard_ui.load(
+            fn=refresh_benchmarks,
+            inputs=[],
+            outputs=[benchmark_filter]
+        )
+        
+        leaderboard_ui.load(
+            fn=lambda: update_leaderboard("All", "all", "bar", "Table"),
+            inputs=[],
+            outputs=[
+                leaderboard_table,
+                performance_chart,
+                dashboard_performance_chart,
+                tag_distribution_chart,
+                benchmark_comparison_chart,
+                leaderboard_table,
+                chart_view,
+                dashboard_view
+            ]
+        )
+    
+    return leaderboard_ui

model_config.py

@@ -0,0 +1,874 @@
+"""
+Community framework documentation and model configuration system for Dynamic Highscores.
+
+This module provides information about the framework and implements a modular
+system for model configurations.
+"""
+
+import os
+import json
+import gradio as gr
+from huggingface_hub import HfApi
+
+class ModelConfigManager:
+    """Manages model configurations for evaluation."""
+    
+    def __init__(self, db_manager):
+        """Initialize the model configuration manager.
+        
+        Args:
+            db_manager: Database manager instance
+        """
+        self.db_manager = db_manager
+        self.config_dir = "model_configs"
+        
+        # Ensure config directory exists
+        os.makedirs(self.config_dir, exist_ok=True)
+        
+        # Default configurations for popular models
+        self.default_configs = {
+            "gemma": {
+                "name": "Gemma",
+                "description": "Configuration for Gemma models",
+                "parameters": {
+                    "temperature": 1.0,
+                    "top_k": 64,
+                    "min_p": 0.01,
+                    "top_p": 0.95,
+                    "repetition_penalty": 1.0
+                }
+            },
+            "llama": {
+                "name": "LLaMA",
+                "description": "Configuration for LLaMA models",
+                "parameters": {
+                    "temperature": 0.8,
+                    "top_k": 40,
+                    "top_p": 0.9,
+                    "repetition_penalty": 1.1
+                }
+            },
+            "mistral": {
+                "name": "Mistral",
+                "description": "Configuration for Mistral models",
+                "parameters": {
+                    "temperature": 0.7,
+                    "top_k": 50,
+                    "top_p": 0.9,
+                    "repetition_penalty": 1.1
+                }
+            },
+            "phi": {
+                "name": "Phi",
+                "description": "Configuration for Phi models",
+                "parameters": {
+                    "temperature": 0.7,
+                    "top_k": 40,
+                    "top_p": 0.9,
+                    "repetition_penalty": 1.05
+                }
+            },
+            "gpt": {
+                "name": "GPT",
+                "description": "Configuration for GPT models",
+                "parameters": {
+                    "temperature": 0.9,
+                    "top_k": 0,
+                    "top_p": 0.9,
+                    "repetition_penalty": 1.0
+                }
+            }
+        }
+        
+        # Initialize default configs if they don't exist
+        self._initialize_default_configs()
+    
+    def _initialize_default_configs(self):
+        """Initialize default configurations if they don't exist."""
+        for model_type, config in self.default_configs.items():
+            config_path = os.path.join(self.config_dir, f"{model_type}.json")
+            if not os.path.exists(config_path):
+                with open(config_path, "w") as f:
+                    json.dump(config, f, indent=2)
+    
+    def get_available_configs(self):
+        """Get all available model configurations.
+        
+        Returns:
+            list: List of configuration information dictionaries
+        """
+        configs = []
+        
+        # Read all JSON files in the config directory
+        for filename in os.listdir(self.config_dir):
+            if filename.endswith(".json"):
+                config_path = os.path.join(self.config_dir, filename)
+                try:
+                    with open(config_path, "r") as f:
+                        config = json.load(f)
+                    
+                    # Add filename (without extension) as ID
+                    config_id = os.path.splitext(filename)[0]
+                    config["id"] = config_id
+                    
+                    configs.append(config)
+                except Exception as e:
+                    print(f"Error loading config {filename}: {e}")
+        
+        return configs
+    
+    def get_config(self, config_id):
+        """Get a specific model configuration.
+        
+        Args:
+            config_id: Configuration ID (filename without extension)
+            
+        Returns:
+            dict: Configuration information or None if not found
+        """
+        config_path = os.path.join(self.config_dir, f"{config_id}.json")
+        
+        if os.path.exists(config_path):
+            try:
+                with open(config_path, "r") as f:
+                    config = json.load(f)
+                
+                # Add ID to config
+                config["id"] = config_id
+                
+                return config
+            except Exception as e:
+                print(f"Error loading config {config_id}: {e}")
+        
+        return None
+    
+    def add_config(self, name, description, parameters):
+        """Add a new model configuration.
+        
+        Args:
+            name: Configuration name
+            description: Configuration description
+            parameters: Dictionary of configuration parameters
+            
+        Returns:
+            str: Configuration ID if successful, None otherwise
+        """
+        try:
+            # Create a sanitized ID from the name
+            config_id = name.lower().replace(" ", "_").replace("-", "_")
+            
+            # Create config object
+            config = {
+                "name": name,
+                "description": description,
+                "parameters": parameters
+            }
+            
+            # Save to file
+            config_path = os.path.join(self.config_dir, f"{config_id}.json")
+            with open(config_path, "w") as f:
+                json.dump(config, f, indent=2)
+            
+            return config_id
+        except Exception as e:
+            print(f"Error adding config: {e}")
+            return None
+    
+    def update_config(self, config_id, name=None, description=None, parameters=None):
+        """Update an existing model configuration.
+        
+        Args:
+            config_id: Configuration ID to update
+            name: New configuration name (optional)
+            description: New configuration description (optional)
+            parameters: New configuration parameters (optional)
+            
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            # Get existing config
+            config = self.get_config(config_id)
+            
+            if not config:
+                return False
+            
+            # Update fields if provided
+            if name:
+                config["name"] = name
+            
+            if description:
+                config["description"] = description
+            
+            if parameters:
+                config["parameters"] = parameters
+            
+            # Remove ID field before saving
+            if "id" in config:
+                del config["id"]
+            
+            # Save to file
+            config_path = os.path.join(self.config_dir, f"{config_id}.json")
+            with open(config_path, "w") as f:
+                json.dump(config, f, indent=2)
+            
+            return True
+        except Exception as e:
+            print(f"Error updating config {config_id}: {e}")
+            return False
+    
+    def delete_config(self, config_id):
+        """Delete a model configuration.
+        
+        Args:
+            config_id: Configuration ID to delete
+            
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            # Check if config exists
+            config_path = os.path.join(self.config_dir, f"{config_id}.json")
+            
+            if not os.path.exists(config_path):
+                return False
+            
+            # Delete file
+            os.remove(config_path)
+            
+            return True
+        except Exception as e:
+            print(f"Error deleting config {config_id}: {e}")
+            return False
+    
+    def apply_config_to_model_params(self, model_params, config_id):
+        """Apply a configuration to model parameters.
+        
+        Args:
+            model_params: Dictionary of model parameters
+            config_id: Configuration ID to apply
+            
+        Returns:
+            dict: Updated model parameters
+        """
+        # Get the configuration
+        config = self.get_config(config_id)
+        
+        if not config or "parameters" not in config:
+            return model_params
+        
+        # Apply configuration parameters
+        for param, value in config["parameters"].items():
+            model_params[param] = value
+        
+        return model_params
+
+def create_community_framework_ui(model_config_manager):
+    """Create the community framework UI components.
+    
+    Args:
+        model_config_manager: Model configuration manager instance
+        
+    Returns:
+        gr.Blocks: Gradio Blocks component with community framework UI
+    """
+    with gr.Blocks() as community_ui:
+        gr.Markdown("# 🌐 Dynamic Highscores Community Framework")
+        
+        with gr.Tabs() as tabs:
+            with gr.TabItem("About the Framework", id=0):
+                gr.Markdown("""
+                ## About Dynamic Highscores
+
+                Dynamic Highscores is an open-source community benchmark system for evaluating language models on any dataset. This project was created to fill the gap left by the retirement of HuggingFace's "Open LLM Leaderboards" which were discontinued due to outdated benchmarks.
+
+                ### Key Features
+
+                - **Flexible Benchmarking**: Test models against any HuggingFace dataset, not just predefined benchmarks
+                - **Community-Driven**: Anyone can add new benchmarks and submit models for evaluation
+                - **Modern Evaluation**: Focus on contemporary benchmarks that better reflect current model capabilities
+                - **CPU-Only Evaluation**: Ensures fair comparisons across different models
+                - **Daily Submission Limits**: Prevents system abuse (one benchmark per day per user)
+                - **Model Tagging**: Categorize models as Merge, Agent, Reasoning, Coding, etc.
+                - **Unified Leaderboard**: View all models with filtering capabilities by tags
+
+                ### Why This Project Matters
+
+                When HuggingFace retired their "Open LLM Leaderboards," the community lost a valuable resource for comparing model performance. The benchmarks used had become outdated and didn't reflect the rapid advances in language model capabilities.
+
+                Dynamic Highscores addresses this issue by allowing users to select from any benchmark on HuggingFace, including the most recent and relevant datasets. This ensures that models are evaluated on tasks that matter for current applications.
+
+                ## How It Works
+
+                1. **Add Benchmarks**: Users can add any dataset from HuggingFace as a benchmark
+                2. **Submit Models**: Submit your HuggingFace model for evaluation against selected benchmarks
+                3. **View Results**: All results appear on the leaderboard, filterable by model type and benchmark
+                4. **Compare Performance**: See how different models perform across various tasks
+
+                ## Project Structure
+
+                The codebase is organized into several key components:
+
+                - **app.py**: Main application integrating all components
+                - **auth.py**: Authentication system for HuggingFace login
+                - **benchmark_selection.py**: UI and logic for selecting and adding benchmarks
+                - **database_schema.py**: SQLite database schema for storing benchmarks, models, and results
+                - **evaluation_queue.py**: Queue system for processing model evaluations
+                - **leaderboard.py**: Unified leaderboard with filtering capabilities
+                - **sample_benchmarks.py**: Initial benchmark examples
+                - **model_config.py**: Modular system for model configurations
+
+                ## Getting Started
+
+                To use Dynamic Highscores:
+
+                1. Log in with your HuggingFace account
+                2. Browse available benchmarks or add your own
+                3. Submit your model for evaluation
+                4. View results on the leaderboard
+
+                ## Contributing to the Project
+
+                We welcome contributions from the community! If you'd like to improve Dynamic Highscores, here are some ways to get involved:
+
+                - **Add New Features**: Enhance the platform with additional functionality
+                - **Improve Evaluation Methods**: Help make model evaluations more accurate and efficient
+                - **Fix Bugs**: Address issues in the codebase
+                - **Enhance Documentation**: Make the project more accessible to new users
+                - **Add Model Configurations**: Contribute optimal configurations for different model types
+
+                To contribute, fork the repository, make your changes, and submit a pull request. We appreciate all contributions, big or small!
+                """)
+            
+            with gr.TabItem("Model Configurations", id=1):
+                gr.Markdown("""
+                ## Model Configuration System
+
+                The model configuration system allows users to create and apply predefined configurations for different model types. This ensures consistent evaluation settings and helps achieve optimal performance for each model architecture.
+
+                ### What Are Model Configurations?
+
+                Model configurations define parameters such as:
+
+                - **Temperature**: Controls randomness in generation
+                - **Top-K**: Limits token selection to top K most likely tokens
+                - **Top-P (nucleus sampling)**: Selects from tokens comprising the top P probability mass
+                - **Min-P**: Sets a minimum probability threshold for token selection
+                - **Repetition Penalty**: Discourages repetitive text
+
+                Different model architectures perform best with different parameter settings. For example, Gemma models typically work well with:
+
+                ```
+                Temperature: 1.0
+                Top_K: 64
+                Min_P: 0.01
+                Top_P: 0.95
+                Repetition Penalty: 1.0
+                ```
+
+                ### Using Model Configurations
+
+                When submitting a model for evaluation, you can select a predefined configuration or create a custom one. The system will apply these parameters during the evaluation process.
+                """)
+                
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### Available Configurations")
+                        config_list = gr.Dataframe(
+                            headers=["Name", "Description"],
+                            label="Available Configurations",
+                            interactive=True
+                        )
+                        
+                        refresh_configs_button = gr.Button("Refresh Configurations")
+                    
+                    with gr.Column():
+                        selected_config = gr.JSON(label="Configuration Details")
+                
+                with gr.Accordion("Add New Configuration", open=False):
+                    with gr.Row():
+                        with gr.Column():
+                            config_name = gr.Textbox(
+                                placeholder="Enter a name for this configuration",
+                                label="Configuration Name"
+                            )
+                            
+                            config_description = gr.Textbox(
+                                placeholder="Enter a description for this configuration",
+                                label="Description",
+                                lines=2
+                            )
+                        
+                        with gr.Column():
+                            temperature = gr.Slider(
+                                minimum=0.0,
+                                maximum=2.0,
+                                value=0.7,
+                                step=0.1,
+                                label="Temperature"
+                            )
+                            
+                            top_k = gr.Slider(
+                                minimum=0,
+                                maximum=100,
+                                value=50,
+                                step=1,
+                                label="Top-K"
+                            )
+                            
+                            top_p = gr.Slider(
+                                minimum=0.0,
+                                maximum=1.0,
+                                value=0.9,
+                                step=0.01,
+                                label="Top-P"
+                            )
+                            
+                            min_p = gr.Slider(
+                                minimum=0.0,
+                                maximum=0.5,
+                                value=0.01,
+                                step=0.01,
+                                label="Min-P"
+                            )
+                            
+                            repetition_penalty = gr.Slider(
+                                minimum=1.0,
+                                maximum=2.0,
+                                value=1.1,
+                                step=0.05,
+                                label="Repetition Penalty"
+                            )
+                    
+                    add_config_button = gr.Button("Add Configuration")
+                    add_config_status = gr.Markdown("")
+                
+                with gr.Accordion("Edit Configuration", open=False):
+                    with gr.Row():
+                        with gr.Column():
+                            edit_config_id = gr.Dropdown(
+                                choices=[],
+                                label="Select Configuration to Edit"
+                            )
+                            
+                            edit_config_name = gr.Textbox(
+                                label="Configuration Name"
+                            )
+                            
+                            edit_config_description = gr.Textbox(
+                                label="Description",
+                                lines=2
+                            )
+                        
+                        with gr.Column():
+                            edit_temperature = gr.Slider(
+                                minimum=0.0,
+                                maximum=2.0,
+                                step=0.1,
+                                label="Temperature"
+                            )
+                            
+                            edit_top_k = gr.Slider(
+                                minimum=0,
+                                maximum=100,
+                                step=1,
+                                label="Top-K"
+                            )
+                            
+                            edit_top_p = gr.Slider(
+                                minimum=0.0,
+                                maximum=1.0,
+                                step=0.01,
+                                label="Top-P"
+                            )
+                            
+                            edit_min_p = gr.Slider(
+                                minimum=0.0,
+                                maximum=0.5,
+                                step=0.01,
+                                label="Min-P"
+                            )
+                            
+                            edit_repetition_penalty = gr.Slider(
+                                minimum=1.0,
+                                maximum=2.0,
+                                step=0.05,
+                                label="Repetition Penalty"
+                            )
+                    
+                    with gr.Row():
+                        update_config_button = gr.Button("Update Configuration")
+                        delete_config_button = gr.Button("Delete Configuration", variant="stop")
+                    
+                    edit_config_status = gr.Markdown("")
+            
+            with gr.TabItem("Setup Guide", id=2):
+                gr.Markdown("""
+                ## Setting Up Dynamic Highscores
+
+                This guide will help you set up your own instance of Dynamic Highscores, whether you're duplicating the Space or running it locally.
+
+                ### Duplicating the Space
+
+                The easiest way to get started is to duplicate the HuggingFace Space:
+
+                1. Navigate to the original Dynamic Highscores Space
+                2. Click the "Duplicate this Space" button
+                3. Choose a name for your Space
+                4. Wait for the Space to be created and deployed
+
+                That's it! The system is designed to work out-of-the-box without additional configuration.
+
+                ### Running Locally
+
+                To run Dynamic Highscores locally:
+
+                1. Clone the repository:
+                   ```bash
+                   git clone https://huggingface.co/spaces/username/dynamic-highscores
+                   cd dynamic-highscores
+                   ```
+
+                2. Install dependencies:
+                   ```bash
+                   pip install -r requirements.txt
+                   ```
+
+                3. Run the application:
+                   ```bash
+                   python app.py
+                   ```
+
+                4. Open your browser and navigate to `http://localhost:7860`
+
+                ### Configuration Options
+
+                Dynamic Highscores can be configured through environment variables:
+
+                - `ADMIN_USERNAME`: Username for admin access (default: "Quazim0t0")
+                - `DB_PATH`: Path to SQLite database file (default: "dynamic_highscores.db")
+                - `MEMORY_LIMIT_GB`: Memory limit for model evaluation in GB (default: 14)
+
+                ### Adding Sample Benchmarks
+
+                The system comes with sample benchmarks, but you can add more:
+
+                1. Navigate to the "Benchmarks" tab
+                2. Click "Add New Benchmark"
+                3. Enter a HuggingFace dataset ID (e.g., "cais/mmlu", "openai/humaneval")
+                4. Add a name and description
+                5. Select evaluation metrics
+                6. Click "Add as Benchmark"
+
+                ### Setting Up OAuth (Advanced)
+
+                If you're running your own instance outside of HuggingFace Spaces, you'll need to set up OAuth:
+
+                1. Create a HuggingFace application at https://huggingface.co/settings/applications
+                2. Set the redirect URI to your application's URL
+                3. Set the following environment variables:
+                   ```
+                   HF_CLIENT_ID=your_client_id
+                   HF_CLIENT_SECRET=your_client_secret
+                   HF_REDIRECT_URI=your_redirect_uri
+                   ```
+
+                ## Troubleshooting
+
+                ### Login Issues
+
+                - Ensure you're logged in to HuggingFace
+                - Check browser console for any errors
+                - Try clearing cookies and cache
+
+                ### Evaluation Failures
+
+                - Check model size (must be under memory limit)
+                - Verify dataset exists and is accessible
+                - Check logs for specific error messages
+
+                ### Database Issues
+
+                - Ensure the database file is writable
+                - Check for disk space issues
+                - Try backing up and recreating the database
+                """)
+            
+            with gr.TabItem("Development Guide", id=3):
+                gr.Markdown("""
+                ## Development Guide
+
+                This guide is for developers who want to contribute to the Dynamic Highscores project or extend its functionality.
+
+                ### Project Architecture
+
+                Dynamic Highscores follows a modular architecture:
+
+                - **Frontend**: Gradio-based UI components
+                - **Backend**: Python modules for business logic
+                - **Database**: SQLite for data storage
+                - **Evaluation**: CPU-based model evaluation system
+
+                ### Key Components
+
+                1. **Authentication System** (auth.py)
+                   - Handles HuggingFace OAuth
+                   - Manages user sessions
+                   - Controls access to features
+
+                2. **Database Schema** (database_schema.py)
+                   - Defines tables for benchmarks, models, users, and evaluations
+                   - Provides CRUD operations for data management
+
+                3. **Benchmark Selection** (benchmark_selection.py)
+                   - UI for browsing and adding benchmarks
+                   - Integration with HuggingFace datasets
+
+                4. **Evaluation Queue** (evaluation_queue.py)
+                   - Manages model evaluation jobs
+                   - Handles CPU-only processing
+                   - Implements progress tracking
+
+                5. **Leaderboard** (leaderboard.py)
+                   - Displays evaluation results
+                   - Provides filtering and sorting
+                   - Visualizes performance metrics
+
+                6. **Model Configuration** (model_config.py)
+                   - Manages model-specific configurations
+                   - Provides parameter presets for different architectures
+
+                ### Development Workflow
+
+                1. **Setup Development Environment**
+                   ```bash
+                   git clone https://huggingface.co/spaces/username/dynamic-highscores
+                   cd dynamic-highscores
+                   pip install -r requirements.txt
+                   ```
+
+                2. **Make Changes**
+                   - Modify code as needed
+                   - Add new features or fix bugs
+                   - Update documentation
+
+                3. **Test Changes**
+                   ```bash
+                   python test_app.py  # Run test suite
+                   python app.py       # Run application locally
+                   ```
+
+                4. **Submit Changes**
+                   - If you have access, push directly to the repository
+                   - Otherwise, submit a pull request with your changes
+
+                ### Adding New Features
+
+                To add a new feature to Dynamic Highscores:
+
+                1. **Identify the Component**: Determine which component should contain your feature
+                2. **Implement Backend Logic**: Add necessary functions and classes
+                3. **Create UI Components**: Add Gradio UI elements
+                4. **Connect UI to Backend**: Wire up event handlers
+                5. **Update Documentation**: Document your new feature
+                6. **Test Thoroughly**: Ensure everything works as expected
+
+                ### Extending Model Configurations
+
+                To add support for a new model architecture:
+
+                1. Add a new configuration file in the `model_configs` directory
+                2. Define optimal parameters for the architecture
+                3. Update the UI to include the new configuration option
+
+                ### Implementing Custom Evaluation Methods
+
+                To add a new evaluation method:
+
+                1. Add a new method to the `EvaluationQueue` class
+                2. Implement the evaluation logic
+                3. Update the `_run_evaluation` method to use your new method
+                4. Add appropriate metrics to the results
+
+                ### Best Practices
+
+                - **Keep It Simple**: Favor simplicity over complexity
+                - **Document Everything**: Add docstrings and comments
+                - **Write Tests**: Ensure your code works as expected
+                - **Follow Conventions**: Maintain consistent coding style
+                - **Consider Performance**: Optimize for CPU-based evaluation
+                - **Think About Security**: Protect user data and tokens
+
+                ### Getting Help
+
+                If you need assistance with development:
+
+                - Check the existing documentation
+                - Look at the code for similar features
+                - Reach out to the project maintainers
+                - Ask questions in the community forum
+
+                We welcome all contributions and are happy to help new developers get started!
+                """)
+        
+        # Event handlers
+        def refresh_configs():
+            configs = model_config_manager.get_available_configs()
+            
+            # Format for dataframe
+            formatted_configs = []
+            for config in configs:
+                formatted_configs.append([
+                    config["name"],
+                    config["description"]
+                ])
+            
+            # Update dropdown choices for edit
+            config_choices = [(c["id"], c["name"]) for c in configs]
+            
+            return formatted_configs, gr.update(choices=config_choices)
+        
+        def view_config(evt: gr.SelectData, configs):
+            if evt.index[0] < len(configs):
+                config_name = configs[evt.index[0]][0]
+                
+                # Find config by name
+                all_configs = model_config_manager.get_available_configs()
+                selected = None
+                
+                for config in all_configs:
+                    if config["name"] == config_name:
+                        selected = config
+                        break
+                
+                if selected:
+                    return selected
+            
+            return None
+        
+        def add_config_handler(name, description, temperature, top_k, top_p, min_p, repetition_penalty):
+            if not name:
+                return "Please enter a name for the configuration."
+            
+            # Create parameters dictionary
+            parameters = {
+                "temperature": temperature,
+                "top_k": top_k,
+                "top_p": top_p,
+                "min_p": min_p,
+                "repetition_penalty": repetition_penalty
+            }
+            
+            # Add configuration
+            config_id = model_config_manager.add_config(name, description, parameters)
+            
+            if config_id:
+                return f"✅ Configuration '{name}' added successfully."
+            else:
+                return "❌ Failed to add configuration."
+        
+        def load_config_for_edit(config_id):
+            if not config_id:
+                return [gr.update() for _ in range(7)]
+            
+            config = model_config_manager.get_config(config_id)
+            
+            if not config:
+                return [gr.update() for _ in range(7)]
+            
+            # Extract parameters with defaults
+            params = config.get("parameters", {})
+            temperature = params.get("temperature", 0.7)
+            top_k = params.get("top_k", 50)
+            top_p = params.get("top_p", 0.9)
+            min_p = params.get("min_p", 0.01)
+            repetition_penalty = params.get("repetition_penalty", 1.1)
+            
+            return [
+                gr.update(value=config["name"]),
+                gr.update(value=config.get("description", "")),
+                gr.update(value=temperature),
+                gr.update(value=top_k),
+                gr.update(value=top_p),
+                gr.update(value=min_p),
+                gr.update(value=repetition_penalty)
+            ]
+        
+        def update_config_handler(config_id, name, description, temperature, top_k, top_p, min_p, repetition_penalty):
+            if not config_id:
+                return "Please select a configuration to update."
+            
+            # Create parameters dictionary
+            parameters = {
+                "temperature": temperature,
+                "top_k": top_k,
+                "top_p": top_p,
+                "min_p": min_p,
+                "repetition_penalty": repetition_penalty
+            }
+            
+            # Update configuration
+            success = model_config_manager.update_config(config_id, name, description, parameters)
+            
+            if success:
+                return f"✅ Configuration '{name}' updated successfully."
+            else:
+                return "❌ Failed to update configuration."
+        
+        def delete_config_handler(config_id):
+            if not config_id:
+                return "Please select a configuration to delete."
+            
+            # Delete configuration
+            success = model_config_manager.delete_config(config_id)
+            
+            if success:
+                return f"✅ Configuration deleted successfully."
+            else:
+                return "❌ Failed to delete configuration."
+        
+        # Connect event handlers
+        refresh_configs_button.click(
+            fn=refresh_configs,
+            inputs=[],
+            outputs=[config_list, edit_config_id]
+        )
+        
+        config_list.select(
+            fn=view_config,
+            inputs=[config_list],
+            outputs=[selected_config]
+        )
+        
+        add_config_button.click(
+            fn=add_config_handler,
+            inputs=[config_name, config_description, temperature, top_k, top_p, min_p, repetition_penalty],
+            outputs=[add_config_status]
+        )
+        
+        edit_config_id.change(
+            fn=load_config_for_edit,
+            inputs=[edit_config_id],
+            outputs=[edit_config_name, edit_config_description, edit_temperature, edit_top_k, edit_top_p, edit_min_p, edit_repetition_penalty]
+        )
+        
+        update_config_button.click(
+            fn=update_config_handler,
+            inputs=[edit_config_id, edit_config_name, edit_config_description, edit_temperature, edit_top_k, edit_top_p, edit_min_p, edit_repetition_penalty],
+            outputs=[edit_config_status]
+        )
+        
+        delete_config_button.click(
+            fn=delete_config_handler,
+            inputs=[edit_config_id],
+            outputs=[edit_config_status]
+        )
+        
+        # Load configurations on page load
+        community_ui.load(
+            fn=refresh_configs,
+            inputs=[],
+            outputs=[config_list, edit_config_id]
+        )
+    
+    return community_ui

model_configs/gemma.json

@@ -0,0 +1,11 @@
+{
+  "name": "Gemma",
+  "description": "Configuration for Gemma models",
+  "parameters": {
+    "temperature": 1.0,
+    "top_k": 64,
+    "min_p": 0.01,
+    "top_p": 0.95,
+    "repetition_penalty": 1.0
+  }
+}

model_configs/llama.json

@@ -0,0 +1,10 @@
+{
+  "name": "LLaMA",
+  "description": "Configuration for LLaMA models",
+  "parameters": {
+    "temperature": 0.8,
+    "top_k": 40,
+    "top_p": 0.9,
+    "repetition_penalty": 1.1
+  }
+}

model_configs/mistral.json

@@ -0,0 +1,10 @@
+{
+  "name": "Mistral",
+  "description": "Configuration for Mistral models",
+  "parameters": {
+    "temperature": 0.7,
+    "top_k": 50,
+    "top_p": 0.9,
+    "repetition_penalty": 1.1
+  }
+}

requirements.txt

@@ -0,0 +1,12 @@
+gradio>=4.4.0
+huggingface-hub>=0.27.1
+datasets>=2.14.5
+transformers>=4.35.2
+torch>=2.0.0
+pandas>=2.0.0
+numpy>=1.24.2
+plotly>=5.13.0
+APScheduler>=3.10.1
+tqdm>=4.65.0
+requests>=2.28.2
+python-dateutil>=2.8.2

sample_benchmarks.py

@@ -0,0 +1,72 @@
+"""
+Sample benchmarks initialization for Dynamic Highscores system.
+
+This script adds sample benchmarks to the database to provide initial options for users.
+"""
+
+from database_schema import DynamicHighscoresDB
+
+def add_sample_benchmarks():
+    """Add sample benchmarks to the database."""
+    # Initialize database
+    db = DynamicHighscoresDB()
+    
+    # Sample benchmarks to add
+    sample_benchmarks = [
+        {
+            "name": "MMLU (Massive Multitask Language Understanding)",
+            "dataset_id": "cais/mmlu",
+            "description": "A benchmark for measuring massive multitask language understanding across 57 tasks including elementary mathematics, US history, computer science, law, and more.",
+            "metrics": {"accuracy": 1.0, "consistency": 1.0}
+        },
+        {
+            "name": "HumanEval (Code Generation)",
+            "dataset_id": "openai/humaneval",
+            "description": "A benchmark for evaluating language models on code generation tasks. It consists of 164 programming problems with unit tests.",
+            "metrics": {"pass@1": 1.0, "functional_correctness": 1.0}
+        },
+        {
+            "name": "HellaSwag (Commonsense Reasoning)",
+            "dataset_id": "hellaswag",
+            "description": "A challenge dataset for evaluating commonsense natural language inference. It consists of multiple-choice questions about grounded situations.",
+            "metrics": {"accuracy": 1.0}
+        },
+        {
+            "name": "GSM8K (Grade School Math)",
+            "dataset_id": "gsm8k",
+            "description": "A dataset of 8.5K high quality grade school math word problems. These problems take between 2 and 8 steps to solve, and solutions primarily involve performing a sequence of elementary calculations using basic arithmetic operations.",
+            "metrics": {"accuracy": 1.0, "correct_steps": 1.0}
+        },
+        {
+            "name": "TruthfulQA",
+            "dataset_id": "truthful_qa",
+            "description": "A benchmark to measure whether a language model is truthful in generating answers to questions. The benchmark comprises 817 questions that span 38 categories, including health, law, finance and politics.",
+            "metrics": {"accuracy": 1.0, "truthfulness": 1.0}
+        }
+    ]
+    
+    # Add each benchmark to the database
+    added_count = 0
+    for benchmark in sample_benchmarks:
+        try:
+            benchmark_id = db.add_benchmark(
+                name=benchmark["name"],
+                dataset_id=benchmark["dataset_id"],
+                description=benchmark["description"],
+                metrics=benchmark["metrics"]
+            )
+            
+            if benchmark_id:
+                print(f"Added benchmark '{benchmark['name']}' with ID: {benchmark_id}")
+                added_count += 1
+        except Exception as e:
+            print(f"Error adding benchmark '{benchmark['name']}': {e}")
+    
+    # Close database connection
+    db.close()
+    
+    return added_count
+
+if __name__ == "__main__":
+    num_added = add_sample_benchmarks()
+    print(f"Added {num_added} sample benchmarks to the database.")

space.yaml

@@ -0,0 +1,30 @@
+"""
+Deployment configuration for Dynamic Highscores on HuggingFace Spaces.
+
+This file configures the application for deployment on HuggingFace Spaces.
+"""
+
+sdk_version: 3.0.0
+app_file: app.py
+models:
+  - huggingface-hub
+  - transformers
+  - datasets
+  - torch
+  - gradio
+  - pandas
+  - plotly
+  - apscheduler
+  - tqdm
+  - requests
+  - python-dateutil
+  - numpy
+python_version: 3.10.12
+hf_oauth: true
+env:
+  - OAUTH_CLIENT_ID=$OAUTH_CLIENT_ID
+  - OAUTH_CLIENT_SECRET=$OAUTH_CLIENT_SECRET
+  - OAUTH_AUTHORIZATION_URL=$OAUTH_AUTHORIZATION_URL
+  - OAUTH_TOKEN_URL=$OAUTH_TOKEN_URL
+oauth_scopes:
+  - inference

test_app.py

@@ -0,0 +1,237 @@
+"""
+Test script for Dynamic Highscores application.
+
+This script tests the key functionality of the Dynamic Highscores application
+to ensure everything works as expected before deployment.
+"""
+
+import os
+import unittest
+import tempfile
+import sqlite3
+from unittest.mock import MagicMock, patch
+
+# Import components to test
+from database_schema import DynamicHighscoresDB
+from auth import HuggingFaceAuth
+from benchmark_selection import BenchmarkSelector
+from evaluation_queue import EvaluationQueue
+from leaderboard import Leaderboard
+
+class TestDynamicHighscores(unittest.TestCase):
+    """Test cases for Dynamic Highscores application."""
+    
+    def setUp(self):
+        """Set up test environment."""
+        # Create temporary database
+        self.db_fd, self.db_path = tempfile.mkstemp()
+        self.db = DynamicHighscoresDB(self.db_path)
+        
+        # Mock auth manager
+        self.auth_manager = HuggingFaceAuth(self.db)
+        
+        # Mock components
+        self.benchmark_selector = BenchmarkSelector(self.db, self.auth_manager)
+        self.evaluation_queue = EvaluationQueue(self.db, self.auth_manager)
+        self.leaderboard = Leaderboard(self.db)
+    
+    def tearDown(self):
+        """Clean up test environment."""
+        os.close(self.db_fd)
+        os.unlink(self.db_path)
+    
+    def test_database_schema(self):
+        """Test database schema creation."""
+        # Check if tables were created
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        
+        # Get list of tables
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        tables = cursor.fetchall()
+        table_names = [table[0] for table in tables]
+        
+        # Check if all expected tables exist
+        expected_tables = ['users', 'benchmarks', 'models', 'evaluations', 'queue']
+        for table in expected_tables:
+            self.assertIn(table, table_names)
+        
+        conn.close()
+    
+    def test_user_management(self):
+        """Test user management functionality."""
+        # Add a test user
+        user_id = self.db.add_user("test_user", "test_hf_id", False)
+        self.assertIsNotNone(user_id)
+        
+        # Add an admin user
+        admin_id = self.db.add_user("admin_user", "admin_hf_id", True)
+        self.assertIsNotNone(admin_id)
+        
+        # Test submission limits
+        self.assertTrue(self.db.can_submit_today(user_id))
+        self.db.update_submission_date(user_id)
+        self.assertFalse(self.db.can_submit_today(user_id))
+        
+        # Admin should always be able to submit
+        self.assertTrue(self.db.can_submit_today(admin_id))
+    
+    def test_benchmark_management(self):
+        """Test benchmark management functionality."""
+        # Add a test benchmark
+        benchmark_id = self.db.add_benchmark(
+            name="Test Benchmark",
+            dataset_id="test/dataset",
+            description="Test description",
+            metrics={"accuracy": 1.0}
+        )
+        self.assertIsNotNone(benchmark_id)
+        
+        # Get benchmarks
+        benchmarks = self.db.get_benchmarks()
+        self.assertEqual(len(benchmarks), 1)
+        self.assertEqual(benchmarks[0]["name"], "Test Benchmark")
+    
+    def test_model_management(self):
+        """Test model management functionality."""
+        # Add a test user
+        user_id = self.db.add_user("test_user", "test_hf_id", False)
+        
+        # Add a test model
+        model_id = self.db.add_model(
+            name="Test Model",
+            hf_model_id="test/model",
+            user_id=user_id,
+            tag="Reasoning",
+            parameters="7B",
+            description="Test model description"
+        )
+        self.assertIsNotNone(model_id)
+        
+        # Get models
+        models = self.db.get_models()
+        self.assertEqual(len(models), 1)
+        self.assertEqual(models[0]["name"], "Test Model")
+        
+        # Get models by tag
+        models = self.db.get_models(tag="Reasoning")
+        self.assertEqual(len(models), 1)
+        self.assertEqual(models[0]["tag"], "Reasoning")
+    
+    def test_evaluation_management(self):
+        """Test evaluation management functionality."""
+        # Add a test user
+        user_id = self.db.add_user("test_user", "test_hf_id", False)
+        
+        # Add a test model
+        model_id = self.db.add_model(
+            name="Test Model",
+            hf_model_id="test/model",
+            user_id=user_id,
+            tag="Reasoning"
+        )
+        
+        # Add a test benchmark
+        benchmark_id = self.db.add_benchmark(
+            name="Test Benchmark",
+            dataset_id="test/dataset"
+        )
+        
+        # Add a test evaluation
+        evaluation_id = self.db.add_evaluation(
+            model_id=model_id,
+            benchmark_id=benchmark_id
+        )
+        self.assertIsNotNone(evaluation_id)
+        
+        # Update evaluation status
+        self.db.update_evaluation_status(
+            evaluation_id=evaluation_id,
+            status="running"
+        )
+        
+        # Get next in queue
+        next_eval = self.db.get_next_in_queue()
+        self.assertIsNotNone(next_eval)
+        self.assertEqual(next_eval["evaluation_id"], evaluation_id)
+        
+        # Complete evaluation
+        self.db.update_evaluation_status(
+            evaluation_id=evaluation_id,
+            status="completed",
+            results={"accuracy": 0.85},
+            score=85.0
+        )
+        
+        # Get evaluation results
+        results = self.db.get_evaluation_results()
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["score"], 85.0)
+    
+    def test_leaderboard(self):
+        """Test leaderboard functionality."""
+        # Add test data
+        user_id = self.db.add_user("test_user", "test_hf_id", False)
+        
+        # Add models with different tags
+        model1_id = self.db.add_model(
+            name="Model 1",
+            hf_model_id="test/model1",
+            user_id=user_id,
+            tag="Reasoning"
+        )
+        
+        model2_id = self.db.add_model(
+            name="Model 2",
+            hf_model_id="test/model2",
+            user_id=user_id,
+            tag="Coding"
+        )
+        
+        # Add a benchmark
+        benchmark_id = self.db.add_benchmark(
+            name="Test Benchmark",
+            dataset_id="test/dataset"
+        )
+        
+        # Add evaluations
+        eval1_id = self.db.add_evaluation(
+            model_id=model1_id,
+            benchmark_id=benchmark_id
+        )
+        
+        eval2_id = self.db.add_evaluation(
+            model_id=model2_id,
+            benchmark_id=benchmark_id
+        )
+        
+        # Complete evaluations
+        self.db.update_evaluation_status(
+            evaluation_id=eval1_id,
+            status="completed",
+            results={"accuracy": 0.9},
+            score=90.0
+        )
+        
+        self.db.update_evaluation_status(
+            evaluation_id=eval2_id,
+            status="completed",
+            results={"accuracy": 0.8},
+            score=80.0
+        )
+        
+        # Get leaderboard data
+        df = self.leaderboard.get_leaderboard_data()
+        self.assertEqual(len(df), 2)
+        
+        # Test filtering by tag
+        df_reasoning = self.leaderboard.get_leaderboard_data(tag="Reasoning")
+        self.assertEqual(len(df_reasoning), 1)
+        self.assertEqual(df_reasoning.iloc[0]["score"], 90.0)
+        
+        df_coding = self.leaderboard.get_leaderboard_data(tag="Coding")
+        self.assertEqual(len(df_coding), 1)
+        self.assertEqual(df_coding.iloc[0]["score"], 80.0)
+
+if __name__ == "__main__":
+    unittest.main()

todo.md

@@ -0,0 +1,48 @@
+# Dynamic Highscores - Todo List
+
+## Analysis and Planning
+- [x] Extract and analyze uploaded framework files
+- [x] Examine leaderboard component structure and functionality
+- [x] Examine dashboard component structure and functionality
+- [x] Analyze requirements.txt files for dependencies
+
+## Database Schema Design
+- [x] Design schema for user authentication and tracking
+- [x] Design schema for benchmark datasets
+- [x] Design schema for model submissions and evaluations
+- [x] Design schema for tagging system (Merge, Agent, Reasoning, Coding, etc.)
+- [x] Design schema for daily submission limits
+
+## User Authentication System
+- [x] Implement HuggingFace login integration
+- [x] Create user profile management
+- [x] Implement special privileges for admin account
+
+## Benchmark Selection Interface
+- [x] Create interface for browsing HuggingFace datasets
+- [x] Implement dataset loading functionality
+- [x] Create dataset preview and selection UI
+
+## Model Evaluation Queue System
+- [x] Implement CPU-only evaluation system
+- [x] Create queue management for benchmark submissions
+- [x] Implement daily submission limit (1 per day per user)
+- [x] Add admin override for submission limits
+
+## Leaderboard with Filtering
+- [x] Implement unified leaderboard for all models
+- [x] Add tag-based filtering (Merge, Agent, Reasoning, Coding)
+- [x] Implement sorting and searching functionality
+- [x] Create visualization components for benchmark results
+
+## Integration
+- [x] Combine dashboard and leaderboard components
+- [x] Create unified UI with consistent styling
+- [x] Implement navigation between different sections
+- [x] Ensure proper data flow between components
+
+## Testing and Deployment
+- [x] Test user authentication flow
+- [x] Test benchmark selection and submission
+- [x] Test leaderboard filtering and visualization
+- [x] Prepare for deployment on HuggingFace Spaces