app.py

"""
Main application for Dynamic Highscores system.

This file integrates all components into a unified application.
"""

import os
import gradio as gr
import threading
import time
from database_schema import DynamicHighscoresDB
from auth import HuggingFaceAuth
from benchmark_selection import BenchmarkSelector, create_benchmark_selection_ui
from evaluation_queue import EvaluationQueue, create_model_submission_ui
from leaderboard import Leaderboard, create_leaderboard_ui
from sample_benchmarks import add_sample_benchmarks

# Initialize components in main thread
db = DynamicHighscoresDB()
auth_manager = HuggingFaceAuth(db)
benchmark_selector = BenchmarkSelector(db, auth_manager)
evaluation_queue = EvaluationQueue(db, auth_manager)
leaderboard = Leaderboard(db)

# Initialize sample benchmarks if none exist
print("Checking for existing benchmarks...")
benchmarks = db.get_benchmarks()
if not benchmarks or len(benchmarks) == 0:
    print("No benchmarks found. Adding sample benchmarks...")
    try:
        # Make sure the database path is clear
        print(f"Database path: {db.db_path}")
        
        # Import and call the function directly 
        num_added = add_sample_benchmarks()
        print(f"Added {num_added} sample benchmarks.")
    except Exception as e:
        print(f"Error adding sample benchmarks: {str(e)}")
        # Try direct DB insertion as fallback
        try:
            print("Attempting direct benchmark insertion...")
            db.add_benchmark(
                name="MMLU (Massive Multitask Language Understanding)",
                dataset_id="cais/mmlu",
                description="Tests knowledge across 57 subjects"
            )
            print("Added fallback benchmark.")
        except Exception as inner_e:
            print(f"Fallback insertion failed: {str(inner_e)}")
else:
    print(f"Found {len(benchmarks)} existing benchmarks.")

# Custom CSS with theme awareness
css = """
/* Theme-adaptive colored info box */
.info-text {
    background-color: rgba(53, 130, 220, 0.1);
    padding: 12px;
    border-radius: 8px;
    border-left: 4px solid #3498db;
    margin: 12px 0;
}

/* High-contrast text for elements - works in light and dark themes */
.info-text, .header, .footer, .tab-content, 
button, input, textarea, select, option, 
.gradio-container *, .markdown-text {
    color: var(--text-color, inherit) !important;
}

/* Container styling */
.container {
    max-width: 1200px;
    margin: 0 auto;
}

/* Header styling */
.header {
    text-align: center;
    margin-bottom: 20px;
    font-weight: bold;
    font-size: 24px;
}

/* Footer styling */
.footer {
    text-align: center;
    margin-top: 40px;
    padding: 20px;
    border-top: 1px solid var(--border-color-primary, #eee);
}

/* Login section styling */
.login-section {
    padding: 10px;
    margin-bottom: 15px;
    border-radius: 8px;
    background-color: rgba(250, 250, 250, 0.1);
    text-align: center;
}

/* Token input styling */
.token-input {
    margin: 10px 0;
    padding: 8px;
    border-radius: 4px;
    border: 1px solid #ccc;
    width: 100%;
}

/* Force high contrast on specific input areas */
input[type="text"], input[type="password"], textarea {
    background-color: var(--background-fill-primary) !important;
    color: var(--body-text-color) !important;
}

/* Force text visibility in multiple contexts */
.gradio-markdown p, .gradio-markdown h1, .gradio-markdown h2, 
.gradio-markdown h3, .gradio-markdown h4, .gradio-markdown li {
    color: var(--body-text-color) !important;
}

/* Fix dark mode text visibility */
@media (prefers-color-scheme: dark) {
    input, textarea, select {
        color: #ffffff !important;
    }
    
    ::placeholder {
        color: rgba(255, 255, 255, 0.5) !important;
    }
}
"""

# Create token input UI
def create_token_input_ui():
    with gr.Row():
        with gr.Column():
            gr.Markdown("### HuggingFace Token Authentication")
            gr.Markdown("""
            Enter your HuggingFace tokens to use this application. 
            You can find your tokens in your [HuggingFace settings](https://huggingface.co/settings/tokens).
            
            - **Read Token**: Required for accessing models and datasets
            - **Write Token**: Required for submitting evaluation results
            
            Your tokens are stored only in your browser's local storage and are not saved on the server.
            """)
            
            read_token = gr.Textbox(
                label="Read Token", 
                placeholder="Enter your HuggingFace read token", 
                type="password"
            )
            write_token = gr.Textbox(
                label="Write Token", 
                placeholder="Enter your HuggingFace write token", 
                type="password"
            )
            save_button = gr.Button("Save Tokens")
            clear_button = gr.Button("Clear Tokens")
            token_status = gr.Markdown("Not authenticated")
            
            # Hidden field to store the token status
            token_state = gr.State(None)
    
    # JavaScript to handle token storage
    token_js = """
    <script>
    // Function to save tokens to localStorage
    function saveTokens() {
        const readToken = document.querySelector('input[placeholder="Enter your HuggingFace read token"]').value;
        const writeToken = document.querySelector('input[placeholder="Enter your HuggingFace write token"]').value;
        
        if (readToken && writeToken) {
            localStorage.setItem("hf_read_token", readToken);
            localStorage.setItem("hf_write_token", writeToken);
            
            // Set token in cookie for server-side access
            document.cookie = "hf_token=" + readToken + "; path=/; SameSite=Strict";
            
            // Update status
            const statusElement = document.querySelector('div[data-testid="markdown"] p');
            if (statusElement) {
                statusElement.textContent = "Authenticated with tokens";
                statusElement.style.color = "green";
            }
            
            // Reload page to apply tokens
            setTimeout(() => window.location.reload(), 1000);
        } else {
            alert("Please enter both read and write tokens");
        }
    }
    
    // Function to clear tokens from localStorage
    function clearTokens() {
        localStorage.removeItem("hf_read_token");
        localStorage.removeItem("hf_write_token");
        
        // Clear token cookie
        document.cookie = "hf_token=; path=/; max-age=0; SameSite=Strict";
        
        // Update status
        const statusElement = document.querySelector('div[data-testid="markdown"] p');
        if (statusElement) {
            statusElement.textContent = "Not authenticated";
            statusElement.style.color = "red";
        }
        
        // Clear input fields
        document.querySelector('input[placeholder="Enter your HuggingFace read token"]').value = "";
        document.querySelector('input[placeholder="Enter your HuggingFace write token"]').value = "";
        
        // Reload page to apply changes
        setTimeout(() => window.location.reload(), 1000);
    }
    
    // Function to load tokens from localStorage
    function loadTokens() {
        const readToken = localStorage.getItem("hf_read_token");
        const writeToken = localStorage.getItem("hf_write_token");
        
        if (readToken && writeToken) {
            document.querySelector('input[placeholder="Enter your HuggingFace read token"]').value = readToken;
            document.querySelector('input[placeholder="Enter your HuggingFace write token"]').value = writeToken;
            
            // Update status
            const statusElement = document.querySelector('div[data-testid="markdown"] p');
            if (statusElement) {
                statusElement.textContent = "Authenticated with tokens";
                statusElement.style.color = "green";
            }
            
            // Set token in cookie for server-side access if not already set
            if (!document.cookie.includes("hf_token=")) {
                document.cookie = "hf_token=" + readToken + "; path=/; SameSite=Strict";
            }
        }
    }
    
    // Add event listeners once DOM is loaded
    document.addEventListener("DOMContentLoaded", function() {
        // Load tokens from localStorage
        loadTokens();
        
        // Add event listeners to buttons
        const saveButton = document.querySelector('button:nth-of-type(1)');
        const clearButton = document.querySelector('button:nth-of-type(2)');
        
        if (saveButton) {
            saveButton.addEventListener("click", saveTokens);
        }
        
        if (clearButton) {
            clearButton.addEventListener("click", clearTokens);
        }
    });
    </script>
    """
    
    return read_token, write_token, save_button, clear_button, token_status, token_state, token_js

# Simple manual authentication check
def check_user(request: gr.Request):
    if request:
        # Check for token in cookies
        token = request.cookies.get("hf_token")
        
        if token:
            try:
                # Validate token with HuggingFace
                user_info = auth_manager.hf_api.whoami(token=token)
                
                if user_info:
                    username = user_info.get("name", "")
                    print(f"User authenticated via token: {username}")
                    
                    # Check if user exists in our database, create if not
                    user = db.get_user_by_username(username)
                    if not user:
                        # Create user if they don't exist
                        print(f"Creating new user: {username}")
                        is_admin = (username == "Quazim0t0")
                        db.add_user(username, username, is_admin)
                        user = db.get_user_by_username(username)
                    
                    return username
            except Exception as e:
                print(f"Token validation error: {e}")
    
    return None

# Start evaluation queue worker
def start_queue_worker():
    # Wait a moment to ensure app is initialized
    time.sleep(2)
    try:
        print("Starting evaluation queue worker...")
        evaluation_queue.start_worker()
    except Exception as e:
        print(f"Error starting queue worker: {e}")

# Create Gradio app
with gr.Blocks(css=css, title="Dynamic Highscores") as app:
    # State to track user
    user_state = gr.State(None)
    
    # Token input UI
    read_token, write_token, save_button, clear_button, token_status, token_state, token_js = create_token_input_ui()
    
    # Add the token handling JavaScript
    gr.HTML(token_js)
    
    gr.Markdown("# 🏆 Dynamic Highscores", elem_classes=["header"])
    gr.Markdown("""
    Welcome to Dynamic Highscores - a community benchmark platform for evaluating and comparing language models.
    
    - **Add your own benchmarks** from HuggingFace datasets
    - **Submit your models** for CPU-only evaluation
    - **Compare performance** across different models and benchmarks
    - **Filter results** by model type (Merge, Agent, Reasoning, Coding, etc.)
    """, elem_classes=["info-text"])
    
    # Main tabs
    with gr.Tabs() as tabs:
        with gr.TabItem("📊 Leaderboard", id=0):
            leaderboard_ui = create_leaderboard_ui(leaderboard, db)
        
        with gr.TabItem("🚀 Submit Model", id=1):
            submission_ui = create_model_submission_ui(evaluation_queue, auth_manager, db)
        
        with gr.TabItem("🔍 Benchmarks", id=2):
            benchmark_ui = create_benchmark_selection_ui(benchmark_selector, auth_manager)
        
        with gr.TabItem("🌐 Community Framework", id=3):
            # Create a simple placeholder for the Community Framework tab
            gr.Markdown("""
            # 🌐 Dynamic Highscores Community Framework

            ## About Dynamic Highscores

            Dynamic Highscores is an open-source community benchmark system for evaluating language models on any dataset. This project was created to fill the gap left by the retirement of HuggingFace's "Open LLM Leaderboards" which were discontinued due to outdated benchmarks.

            ### Key Features

            - **Flexible Benchmarking**: Test models against any HuggingFace dataset, not just predefined benchmarks
            - **Community-Driven**: Anyone can add new benchmarks and submit models for evaluation
            - **Modern Evaluation**: Focus on contemporary benchmarks that better reflect current model capabilities
            - **CPU-Only Evaluation**: Ensures fair comparisons across different models
            - **Daily Submission Limits**: Prevents system abuse (one benchmark per day per user)
            - **Model Tagging**: Categorize models as Merge, Agent, Reasoning, Coding, etc.
            - **Unified Leaderboard**: View all models with filtering capabilities by tags

            ### Why This Project Matters

            When HuggingFace retired their "Open LLM Leaderboards," the community lost a valuable resource for comparing model performance. The benchmarks used had become outdated and didn't reflect the rapid advances in language model capabilities.

            Dynamic Highscores addresses this issue by allowing users to select from any benchmark on HuggingFace, including the most recent and relevant datasets. This ensures that models are evaluated on tasks that matter for current applications.

            ## Model Configuration System (Coming Soon)

            We're working on a modular system for model configurations that will allow users to:

            - Create and apply predefined configurations for different model types
            - Define parameters such as Temperature, Top-K, Min-P, Top-P, and Repetition Penalty
            - Share optimal configurations with the community

            ### Example Configuration (Gemma)

            ```
            Temperature: 1.0
            Top_K: 64
            Min_P: 0.01
            Top_P: 0.95
            Repetition Penalty: 1.0
            ```

            ## Contributing to the Project

            We welcome contributions from the community! If you'd like to improve Dynamic Highscores, here are some ways to get involved:

            - **Add New Features**: Enhance the platform with additional functionality
            - **Improve Evaluation Methods**: Help make model evaluations more accurate and efficient
            - **Fix Bugs**: Address issues in the codebase
            - **Enhance Documentation**: Make the project more accessible to new users
            - **Add Model Configurations**: Contribute optimal configurations for different model types

            To contribute, fork the repository, make your changes, and submit a pull request. We appreciate all contributions, big or small!
            """)
    
    gr.Markdown("""
    ### About Dynamic Highscores
    
    This platform allows users to select benchmarks from HuggingFace datasets and evaluate models against them.
    Each user can submit one benchmark per day (admin users are exempt from this limit).
    All evaluations run on CPU only to ensure fair comparisons.
    
    Created by Quazim0t0
    """, elem_classes=["footer"])
    
    # Check login on page load
    app.load(
        fn=check_user,
        inputs=[],
        outputs=[user_state]
    )

# Launch the app
if __name__ == "__main__":
    # Start queue worker in a separate thread
    queue_thread = threading.Thread(target=start_queue_worker)
    queue_thread.daemon = True
    queue_thread.start()
    
    # Launch the app
    app.launch()