model_config.py

"""
Model configuration system for Dynamic Highscores.

This module provides a modular system for model configurations.
"""

import os
import json
import gradio as gr
from huggingface_hub import HfApi

class ModelConfigManager:
    """Manages model configurations for evaluation."""
    
    def __init__(self, db_manager):
        """Initialize the model configuration manager.
        
        Args:
            db_manager: Database manager instance
        """
        self.db_manager = db_manager
        self.config_dir = "model_configs"
        
        # Ensure config directory exists
        os.makedirs(self.config_dir, exist_ok=True)
        
        # Default configurations for popular models
        self.default_configs = {
            "gemma": {
                "name": "Gemma",
                "description": "Configuration for Gemma models",
                "parameters": {
                    "temperature": 1.0,
                    "top_k": 64,
                    "min_p": 0.01,
                    "top_p": 0.95,
                    "repetition_penalty": 1.0
                }
            },
            "llama": {
                "name": "LLaMA",
                "description": "Configuration for LLaMA models",
                "parameters": {
                    "temperature": 0.8,
                    "top_k": 40,
                    "top_p": 0.9,
                    "repetition_penalty": 1.1
                }
            },
            "mistral": {
                "name": "Mistral",
                "description": "Configuration for Mistral models",
                "parameters": {
                    "temperature": 0.7,
                    "top_k": 50,
                    "top_p": 0.9,
                    "repetition_penalty": 1.1
                }
            },
            "phi": {
                "name": "Phi",
                "description": "Configuration for Phi models",
                "parameters": {
                    "temperature": 0.7,
                    "top_k": 40,
                    "top_p": 0.9,
                    "repetition_penalty": 1.05
                }
            },
            "gpt": {
                "name": "GPT",
                "description": "Configuration for GPT models",
                "parameters": {
                    "temperature": 0.9,
                    "top_k": 0,
                    "top_p": 0.9,
                    "repetition_penalty": 1.0
                }
            }
        }
        
        # Initialize default configs if they don't exist
        self._initialize_default_configs()
    
    def _initialize_default_configs(self):
        """Initialize default configurations if they don't exist."""
        for model_type, config in self.default_configs.items():
            config_path = os.path.join(self.config_dir, f"{model_type}.json")
            if not os.path.exists(config_path):
                with open(config_path, "w") as f:
                    json.dump(config, f, indent=2)
    
    def get_available_configs(self):
        """Get all available model configurations.
        
        Returns:
            list: List of configuration information dictionaries
        """
        configs = []
        
        # Read all JSON files in the config directory
        if os.path.exists(self.config_dir):
            for filename in os.listdir(self.config_dir):
                if filename.endswith(".json"):
                    config_path = os.path.join(self.config_dir, filename)
                    try:
                        with open(config_path, "r") as f:
                            config = json.load(f)
                        
                        # Add filename (without extension) as ID
                        config_id = os.path.splitext(filename)[0]
                        config["id"] = config_id
                        
                        configs.append(config)
                    except Exception as e:
                        print(f"Error loading config {filename}: {e}")
        
        return configs
    
    def get_config(self, config_id):
        """Get a specific model configuration.
        
        Args:
            config_id: Configuration ID (filename without extension)
            
        Returns:
            dict: Configuration information or None if not found
        """
        config_path = os.path.join(self.config_dir, f"{config_id}.json")
        
        if os.path.exists(config_path):
            try:
                with open(config_path, "r") as f:
                    config = json.load(f)
                
                # Add ID to config
                config["id"] = config_id
                
                return config
            except Exception as e:
                print(f"Error loading config {config_id}: {e}")
        
        return None
    
    def add_config(self, name, description, parameters):
        """Add a new model configuration.
        
        Args:
            name: Configuration name
            description: Configuration description
            parameters: Dictionary of configuration parameters
            
        Returns:
            str: Configuration ID if successful, None otherwise
        """
        try:
            # Create a sanitized ID from the name
            config_id = name.lower().replace(" ", "_").replace("-", "_")
            
            # Create config object
            config = {
                "name": name,
                "description": description,
                "parameters": parameters
            }
            
            # Save to file
            config_path = os.path.join(self.config_dir, f"{config_id}.json")
            with open(config_path, "w") as f:
                json.dump(config, f, indent=2)
            
            return config_id
        except Exception as e:
            print(f"Error adding config: {e}")
            return None
    
    def update_config(self, config_id, name=None, description=None, parameters=None):
        """Update an existing model configuration.
        
        Args:
            config_id: Configuration ID to update
            name: New configuration name (optional)
            description: New configuration description (optional)
            parameters: New configuration parameters (optional)
            
        Returns:
            bool: True if successful, False otherwise
        """
        try:
            # Get existing config
            config = self.get_config(config_id)
            
            if not config:
                return False
            
            # Update fields if provided
            if name:
                config["name"] = name
            
            if description:
                config["description"] = description
            
            if parameters:
                config["parameters"] = parameters
            
            # Remove ID field before saving
            if "id" in config:
                del config["id"]
            
            # Save to file
            config_path = os.path.join(self.config_dir, f"{config_id}.json")
            with open(config_path, "w") as f:
                json.dump(config, f, indent=2)
            
            return True
        except Exception as e:
            print(f"Error updating config: {e}")
            return False
    
    def delete_config(self, config_id):
        """Delete a model configuration.
        
        Args:
            config_id: Configuration ID to delete
            
        Returns:
            bool: True if successful, False otherwise
        """
        try:
            # Check if this is a default config
            if config_id in self.default_configs:
                print(f"Cannot delete default config: {config_id}")
                return False
            
            # Delete file
            config_path = os.path.join(self.config_dir, f"{config_id}.json")
            if os.path.exists(config_path):
                os.remove(config_path)
                return True
            
            return False
        except Exception as e:
            print(f"Error deleting config: {e}")
            return False
    
    def apply_config_to_model_params(self, model_params, config_id):
        """Apply a configuration to model parameters.
        
        Args:
            model_params: Dictionary of model parameters to update
            config_id: Configuration ID to apply
            
        Returns:
            dict: Updated model parameters
        """
        config = self.get_config(config_id)
        
        if not config or "parameters" not in config:
            return model_params
        
        # Apply configuration parameters
        for param, value in config["parameters"].items():
            model_params[param] = value
        
        return model_params

def create_community_framework_ui(model_config_manager):
    """Create the community framework UI components.
    
    Args:
        model_config_manager: Model configuration manager instance
        
    Returns:
        gr.Blocks: Gradio Blocks component with community framework UI
    """
    with gr.Blocks() as community_ui:
        gr.Markdown("# 🌐 Dynamic Highscores Community Framework")
        
        with gr.Tabs() as tabs:
            with gr.TabItem("About the Framework", id=0):
                gr.Markdown("""
                ## About Dynamic Highscores

                Dynamic Highscores is an open-source community benchmark system for evaluating language models on any dataset. This project was created to fill the gap left by the retirement of HuggingFace's "Open LLM Leaderboards" which were discontinued due to outdated benchmarks.

                ### Key Features

                - **Flexible Benchmarking**: Test models against any HuggingFace dataset, not just predefined benchmarks
                - **Community-Driven**: Anyone can add new benchmarks and submit models for evaluation
                - **Modern Evaluation**: Focus on contemporary benchmarks that better reflect current model capabilities
                - **CPU-Only Evaluation**: Ensures fair comparisons across different models
                - **Daily Submission Limits**: Prevents system abuse (one benchmark per day per user)
                - **Model Tagging**: Categorize models as Merge, Agent, Reasoning, Coding, etc.
                - **Unified Leaderboard**: View all models with filtering capabilities by tags

                ### Why This Project Matters

                When HuggingFace retired their "Open LLM Leaderboards," the community lost a valuable resource for comparing model performance. The benchmarks used had become outdated and didn't reflect the rapid advances in language model capabilities.

                Dynamic Highscores addresses this issue by allowing users to select from any benchmark on HuggingFace, including the most recent and relevant datasets. This ensures that models are evaluated on tasks that matter for current applications.

                ## How It Works

                1. **Add Benchmarks**: Users can add any dataset from HuggingFace as a benchmark
                2. **Submit Models**: Submit your HuggingFace model for evaluation against selected benchmarks
                3. **View Results**: All results appear on the leaderboard, filterable by model type and benchmark
                4. **Compare Performance**: See how different models perform across various tasks

                ## Project Structure

                The codebase is organized into several key components:

                - **app.py**: Main application integrating all components
                - **auth.py**: Authentication system for HuggingFace login
                - **benchmark_selection.py**: UI and logic for selecting and adding benchmarks
                - **database_schema.py**: SQLite database schema for storing benchmarks, models, and results
                - **evaluation_queue.py**: Queue system for processing model evaluations
                - **leaderboard.py**: Unified leaderboard with filtering capabilities
                - **sample_benchmarks.py**: Initial benchmark examples
                - **model_config.py**: Modular system for model configurations

                ## Getting Started

                To use Dynamic Highscores:

                1. Log in with your HuggingFace account
                2. Browse available benchmarks or add your own
                3. Submit your model for evaluation
                4. View results on the leaderboard

                ## Contributing to the Project

                We welcome contributions from the community! If you'd like to improve Dynamic Highscores, here are some ways to get involved:

                - **Add New Features**: Enhance the platform with additional functionality
                - **Improve Evaluation Methods**: Help make model evaluations more accurate and efficient
                - **Fix Bugs**: Address issues in the codebase
                - **Enhance Documentation**: Make the project more accessible to new users
                - **Add Model Configurations**: Contribute optimal configurations for different model types

                To contribute, fork the repository, make your changes, and submit a pull request. We appreciate all contributions, big or small!
                """)
            
            with gr.TabItem("Model Configurations", id=1):
                gr.Markdown("""
                ## Model Configuration System

                The model configuration system allows users to create and apply predefined configurations for different model types. This ensures consistent evaluation settings and helps achieve optimal performance for each model architecture.

                ### What Are Model Configurations?

                Model configurations define parameters such as:

                - **Temperature**: Controls randomness in generation
                - **Top-K**: Limits token selection to top K most likely tokens
                - **Top-P (nucleus sampling)**: Selects from tokens comprising the top P probability mass
                - **Min-P**: Sets a minimum probability threshold for token selection
                - **Repetition Penalty**: Discourages repetitive text

                Different model architectures perform best with different parameter settings. For example, Gemma models typically work well with:

                ```
                Temperature: 1.0
                Top_K: 64
                Min_P: 0.01
                Top_P: 0.95
                Repetition Penalty: 1.0
                ```

                ### Using Model Configurations

                When submitting a model for evaluation, you can select a predefined configuration or create a custom one. The system will apply these parameters during the evaluation process.
                """)
                
                with gr.Row():
                    with gr.Column():
                        gr.Markdown("### Available Configurations")
                        config_list = gr.Dataframe(
                            headers=["Name", "Description"],
                            label="Available Configurations",
                            interactive=True
                        )
                        
                        refresh_configs_button = gr.Button("Refresh Configurations")
                    
                    with gr.Column():
                        selected_config = gr.JSON(label="Configuration Details")
                
                with gr.Accordion("Add New Configuration", open=False):
                    with gr.Row():
                        with gr.Column():
                            config_name = gr.Textbox(
                                placeholder="Enter a name for this configuration",
                                label="Configuration Name"
                            )
                            
                            config_description = gr.Textbox(
                                placeholder="Enter a description for this configuration",
                                label="Description",
                                lines=2
                            )
                        
                        with gr.Column():
                            temperature = gr.Slider(
                                minimum=0.0,
                                maximum=2.0,
                                value=0.7,
                                step=0.1,
                                label="Temperature"
                            )
                            
                            top_k = gr.Slider(
                                minimum=0,
                                maximum=100,
                                value=50,
                                step=1,
                                label="Top-K"
                            )
                            
                            top_p = gr.Slider(
                                minimum=0.0,
                                maximum=1.0,
                                value=0.9,
                                step=0.01,
                                label="Top-P"
                            )
                            
                            min_p = gr.Slider(
                                minimum=0.0,
                                maximum=0.5,
                                value=0.01,
                                step=0.01,
                                label="Min-P"
                            )
                            
                            repetition_penalty = gr.Slider(
                                minimum=1.0,
                                maximum=2.0,
                                value=1.1,
                                step=0.05,
                                label="Repetition Penalty"
                            )
                    
                    add_config_button = gr.Button("Add Configuration")
                    add_config_status = gr.Markdown("")
                
                with gr.Accordion("Edit Configuration", open=False):
                    with gr.Row():
                        with gr.Column():
                            edit_config_id = gr.Dropdown(
                                choices=[],
                                label="Select Configuration to Edit"
                            )
                            
                            edit_config_name = gr.Textbox(
                                label="Configuration Name"
                            )
                            
                            edit_config_description = gr.Textbox(
                                label="Description",
                                lines=2
                            )
                        
                        with gr.Column():
                            edit_temperature = gr.Slider(
                                minimum=0.0,
                                maximum=2.0,
                                step=0.1,
                                label="Temperature"
                            )
                            
                            edit_top_k = gr.Slider(
                                minimum=0,
                                maximum=100,
                                step=1,
                                label="Top-K"
                            )
                            
                            edit_top_p = gr.Slider(
                                minimum=0.0,
                                maximum=1.0,
                                step=0.01,
                                label="Top-P"
                            )
                            
                            edit_min_p = gr.Slider(
                                minimum=0.0,
                                maximum=0.5,
                                step=0.01,
                                label="Min-P"
                            )
                            
                            edit_repetition_penalty = gr.Slider(
                                minimum=1.0,
                                maximum=2.0,
                                step=0.05,
                                label="Repetition Penalty"
                            )
                    
                    with gr.Row():
                        update_config_button = gr.Button("Update Configuration")
                        delete_config_button = gr.Button("Delete Configuration", variant="stop")
                    
                    edit_config_status = gr.Markdown("")
            
            with gr.TabItem("Setup Guide", id=2):
                gr.Markdown("""
                ## Setting Up Dynamic Highscores

                This guide will help you set up your own instance of Dynamic Highscores, whether you're duplicating the Space or running it locally.

                ### Duplicating the Space

                The easiest way to get started is to duplicate the HuggingFace Space:

                1. Navigate to the original Dynamic Highscores Space
                2. Click the "Duplicate this Space" button
                3. Choose a name for your Space
                4. Wait for the Space to be created and deployed

                That's it! The system is designed to work out-of-the-box without additional configuration.

                ### Running Locally

                To run Dynamic Highscores locally:

                1. Clone the repository:
                   ```bash
                   git clone https://huggingface.co/spaces/username/dynamic-highscores
                   cd dynamic-highscores
                   ```

                2. Install dependencies:
                   ```bash
                   pip install -r requirements.txt
                   ```

                3. Run the application:
                   ```bash
                   python app.py
                   ```

                4. Open your browser and navigate to `http://localhost:7860`

                ### Configuration Options

                Dynamic Highscores can be configured through environment variables:

                - `ADMIN_USERNAME`: Username for admin access (default: "Quazim0t0")
                - `DB_PATH`: Path to SQLite database file (default: "dynamic_highscores.db")
                - `MEMORY_LIMIT_GB`: Memory limit for model evaluation in GB (default: 14)

                ### Adding Sample Benchmarks

                The system comes with sample benchmarks, but you can add more:

                1. Navigate to the "Benchmarks" tab
                2. Click "Add New Benchmark"
                3. Enter a HuggingFace dataset ID (e.g., "cais/mmlu", "openai/humaneval")
                4. Add a name and description
                5. Select evaluation metrics
                6. Click "Add as Benchmark"

                ### Setting Up OAuth (Advanced)

                If you're running your own instance outside of HuggingFace Spaces, you'll need to set up OAuth:

                1. Create a HuggingFace application at https://huggingface.co/settings/applications
                2. Set the redirect URI to your application's URL
                3. Set the following environment variables:
                   ```
                   HF_CLIENT_ID=your_client_id
                   HF_CLIENT_SECRET=your_client_secret
                   HF_REDIRECT_URI=your_redirect_uri
                   ```

                ## Troubleshooting

                ### Login Issues

                - Ensure you're logged in to HuggingFace
                - Check browser console for any errors
                - Try clearing cookies and cache

                ### Evaluation Failures

                - Check model size (must be under memory limit)
                - Verify dataset exists and is accessible
                - Check logs for specific error messages

                ### Database Issues

                - Ensure the database file is writable
                - Check for disk space issues
                - Try backing up and recreating the database
                """)
            
            with gr.TabItem("Development Guide", id=3):
                gr.Markdown("""
                ## Development Guide

                This guide is for developers who want to contribute to the Dynamic Highscores project or extend its functionality.

                ### Project Architecture

                Dynamic Highscores follows a modular architecture:

                - **Frontend**: Gradio-based UI components
                - **Backend**: Python modules for business logic
                - **Database**: SQLite for data storage
                - **Evaluation**: CPU-based model evaluation system

                ### Key Components

                1. **Authentication System** (auth.py)
                   - Handles HuggingFace OAuth
                   - Manages user sessions
                   - Controls access to features

                2. **Database Schema** (database_schema.py)
                   - Defines tables for benchmarks, models, users, and evaluations
                   - Provides CRUD operations for data management

                3. **Benchmark Selection** (benchmark_selection.py)
                   - UI for browsing and adding benchmarks
                   - Integration with HuggingFace datasets

                4. **Evaluation Queue** (evaluation_queue.py)
                   - Manages model evaluation jobs
                   - Handles CPU-only processing
                   - Implements progress tracking

                5. **Leaderboard** (leaderboard.py)
                   - Displays evaluation results
                   - Provides filtering and sorting
                   - Visualizes performance metrics

                6. **Model Configuration** (model_config.py)
                   - Manages model-specific configurations
                   - Provides parameter presets for different architectures

                ### Development Workflow

                1. **Setup Development Environment**
                   ```bash
                   git clone https://huggingface.co/spaces/username/dynamic-highscores
                   cd dynamic-highscores
                   pip install -r requirements.txt
                   ```

                2. **Make Changes**
                   - Modify code as needed
                   - Add new features or fix bugs
                   - Update documentation

                3. **Test Changes**
                   ```bash
                   python test_app.py  # Run test suite
                   python app.py       # Run application locally
                   ```

                4. **Submit Changes**
                   - If you have access, push directly to the repository
                   - Otherwise, submit a pull request with your changes

                ### Adding New Features

                To add a new feature to Dynamic Highscores:

                1. **Identify the Component**: Determine which component should contain your feature
                2. **Implement Backend Logic**: Add necessary functions and classes
                3. **Create UI Components**: Add Gradio UI elements
                4. **Connect UI to Backend**: Wire up event handlers
                5. **Update Documentation**: Document your new feature
                6. **Test Thoroughly**: Ensure everything works as expected

                ### Extending Model Configurations

                To add support for a new model architecture:

                1. Add a new configuration file in the `model_configs` directory
                2. Define optimal parameters for the architecture
                3. Update the UI to include the new configuration option

                ### Implementing Custom Evaluation Methods

                To add a new evaluation method:

                1. Add a new method to the `EvaluationQueue` class
                2. Implement the evaluation logic
                3. Update the `_run_evaluation` method to use your new method
                4. Add appropriate metrics to the results

                ### Best Practices

                - **Keep It Simple**: Favor simplicity over complexity
                - **Document Everything**: Add docstrings and comments
                - **Write Tests**: Ensure your code works as expected
                - **Follow Conventions**: Maintain consistent coding style
                - **Consider Performance**: Optimize for CPU-based evaluation
                - **Think About Security**: Protect user data and tokens

                ### Getting Help

                If you need assistance with development:

                - Check the existing documentation
                - Look at the code for similar features
                - Reach out to the project maintainers
                - Ask questions in the community forum

                We welcome all contributions and are happy to help new developers get started!
                """)
        
        # Event handlers
        def refresh_configs():
            configs = model_config_manager.get_available_configs()
            
            # Format for dataframe
            formatted_configs = []
            for config in configs:
                formatted_configs.append([
                    config["name"],
                    config["description"]
                ])
            
            # Update dropdown choices for edit
            config_choices = [(c["id"], c["name"]) for c in configs]
            
            return formatted_configs, gr.update(choices=config_choices)
        
        def view_config(evt: gr.SelectData, configs):
            if evt.index[0] < len(configs):
                config_name = configs[evt.index[0]][0]
                
                # Find config by name
                all_configs = model_config_manager.get_available_configs()
                selected = None
                
                for config in all_configs:
                    if config["name"] == config_name:
                        selected = config
                        break
                
                if selected:
                    return selected
            
            return None
        
        def add_config_handler(name, description, temperature, top_k, top_p, min_p, repetition_penalty):
            if not name:
                return "Please enter a name for the configuration."
            
            # Create parameters dictionary
            parameters = {
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "min_p": min_p,
                "repetition_penalty": repetition_penalty
            }
            
            # Add configuration
            config_id = model_config_manager.add_config(name, description, parameters)
            
            if config_id:
                return f"✅ Configuration '{name}' added successfully."
            else:
                return "❌ Failed to add configuration."
        
        def load_config_for_edit(config_id):
            if not config_id:
                return [gr.update() for _ in range(7)]
            
            config = model_config_manager.get_config(config_id)
            
            if not config:
                return [gr.update() for _ in range(7)]
            
            # Extract parameters with defaults
            params = config.get("parameters", {})
            temperature = params.get("temperature", 0.7)
            top_k = params.get("top_k", 50)
            top_p = params.get("top_p", 0.9)
            min_p = params.get("min_p", 0.01)
            repetition_penalty = params.get("repetition_penalty", 1.1)
            
            return [
                gr.update(value=config["name"]),
                gr.update(value=config.get("description", "")),
                gr.update(value=temperature),
                gr.update(value=top_k),
                gr.update(value=top_p),
                gr.update(value=min_p),
                gr.update(value=repetition_penalty)
            ]
        
        def update_config_handler(config_id, name, description, temperature, top_k, top_p, min_p, repetition_penalty):
            if not config_id:
                return "Please select a configuration to update."
            
            # Create parameters dictionary
            parameters = {
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "min_p": min_p,
                "repetition_penalty": repetition_penalty
            }
            
            # Update configuration
            success = model_config_manager.update_config(config_id, name, description, parameters)
            
            if success:
                return f"✅ Configuration '{name}' updated successfully."
            else:
                return "❌ Failed to update configuration."
        
        def delete_config_handler(config_id):
            if not config_id:
                return "Please select a configuration to delete."
            
            # Delete configuration
            success = model_config_manager.delete_config(config_id)
            
            if success:
                return f"✅ Configuration deleted successfully."
            else:
                return "❌ Failed to delete configuration."
        
        # Connect event handlers
        refresh_configs_button.click(
            fn=refresh_configs,
            inputs=[],
            outputs=[config_list, edit_config_id]
        )
        
        config_list.select(
            fn=view_config,
            inputs=[config_list],
            outputs=[selected_config]
        )
        
        add_config_button.click(
            fn=add_config_handler,
            inputs=[config_name, config_description, temperature, top_k, top_p, min_p, repetition_penalty],
            outputs=[add_config_status]
        )
        
        edit_config_id.change(
            fn=load_config_for_edit,
            inputs=[edit_config_id],
            outputs=[edit_config_name, edit_config_description, edit_temperature, edit_top_k, edit_top_p, edit_min_p, edit_repetition_penalty]
        )
        
        update_config_button.click(
            fn=update_config_handler,
            inputs=[edit_config_id, edit_config_name, edit_config_description, edit_temperature, edit_top_k, edit_top_p, edit_min_p, edit_repetition_penalty],
            outputs=[edit_config_status]
        )
        
        delete_config_button.click(
            fn=delete_config_handler,
            inputs=[edit_config_id],
            outputs=[edit_config_status]
        )
        
        # Load configurations on page load
        community_ui.load(
            fn=refresh_configs,
            inputs=[],
            outputs=[config_list, edit_config_id]
        )
    
    return community_ui