Delete database_schema.py

database_schema.py

@@ -1,393 +0,0 @@
-"""
-Database schema for Dynamic Highscores system.
-
-This module defines the SQLite database schema for the Dynamic Highscores system,
-which integrates benchmark selection, model evaluation, and leaderboard functionality.
-"""
-
-import sqlite3
-import os
-import json
-from datetime import datetime, timedelta
-import pandas as pd
-
-class DynamicHighscoresDB:
-    """Database manager for the Dynamic Highscores system."""
-    
-    def __init__(self, db_path="dynamic_highscores.db"):
-        """Initialize the database connection and create tables if they don't exist."""
-        self.db_path = db_path
-        self.conn = None
-        self.cursor = None
-        self.connect()
-        self.create_tables()
-    
-    def connect(self):
-        """Connect to the SQLite database."""
-        self.conn = sqlite3.connect(self.db_path)
-        self.conn.row_factory = sqlite3.Row
-        self.cursor = self.conn.cursor()
-    
-    def close(self):
-        """Close the database connection."""
-        if self.conn:
-            self.conn.close()
-    
-    def create_tables(self):
-        """Create all necessary tables if they don't exist."""
-        # Users table - stores user information
-        self.cursor.execute('''
-        CREATE TABLE IF NOT EXISTS users (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            username TEXT UNIQUE NOT NULL,
-            hf_user_id TEXT UNIQUE NOT NULL,
-            is_admin BOOLEAN DEFAULT 0,
-            last_submission_date TEXT,
-            created_at TEXT DEFAULT CURRENT_TIMESTAMP
-        )
-        ''')
-        
-        # Benchmarks table - stores information about available benchmarks
-        self.cursor.execute('''
-        CREATE TABLE IF NOT EXISTS benchmarks (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            name TEXT NOT NULL,
-            dataset_id TEXT NOT NULL,
-            description TEXT,
-            metrics TEXT,  -- JSON string of metrics
-            created_at TEXT DEFAULT CURRENT_TIMESTAMP
-        )
-        ''')
-        
-        # Models table - stores information about submitted models
-        self.cursor.execute('''
-        CREATE TABLE IF NOT EXISTS models (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            name TEXT NOT NULL,
-            hf_model_id TEXT NOT NULL,
-            user_id INTEGER NOT NULL,
-            tag TEXT NOT NULL,  -- One of: Merge, Agent, Reasoning, Coding, etc.
-            parameters TEXT,  -- Number of parameters (can be NULL)
-            description TEXT,
-            created_at TEXT DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (user_id) REFERENCES users (id),
-            UNIQUE (hf_model_id, user_id)
-        )
-        ''')
-        
-        # Evaluations table - stores evaluation results
-        self.cursor.execute('''
-        CREATE TABLE IF NOT EXISTS evaluations (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            model_id INTEGER NOT NULL,
-            benchmark_id INTEGER NOT NULL,
-            status TEXT NOT NULL,  -- pending, running, completed, failed
-            results TEXT,  -- JSON string of results
-            score REAL,  -- Overall score (can be NULL)
-            submitted_at TEXT DEFAULT CURRENT_TIMESTAMP,
-            completed_at TEXT,
-            FOREIGN KEY (model_id) REFERENCES models (id),
-            FOREIGN KEY (benchmark_id) REFERENCES benchmarks (id)
-        )
-        ''')
-        
-        # Queue table - stores evaluation queue
-        self.cursor.execute('''
-        CREATE TABLE IF NOT EXISTS queue (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            evaluation_id INTEGER NOT NULL,
-            priority INTEGER DEFAULT 0,  -- Higher number = higher priority
-            added_at TEXT DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (evaluation_id) REFERENCES evaluations (id)
-        )
-        ''')
-        
-        self.conn.commit()
-    
-    # User management methods
-    def add_user(self, username, hf_user_id, is_admin=False):
-        """Add a new user to the database."""
-        try:
-            self.cursor.execute(
-                "INSERT INTO users (username, hf_user_id, is_admin) VALUES (?, ?, ?)",
-                (username, hf_user_id, is_admin)
-            )
-            self.conn.commit()
-            return self.cursor.lastrowid
-        except sqlite3.IntegrityError:
-            # User already exists
-            self.cursor.execute(
-                "SELECT id FROM users WHERE hf_user_id = ?",
-                (hf_user_id,)
-            )
-            return self.cursor.fetchone()[0]
-    
-    def get_user(self, hf_user_id):
-        """Get user information by HuggingFace user ID."""
-        self.cursor.execute(
-            "SELECT * FROM users WHERE hf_user_id = ?",
-            (hf_user_id,)
-        )
-        return dict(self.cursor.fetchone()) if self.cursor.fetchone() else None
-    
-    def can_submit_today(self, user_id):
-        """Check if a user can submit a benchmark evaluation today."""
-        self.cursor.execute(
-            "SELECT is_admin, last_submission_date FROM users WHERE id = ?",
-            (user_id,)
-        )
-        result = self.cursor.fetchone()
-        
-        if not result:
-            return False
-        
-        user_data = dict(result)
-        
-        # Admin can always submit
-        if user_data['is_admin']:
-            return True
-        
-        # If no previous submission, user can submit
-        if not user_data['last_submission_date']:
-            return True
-        
-        # Check if last submission was before today
-        last_date = datetime.fromisoformat(user_data['last_submission_date'])
-        today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
-        
-        return last_date < today
-    
-    def update_submission_date(self, user_id):
-        """Update the last submission date for a user."""
-        current_time = datetime.now().isoformat()
-        self.cursor.execute(
-            "UPDATE users SET last_submission_date = ? WHERE id = ?",
-            (current_time, user_id)
-        )
-        self.conn.commit()
-    
-    # Benchmark management methods
-    def add_benchmark(self, name, dataset_id, description="", metrics=None):
-        """Add a new benchmark to the database."""
-        if metrics is None:
-            metrics = {}
-        
-        metrics_json = json.dumps(metrics)
-        
-        try:
-            self.cursor.execute(
-                "INSERT INTO benchmarks (name, dataset_id, description, metrics) VALUES (?, ?, ?, ?)",
-                (name, dataset_id, description, metrics_json)
-            )
-            self.conn.commit()
-            return self.cursor.lastrowid
-        except sqlite3.IntegrityError:
-            # Benchmark already exists with this dataset_id
-            self.cursor.execute(
-                "SELECT id FROM benchmarks WHERE dataset_id = ?",
-                (dataset_id,)
-            )
-            return self.cursor.fetchone()[0]
-    
-    def get_benchmarks(self):
-        """Get all available benchmarks."""
-        self.cursor.execute("SELECT * FROM benchmarks")
-        benchmarks = [dict(row) for row in self.cursor.fetchall()]
-        
-        # Parse metrics JSON
-        for benchmark in benchmarks:
-            benchmark['metrics'] = json.loads(benchmark['metrics'])
-        
-        return benchmarks
-    
-    def get_benchmark(self, benchmark_id):
-        """Get benchmark information by ID."""
-        self.cursor.execute(
-            "SELECT * FROM benchmarks WHERE id = ?",
-            (benchmark_id,)
-        )
-        benchmark = dict(self.cursor.fetchone()) if self.cursor.fetchone() else None
-        
-        if benchmark:
-            benchmark['metrics'] = json.loads(benchmark['metrics'])
-        
-        return benchmark
-    
-    # Model management methods
-    def add_model(self, name, hf_model_id, user_id, tag, parameters=None, description=""):
-        """Add a new model to the database."""
-        try:
-            self.cursor.execute(
-                "INSERT INTO models (name, hf_model_id, user_id, tag, parameters, description) VALUES (?, ?, ?, ?, ?, ?)",
-                (name, hf_model_id, user_id, tag, parameters, description)
-            )
-            self.conn.commit()
-            return self.cursor.lastrowid
-        except sqlite3.IntegrityError:
-            # Model already exists for this user
-            self.cursor.execute(
-                "SELECT id FROM models WHERE hf_model_id = ? AND user_id = ?",
-                (hf_model_id, user_id)
-            )
-            return self.cursor.fetchone()[0]
-    
-    def get_models(self, tag=None):
-        """Get all models, optionally filtered by tag."""
-        if tag:
-            self.cursor.execute(
-                "SELECT * FROM models WHERE tag = ?",
-                (tag,)
-            )
-        else:
-            self.cursor.execute("SELECT * FROM models")
-        
-        return [dict(row) for row in self.cursor.fetchall()]
-    
-    def get_model(self, model_id):
-        """Get model information by ID."""
-        self.cursor.execute(
-            "SELECT * FROM models WHERE id = ?",
-            (model_id,)
-        )
-        return dict(self.cursor.fetchone()) if self.cursor.fetchone() else None
-    
-    # Evaluation management methods
-    def add_evaluation(self, model_id, benchmark_id, priority=0):
-        """Add a new evaluation to the database and queue."""
-        # First, add the evaluation
-        self.cursor.execute(
-            "INSERT INTO evaluations (model_id, benchmark_id, status) VALUES (?, ?, 'pending')",
-            (model_id, benchmark_id)
-        )
-        evaluation_id = self.cursor.lastrowid
-        
-        # Then, add it to the queue
-        self.cursor.execute(
-            "INSERT INTO queue (evaluation_id, priority) VALUES (?, ?)",
-            (evaluation_id, priority)
-        )
-        
-        self.conn.commit()
-        return evaluation_id
-    
-    def update_evaluation_status(self, evaluation_id, status, results=None, score=None):
-        """Update the status of an evaluation."""
-        params = [status, evaluation_id]
-        sql = "UPDATE evaluations SET status = ?"
-        
-        if results is not None:
-            sql += ", results = ?"
-            params.insert(1, json.dumps(results))
-        
-        if score is not None:
-            sql += ", score = ?"
-            params.insert(1 if results is None else 2, score)
-        
-        if status in ['completed', 'failed']:
-            sql += ", completed_at = ?"
-            params.insert(1 if results is None and score is None else (2 if results is None or score is None else 3), 
-                         datetime.now().isoformat())
-        
-        sql += " WHERE id = ?"
-        
-        self.cursor.execute(sql, params)
-        self.conn.commit()
-        
-        # If completed or failed, remove from queue
-        if status in ['completed', 'failed']:
-            self.cursor.execute(
-                "DELETE FROM queue WHERE evaluation_id = ?",
-                (evaluation_id,)
-            )
-            self.conn.commit()
-    
-    def get_next_in_queue(self):
-        """Get the next evaluation in the queue."""
-        self.cursor.execute("""
-            SELECT q.id as queue_id, q.evaluation_id, e.model_id, e.benchmark_id, m.hf_model_id, b.dataset_id
-            FROM queue q
-            JOIN evaluations e ON q.evaluation_id = e.id
-            JOIN models m ON e.model_id = m.id
-            JOIN benchmarks b ON e.benchmark_id = b.id
-            WHERE e.status = 'pending'
-            ORDER BY q.priority DESC, q.added_at ASC
-            LIMIT 1
-        """)
-        
-        result = self.cursor.fetchone()
-        return dict(result) if result else None
-    
-    def get_evaluation_results(self, model_id=None, benchmark_id=None, tag=None):
-        """Get evaluation results, optionally filtered by model, benchmark, or tag."""
-        sql = """
-            SELECT e.id, e.model_id, e.benchmark_id, e.status, e.results, e.score, 
-                   e.submitted_at, e.completed_at, m.name as model_name, m.tag, 
-                   b.name as benchmark_name
-            FROM evaluations e
-            JOIN models m ON e.model_id = m.id
-            JOIN benchmarks b ON e.benchmark_id = b.id
-            WHERE e.status = 'completed'
-        """
-        
-        params = []
-        
-        if model_id:
-            sql += " AND e.model_id = ?"
-            params.append(model_id)
-        
-        if benchmark_id:
-            sql += " AND e.benchmark_id = ?"
-            params.append(benchmark_id)
-        
-        if tag:
-            sql += " AND m.tag = ?"
-            params.append(tag)
-        
-        sql += " ORDER BY e.completed_at DESC"
-        
-        self.cursor.execute(sql, params)
-        results = [dict(row) for row in self.cursor.fetchall()]
-        
-        # Parse results JSON
-        for result in results:
-            if result['results']:
-                result['results'] = json.loads(result['results'])
-        
-        return results
-    
-    def get_leaderboard_df(self, tag=None):
-        """Get a pandas DataFrame of the leaderboard, optionally filtered by tag."""
-        results = self.get_evaluation_results(tag=tag)
-        
-        if not results:
-            return pd.DataFrame()
-        
-        # Create a list of dictionaries for the DataFrame
-        leaderboard_data = []
-        
-        for result in results:
-            entry = {
-                'model_name': result['model_name'],
-                'model_id': result['model_id'],
-                'benchmark_name': result['benchmark_name'],
-                'benchmark_id': result['benchmark_id'],
-                'tag': result['tag'],
-                'score': result['score'],
-                'completed_at': result['completed_at']
-            }
-            
-            # Add individual metrics from results
-            if result['results'] and isinstance(result['results'], dict):
-                for metric, value in result['results'].items():
-                    if isinstance(value, (int, float)):
-                        entry[f'metric_{metric}'] = value
-            
-            leaderboard_data.append(entry)
-        
-        return pd.DataFrame(leaderboard_data)
-
-# Initialize the database
-def init_db(db_path="dynamic_highscores.db"):
-    """Initialize the database and return the database manager."""
-    db = DynamicHighscoresDB(db_path)
-    return db