File size: 5,009 Bytes
d1078a3 968f2d4 d1078a3 968f2d4 d1078a3 968f2d4 d1078a3 968f2d4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 |
"""
Leaderboard module for Dynamic Highscores system.
This module implements the unified leaderboard with tag-based filtering
for displaying all evaluated models.
"""
import os
import json
import pandas as pd
import gradio as gr
import plotly.express as px
import plotly.graph_objects as go
class Leaderboard:
"""Manages the unified leaderboard with filtering capabilities."""
def __init__(self, db_manager):
"""Initialize the leaderboard manager.
Args:
db_manager: Database manager instance
"""
self.db_manager = db_manager
self.model_tags = ["All", "Merge", "Agent", "Reasoning", "Coding", "General", "Specialized", "Instruction", "Chat"]
# Define color scheme for tags
self.tag_colors = {
"Merge": "#FF6B6B",
"Agent": "#4ECDC4",
"Reasoning": "#FFD166",
"Coding": "#6B5B95",
"General": "#88D8B0",
"Specialized": "#FF8C42",
"Instruction": "#5D9CEC",
"Chat": "#AC92EB"
}
def get_leaderboard_data(self, tag=None, benchmark_id=None):
"""Get leaderboard data, optionally filtered by tag or benchmark.
Args:
tag: Model tag to filter by (None for all)
benchmark_id: Benchmark ID to filter by (None for all)
Returns:
pd.DataFrame: Leaderboard data
"""
# Get evaluation results from database
if tag and tag != "All":
df = self.db_manager.get_leaderboard_df(tag=tag, benchmark_id=benchmark_id)
else:
df = self.db_manager.get_leaderboard_df(benchmark_id=benchmark_id)
return df
def format_leaderboard_for_display(self, df):
"""Format leaderboard data for display.
Args:
df: Leaderboard DataFrame
Returns:
pd.DataFrame: Formatted leaderboard for display
"""
if df.empty:
return pd.DataFrame(columns=['Model', 'Benchmark', 'Tag', 'Score', 'Completed'])
# Select and rename columns for display
display_df = df[['model_name', 'benchmark_name', 'tag', 'score', 'completed_at']].copy()
display_df.columns = ['Model', 'Benchmark', 'Tag', 'Score', 'Completed']
# Round score to 2 decimal places
display_df['Score'] = display_df['Score'].round(2)
# Sort by score (descending)
display_df = display_df.sort_values('Score', ascending=False)
return display_df
def create_performance_chart(self, df, chart_type="bar"):
"""Create a performance chart from leaderboard data.
Args:
df: Leaderboard DataFrame
chart_type: Type of chart to create ("bar" or "scatter")
Returns:
plotly.graph_objects.Figure: Performance chart
"""
if df.empty:
# Return empty figure
fig = go.Figure()
fig.update_layout(
title="No data available",
xaxis_title="Model",
yaxis_title="Score"
)
return fig
# Prepare data for visualization
plot_df = df[['model_name', 'benchmark_name', 'tag', 'score']].copy()
plot_df.columns = ['Model', 'Benchmark', 'Tag', 'Score']
# Create chart based on type
if chart_type == "scatter":
fig = px.scatter(
plot_df,
x="Model",
y="Score",
color="Tag",
symbol="Benchmark",
size="Score",
hover_data=["Model", "Benchmark", "Score"],
color_discrete_map=self.tag_colors
)
else: # Default to bar chart
fig = px.bar(
plot_df,
x="Model",
y="Score",
color="Tag",
barmode="group",
hover_data=["Model", "Benchmark", "Score"],
color_discrete_map=self.tag_colors
)
# Customize layout
fig.update_layout(
title="Model Performance Comparison",
xaxis_title="Model",
yaxis_title="Score",
legend_title="Tag",
font=dict(size=12)
)
return fig
def create_tag_distribution_chart(self, df):
"""Create a chart showing distribution of models by tag.
Args:
df: Leaderboard DataFrame
Returns:
plotly.graph_objects.Figure: Tag distribution chart
"""
if df.empty:
# Return empty figure
fig = go.Figure()
fig.update_layout(
title="No data available",
xaxis_title="Tag",
yaxis_title="Count"
)
return fig |