|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import gradio as gr |
|
import pandas as pd |
|
import os |
|
import re |
|
from datetime import datetime |
|
from huggingface_hub import hf_hub_download |
|
from huggingface_hub import HfApi, HfFolder |
|
|
|
LEADERBOARD_FILE = "leaderboard.csv" |
|
GROUND_TRUTH_FILE = "ground_truth.csv" |
|
LAST_UPDATED = datetime.now().strftime("%B %d, %Y") |
|
|
|
|
|
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1" |
|
HF_TOKEN = os.getenv("HF_TOKEN") |
|
if not HF_TOKEN: |
|
raise ValueError("HF_TOKEN environment variable is not set or invalid.") |
|
|
|
def initialize_leaderboard_file(): |
|
""" |
|
Ensure the leaderboard file exists and has the correct headers. |
|
""" |
|
if not os.path.exists(LEADERBOARD_FILE): |
|
pd.DataFrame(columns=[ |
|
"Model Name", "Overall Accuracy", "Valid Accuracy", |
|
"Correct Predictions", "Total Questions", "Timestamp" |
|
]).to_csv(LEADERBOARD_FILE, index=False) |
|
elif os.stat(LEADERBOARD_FILE).st_size == 0: |
|
pd.DataFrame(columns=[ |
|
"Model Name", "Overall Accuracy", "Valid Accuracy", |
|
"Correct Predictions", "Total Questions", "Timestamp" |
|
]).to_csv(LEADERBOARD_FILE, index=False) |
|
|
|
def clean_answer(answer): |
|
if pd.isna(answer): |
|
return None |
|
answer = str(answer) |
|
clean = re.sub(r'[^A-Da-d]', '', answer) |
|
return clean[0].upper() if clean else None |
|
|
|
|
|
def update_leaderboard(results): |
|
""" |
|
Append new submission results to the leaderboard file and push updates to the Hugging Face repository. |
|
""" |
|
new_entry = { |
|
"Model Name": results['model_name'], |
|
"Overall Accuracy": round(results['overall_accuracy'] * 100, 2), |
|
"Valid Accuracy": round(results['valid_accuracy'] * 100, 2), |
|
"Correct Predictions": results['correct_predictions'], |
|
"Total Questions": results['total_questions'], |
|
"Timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), |
|
} |
|
|
|
try: |
|
|
|
new_entry_df = pd.DataFrame([new_entry]) |
|
file_exists = os.path.exists(LEADERBOARD_FILE) |
|
|
|
new_entry_df.to_csv( |
|
LEADERBOARD_FILE, |
|
mode='a', |
|
index=False, |
|
header=not file_exists |
|
) |
|
print(f"Leaderboard updated successfully at {LEADERBOARD_FILE}") |
|
|
|
|
|
api = HfApi() |
|
token = HfFolder.get_token() |
|
|
|
api.upload_file( |
|
path_or_fileobj=LEADERBOARD_FILE, |
|
path_in_repo="leaderboard.csv", |
|
repo_id="SondosMB/ss", |
|
repo_type="space", |
|
token=token |
|
) |
|
print("Leaderboard changes pushed to Hugging Face repository.") |
|
|
|
except Exception as e: |
|
print(f"Error updating leaderboard file: {e}") |
|
|
|
|
|
|
|
def load_leaderboard(): |
|
if not os.path.exists(LEADERBOARD_FILE) or os.stat(LEADERBOARD_FILE).st_size == 0: |
|
return pd.DataFrame({ |
|
"Model Name": [], |
|
"Overall Accuracy": [], |
|
"Valid Accuracy": [], |
|
"Correct Predictions": [], |
|
"Total Questions": [], |
|
"Timestamp": [], |
|
}) |
|
return pd.read_csv(LEADERBOARD_FILE) |
|
|
|
def evaluate_predictions(prediction_file, model_name, add_to_leaderboard): |
|
try: |
|
ground_truth_path = hf_hub_download( |
|
repo_id="SondosMB/ground-truth-dataset", |
|
filename="ground_truth.csv", |
|
repo_type="dataset", |
|
use_auth_token=True |
|
) |
|
ground_truth_df = pd.read_csv(ground_truth_path) |
|
except FileNotFoundError: |
|
return "Ground truth file not found in the dataset repository.", load_leaderboard() |
|
except Exception as e: |
|
return f"Error loading ground truth: {e}", load_leaderboard() |
|
|
|
if not prediction_file: |
|
return "Prediction file not uploaded.", load_leaderboard() |
|
|
|
try: |
|
predictions_df = pd.read_csv(prediction_file.name) |
|
merged_df = pd.merge(predictions_df, ground_truth_df, on='question_id', how='inner') |
|
merged_df['pred_answer'] = merged_df['predicted_answer'].apply(clean_answer) |
|
|
|
valid_predictions = merged_df.dropna(subset=['pred_answer']) |
|
correct_predictions = (valid_predictions['pred_answer'] == valid_predictions['Answer']).sum() |
|
total_predictions = len(merged_df) |
|
total_valid_predictions = len(valid_predictions) |
|
|
|
overall_accuracy = correct_predictions / total_predictions if total_predictions > 0 else 0 |
|
valid_accuracy = correct_predictions / total_valid_predictions if total_valid_predictions > 0 else 0 |
|
|
|
results = { |
|
'model_name': model_name if model_name else "Unknown Model", |
|
'overall_accuracy': overall_accuracy, |
|
'valid_accuracy': valid_accuracy, |
|
'correct_predictions': correct_predictions, |
|
'total_questions': total_predictions, |
|
} |
|
|
|
if add_to_leaderboard: |
|
update_leaderboard(results) |
|
return "Evaluation completed and added to leaderboard.", load_leaderboard() |
|
else: |
|
return "Evaluation completed but not added to leaderboard.", load_leaderboard() |
|
|
|
except Exception as e: |
|
return f"Error during evaluation: {str(e)}", load_leaderboard() |
|
|
|
initialize_leaderboard_file() |
|
|
|
with gr.Blocks() as demo: |
|
gr.Markdown(""" |
|
# Competition Title |
|
### Welcome to the Competition Overview |
|
 |
|
|
|
) |
|
Here you can submit your predictions, view the leaderboard, and track your performance! |
|
""") |
|
|
|
with gr.Tabs(): |
|
with gr.TabItem("📖 Overview"): |
|
gr.Markdown(""" |
|
## Overview |
|
# Welcome to the Mobile-MMLU Benchmark Competition |
|
|
|
Evaluate the performance of mobile-compatible Large Language Models (LLMs) on 16,186 scenario-based and factual questions across 80 fields. Compete to showcase your model’s accuracy for real-world mobile scenarios. |
|
|
|
## What is Mobile-MMLU? |
|
|
|
Mobile-MMLU is a benchmark designed to test the capabilities of LLMs optimized for mobile use. By participating in this competition, you contribute to advancing mobile intelligence benchmarks and shaping the future of mobile-compatible AI systems. |
|
|
|
--- |
|
|
|
## How It Works |
|
|
|
1. **Download the Dataset** |
|
Access the dataset and detailed generation instructions on our [GitHub page](https://github.com/your-github-repo). |
|
|
|
2. **Generate Predictions** |
|
Use your LLM to answer the questions and format your predictions as a CSV file with the following structure as written on our GitHub page : |
|
|
|
3. **Submit Predictions** |
|
Upload your predictions via the submission portal. |
|
|
|
4. **Evaluation** |
|
Your submission will be scored on accuracy |
|
|
|
5. **Leaderboard** |
|
Compare your results against other participants on the live leaderboard. |
|
|
|
--- |
|
|
|
## Competition Tasks |
|
|
|
Participants are tasked with generating predictions for the dataset and optimizing their models for: |
|
- **Accuracy**: Correctly answering questions across diverse fields. |
|
--- |
|
|
|
|
|
## Get Started |
|
|
|
1. **Prepare Your Model** |
|
Refer to our [GitHub page](https://github.com/your-github-repo) for dataset access and response generation instructions. |
|
|
|
2. **Submit Predictions** |
|
Format your submission as specified in the rules. |
|
|
|
3. **Track Progress** |
|
Check the leaderboard for real-time rankings. |
|
|
|
--- |
|
|
|
## Contact Us |
|
|
|
For questions or support, contact us at: [Insert Email Address] |
|
""") |
|
|
|
with gr.TabItem("📤 Submission"): |
|
with gr.Row(): |
|
file_input = gr.File(label="Upload Prediction CSV", file_types=[".csv"], interactive=True) |
|
model_name_input = gr.Textbox(label="Model Name", placeholder="Enter your model name") |
|
|
|
with gr.Row(): |
|
overall_accuracy_display = gr.Number(label="Overall Accuracy", interactive=False) |
|
add_to_leaderboard_checkbox = gr.Checkbox(label="Add to Leaderboard?", value=True) |
|
|
|
eval_button = gr.Button("Evaluate") |
|
eval_status = gr.Textbox(label="Evaluation Status", interactive=False) |
|
|
|
def handle_evaluation(file, model_name, add_to_leaderboard): |
|
status, leaderboard = evaluate_predictions(file, model_name, add_to_leaderboard) |
|
if leaderboard.empty: |
|
overall_accuracy = 0 |
|
else: |
|
overall_accuracy = leaderboard.iloc[-1]["Overall Accuracy"] |
|
return status, overall_accuracy |
|
|
|
eval_button.click( |
|
handle_evaluation, |
|
inputs=[file_input, model_name_input, add_to_leaderboard_checkbox], |
|
outputs=[eval_status, overall_accuracy_display], |
|
) |
|
|
|
with gr.TabItem("🏅 Leaderboard"): |
|
leaderboard_table = gr.Dataframe( |
|
value=load_leaderboard(), |
|
label="Leaderboard", |
|
interactive=False, |
|
wrap=True, |
|
) |
|
refresh_button = gr.Button("Refresh Leaderboard") |
|
refresh_button.click( |
|
lambda: load_leaderboard(), |
|
inputs=[], |
|
outputs=[leaderboard_table], |
|
) |
|
|
|
gr.Markdown(f"Last updated on **{LAST_UPDATED}**") |
|
|
|
demo.launch() |
|
|
|
|
