|
import gradio as gr |
|
import pandas as pd |
|
from datasets import load_dataset |
|
from jiwer import wer, cer |
|
import os |
|
from datetime import datetime |
|
import re |
|
|
|
from huggingface_hub import login |
|
|
|
|
|
token = os.environ.get("HG_TOKEN") |
|
if token: |
|
login(token) |
|
|
|
|
|
try: |
|
dataset = load_dataset("sudoping01/bambara-speech-recognition-benchmark", name="default")["eval"] |
|
references = {row["id"]: row["text"] for row in dataset} |
|
print(f"Loaded {len(references)} reference transcriptions") |
|
except Exception as e: |
|
print(f"Error loading dataset: {str(e)}") |
|
references = {} |
|
|
|
|
|
leaderboard_file = "leaderboard.csv" |
|
if not os.path.exists(leaderboard_file): |
|
|
|
sample_data = [ |
|
["test_1", 0.2264, 0.1094, 0.1922, "2025-03-15 10:30:45"], |
|
["test_2", 0.3264, 0.1094, 0.1922, "2025-03-15 10:30:45"], |
|
] |
|
pd.DataFrame(sample_data, |
|
columns=["Model_Name", "WER", "CER", "Combined_Score", "timestamp"]).to_csv(leaderboard_file, index=False) |
|
print(f"Created new leaderboard file with sample data") |
|
else: |
|
leaderboard_df = pd.read_csv(leaderboard_file) |
|
|
|
|
|
if "Combined_Score" not in leaderboard_df.columns: |
|
leaderboard_df["Combined_Score"] = leaderboard_df["WER"] * 0.7 + leaderboard_df["CER"] * 0.3 |
|
leaderboard_df.to_csv(leaderboard_file, index=False) |
|
print(f"Added Combined_Score column to existing leaderboard") |
|
print(f"Loaded leaderboard with {len(leaderboard_df)} entries") |
|
|
|
def normalize_text(text): |
|
"""Normalize text for WER/CER calculation""" |
|
if not isinstance(text, str): |
|
text = str(text) |
|
|
|
text = text.lower() |
|
text = re.sub(r'[^\w\s]', '', text) |
|
text = re.sub(r'\s+', ' ', text).strip() |
|
return text |
|
|
|
def calculate_metrics(predictions_df): |
|
"""Calculate WER and CER for predictions.""" |
|
results = [] |
|
total_ref_words = 0 |
|
total_ref_chars = 0 |
|
|
|
for _, row in predictions_df.iterrows(): |
|
id_val = row["id"] |
|
if id_val not in references: |
|
continue |
|
|
|
reference = normalize_text(references[id_val]) |
|
hypothesis = normalize_text(row["text"]) |
|
|
|
if not reference or not hypothesis: |
|
continue |
|
|
|
reference_words = reference.split() |
|
hypothesis_words = hypothesis.split() |
|
reference_chars = list(reference) |
|
|
|
try: |
|
sample_wer = wer(reference, hypothesis) |
|
sample_cer = cer(reference, hypothesis) |
|
|
|
sample_wer = min(sample_wer, 2.0) |
|
sample_cer = min(sample_cer, 2.0) |
|
|
|
total_ref_words += len(reference_words) |
|
total_ref_chars += len(reference_chars) |
|
|
|
results.append({ |
|
"id": id_val, |
|
"reference": reference, |
|
"hypothesis": hypothesis, |
|
"ref_word_count": len(reference_words), |
|
"ref_char_count": len(reference_chars), |
|
"wer": sample_wer, |
|
"cer": sample_cer |
|
}) |
|
except Exception as e: |
|
print(f"Error processing sample {id_val}: {str(e)}") |
|
pass |
|
|
|
if not results: |
|
raise ValueError("No valid samples for WER/CER calculation") |
|
|
|
avg_wer = sum(item["wer"] for item in results) / len(results) |
|
avg_cer = sum(item["cer"] for item in results) / len(results) |
|
|
|
|
|
weighted_wer = sum(item["wer"] * item["ref_word_count"] for item in results) / total_ref_words |
|
weighted_cer = sum(item["cer"] * item["ref_char_count"] for item in results) / total_ref_chars |
|
|
|
return avg_wer, avg_cer, weighted_wer, weighted_cer, results |
|
|
|
def format_as_percentage(value): |
|
"""Convert decimal to percentage with 2 decimal places""" |
|
return f"{value * 100:.2f}%" |
|
|
|
def prepare_leaderboard_for_display(df, sort_by="Combined_Score"): |
|
"""Format leaderboard for display with ranking and percentages""" |
|
if df is None or len(df) == 0: |
|
return pd.DataFrame(columns=["Rank", "Model_Name", "WER (%)", "CER (%)", "Combined_Score (%)", "timestamp"]) |
|
|
|
|
|
display_df = df.copy() |
|
|
|
|
|
display_df = display_df.sort_values(sort_by) |
|
|
|
display_df.insert(0, "Rank", range(1, len(display_df) + 1)) |
|
|
|
for col in ["WER", "CER", "Combined_Score"]: |
|
if col in display_df.columns: |
|
display_df[f"{col} (%)"] = display_df[col].apply(lambda x: f"{x * 100:.2f}") |
|
|
|
|
|
|
|
return display_df |
|
|
|
def update_ranking(method): |
|
"""Update leaderboard ranking based on selected method""" |
|
try: |
|
current_lb = pd.read_csv(leaderboard_file) |
|
|
|
if "Combined_Score" not in current_lb.columns: |
|
current_lb["Combined_Score"] = current_lb["WER"] * 0.7 + current_lb["CER"] * 0.3 |
|
|
|
sort_column = "Combined_Score" |
|
if method == "WER Only": |
|
sort_column = "WER" |
|
elif method == "CER Only": |
|
sort_column = "CER" |
|
|
|
return prepare_leaderboard_for_display(current_lb, sort_column) |
|
|
|
except Exception as e: |
|
print(f"Error updating ranking: {str(e)}") |
|
return pd.DataFrame(columns=["Rank", "Model_Name", "WER (%)", "CER (%)", "Combined_Score (%)", "timestamp"]) |
|
|
|
def process_submission(model_name, csv_file): |
|
"""Process a new model submission""" |
|
if not model_name or not model_name.strip(): |
|
return "Error: Please provide a model name.", None |
|
|
|
if not csv_file: |
|
return "Error: Please upload a CSV file.", None |
|
|
|
try: |
|
df = pd.read_csv(csv_file) |
|
|
|
if len(df) == 0: |
|
return "Error: Uploaded CSV is empty.", None |
|
|
|
if set(df.columns) != {"id", "text"}: |
|
return f"Error: CSV must contain exactly 'id' and 'text' columns. Found: {', '.join(df.columns)}", None |
|
|
|
if df["id"].duplicated().any(): |
|
dup_ids = df[df["id"].duplicated()]["id"].unique() |
|
return f"Error: Duplicate IDs found: {', '.join(map(str, dup_ids[:5]))}", None |
|
|
|
missing_ids = set(references.keys()) - set(df["id"]) |
|
extra_ids = set(df["id"]) - set(references.keys()) |
|
|
|
if missing_ids: |
|
return f"Error: Missing {len(missing_ids)} IDs in submission. First few missing: {', '.join(map(str, list(missing_ids)[:5]))}", None |
|
|
|
if extra_ids: |
|
return f"Error: Found {len(extra_ids)} extra IDs not in reference dataset. First few extra: {', '.join(map(str, list(extra_ids)[:5]))}", None |
|
|
|
try: |
|
avg_wer, avg_cer, weighted_wer, weighted_cer, detailed_results = calculate_metrics(df) |
|
|
|
|
|
if avg_wer < 0.001: |
|
return "Error: WER calculation yielded suspicious results (near-zero). Please check your submission CSV.", None |
|
|
|
except Exception as e: |
|
return f"Error calculating metrics: {str(e)}", None |
|
|
|
|
|
leaderboard = pd.read_csv(leaderboard_file) |
|
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
|
|
|
combined_score = avg_wer * 0.7 + avg_cer * 0.3 |
|
|
|
if model_name in leaderboard["Model_Name"].values: |
|
idx = leaderboard[leaderboard["Model_Name"] == model_name].index |
|
leaderboard.loc[idx, "WER"] = avg_wer |
|
leaderboard.loc[idx, "CER"] = avg_cer |
|
leaderboard.loc[idx, "Combined_Score"] = combined_score |
|
leaderboard.loc[idx, "timestamp"] = timestamp |
|
updated_leaderboard = leaderboard |
|
else: |
|
new_entry = pd.DataFrame( |
|
[[model_name, avg_wer, avg_cer, combined_score, timestamp]], |
|
columns=["Model_Name", "WER", "CER", "Combined_Score", "timestamp"] |
|
) |
|
updated_leaderboard = pd.concat([leaderboard, new_entry]) |
|
|
|
updated_leaderboard = updated_leaderboard.sort_values("Combined_Score") |
|
updated_leaderboard.to_csv(leaderboard_file, index=False) |
|
|
|
display_leaderboard = prepare_leaderboard_for_display(updated_leaderboard) |
|
|
|
return f"Submission processed successfully! WER: {format_as_percentage(avg_wer)}, CER: {format_as_percentage(avg_cer)}, Combined Score: {format_as_percentage(combined_score)}", display_leaderboard |
|
|
|
except Exception as e: |
|
return f"Error processing submission: {str(e)}", None |
|
|
|
def get_current_leaderboard(): |
|
"""Get the current leaderboard data for display""" |
|
try: |
|
if os.path.exists(leaderboard_file): |
|
current_leaderboard = pd.read_csv(leaderboard_file) |
|
|
|
if "Combined_Score" not in current_leaderboard.columns: |
|
current_leaderboard["Combined_Score"] = current_leaderboard["WER"] * 0.7 + current_leaderboard["CER"] * 0.3 |
|
current_leaderboard.to_csv(leaderboard_file, index=False) |
|
|
|
return current_leaderboard |
|
else: |
|
return pd.DataFrame(columns=["Model_Name", "WER", "CER", "Combined_Score", "timestamp"]) |
|
except Exception as e: |
|
print(f"Error getting leaderboard: {str(e)}") |
|
return pd.DataFrame(columns=["Model_Name", "WER", "CER", "Combined_Score", "timestamp"]) |
|
|
|
def create_leaderboard_table(): |
|
"""Create and format the leaderboard table for display""" |
|
leaderboard_data = get_current_leaderboard() |
|
return prepare_leaderboard_for_display(leaderboard_data) |
|
|
|
with gr.Blocks(title="Bambara ASR Leaderboard") as demo: |
|
gr.Markdown( |
|
""" |
|
# 🇲🇱 Bambara ASR Leaderboard |
|
|
|
This leaderboard tracks and evaluates speech recognition models for the Bambara language. |
|
Models are ranked based on Word Error Rate (WER), Character Error Rate (CER), and a combined score. |
|
|
|
## Current Models Performance |
|
""" |
|
) |
|
|
|
current_data = get_current_leaderboard() |
|
|
|
|
|
if len(current_data) > 0: |
|
best_model = current_data.sort_values("Combined_Score").iloc[0] |
|
gr.Markdown(f""" |
|
### 🏆 Current Best Model: **{best_model['Model_Name']}** |
|
* WER: **{best_model['WER']*100:.2f}%** |
|
* CER: **{best_model['CER']*100:.2f}%** |
|
* Combined Score: **{best_model['Combined_Score']*100:.2f}%** |
|
""") |
|
|
|
with gr.Tabs() as tabs: |
|
with gr.TabItem("🏅 Model Rankings"): |
|
|
|
initial_leaderboard = create_leaderboard_table() |
|
|
|
ranking_method = gr.Radio( |
|
["Combined Score (WER 70%, CER 30%)", "WER Only", "CER Only"], |
|
label="Ranking Method", |
|
value="Combined Score (WER 70%, CER 30%)" |
|
) |
|
|
|
leaderboard_view = gr.DataFrame( |
|
value=initial_leaderboard, |
|
interactive=False, |
|
label="Models are ranked by selected metric - lower is better" |
|
) |
|
|
|
ranking_method.change( |
|
fn=update_ranking, |
|
inputs=[ranking_method], |
|
outputs=[leaderboard_view] |
|
) |
|
|
|
with gr.Accordion("Metrics Explanation", open=False): |
|
gr.Markdown( |
|
""" |
|
## Understanding ASR Metrics |
|
|
|
### Word Error Rate (WER) |
|
WER measures how accurately the ASR system recognizes whole words: |
|
* Lower values indicate better performance |
|
* Calculated as: (Substitutions + Insertions + Deletions) / Total Words |
|
* A WER of 0% means perfect transcription |
|
* A WER of 20% means approximately 1 in 5 words contains an error |
|
|
|
### Character Error Rate (CER) |
|
CER measures accuracy at the character level: |
|
* More fine-grained than WER |
|
* Better at capturing partial word matches |
|
* Particularly useful for agglutinative languages like Bambara |
|
|
|
### Combined Score |
|
* Weighted average: 70% WER + 30% CER |
|
* Provides a balanced evaluation of model performance |
|
* Used as the primary ranking metric |
|
""" |
|
) |
|
|
|
with gr.TabItem("📊 Submit New Results"): |
|
gr.Markdown( |
|
""" |
|
### Submit a new model for evaluation |
|
|
|
Upload a CSV file with the following format: |
|
* Must contain exactly two columns: 'id' and 'text' |
|
* The 'id' column should match the reference dataset IDs |
|
* The 'text' column should contain your model's transcriptions |
|
""" |
|
) |
|
|
|
with gr.Row(): |
|
model_name_input = gr.Textbox( |
|
label="Model Name", |
|
placeholder="e.g., MALIBA-AI/bambara-asr" |
|
) |
|
gr.Markdown("*Use a descriptive name to identify your model*") |
|
|
|
with gr.Row(): |
|
csv_upload = gr.File( |
|
label="Upload CSV File", |
|
file_types=[".csv"] |
|
) |
|
gr.Markdown("*CSV with columns: id, text*") |
|
|
|
submit_btn = gr.Button("Submit", variant="primary") |
|
output_msg = gr.Textbox(label="Status", interactive=False) |
|
leaderboard_display = gr.DataFrame( |
|
label="Updated Leaderboard", |
|
value=initial_leaderboard, |
|
interactive=False |
|
) |
|
|
|
submit_btn.click( |
|
fn=process_submission, |
|
inputs=[model_name_input, csv_upload], |
|
outputs=[output_msg, leaderboard_display] |
|
) |
|
|
|
with gr.TabItem("📝 Benchmark Dataset"): |
|
gr.Markdown( |
|
""" |
|
## About the Benchmark Dataset |
|
|
|
This leaderboard uses the **[sudoping01/bambara-speech-recognition-benchmark](https://huggingface.co/datasets/MALIBA-AI/bambara-speech-recognition-leaderboard)** dataset: |
|
|
|
* Contains diverse Bambara speech samples |
|
* Includes various speakers, accents, and dialects |
|
* Covers different speech styles and recording conditions |
|
* Transcribed and validated |
|
|
|
### How to Generate Predictions |
|
|
|
To submit results to this leaderboard: |
|
|
|
1. Download the audio files from the benchmark dataset |
|
2. Run your ASR model on the audio files |
|
3. Generate a CSV file with 'id' and 'text' columns |
|
4. Submit your results using the form in the "Submit New Results" tab |
|
|
|
### Evaluation Guidelines |
|
|
|
* Text is normalized (lowercase, punctuation removed) before metrics calculation |
|
* Extreme outliers are capped to prevent skewing results |
|
* All submissions are validated for format and completeness |
|
|
|
NB: This work is a collaboration between MALIBA-AI, RobotsMali AI4D-LAB and Djelia |
|
""" |
|
) |
|
|
|
gr.Markdown( |
|
""" |
|
--- |
|
### About MALIBA-AI |
|
|
|
**MALIBA-AI: Empowering Mali's Future Through Community-Driven AI Innovation** |
|
|
|
*"No Malian Language Left Behind"* |
|
|
|
This leaderboard is maintained by the MALIBA-AI initiative to track progress in Bambara speech recognition technology. |
|
For more information, visit [MALIBA-AI on Hugging Face](https://huggingface.co/MALIBA-AI). |
|
""" |
|
) |
|
|
|
if __name__ == "__main__": |
|
demo.launch() |
