Spaces:
Running
Running
Commit
ยท
e8d658f
1
Parent(s):
b2730cf
changed coloring on frontend
Browse files- __pycache__/app.cpython-310.pyc +0 -0
- app.py +90 -242
__pycache__/app.cpython-310.pyc
ADDED
Binary file (2.6 kB). View file
|
|
app.py
CHANGED
@@ -1,281 +1,129 @@
|
|
1 |
import gradio as gr
|
2 |
import pandas as pd
|
3 |
-
import
|
4 |
from pathlib import Path
|
5 |
from datetime import datetime, timezone
|
6 |
-
import
|
|
|
7 |
|
|
|
|
|
8 |
LAST_UPDATED = "Dec 4th 2024"
|
9 |
-
QUEUE_DIR = Path("/Users/arunasrivastava/Koel/IPA-Leaderboard/IPA-Transcription-EN-queue/queue")
|
10 |
-
APP_DIR = Path("./")
|
11 |
-
|
12 |
-
# Modified column names for phonemic transcription metrics
|
13 |
-
column_names = {
|
14 |
-
"MODEL": "Model",
|
15 |
-
"AVG_PER": "Average PER โฌ๏ธ",
|
16 |
-
"AVG_PWED": "Average PWED โฌ๏ธ",
|
17 |
-
"GITHUB_URL": "GitHub",
|
18 |
-
"DATE": "Submission Date"
|
19 |
-
}
|
20 |
-
|
21 |
-
def load_json_file(file_path: Path, default=None):
|
22 |
-
"""Safely load a JSON file or return default if file doesn't exist"""
|
23 |
-
if default is None:
|
24 |
-
default = []
|
25 |
-
|
26 |
-
if not file_path.exists():
|
27 |
-
return default
|
28 |
-
|
29 |
-
try:
|
30 |
-
with open(file_path, 'r') as f:
|
31 |
-
return json.load(f)
|
32 |
-
except json.JSONDecodeError:
|
33 |
-
return default
|
34 |
-
|
35 |
-
def save_json_file(file_path: Path, data):
|
36 |
-
"""Safely save data to a JSON file"""
|
37 |
-
file_path.parent.mkdir(parents=True, exist_ok=True)
|
38 |
-
with open(file_path, 'w') as f:
|
39 |
-
json.dump(data, f, indent=2, ensure_ascii=False)
|
40 |
|
41 |
def load_leaderboard_data():
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
|
|
|
|
|
|
46 |
|
47 |
def format_leaderboard_df(df):
|
48 |
-
"""Format leaderboard dataframe for display"""
|
49 |
if df.empty:
|
50 |
return df
|
51 |
-
|
52 |
-
# Select and rename only the columns we want to display
|
53 |
display_df = pd.DataFrame({
|
54 |
-
"
|
55 |
-
"
|
56 |
-
"
|
57 |
-
"
|
58 |
-
"
|
59 |
})
|
60 |
|
61 |
-
|
62 |
-
display_df["AVG_PER"] = display_df["AVG_PER"].apply(lambda x: f"{x:.4f}")
|
63 |
-
display_df["AVG_PWED"] = display_df["AVG_PWED"].apply(lambda x: f"{x:.4f}")
|
64 |
-
|
65 |
-
# Make GitHub URLs clickable
|
66 |
-
display_df["GITHUB_URL"] = display_df["GITHUB_URL"].apply(
|
67 |
-
lambda x: f'<a href="{x}" target="_blank">Repository</a>' if x else "N/A"
|
68 |
-
)
|
69 |
-
|
70 |
-
# Sort by PER (ascending)
|
71 |
-
display_df.sort_values(by="AVG_PER", inplace=True)
|
72 |
-
|
73 |
-
return display_df
|
74 |
|
75 |
-
def
|
76 |
-
"""Submit new evaluation request"""
|
77 |
if not model_name or not submission_name:
|
78 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
79 |
|
80 |
try:
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
# Create new task
|
89 |
-
new_task = {
|
90 |
-
"id": str(uuid.uuid4()),
|
91 |
-
"transcription_model": model_name,
|
92 |
-
"subset": subset,
|
93 |
-
"max_samples": max_samples,
|
94 |
-
"submission_name": submission_name,
|
95 |
-
"github_url": github_url or "",
|
96 |
-
"status": "queued",
|
97 |
-
"submitted_at": datetime.now(timezone.utc).isoformat()
|
98 |
-
}
|
99 |
-
|
100 |
-
# Add new task to existing tasks
|
101 |
-
tasks.append(new_task)
|
102 |
-
|
103 |
-
# Save updated tasks
|
104 |
-
save_json_file(tasks_file, tasks)
|
105 |
-
|
106 |
-
return gr.Markdown("โ
Evaluation request submitted successfully! Your results will appear on the leaderboard once processing is complete.")
|
107 |
-
|
108 |
-
except Exception as e:
|
109 |
-
return gr.Markdown(f"โ Error submitting request: {str(e)}")
|
110 |
|
111 |
-
def
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
return
|
120 |
-
|
121 |
-
# Get the most recent result
|
122 |
-
latest_result = max(model_results, key=lambda x: x["timestamp"])
|
123 |
-
return latest_result
|
124 |
|
125 |
def create_html_table(df):
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
136 |
-
background-color: #1a1a1a;
|
137 |
-
}
|
138 |
-
th, td {
|
139 |
-
padding: 8px;
|
140 |
-
text-align: left;
|
141 |
-
border: 1px solid #333;
|
142 |
-
}
|
143 |
-
th {
|
144 |
-
background-color: #2a2a2a;
|
145 |
-
color: white;
|
146 |
}
|
147 |
-
|
148 |
-
background-color: #
|
|
|
|
|
149 |
}
|
150 |
-
|
151 |
-
|
|
|
152 |
}
|
153 |
-
|
154 |
-
|
155 |
-
text-decoration: none;
|
156 |
}
|
157 |
-
|
158 |
-
|
159 |
}
|
160 |
-
|
161 |
-
<table>
|
162 |
-
<thead>
|
163 |
-
<tr>
|
164 |
-
"""
|
165 |
-
|
166 |
-
# Add headers
|
167 |
-
for header in column_names.values():
|
168 |
-
html += f"<th>{header}</th>"
|
169 |
-
|
170 |
-
html += "</tr></thead><tbody>"
|
171 |
-
|
172 |
-
# Add rows
|
173 |
-
for _, row in df.iterrows():
|
174 |
-
html += "<tr>"
|
175 |
-
for col in df.columns:
|
176 |
-
if col == "GITHUB_URL":
|
177 |
-
html += f"<td>{row[col]}</td>" # URL is already formatted as HTML
|
178 |
-
else:
|
179 |
-
html += f"<td>{row[col]}</td>"
|
180 |
-
html += "</tr>"
|
181 |
-
|
182 |
-
html += "</tbody></table>"
|
183 |
-
return html
|
184 |
-
|
185 |
-
# Create Gradio interface
|
186 |
-
with gr.Blocks() as demo:
|
187 |
gr.Markdown("# ๐ฏ Phonemic Transcription Model Evaluation Leaderboard")
|
188 |
-
gr.Markdown("""
|
189 |
-
Compare the performance of different phonemic transcription models on speech-to-IPA transcription tasks for English.
|
190 |
-
|
191 |
-
**Metrics:**
|
192 |
-
- **PER (Phoneme Error Rate)**: Measures the edit distance between predicted and ground truth phonemes (lower is better)
|
193 |
-
- **PWED (Phoneme Weighted Edit Distance)**: Measures a weighted difference in phonemes using phonemic features (lower is better)
|
194 |
-
|
195 |
-
**Datasets:**
|
196 |
-
- **[TIMIT](https://www.kaggle.com/datasets/mfekadu/darpa-timit-acousticphonetic-continuous-speech)**: A phonemic transcription dataset for English speech recognition
|
197 |
-
|
198 |
-
To learn more about the evaluation metrics, check out our blog post [here](https://huggingface.co/spaces/evaluate-metric/wer).
|
199 |
-
""")
|
200 |
|
201 |
-
with gr.Tabs()
|
202 |
with gr.TabItem("๐ Leaderboard"):
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
leaderboard_table = gr.HTML(
|
207 |
-
value=create_html_table(formatted_df)
|
208 |
-
)
|
209 |
-
|
210 |
-
refresh_btn = gr.Button("๐ Refresh Leaderboard")
|
211 |
refresh_btn.click(
|
212 |
-
lambda:
|
|
|
213 |
)
|
214 |
-
|
215 |
-
with gr.TabItem("๐ Submit Model"):
|
216 |
-
with gr.Column():
|
217 |
-
model_input = gr.Textbox(
|
218 |
-
label="Model Name",
|
219 |
-
placeholder="facebook/wav2vec2-lv-60-espeak-cv-ft",
|
220 |
-
info="Enter the Hugging Face model ID"
|
221 |
-
)
|
222 |
-
submission_name = gr.Textbox(
|
223 |
-
label="Submission Name",
|
224 |
-
placeholder="My Awesome Model v1.0",
|
225 |
-
info="Give your submission a descriptive name"
|
226 |
-
)
|
227 |
-
github_url = gr.Textbox(
|
228 |
-
label="GitHub Repository URL (optional)",
|
229 |
-
placeholder="https://github.com/username/repo",
|
230 |
-
info="Link to your model's code repository"
|
231 |
-
)
|
232 |
-
|
233 |
-
submit_btn = gr.Button("๐ Submit for Evaluation")
|
234 |
-
result_text = gr.Markdown()
|
235 |
-
|
236 |
-
def submit_and_clear(model_name, submission_name, github_url):
|
237 |
-
result = request_evaluation(model_name, submission_name, github_url)
|
238 |
-
# If submission was successful, clear the form
|
239 |
-
if "โ
" in result.value:
|
240 |
-
return {
|
241 |
-
model_input: "",
|
242 |
-
submission_name: "",
|
243 |
-
github_url: "",
|
244 |
-
result_text: result
|
245 |
-
}
|
246 |
-
# If there was an error, keep the form data and show error
|
247 |
-
return {
|
248 |
-
model_input: model_name,
|
249 |
-
submission_name: submission_name,
|
250 |
-
github_url: github_url,
|
251 |
-
result_text: result
|
252 |
-
}
|
253 |
-
|
254 |
-
submit_btn.click(
|
255 |
-
submit_and_clear,
|
256 |
-
inputs=[model_input, submission_name, github_url],
|
257 |
-
outputs=[model_input, submission_name, github_url, result_text]
|
258 |
-
)
|
259 |
|
260 |
-
with gr.TabItem("
|
261 |
-
|
262 |
-
|
263 |
-
|
264 |
-
)
|
265 |
-
|
266 |
-
results_json = gr.JSON(label="Detailed Results")
|
267 |
|
268 |
-
|
269 |
-
|
270 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
271 |
|
272 |
-
|
273 |
-
|
274 |
-
inputs=
|
275 |
-
outputs=
|
276 |
)
|
277 |
-
|
278 |
-
gr.Markdown(f"Last updated: {LAST_UPDATED}")
|
279 |
|
280 |
if __name__ == "__main__":
|
281 |
demo.launch()
|
|
|
1 |
import gradio as gr
|
2 |
import pandas as pd
|
3 |
+
import requests
|
4 |
from pathlib import Path
|
5 |
from datetime import datetime, timezone
|
6 |
+
import time
|
7 |
+
import logging
|
8 |
|
9 |
+
logging.basicConfig(level=logging.INFO)
|
10 |
+
BACKEND_URL = "http://127.0.0.1:8000"
|
11 |
LAST_UPDATED = "Dec 4th 2024"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
|
13 |
def load_leaderboard_data():
|
14 |
+
try:
|
15 |
+
response = requests.get(f"{BACKEND_URL}/leaderboard")
|
16 |
+
response.raise_for_status()
|
17 |
+
return pd.DataFrame(response.json())
|
18 |
+
except requests.RequestException as e:
|
19 |
+
logging.error(f"Error loading leaderboard: {e}")
|
20 |
+
return pd.DataFrame()
|
21 |
|
22 |
def format_leaderboard_df(df):
|
|
|
23 |
if df.empty:
|
24 |
return df
|
25 |
+
|
|
|
26 |
display_df = pd.DataFrame({
|
27 |
+
"Model": df["model"],
|
28 |
+
"Average PER โฌ๏ธ": df["average_per"].apply(lambda x: f"{x:.4f}"),
|
29 |
+
"Average PWED โฌ๏ธ": df["average_pwed"].apply(lambda x: f"{x:.4f}"),
|
30 |
+
"GitHub": df["github_url"].apply(lambda x: f'<a href="{x}" target="_blank">Repository</a>' if x else "N/A"),
|
31 |
+
"Submission Date": pd.to_datetime(df["submission_date"]).dt.strftime("%Y-%m-%d")
|
32 |
})
|
33 |
|
34 |
+
return display_df.sort_values("Average PER โฌ๏ธ")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
35 |
|
36 |
+
def submit_evaluation(model_name, submission_name, github_url):
|
|
|
37 |
if not model_name or not submission_name:
|
38 |
+
return "โ ๏ธ Please provide both model name and submission name."
|
39 |
+
|
40 |
+
request_data = {
|
41 |
+
"transcription_model": model_name,
|
42 |
+
"subset": "test",
|
43 |
+
"submission_name": submission_name,
|
44 |
+
"github_url": github_url if github_url else None
|
45 |
+
}
|
46 |
|
47 |
try:
|
48 |
+
response = requests.post(f"{BACKEND_URL}/evaluate", json=request_data)
|
49 |
+
response.raise_for_status()
|
50 |
+
task_id = response.json()["task_id"]
|
51 |
+
return f"โ
Evaluation submitted successfully! Task ID: {task_id}"
|
52 |
+
except requests.RequestException as e:
|
53 |
+
return f"โ Error: {str(e)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
54 |
|
55 |
+
def check_task_status(task_id):
|
56 |
+
if not task_id:
|
57 |
+
return "Please enter a task ID"
|
58 |
+
try:
|
59 |
+
response = requests.get(f"{BACKEND_URL}/tasks/{task_id}")
|
60 |
+
response.raise_for_status()
|
61 |
+
return response.json()
|
62 |
+
except requests.RequestException as e:
|
63 |
+
return f"Error checking status: {str(e)}"
|
|
|
|
|
|
|
|
|
64 |
|
65 |
def create_html_table(df):
|
66 |
+
return df.to_html(escape=False, index=False, classes="styled-table")
|
67 |
+
|
68 |
+
with gr.Blocks(css="""
|
69 |
+
.styled-table {
|
70 |
+
width: 100%;
|
71 |
+
border-collapse: collapse;
|
72 |
+
margin: 25px 0;
|
73 |
+
font-size: 0.9em;
|
74 |
+
font-family: sans-serif;
|
75 |
+
box-shadow: 0 0 20px rgba(0, 0, 0, 0.15);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
76 |
}
|
77 |
+
.styled-table thead tr {
|
78 |
+
background-color: #009879;
|
79 |
+
color: #ffffff;
|
80 |
+
text-align: left;
|
81 |
}
|
82 |
+
.styled-table th,
|
83 |
+
.styled-table td {
|
84 |
+
padding: 12px 15px;
|
85 |
}
|
86 |
+
.styled-table tbody tr {
|
87 |
+
border-bottom: 1px solid #dddddd;
|
|
|
88 |
}
|
89 |
+
.styled-table tbody tr:nth-of-type(even) {
|
90 |
+
background-color: #000000;
|
91 |
}
|
92 |
+
""") as demo:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
93 |
gr.Markdown("# ๐ฏ Phonemic Transcription Model Evaluation Leaderboard")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
94 |
|
95 |
+
with gr.Tabs():
|
96 |
with gr.TabItem("๐ Leaderboard"):
|
97 |
+
leaderboard_html = gr.HTML(create_html_table(format_leaderboard_df(load_leaderboard_data())))
|
98 |
+
refresh_btn = gr.Button("๐ Refresh")
|
|
|
|
|
|
|
|
|
|
|
|
|
99 |
refresh_btn.click(
|
100 |
+
lambda: create_html_table(format_leaderboard_df(load_leaderboard_data())),
|
101 |
+
outputs=leaderboard_html
|
102 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
|
104 |
+
with gr.TabItem("๐ Submit Model"):
|
105 |
+
model_name = gr.Textbox(label="Model Name", placeholder="facebook/wav2vec2-lv-60-espeak-cv-ft")
|
106 |
+
submission_name = gr.Textbox(label="Submission Name", placeholder="My Model v1.0")
|
107 |
+
github_url = gr.Textbox(label="GitHub URL (optional)", placeholder="https://github.com/username/repo")
|
108 |
+
submit_btn = gr.Button("Submit")
|
109 |
+
result = gr.Textbox(label="Submission Status")
|
|
|
110 |
|
111 |
+
submit_btn.click(
|
112 |
+
submit_evaluation,
|
113 |
+
inputs=[model_name, submission_name, github_url],
|
114 |
+
outputs=result
|
115 |
+
)
|
116 |
+
|
117 |
+
with gr.TabItem("๐ Task Status"):
|
118 |
+
task_id = gr.Textbox(label="Task ID")
|
119 |
+
status_btn = gr.Button("Check Status")
|
120 |
+
status_output = gr.JSON(label="Status")
|
121 |
|
122 |
+
status_btn.click(
|
123 |
+
check_task_status,
|
124 |
+
inputs=task_id,
|
125 |
+
outputs=status_output
|
126 |
)
|
|
|
|
|
127 |
|
128 |
if __name__ == "__main__":
|
129 |
demo.launch()
|