arunasrivastava commited on
Commit
e8d658f
ยท
1 Parent(s): b2730cf

changed coloring on frontend

Browse files
Files changed (2) hide show
  1. __pycache__/app.cpython-310.pyc +0 -0
  2. app.py +90 -242
__pycache__/app.cpython-310.pyc ADDED
Binary file (2.6 kB). View file
 
app.py CHANGED
@@ -1,281 +1,129 @@
1
  import gradio as gr
2
  import pandas as pd
3
- import json
4
  from pathlib import Path
5
  from datetime import datetime, timezone
6
- import uuid
 
7
 
 
 
8
  LAST_UPDATED = "Dec 4th 2024"
9
- QUEUE_DIR = Path("/Users/arunasrivastava/Koel/IPA-Leaderboard/IPA-Transcription-EN-queue/queue")
10
- APP_DIR = Path("./")
11
-
12
- # Modified column names for phonemic transcription metrics
13
- column_names = {
14
- "MODEL": "Model",
15
- "AVG_PER": "Average PER โฌ‡๏ธ",
16
- "AVG_PWED": "Average PWED โฌ‡๏ธ",
17
- "GITHUB_URL": "GitHub",
18
- "DATE": "Submission Date"
19
- }
20
-
21
- def load_json_file(file_path: Path, default=None):
22
- """Safely load a JSON file or return default if file doesn't exist"""
23
- if default is None:
24
- default = []
25
-
26
- if not file_path.exists():
27
- return default
28
-
29
- try:
30
- with open(file_path, 'r') as f:
31
- return json.load(f)
32
- except json.JSONDecodeError:
33
- return default
34
-
35
- def save_json_file(file_path: Path, data):
36
- """Safely save data to a JSON file"""
37
- file_path.parent.mkdir(parents=True, exist_ok=True)
38
- with open(file_path, 'w') as f:
39
- json.dump(data, f, indent=2, ensure_ascii=False)
40
 
41
  def load_leaderboard_data():
42
- """Load and parse leaderboard data"""
43
- leaderboard_path = QUEUE_DIR / "leaderboard.json"
44
- data = load_json_file(leaderboard_path)
45
- return pd.DataFrame(data) if data else pd.DataFrame()
 
 
 
46
 
47
  def format_leaderboard_df(df):
48
- """Format leaderboard dataframe for display"""
49
  if df.empty:
50
  return df
51
-
52
- # Select and rename only the columns we want to display
53
  display_df = pd.DataFrame({
54
- "MODEL": df["model"],
55
- "AVG_PER": df["average_per"],
56
- "AVG_PWED": df["average_pwed"],
57
- "GITHUB_URL": df["github_url"],
58
- "DATE": pd.to_datetime(df["submission_date"]).dt.strftime("%Y-%m-%d")
59
  })
60
 
61
- # Format numeric columns
62
- display_df["AVG_PER"] = display_df["AVG_PER"].apply(lambda x: f"{x:.4f}")
63
- display_df["AVG_PWED"] = display_df["AVG_PWED"].apply(lambda x: f"{x:.4f}")
64
-
65
- # Make GitHub URLs clickable
66
- display_df["GITHUB_URL"] = display_df["GITHUB_URL"].apply(
67
- lambda x: f'<a href="{x}" target="_blank">Repository</a>' if x else "N/A"
68
- )
69
-
70
- # Sort by PER (ascending)
71
- display_df.sort_values(by="AVG_PER", inplace=True)
72
-
73
- return display_df
74
 
75
- def request_evaluation(model_name, submission_name, github_url, subset="test", max_samples=None):
76
- """Submit new evaluation request"""
77
  if not model_name or not submission_name:
78
- return gr.Markdown("โš ๏ธ Please provide both model name and submission name.")
 
 
 
 
 
 
 
79
 
80
  try:
81
- # Ensure queue directory exists
82
- QUEUE_DIR.mkdir(parents=True, exist_ok=True)
83
-
84
- # Load existing tasks
85
- tasks_file = QUEUE_DIR / "tasks.json"
86
- tasks = load_json_file(tasks_file)
87
-
88
- # Create new task
89
- new_task = {
90
- "id": str(uuid.uuid4()),
91
- "transcription_model": model_name,
92
- "subset": subset,
93
- "max_samples": max_samples,
94
- "submission_name": submission_name,
95
- "github_url": github_url or "",
96
- "status": "queued",
97
- "submitted_at": datetime.now(timezone.utc).isoformat()
98
- }
99
-
100
- # Add new task to existing tasks
101
- tasks.append(new_task)
102
-
103
- # Save updated tasks
104
- save_json_file(tasks_file, tasks)
105
-
106
- return gr.Markdown("โœ… Evaluation request submitted successfully! Your results will appear on the leaderboard once processing is complete.")
107
-
108
- except Exception as e:
109
- return gr.Markdown(f"โŒ Error submitting request: {str(e)}")
110
 
111
- def load_results_for_model(model_name):
112
- """Load detailed results for a specific model"""
113
- results_path = QUEUE_DIR / "results.json"
114
- results = load_json_file(results_path)
115
-
116
- # Filter results for the specific model
117
- model_results = [r for r in results if r["model"] == model_name]
118
- if not model_results:
119
- return None
120
-
121
- # Get the most recent result
122
- latest_result = max(model_results, key=lambda x: x["timestamp"])
123
- return latest_result
124
 
125
  def create_html_table(df):
126
- """Create HTML table with dark theme styling"""
127
- if df.empty:
128
- return "<p>No data available</p>"
129
-
130
- html = """
131
- <style>
132
- table {
133
- width: 100%;
134
- border-collapse: collapse;
135
- color: white;
136
- background-color: #1a1a1a;
137
- }
138
- th, td {
139
- padding: 8px;
140
- text-align: left;
141
- border: 1px solid #333;
142
- }
143
- th {
144
- background-color: #2a2a2a;
145
- color: white;
146
  }
147
- tr:nth-child(even) {
148
- background-color: #252525;
 
 
149
  }
150
- tr:hover {
151
- background-color: #303030;
 
152
  }
153
- a {
154
- color: #6ea8fe;
155
- text-decoration: none;
156
  }
157
- a:hover {
158
- text-decoration: underline;
159
  }
160
- </style>
161
- <table>
162
- <thead>
163
- <tr>
164
- """
165
-
166
- # Add headers
167
- for header in column_names.values():
168
- html += f"<th>{header}</th>"
169
-
170
- html += "</tr></thead><tbody>"
171
-
172
- # Add rows
173
- for _, row in df.iterrows():
174
- html += "<tr>"
175
- for col in df.columns:
176
- if col == "GITHUB_URL":
177
- html += f"<td>{row[col]}</td>" # URL is already formatted as HTML
178
- else:
179
- html += f"<td>{row[col]}</td>"
180
- html += "</tr>"
181
-
182
- html += "</tbody></table>"
183
- return html
184
-
185
- # Create Gradio interface
186
- with gr.Blocks() as demo:
187
  gr.Markdown("# ๐ŸŽฏ Phonemic Transcription Model Evaluation Leaderboard")
188
- gr.Markdown("""
189
- Compare the performance of different phonemic transcription models on speech-to-IPA transcription tasks for English.
190
-
191
- **Metrics:**
192
- - **PER (Phoneme Error Rate)**: Measures the edit distance between predicted and ground truth phonemes (lower is better)
193
- - **PWED (Phoneme Weighted Edit Distance)**: Measures a weighted difference in phonemes using phonemic features (lower is better)
194
-
195
- **Datasets:**
196
- - **[TIMIT](https://www.kaggle.com/datasets/mfekadu/darpa-timit-acousticphonetic-continuous-speech)**: A phonemic transcription dataset for English speech recognition
197
-
198
- To learn more about the evaluation metrics, check out our blog post [here](https://huggingface.co/spaces/evaluate-metric/wer).
199
- """)
200
 
201
- with gr.Tabs() as tabs:
202
  with gr.TabItem("๐Ÿ† Leaderboard"):
203
- leaderboard_df = load_leaderboard_data()
204
- formatted_df = format_leaderboard_df(leaderboard_df)
205
-
206
- leaderboard_table = gr.HTML(
207
- value=create_html_table(formatted_df)
208
- )
209
-
210
- refresh_btn = gr.Button("๐Ÿ”„ Refresh Leaderboard")
211
  refresh_btn.click(
212
- lambda: gr.HTML(value=create_html_table(format_leaderboard_df(load_leaderboard_data())))
 
213
  )
214
-
215
- with gr.TabItem("๐Ÿ“ Submit Model"):
216
- with gr.Column():
217
- model_input = gr.Textbox(
218
- label="Model Name",
219
- placeholder="facebook/wav2vec2-lv-60-espeak-cv-ft",
220
- info="Enter the Hugging Face model ID"
221
- )
222
- submission_name = gr.Textbox(
223
- label="Submission Name",
224
- placeholder="My Awesome Model v1.0",
225
- info="Give your submission a descriptive name"
226
- )
227
- github_url = gr.Textbox(
228
- label="GitHub Repository URL (optional)",
229
- placeholder="https://github.com/username/repo",
230
- info="Link to your model's code repository"
231
- )
232
-
233
- submit_btn = gr.Button("๐Ÿš€ Submit for Evaluation")
234
- result_text = gr.Markdown()
235
-
236
- def submit_and_clear(model_name, submission_name, github_url):
237
- result = request_evaluation(model_name, submission_name, github_url)
238
- # If submission was successful, clear the form
239
- if "โœ…" in result.value:
240
- return {
241
- model_input: "",
242
- submission_name: "",
243
- github_url: "",
244
- result_text: result
245
- }
246
- # If there was an error, keep the form data and show error
247
- return {
248
- model_input: model_name,
249
- submission_name: submission_name,
250
- github_url: github_url,
251
- result_text: result
252
- }
253
-
254
- submit_btn.click(
255
- submit_and_clear,
256
- inputs=[model_input, submission_name, github_url],
257
- outputs=[model_input, submission_name, github_url, result_text]
258
- )
259
 
260
- with gr.TabItem("โ„น๏ธ Detailed Results"):
261
- model_selector = gr.Textbox(
262
- label="Enter Model Name to View Details",
263
- placeholder="facebook/wav2vec2-lv-60-espeak-cv-ft"
264
- )
265
- view_btn = gr.Button("View Results")
266
- results_json = gr.JSON(label="Detailed Results")
267
 
268
- def show_model_results(model_name):
269
- results = load_results_for_model(model_name)
270
- return results or {"error": "No results found for this model"}
 
 
 
 
 
 
 
271
 
272
- view_btn.click(
273
- show_model_results,
274
- inputs=[model_selector],
275
- outputs=[results_json]
276
  )
277
-
278
- gr.Markdown(f"Last updated: {LAST_UPDATED}")
279
 
280
  if __name__ == "__main__":
281
  demo.launch()
 
1
  import gradio as gr
2
  import pandas as pd
3
+ import requests
4
  from pathlib import Path
5
  from datetime import datetime, timezone
6
+ import time
7
+ import logging
8
 
9
+ logging.basicConfig(level=logging.INFO)
10
+ BACKEND_URL = "http://127.0.0.1:8000"
11
  LAST_UPDATED = "Dec 4th 2024"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
  def load_leaderboard_data():
14
+ try:
15
+ response = requests.get(f"{BACKEND_URL}/leaderboard")
16
+ response.raise_for_status()
17
+ return pd.DataFrame(response.json())
18
+ except requests.RequestException as e:
19
+ logging.error(f"Error loading leaderboard: {e}")
20
+ return pd.DataFrame()
21
 
22
  def format_leaderboard_df(df):
 
23
  if df.empty:
24
  return df
25
+
 
26
  display_df = pd.DataFrame({
27
+ "Model": df["model"],
28
+ "Average PER โฌ‡๏ธ": df["average_per"].apply(lambda x: f"{x:.4f}"),
29
+ "Average PWED โฌ‡๏ธ": df["average_pwed"].apply(lambda x: f"{x:.4f}"),
30
+ "GitHub": df["github_url"].apply(lambda x: f'<a href="{x}" target="_blank">Repository</a>' if x else "N/A"),
31
+ "Submission Date": pd.to_datetime(df["submission_date"]).dt.strftime("%Y-%m-%d")
32
  })
33
 
34
+ return display_df.sort_values("Average PER โฌ‡๏ธ")
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
+ def submit_evaluation(model_name, submission_name, github_url):
 
37
  if not model_name or not submission_name:
38
+ return "โš ๏ธ Please provide both model name and submission name."
39
+
40
+ request_data = {
41
+ "transcription_model": model_name,
42
+ "subset": "test",
43
+ "submission_name": submission_name,
44
+ "github_url": github_url if github_url else None
45
+ }
46
 
47
  try:
48
+ response = requests.post(f"{BACKEND_URL}/evaluate", json=request_data)
49
+ response.raise_for_status()
50
+ task_id = response.json()["task_id"]
51
+ return f"โœ… Evaluation submitted successfully! Task ID: {task_id}"
52
+ except requests.RequestException as e:
53
+ return f"โŒ Error: {str(e)}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
+ def check_task_status(task_id):
56
+ if not task_id:
57
+ return "Please enter a task ID"
58
+ try:
59
+ response = requests.get(f"{BACKEND_URL}/tasks/{task_id}")
60
+ response.raise_for_status()
61
+ return response.json()
62
+ except requests.RequestException as e:
63
+ return f"Error checking status: {str(e)}"
 
 
 
 
64
 
65
  def create_html_table(df):
66
+ return df.to_html(escape=False, index=False, classes="styled-table")
67
+
68
+ with gr.Blocks(css="""
69
+ .styled-table {
70
+ width: 100%;
71
+ border-collapse: collapse;
72
+ margin: 25px 0;
73
+ font-size: 0.9em;
74
+ font-family: sans-serif;
75
+ box-shadow: 0 0 20px rgba(0, 0, 0, 0.15);
 
 
 
 
 
 
 
 
 
 
76
  }
77
+ .styled-table thead tr {
78
+ background-color: #009879;
79
+ color: #ffffff;
80
+ text-align: left;
81
  }
82
+ .styled-table th,
83
+ .styled-table td {
84
+ padding: 12px 15px;
85
  }
86
+ .styled-table tbody tr {
87
+ border-bottom: 1px solid #dddddd;
 
88
  }
89
+ .styled-table tbody tr:nth-of-type(even) {
90
+ background-color: #000000;
91
  }
92
+ """) as demo:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  gr.Markdown("# ๐ŸŽฏ Phonemic Transcription Model Evaluation Leaderboard")
 
 
 
 
 
 
 
 
 
 
 
 
94
 
95
+ with gr.Tabs():
96
  with gr.TabItem("๐Ÿ† Leaderboard"):
97
+ leaderboard_html = gr.HTML(create_html_table(format_leaderboard_df(load_leaderboard_data())))
98
+ refresh_btn = gr.Button("๐Ÿ”„ Refresh")
 
 
 
 
 
 
99
  refresh_btn.click(
100
+ lambda: create_html_table(format_leaderboard_df(load_leaderboard_data())),
101
+ outputs=leaderboard_html
102
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
 
104
+ with gr.TabItem("๐Ÿ“ Submit Model"):
105
+ model_name = gr.Textbox(label="Model Name", placeholder="facebook/wav2vec2-lv-60-espeak-cv-ft")
106
+ submission_name = gr.Textbox(label="Submission Name", placeholder="My Model v1.0")
107
+ github_url = gr.Textbox(label="GitHub URL (optional)", placeholder="https://github.com/username/repo")
108
+ submit_btn = gr.Button("Submit")
109
+ result = gr.Textbox(label="Submission Status")
 
110
 
111
+ submit_btn.click(
112
+ submit_evaluation,
113
+ inputs=[model_name, submission_name, github_url],
114
+ outputs=result
115
+ )
116
+
117
+ with gr.TabItem("๐Ÿ“Š Task Status"):
118
+ task_id = gr.Textbox(label="Task ID")
119
+ status_btn = gr.Button("Check Status")
120
+ status_output = gr.JSON(label="Status")
121
 
122
+ status_btn.click(
123
+ check_task_status,
124
+ inputs=task_id,
125
+ outputs=status_output
126
  )
 
 
127
 
128
  if __name__ == "__main__":
129
  demo.launch()