Update app.py

app.py

@@ -36,7 +36,8 @@ def extract_choice_letter(output):
     match = re.search(r"\b([ABCD])\b", output.strip())
     return match.group(1) if match else None
 
-def evaluate(model_id, sample_count, config_name):
+# Added progress parameter to the evaluate function
+def evaluate(model_id, sample_count, config_name, progress=gr.Progress()):
     if config_name == "ALL":
         subjects = [
             "abstract_algebra", "anatomy", "astronomy", "business_ethics", "college_biology",
@@ -57,11 +58,13 @@ def evaluate(model_id, sample_count, config_name):
         total_correct = 0
         total_samples = 0
         all_results = []
-        for subject in subjects:
+        # Use progress for subject iteration
+        for i, subject in enumerate(progress.tqdm(subjects, desc="Evaluating subjects")):
             dataset = load_dataset("cais/mmlu", subject, token=HF_TOKEN)["test"]
             dataset = dataset.shuffle(seed=42).select(range(min(sample_count, len(dataset))))
             correct = 0
-            for item in dataset:
+            # Use progress for sample iteration within each subject
+            for j, item in enumerate(progress.tqdm(dataset, desc=f"Processing {subject} samples")):
                 prompt, answer = format_prompt(item)
                 output = gen(prompt, max_new_tokens=20, do_sample=False)[0]["generated_text"]
                 output_letter = extract_choice_letter(output)
@@ -70,7 +73,7 @@ def evaluate(model_id, sample_count, config_name):
             accuracy = correct / len(dataset) * 100
             record = {"model_id": model_id, "subject": subject, "accuracy": accuracy}
             with open("eval.jsonl", "a") as f:
-                f.write(json.dumps(record) + "\n") # Fixed: added closing double quote and newline
+                f.write(json.dumps(record) + "\n")
             total_correct += correct
             total_samples += len(dataset)
         avg_accuracy = total_correct / total_samples * 100
@@ -82,7 +85,8 @@ def evaluate(model_id, sample_count, config_name):
     correct = 0
     results = []
 
-    for item in dataset:
+    # Use progress for sample iteration
+    for i, item in enumerate(progress.tqdm(dataset, desc=f"Processing {config_name} samples")):
         prompt, answer = format_prompt(item)
         output = gen(prompt, max_new_tokens=20, do_sample=False)[0]["generated_text"]
         output_letter = extract_choice_letter(output)
@@ -93,8 +97,9 @@ def evaluate(model_id, sample_count, config_name):
     accuracy = correct / len(dataset) * 100
     return f"Accuracy: {accuracy:.2f}%, out of {len(dataset)} samples", results
 
-def run(model_id, sample_count, config_name):
-    score, details = evaluate(model_id, sample_count, config_name)
+# Pass progress to evaluate function
+def run(model_id, sample_count, config_name, progress=gr.Progress()):
+    score, details = evaluate(model_id, sample_count, config_name, progress)
     formatted = "\n\n".join([
         f"### Question:\n{q}\n\n**Model Answer:** {o}\n**Expected:** {a}\n**Predicted:** {g}\n**Correct:** {c}"
         for q, o, a, g, c in details
@@ -102,7 +107,7 @@ def run(model_id, sample_count, config_name):
     accuracy_value = float(score.split()[1][:-1])
     record = {"model_id": model_id, "subject": config_name, "accuracy": accuracy_value}
     with open("eval.jsonl", "a") as f:
-        f.write(json.dumps(record) + "\n") # Fixed: added closing double quote and newline
+        f.write(json.dumps(record) + "\n")
     return score, formatted
 
 def save_text(text):
@@ -133,6 +138,7 @@ with gr.Blocks(css="body {font-family: Inter, sans-serif; padding: 1em; max-widt
     detail_output = gr.Textbox(label="Evaluation Details", lines=20, interactive=False)
     download_button = gr.Button("📥 Download Full Evaluation")
 
+    # Pass progress to the run function
     run_button.click(run, inputs=[model_id, sample_count, config_name], outputs=[acc_output, detail_output])
     download_button.click(save_text, inputs=detail_output, outputs=gr.File())
 
@@ -157,8 +163,8 @@ with gr.Blocks(css="body {font-family: Inter, sans-serif; padding: 1em; max-widt
             return fig, df_sorted
         except Exception as e:
             # Handle the case where eval.jsonl might not exist yet
-            return plt.figure(), pd.DataFrame(columns=["model_id", "average_accuracy"]) # Corrected columns
+            return plt.figure(), pd.DataFrame(columns=["model_id", "average_accuracy"])
 
     demo.load(load_leaderboard, inputs=[], outputs=[leaderboard_plot, leaderboard_table])
 
-demo.launch()
+demo.launch()