Update app.py

app.py

@@ -36,7 +36,7 @@ def extract_choice_letter(output):
     match = re.search(r"\b([ABCD])\b", output.strip())
     return match.group(1) if match else None
 
-# Added progress parameter to the evaluate function
+# Modified evaluate function to return accuracy as a float directly
 def evaluate(model_id, sample_count, config_name, progress=gr.Progress()):
     if config_name == "ALL":
         subjects = [
@@ -58,26 +58,22 @@ def evaluate(model_id, sample_count, config_name, progress=gr.Progress()):
         total_correct = 0
         total_samples = 0
         all_results = []
-        # Use progress for subject iteration
         for i, subject in enumerate(progress.tqdm(subjects, desc="Evaluating subjects")):
             dataset = load_dataset("cais/mmlu", subject, token=HF_TOKEN)["test"]
             dataset = dataset.shuffle(seed=42).select(range(min(sample_count, len(dataset))))
             correct = 0
-            # Use progress for sample iteration within each subject
             for j, item in enumerate(progress.tqdm(dataset, desc=f"Processing {subject} samples")):
                 prompt, answer = format_prompt(item)
                 output = gen(prompt, max_new_tokens=20, do_sample=False)[0]["generated_text"]
                 output_letter = extract_choice_letter(output)
                 correct += output_letter == answer
                 all_results.append((prompt, output.strip(), answer, output_letter, output_letter == answer))
-            accuracy = correct / len(dataset) * 100
-            record = {"model_id": model_id, "subject": subject, "accuracy": accuracy}
-            with open("eval.jsonl", "a") as f:
-                f.write(json.dumps(record) + "\n")
+            # No need to write subject-level record here, only aggregate
             total_correct += correct
             total_samples += len(dataset)
         avg_accuracy = total_correct / total_samples * 100
-        return f"Average Accuracy: {avg_accuracy:.2f}% across all subjects", all_results
+        # Return the float accuracy value
+        return avg_accuracy, all_results
     gen = load_model(model_id)
     dataset = load_dataset("cais/mmlu", config_name, token=HF_TOKEN)["test"]
     dataset = dataset.shuffle(seed=42).select(range(min(sample_count, len(dataset))))
@@ -85,7 +81,6 @@ def evaluate(model_id, sample_count, config_name, progress=gr.Progress()):
     correct = 0
     results = []
 
-    # Use progress for sample iteration
     for i, item in enumerate(progress.tqdm(dataset, desc=f"Processing {config_name} samples")):
         prompt, answer = format_prompt(item)
         output = gen(prompt, max_new_tokens=20, do_sample=False)[0]["generated_text"]
@@ -95,20 +90,30 @@ def evaluate(model_id, sample_count, config_name, progress=gr.Progress()):
         results.append((prompt, output.strip(), answer, output_letter, is_correct))
 
     accuracy = correct / len(dataset) * 100
-    return f"Accuracy: {accuracy:.2f}%, out of {len(dataset)} samples", results
+    # Return the float accuracy value
+    return accuracy, results
 
 # Pass progress to evaluate function
 def run(model_id, sample_count, config_name, progress=gr.Progress()):
-    score, details = evaluate(model_id, sample_count, config_name, progress)
+    # Receive accuracy_value directly as a float
+    accuracy_value, details = evaluate(model_id, sample_count, config_name, progress)
+
     formatted = "\n\n".join([
         f"### Question:\n{q}\n\n**Model Answer:** {o}\n**Expected:** {a}\n**Predicted:** {g}\n**Correct:** {c}"
         for q, o, a, g, c in details
     ])
-    accuracy_value = float(score.split()[1][:-1])
+    
+    # Format the score string based on config_name
+    if config_name == "ALL":
+        score_string = f"Average Accuracy: {accuracy_value:.2f}% across all subjects"
+    else:
+        # Assuming len(details) corresponds to the number of samples processed for a single subject
+        score_string = f"Accuracy: {accuracy_value:.2f}%, out of {len(details)} samples"
+
     record = {"model_id": model_id, "subject": config_name, "accuracy": accuracy_value}
     with open("eval.jsonl", "a") as f:
         f.write(json.dumps(record) + "\n")
-    return score, formatted
+    return score_string, formatted # Return the formatted string and details
 
 def save_text(text):
     return "evaluation_results.txt", text
@@ -167,4 +172,4 @@ with gr.Blocks(css="body {font-family: Inter, sans-serif; padding: 1em; max-widt
 
     demo.load(load_leaderboard, inputs=[], outputs=[leaderboard_plot, leaderboard_table])
 
-demo.launch()
+demo.launch()