update populate

src/populate.py

@@ -3,8 +3,10 @@
 import os
 import pandas as pd
 import json
+import random
 
-from src.display.utils import COLUMNS, EVAL_COLS
+from src.display.utils import COLUMNS, EVAL_COLS, Tasks
+from src.envs import EVAL_RESULTS_PATH, FIXED_QUESTIONS_FILE  # Define FIXED_QUESTIONS_FILE in envs.py
 
 def get_leaderboard_df(eval_results_path, eval_requests_path, cols, benchmark_cols):
     # Initialize an empty DataFrame
@@ -58,3 +60,42 @@ def get_evaluation_queue_df(eval_requests_path, eval_cols):
             pending_df = df[df['status'] == 'pending']
 
     return finished_df, running_df, pending_df
+
+def preselect_fixed_questions(dataset_path, num_questions_per_subject=30, fixed_questions_file='fixed_questions.json'):
+    """
+    Preselects a fixed number of questions per subject and saves them to a JSON file.
+    """
+    # Load the dataset
+    # Assuming the dataset is in CSV format with a 'Subject' column
+    if not os.path.exists(dataset_path):
+        raise FileNotFoundError(f"Dataset file not found at {dataset_path}")
+
+    dataset = pd.read_csv(dataset_path)
+
+    fixed_questions = {}
+
+    for task in Tasks:
+        subject = task.value.benchmark
+        subject_questions = dataset[dataset['Subject'] == subject]
+
+        if len(subject_questions) < num_questions_per_subject:
+            raise ValueError(f"Not enough questions for subject '{subject}'. Required: {num_questions_per_subject}, Available: {len(subject_questions)}")
+
+        # Randomly select fixed number of questions
+        selected_questions = subject_questions.sample(n=num_questions_per_subject, random_state=42)  # random_state for reproducibility
+        fixed_questions[subject] = selected_questions.to_dict(orient='records')
+
+    # Save fixed questions to a JSON file
+    with open(os.path.join(EVAL_RESULTS_PATH, fixed_questions_file), 'w') as f:
+        json.dump(fixed_questions, f, indent=4)
+
+    print(f"Fixed questions preselected and saved to {fixed_questions_file}")
+
+if __name__ == "__main__":
+    # Example usage:
+    # Define the path to your dataset
+    DATASET_PATH = os.path.join(EVAL_RESULTS_PATH, "your_dataset.csv")  # Update with your actual dataset file
+    FIXED_QUESTIONS_FILE = "fixed_questions.json"  # Define the name for fixed questions file
+
+    # Preselect fixed questions
+    preselect_fixed_questions(DATASET_PATH, num_questions_per_subject=30, fixed_questions_file=FIXED_QUESTIONS_FILE)