Add evaluation code

evaluate.py

@@ -63,9 +63,25 @@ def get_solution_code(day: int, model: str) -> str:
 def extract_solutions(df, output_file = "solutions.json"):
     # TODO: better way of getting this?
     solutions = {}
-    for day in range(1, 25):
+    for day in range(1, 26):
         sub_df = df[(df.model == "jerpint") & (df.day == day)]
-        part1, part2 = sub_df.result.to_list()[0].strip("\n").split("\n")
+
+
+        day_solution = sub_df.result.to_list()[0].strip("\n").split("\n")
+        if len(day_solution) == 0:
+            part1 = "N/A"
+            part2 = "N/A"
+        elif len(day_solution) == 1:
+            part1 = day_solution[0]
+            part2 = "N/A"
+        elif len(day_solution) == 2:
+            part1, part2 = day_solution
+        else:
+            print("Something went wrong, check day {day} solution: \n {day_solution}")
+            part1 = "N/A"
+            part2 = "N/A"
+
+
         solutions[day] = [part1, part2]
 
     with open(output_file, "w") as f:
@@ -82,8 +98,7 @@ def evaluate_submissions(all_models, results_file = "results.csv", skip = True):
     else:
         df = pd.DataFrame(columns=["day", "model", "result", "total_time"])
 
-    # for day in range(1, 26):
-    for day in range(1, 11):
+    for day in range(1, 26):
         print("*" * 80)
         print(f"Evaluating day {day}")
         for provider in all_models:
@@ -98,9 +113,13 @@ def evaluate_submissions(all_models, results_file = "results.csv", skip = True):
                 result = evaluate_submission(day, model)
                 df = pd.concat([df, pd.DataFrame({"day": [day], "model": [model], "result": [result["result"]], "total_time": [result["total_time"]]})], ignore_index=True)
 
+                # Save incrementally
                 df.to_csv("results.csv", index=False)
                 print("-" * 80)
+
         print("*" * 80)
+        df = df.sort_values(by="day")
+        df.to_csv("results.csv", index=False)
     return df
 
 
@@ -109,35 +128,5 @@ if __name__ == "__main__":
     all_models["human"] = ["jerpint"]
     df = evaluate_submissions(all_models, results_file="results.csv")
 
-    # For now, only evaluate first 9 days
-    # TODO: All days
-    df = df[df.day < 10]
-
     # Run once to save results
-    # solutions = extract_solutions(df)
-
-    with open("solutions.json") as f:
-        solutions = json.load(f)
-
-    def score_submissions(row):
-        result = row["result"]
-        day = row["day"]
-        solution = solutions[str(day)]
-
-        score_1 = solution[0] in result
-        score_2 = solution[1] in result
-        return [score_1, score_2]
-
-
-    df["scores"] = df.apply(score_submissions, axis=1)
-
-    df["part_1"] = df["scores"].apply(lambda x: x[0])
-    df["part_2"] = df["scores"].apply(lambda x: x[1])
-
-    for model in df.model.unique():
-        df_model = df[df.model == model]
-        silver_stars = df_model.part_1.sum()
-        gold_stars = df_model.part_2.sum()
-        total_stars = silver_stars + gold_stars
-
-        print(model, total_stars)
+    solutions = extract_solutions(df, output_file="solutions.json")