cache

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 results.duckdb filter=lfs diff=lfs merge=lfs -text
 puzzles_cleaned.csv filter=lfs diff=lfs merge=lfs -text
+answer_cache/cache.db filter=lfs diff=lfs merge=lfs -text

answer_cache/cache.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e1a6adfdf2de2f7718c1284bf810dc64d191ee415a0975949b01c6954be1b2
+size 196784128

metrics.py

@@ -6,6 +6,11 @@ import argparse
 import unicodedata
 import re
 
+
+import diskcache as dc
+
+cache = dc.Cache("answer_cache")
+
 def normalize_text(text: str) -> str:
     """Normalize text to remove accents, convert to lowercase, and strip spaces."""
     text = unicodedata.normalize("NFKD", text)  # Decomposes letters with accents (e.g., é → e + ́)
@@ -42,11 +47,16 @@ def _answer_without_thoughts(completion: str) -> str:
     return completion
 
 
+
 def _check_answer(completion: str, answer: str) -> bool:
     """
     Check that all the phrases that must appear in the answer appear in the
     completion. We ignore "thoughts", capitalization, and punctuation.
     """
+    key = (completion, answer)
+    if key in cache:
+        return cache[key]
+    
     completion = _answer_without_thoughts(completion).lower()
     completion = completion.replace("**","")
     completion  = re.sub(r'[^\w\s]', ' ', completion) # this replaces punctuations with space, aligning with the _parse_answer function's ' '.join
@@ -56,9 +66,12 @@ def _check_answer(completion: str, answer: str) -> bool:
     for answer_phrases in alternative_answers:
         # if all(phrase in completion for phrase in answer_phrases):
         if all(re.search(rf'\b{re.escape(phrase)}\b', completion) for phrase in answer_phrases):
+            cache[key] = True
             return True
+    cache[key] = False
     return False
 
+
 def _clip_text(text: str, width: int) -> str:
     return text if len(text) <= width else text[:width] + "..."