Upload from GitHub Actions: Add Todos for using existing machine-translated datasets rather than our own ones

evals/datasets_/arc.py

@@ -1,11 +1,10 @@
 import random
-from collections import Counter, defaultdict
 
-from langcodes import Language, standardize_tag
+from langcodes import standardize_tag
 from rich import print
 from models import translate_google, get_google_supported_languages
 from tqdm import tqdm
-from datasets import load_dataset
+from datasets import load_dataset, Dataset
 import asyncio
 from tqdm.asyncio import tqdm_asyncio
 import os
@@ -62,7 +61,6 @@ def load_uhura_arc_easy(language_bcp_47, nr):
         task = ds["test"].filter(lambda x: x["id"] == common_ids_test[nr])[0]
         return "fair-forward/arc-easy-autotranslated", task, "machine"
     else:
-        # ARC does not support on-the-fly translation currently
         return None, None, None
 
 

evals/datasets_/mgsm.py

@@ -49,13 +49,6 @@ def load_mgsm(language_bcp_47, nr):
             slug_afrimgsm, subset=tags_afrimgsm[language_bcp_47], split="test"
         )
         return slug_afrimgsm, ds[nr], "human"
-    elif language_bcp_47 in tags_gsm_autotranslated.keys():
-        ds = _load_dataset(
-            slug_gsm_autotranslated,
-            subset=tags_gsm_autotranslated[language_bcp_47],
-            split="test",
-        )
-        return slug_gsm_autotranslated, ds[nr], "machine"
     elif language_bcp_47 in tags_gsm8kx.keys():
         row = _load_dataset(
             slug_gsm8kx,
@@ -64,7 +57,14 @@ def load_mgsm(language_bcp_47, nr):
             trust_remote_code=True,
         )[nr]
         row["answer_number"] = row["answer"].split("####")[1].strip()
-        return slug_gsm8kx, row, "human" # Assuming Eurolingua is human-translated
+        return slug_gsm8kx, row, "machine"
+    elif language_bcp_47 in tags_gsm_autotranslated.keys():
+        ds = _load_dataset(
+            slug_gsm_autotranslated,
+            subset=tags_gsm_autotranslated[language_bcp_47],
+            split="test",
+        )
+        return slug_gsm_autotranslated, ds[nr], "machine"
     else:
         return None, None, None
 

evals/datasets_/mmlu.py

@@ -164,65 +164,13 @@ async def load_mmlu(language_bcp_47, nr):
         ds = ds.map(add_choices)
         task = ds["test"].filter(lambda x: x["subject"] == category)[nr]
         return "CohereForAI/Global-MMLU", task, "human"
+    # TODO: add in Okapi, MMLUX @Jonas
     elif language_bcp_47 in tags_mmlu_autotranslated:
         ds = _load_dataset("fair-forward/mmlu-autotranslated", language_bcp_47)
         filtered = ds["test"].filter(lambda x: x["subject"] == category)
-        if nr < len(filtered):
-            task = filtered[nr]
-            return "fair-forward/mmlu-autotranslated", task, "machine"
-        # Requested index exceeds stored sample count → fallback to on-the-fly
-        return await load_mmlu_translated(language_bcp_47, nr)
+        task = filtered[nr]
+        return "fair-forward/mmlu-autotranslated", task, "machine"
     else:
-        # Fallback to on-the-fly translation for missing languages
-        return await load_mmlu_translated(language_bcp_47, nr)
-
-
-async def load_mmlu_translated(language_bcp_47, nr):
-    """
-    Load MMLU data with on-the-fly Google translation for languages
-    without native or stored auto-translated MMLU, or when more samples are requested.
-    """
-    supported_languages = get_google_supported_languages()
-    if language_bcp_47 not in supported_languages:
-        return None, None, None
-
-    print(f"🔄 Translating MMLU data to {language_bcp_47} on-the-fly...")
-
-    try:
-        # Load English MMLU base (AfriMMLU English split for category alignment)
-        category = categories[nr % len(categories)]
-        ds = _load_dataset("masakhane/afrimmlu", "eng")
-        ds = ds.map(parse_choices)
-        filtered = ds["test"].filter(lambda x: x["subject"] == category)
-        if len(filtered) == 0:
-            return None, None, None
-        
-        # Use the same 20 samples that the evaluation pipeline uses (indices 0-19)
-        if nr < 20:
-            task = filtered[nr]  # Direct mapping to same sample
-        else:
-            # Fallback to sequential if nr exceeds our sample count
-            task = filtered[nr % len(filtered)]
-
-        # Translate question and choices
-        question_translated = await translate_google(task["question"], "en", language_bcp_47)
-        choices_translated = []
-        for choice in task["choices"]:
-            choice_translated = await translate_google(choice, "en", language_bcp_47)
-            choices_translated.append(choice_translated)
-
-        # Create translated task
-        translated_task = {
-            "question": question_translated,
-            "choices": choices_translated,
-            "answer": task["answer"],  # Keep original answer index
-            "subject": task["subject"],
-        }
-
-        return f"mmlu-translated-{language_bcp_47}", translated_task, "machine"
-
-    except Exception as e:
-        print(f"❌ Translation failed for {language_bcp_47}: {e}")
         return None, None, None
 
 

evals/datasets_/truthfulqa.py

@@ -48,58 +48,12 @@ async def load_truthfulqa(language_bcp_47, nr):
         # Load from auto-translated dataset (same samples as translation)
         ds = _load_dataset(slug_truthfulqa_autotranslated, language_bcp_47)
         test_split = ds["test"] if "test" in ds else ds
-        if nr < len(test_split):
-            task = test_split[nr]
-            return slug_truthfulqa_autotranslated, task, "machine"
-        # If requested index exceeds stored sample count, fall back to on-the-fly
-        return await load_truthfulqa_translated(language_bcp_47, nr)
+        task = test_split[nr]
+        return slug_truthfulqa_autotranslated, task, "machine"
+    # TODO: add Okapi, TruthfulQA-X @Jonas
     else:
-        # Fallback to on-the-fly translation for missing languages/samples
-        return await load_truthfulqa_translated(language_bcp_47, nr)
-
-async def load_truthfulqa_translated(language_bcp_47, nr):
-    """
-    Load TruthfulQA data with on-the-fly Google translation.
-    """
-    supported_languages = get_google_supported_languages()
-    if language_bcp_47 not in supported_languages:
         return None, None, None
 
-    print(f"🔄 Translating TruthfulQA data to {language_bcp_47} on-the-fly...")
-
-    try:
-        # Load English TruthfulQA data
-        ds = _load_dataset(slug_uhura_truthfulqa, tags_uhura_truthfulqa["en"])
-        ds = ds.map(add_choices)
-        
-        # Use the same 20 samples that the evaluation pipeline uses (indices 0-19)
-        if nr < 20:
-            task = ds["test"][nr]  # Direct mapping to same sample
-        else:
-            # Fallback to sequential if nr exceeds our sample count
-            task = ds["test"][nr % len(ds["test"])]
-
-        # Translate question and choices
-        question_translated = await translate_google(task["question"], "en", language_bcp_47)
-        choices_translated = []
-        for choice in task["choices"]:
-            choice_translated = await translate_google(choice, "en", language_bcp_47)
-            choices_translated.append(choice_translated)
-
-        translated_task = {
-            "question": question_translated,
-            "choices": choices_translated,
-            "labels": task["labels"], # Keep original labels
-        }
-
-        return f"truthfulqa-translated-{language_bcp_47}", translated_task, "machine"
-
-    except Exception as e:
-        print(f"❌ Translation failed for {language_bcp_47}: {e}")
-        return None, None, None
-
-
-
 def translate_truthfulqa(languages):
     human_translated = [*tags_uhura_truthfulqa.keys()]
     untranslated = [

evals/tasks.py

@@ -120,32 +120,22 @@ Reply with only the topic name.
 Text:
 {test_paragraph.text}
 """
-
-    # some models have poor tokenization for some languages, and the prompt for this task is relatively long, so it sometimes exceeds the context window
-    # this is not just to blame on the context window but mostly on the model's tokenization, so we assign 0 accuracy in this case
-    try:
-        pred = await complete(
-            model=model,
-            messages=[{"role": "user", "content": prompt}],
-            temperature=0,
-            max_tokens=30,
-        )
-        true = test_paragraph.topic
-        others = [t for t in top_topics if t != true]
-        acc = (
-            int(
-                pred.startswith(true)
-                or (true in pred and not any(o in pred for o in others))
-            )
-            if pred
-            else 0
+    pred = await complete(
+        model=model,
+        messages=[{"role": "user", "content": prompt}],
+        temperature=0,
+        max_tokens=30,
+    ).lower().strip()
+    true = test_paragraph.topic.lower().strip()
+    others = [t for t in top_topics if t != true]
+    acc = (
+        int(
+            pred.startswith(true)
+            or (true in pred and not any(o in pred for o in others))
         )
-    except Exception as e:
-        if "`inputs` tokens + `max_new_tokens` must be <= 4097" in str(e):
-            print(f"Max tokens exceeded for {model} in {bcp_47}")
-            acc = 0
-        else:
-            raise e
+        if pred
+        else 0
+    )
     return [
         {
             "model": model,
@@ -331,7 +321,6 @@ def format_multiple_choice_truthfulqa(item):
     text = item["question"] + "\n\n"
     for i, choice in enumerate(item["choices"]):
         text += f"{letters[i]}: {choice}\n"
-    text += "|".join(letters[: len(item["choices"])]) + "?"
     return text