Use langcodes for language matching

evals.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import os
+import re
 from os import getenv
 
 import evaluate
@@ -14,17 +15,19 @@ from tqdm.asyncio import tqdm_asyncio
 from transformers import NllbTokenizer
 from datetime import date
 from requests import get
+from language_data.population_data import LANGUAGE_SPEAKING_POPULATION
+from langcodes import standardize_tag, Language
 
 # config
 models = [
-    "openai/gpt-4o-mini", # 0.6$/M tokens
+    "openai/gpt-4o-mini",  # 0.6$/M tokens
     # "anthropic/claude-3.5-haiku", # 4$/M tokens -> too expensive
-    "meta-llama/llama-3.3-70b-instruct", # 0.3$/M tokens
-    "mistralai/mistral-small-24b-instruct-2501", # 0.14$/M tokens
-    "google/gemini-2.0-flash-001", # 0.4$/M tokens
+    "meta-llama/llama-3.3-70b-instruct",  # 0.3$/M tokens
+    "mistralai/mistral-small-24b-instruct-2501",  # 0.14$/M tokens
+    "google/gemini-2.0-flash-001",  # 0.4$/M tokens
     # "qwen/qwen-turbo", # 0.2$/M tokens; recognizes "inappropriate content"
-    "deepseek/deepseek-chat", # 0.9$/M tokens
-    "microsoft/phi-4", # 0.07$/M tokens
+    "deepseek/deepseek-chat",  # 0.9$/M tokens
+    "microsoft/phi-4",  # 0.07$/M tokens
 ]
 fast_model = "meta-llama/llama-3.3-70b-instruct"
 n_sentences = 30
@@ -47,73 +50,79 @@ def reorder(language_name):
         return language_name.split(",")[1] + " " + language_name.split(",")[0]
     return language_name
 
+
+# load general language data
+languages = {
+    lang: pop
+    for lang, pop in LANGUAGE_SPEAKING_POPULATION.items()
+    if not re.match(r".*-[A-Z]{2}$", lang)
+}
+languages = pd.DataFrame(list(languages.items()), columns=["bcp_47", "speakers"])
+languages["name"] = languages["bcp_47"].apply(lambda x: Language.get(x).display_name())
+
+# load script codes and names
+scripts = pd.read_csv("data/ScriptCodes.csv").rename(columns={"Code": "iso15924", "English Name": "script_name"})
+
+def script_name(iso15924):
+    return scripts[scripts["iso15924"] == iso15924]["script_name"].values[0]
+
 # load benchmark languages and scripts
 benchmark_dir = "data/floresp-v2.0-rc.3/dev"
 benchmark_languages = pd.DataFrame(
     [f.split(".")[1].split("_", 1) for f in os.listdir(benchmark_dir)],
-    columns=["language_code", "script_code"],
+    columns=["iso639_3", "iso15924"],
 )
-# hack: drop additional script codes for languages with multiple scripts
-benchmark_languages = benchmark_languages.groupby("language_code").head(1)
-benchmark_languages["in_benchmark"] = True
-
-# load Ethnologue language names
-language_names = (
-    pd.read_csv("data/LanguageCodes.tab", sep="\t")
-    .rename(columns={"LangID": "language_code", "Name": "language_name"})[
-        ["language_code", "language_name"]
-    ]
-    .assign(language_name=lambda df: df["language_name"].apply(reorder).str.strip())
+benchmark_languages["bcp_47"] = benchmark_languages.apply(
+    lambda row: standardize_tag(row["iso639_3"] + "-" + row["iso15924"], macro=True),
+    axis=1,
 )
-
-# load Wikidata speaker stats
-language_stats = (
-    pd.read_csv("data/languages.tsv", sep="\t")
-    .rename(columns={"iso639_3": "language_code", "maxSpeakers": "speakers"})[
-        ["language_code", "speakers", "iso639_1"]
-    ]
-    .dropna(subset=["language_code"])
+# ignore script (language is language)
+benchmark_languages["bcp_47"] = benchmark_languages["bcp_47"].apply(
+    lambda x: re.sub(r"-[A-Z][a-z]+$", "", x)
+)
+benchmark_languages = (
+    benchmark_languages.groupby("bcp_47")
+    .agg({"iso639_3": "first", "iso15924": "first"})
+    .reset_index()
 )
-language_stats["speakers"] = pd.to_numeric(language_stats["speakers"], errors="coerce")
-ignored_languages = [
-    "zho",  # Chinese -> use Mandarin (cmn) instead
-    "ara",  # Arabic -> use Standard Arabic (arb) instead
-    "pus",  # Pashto -> use Nothern / Central / Southern Pashto instead (pbt / pst / pbu)
-    "fas",  # Persian -> use Iranian Persian (pes) instead
-    "msa",  # Malay -> use Indonesian (ind) instead
-]
-language_stats = language_stats[
-    ~language_stats["language_code"].isin(ignored_languages)
-]
-
-# load unicode script names
-script_names = pd.read_csv("data/ScriptCodes.csv").rename(
-    columns={"Code": "script_code", "English Name": "script_name"}
-)[["script_code", "script_name"]]
 
-# merge data
-languages = pd.merge(language_stats, language_names, on="language_code", how="outer")
-languages = pd.merge(benchmark_languages, languages, on="language_code", how="outer")
-languages = pd.merge(languages, script_names, on="script_code", how="left")
-languages["in_benchmark"] = languages["in_benchmark"].fillna(False)
-languages = languages.sort_values(by="speakers", ascending=False)
-languages = languages.iloc[:30]
 
-# retrieve CommonVoice stats
-@cache # cache for 1 day
+# load CommonVoice stats
+@cache  # cache for 1 day
 def get_commonvoice_stats(date: date):
     return get("https://commonvoice.mozilla.org/api/v1/stats/languages").json()
 
-commonvoice_stats = pd.DataFrame(get_commonvoice_stats(date.today()))
+
+commonvoice_stats = pd.DataFrame(get_commonvoice_stats(date.today())).rename(
+    columns={"locale": "bcp_47", "validatedHours": "commonvoice_hours"}
+)[["bcp_47", "commonvoice_hours"]]
+# ignore country (language is language) (in practive this is only relevant to zh-CN/zh-TW/zh-HK)
+commonvoice_stats["bcp_47"] = commonvoice_stats["bcp_47"].apply(
+    lambda x: re.sub(r"-[A-Z]{2}$", "", x)
+)
+commonvoice_stats["bcp_47"] = commonvoice_stats["bcp_47"].apply(
+    lambda x: standardize_tag(x, macro=True)
+)  # this does not really seem to get macrolanguages though, e.g. not for Quechua
+commonvoice_stats = commonvoice_stats.groupby("bcp_47").sum().reset_index()
+
+# merge data
+languages = pd.merge(
+    languages, benchmark_languages, on="bcp_47", how="left"
+)  # "left" because keep it simple for now
+languages = pd.merge(
+    languages, commonvoice_stats, on="bcp_47", how="left"
+)  # "left" because keep it simple for now
+languages["in_benchmark"] = languages["bcp_47"].isin(benchmark_languages["bcp_47"])
+
+languages = languages.sort_values(by="speakers", ascending=False)
+languages = languages.iloc[:10]
 
 # sample languages to translate to
 target_languages = languages[languages["in_benchmark"]].sample(
     n=n_sentences, weights="speakers", replace=True, random_state=42
 )
 # sample languages to analyze with all models
-detailed_languages = languages[languages["in_benchmark"]].sample(
-    n=10, random_state=42
-)
+detailed_languages = languages[languages["in_benchmark"]].sample(n=3, random_state=42)
 
 
 # utils
@@ -140,15 +149,14 @@ async def complete(**kwargs):
         raise Exception(response)
     return response
 
-
-@cache
-async def translate(model, target_language, target_script, sentence):
+async def translate(model, target_language, sentence):
+    script = script_name(target_language.iso15924)
     reply = await complete(
         model=model,
         messages=[
             {
                 "role": "user",
-                "content": f"Translate the following text to the {target_language} language; use the {target_script} script; reply only with the translation:\n\n{sentence}",
+                "content": f"Translate the following text to the {target_language.name} language; use the {script} script; reply only with the translation:\n\n{sentence}",
             }
         ],
         temperature=0,
@@ -162,40 +170,33 @@ def mean(l):
 
 
 def load_sentences(language):
-    return open(
-        f"{benchmark_dir}/dev.{language.language_code}_{language.script_code}"
-    ).readlines()
+    return open(f"{benchmark_dir}/dev.{language.iso639_3}_{language.iso15924}").readlines()
 
 
 # evaluation!
 async def main():
     results = []
     for language in list(languages.itertuples()):
-        name = (
-            language.language_name
-            if not pd.isna(language.language_name)
-            else language.language_code
-        )
-        print(name)
         scores = []
         if language.in_benchmark:
             original_sentences = load_sentences(language)[:n_sentences]
             for model in models:
                 if (
                     model != fast_model
-                    and language.language_code
-                    not in detailed_languages.language_code.values
+                    and language.bcp_47 not in detailed_languages.bcp_47.values
                 ):
                     continue
-                
-                print(model)
                 predictions = [
                     translate(
-                        model, language.language_name, language.script_name, sentence
+                        model,
+                        language,
+                        sentence,
+                    )
+                    for sentence, language in zip(
+                        original_sentences, target_languages.itertuples()
                     )
-                    for sentence, language in zip(original_sentences, target_languages.itertuples())
                 ]
-                predictions = await tqdm_asyncio.gather(*predictions, miniters=1)
+                predictions = await tqdm_asyncio.gather(*predictions, miniters=1, desc=f"{language.name} {model.split('/')[0]}")
                 target_sentences = [
                     load_sentences(lang)[i]
                     for i, lang in enumerate(target_languages.itertuples())
@@ -217,17 +218,15 @@ async def main():
                         # "bert_score": mean(metrics_bert["f1"]),
                     }
                 )
-        commonvoice_hours = commonvoice_stats[commonvoice_stats["locale"] == language.iso639_1]["validatedHours"].values
-        commonvoice_hours = commonvoice_hours[0] if commonvoice_hours.size > 0 else "N/A"
         results.append(
             {
-                "language_name": name,
-                "language_code": language.language_code,
+                "language_name": language.name,
+                "bcp_47": language.bcp_47,
                 "speakers": language.speakers if not pd.isna(language.speakers) else 0,
                 "scores": scores,
                 "bleu": mean([s["bleu"] for s in scores]) if scores else None,
                 # "bert_score": mean([s["bert_score"] for s in scores]),
-                "commonvoice_hours": commonvoice_hours,
+                "commonvoice_hours": language.commonvoice_hours,
             }
         )
     with open("results.json", "w") as f:

pyproject.toml

@@ -16,6 +16,7 @@ dev-dependencies = [
     "bert-score>=0.3.13",
     "evaluate==0.4.0",
     "joblib>=1.4.2",
+    "langcodes>=3.5.0",
     "openai>=1.52.2",
     "protobuf>=5.28.3",
     "python-dotenv>=1.0.1",

results.json

@@ -1,582 +1,192 @@
 [
   {
     "language_name": "English",
-    "language_code": "eng",
-    "speakers": 1132366680.0,
+    "bcp_47": "en",
+    "speakers": 1636485840,
     "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.47104084248165595
-      },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.4207265890491719
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.4642719176436136
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.5237470882988915
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.516570670982587
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.44668905281921456
+        "bleu": 0.4931825583688982
       }
     ],
-    "bleu": 0.47384102687918905,
+    "bleu": 0.4931825583688982,
     "commonvoice_hours": 2649.0
   },
   {
-    "language_name": "Mandarin Chinese",
-    "language_code": "cmn",
-    "speakers": 1074000000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.48254866511762295
-      }
-    ],
-    "bleu": 0.48254866511762295,
-    "commonvoice_hours": "N/A"
-  },
-  {
-    "language_name": "Spanish",
-    "language_code": "spa",
-    "speakers": 485000000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.31606621368361204
-      }
-    ],
-    "bleu": 0.31606621368361204,
-    "commonvoice_hours": 446.0
-  },
-  {
-    "language_name": "Hindi",
-    "language_code": "hin",
-    "speakers": 341000000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.3273225856613046
-      }
-    ],
-    "bleu": 0.3273225856613046,
-    "commonvoice_hours": 16.0
-  },
-  {
-    "language_name": "Bengali",
-    "language_code": "ben",
-    "speakers": 300000000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.23110496173302814
-      }
-    ],
-    "bleu": 0.23110496173302814,
-    "commonvoice_hours": 49.0
-  },
-  {
-    "language_name": "Portuguese",
-    "language_code": "por",
-    "speakers": 254300000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.35032125995743685
-      }
-    ],
-    "bleu": 0.35032125995743685,
-    "commonvoice_hours": 176.0
-  },
-  {
-    "language_name": "French",
-    "language_code": "fra",
-    "speakers": 208157220.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.31625053573185663
-      }
-    ],
-    "bleu": 0.31625053573185663,
-    "commonvoice_hours": 1051.0
-  },
-  {
-    "language_name": "Indonesian",
-    "language_code": "ind",
-    "speakers": 198996550.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.3112185444311794
-      }
-    ],
-    "bleu": 0.3112185444311794,
-    "commonvoice_hours": 33.0
-  },
-  {
-    "language_name": "Russian",
-    "language_code": "rus",
-    "speakers": 171428900.0,
+    "language_name": "Chinese",
+    "bcp_47": "zh",
+    "speakers": 1304678914,
     "scores": [
       {
         "model": "openai/gpt-4o-mini",
-        "bleu": 0.32615858913589074
+        "bleu": 0.4807599914028467
       },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.3244999119385425
+        "bleu": 0.48224897154012053
       },
       {
         "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.315801608032821
+        "bleu": 0.2688927547323512
       },
       {
         "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.3683733679689521
+        "bleu": 0.4876059353172742
       },
       {
         "model": "deepseek/deepseek-chat",
-        "bleu": 0.35988734604889566
+        "bleu": 0.46126489333496423
       },
       {
         "model": "microsoft/phi-4",
-        "bleu": 0.31289371159965956
+        "bleu": 0.43306718920654086
       }
     ],
-    "bleu": 0.3346024224541269,
-    "commonvoice_hours": 241.0
+    "bleu": 0.4356399559223496,
+    "commonvoice_hours": 422.0
   },
   {
-    "language_name": "Japanese",
-    "language_code": "jpn",
-    "speakers": 128000000.0,
+    "language_name": "Hindi",
+    "bcp_47": "hi",
+    "speakers": 546882144,
     "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.28991739992953497
-      },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.2675679907215641
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.21348802780641032
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.3345265427223546
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.3101203037558905
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.2585222780278109
+        "bleu": 0.42910938007537924
       }
     ],
-    "bleu": 0.2790237571605942,
-    "commonvoice_hours": 222.0
+    "bleu": 0.42910938007537924,
+    "commonvoice_hours": 16.0
   },
   {
-    "language_name": "Eastern Punjabi",
-    "language_code": "pan",
-    "speakers": 125000000.0,
+    "language_name": "Spanish",
+    "bcp_47": "es",
+    "speakers": 493528077,
     "scores": [
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.27325501919134315
+        "bleu": 0.3335615012680206
       }
     ],
-    "bleu": 0.27325501919134315,
-    "commonvoice_hours": "N/A"
+    "bleu": 0.3335615012680206,
+    "commonvoice_hours": 446.0
   },
   {
-    "language_name": "Standard German",
-    "language_code": "deu",
-    "speakers": 105000000.0,
+    "language_name": "Arabic",
+    "bcp_47": "ar",
+    "speakers": 351664197,
     "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.39019323183176663
-      },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.37266353070949576
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.3647632576435612
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.4466723425292597
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.4045496243095387
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.36047992103881465
+        "bleu": 0.19072998559991275
       }
     ],
-    "bleu": 0.3898869846770727,
-    "commonvoice_hours": 1357.0
-  },
-  {
-    "language_name": "Egyptian Arabic",
-    "language_code": "arz",
-    "speakers": 100542400.0,
-    "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.2339779422333898
-      },
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.20475486619797384
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.20783660453505234
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.2840808045687292
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.2786287793608212
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.19969813973959594
-      }
-    ],
-    "bleu": 0.23482952277259375,
-    "commonvoice_hours": "N/A"
+    "bleu": 0.19072998559991275,
+    "commonvoice_hours": 91.0
   },
   {
     "language_name": "Urdu",
-    "language_code": "urd",
-    "speakers": 94022900.0,
+    "bcp_47": "ur",
+    "speakers": 290790290,
     "scores": [
       {
         "model": "openai/gpt-4o-mini",
-        "bleu": 0.297325653414119
+        "bleu": 0.3223557428811336
       },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.24593966310665433
+        "bleu": 0.3361392064611452
       },
       {
         "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.21988755291389567
+        "bleu": 0.30361668093990973
       },
       {
         "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.31796430998058983
+        "bleu": 0.38811035932918286
       },
       {
         "model": "deepseek/deepseek-chat",
-        "bleu": 0.3043614136242901
+        "bleu": 0.33221997814253806
       },
       {
         "model": "microsoft/phi-4",
-        "bleu": 0.2285337340113323
+        "bleu": 0.2541447606474814
       }
     ],
-    "bleu": 0.2690020545084802,
+    "bleu": 0.32276445473356513,
     "commonvoice_hours": 76.0
   },
   {
-    "language_name": "Filipino",
-    "language_code": "fil",
-    "speakers": 90000000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.33268969497468076
-      }
-    ],
-    "bleu": 0.33268969497468076,
-    "commonvoice_hours": "N/A"
-  },
-  {
-    "language_name": "Javanese",
-    "language_code": "jav",
-    "speakers": 84308740.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.2528746866064681
-      }
-    ],
-    "bleu": 0.2528746866064681,
-    "commonvoice_hours": 0.0
-  },
-  {
-    "language_name": "Marathi",
-    "language_code": "mar",
-    "speakers": 83100000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.24876051941895777
-      }
-    ],
-    "bleu": 0.24876051941895777,
-    "commonvoice_hours": 20.0
-  },
-  {
-    "language_name": "Swahili",
-    "language_code": "swh",
-    "speakers": 82300000.0,
-    "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.34863560100932933
-      },
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.30524292832054034
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.23580256234118713
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.3871437234807849
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.3476225063617937
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.21803176063271826
-      }
-    ],
-    "bleu": 0.3070798470243923,
-    "commonvoice_hours": "N/A"
-  },
-  {
-    "language_name": "Turkish",
-    "language_code": "tur",
-    "speakers": 82231620.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.29874140544434125
-      }
-    ],
-    "bleu": 0.29874140544434125,
-    "commonvoice_hours": 127.0
-  },
-  {
-    "language_name": "Telugu",
-    "language_code": "tel",
-    "speakers": 82000000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.28869836899054496
-      }
-    ],
-    "bleu": 0.28869836899054496,
-    "commonvoice_hours": 0.3
-  },
-  {
-    "language_name": "Wu Chinese",
-    "language_code": "wuu",
-    "speakers": 81400000.0,
-    "scores": [],
-    "bleu": null,
-    "commonvoice_hours": "N/A"
-  },
-  {
-    "language_name": "Korean",
-    "language_code": "kor",
-    "speakers": 77300000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.2566453806044083
-      }
-    ],
-    "bleu": 0.2566453806044083,
-    "commonvoice_hours": 1.7
-  },
-  {
-    "language_name": "Vietnamese",
-    "language_code": "vie",
-    "speakers": 76000000.0,
-    "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.3104431723374164
-      },
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.3098478561790782
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.28074941515909896
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.37327273228460267
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.3487726531917602
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.18355331419148843
-      }
-    ],
-    "bleu": 0.3011065238905742,
-    "commonvoice_hours": 5.9
-  },
-  {
-    "language_name": "Tamil",
-    "language_code": "tam",
-    "speakers": 75000000.0,
-    "scores": [
-      {
-        "model": "openai/gpt-4o-mini",
-        "bleu": 0.24593649157372188
-      },
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.24009996232522382
-      },
-      {
-        "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.16785828803139252
-      },
-      {
-        "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.3411457686951495
-      },
-      {
-        "model": "deepseek/deepseek-chat",
-        "bleu": 0.2875340171253509
-      },
-      {
-        "model": "microsoft/phi-4",
-        "bleu": 0.12646276530642359
-      }
-    ],
-    "bleu": 0.23483954884287706,
-    "commonvoice_hours": 234.0
-  },
-  {
-    "language_name": "Yue Chinese",
-    "language_code": "yue",
-    "speakers": 73100000.0,
+    "language_name": "French",
+    "bcp_47": "fr",
+    "speakers": 278611507,
     "scores": [
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.2663995648378034
+        "bleu": 0.40595466651226686
       }
     ],
-    "bleu": 0.2663995648378034,
-    "commonvoice_hours": "N/A"
+    "bleu": 0.40595466651226686,
+    "commonvoice_hours": 1051.0
   },
   {
-    "language_name": "Italian",
-    "language_code": "ita",
-    "speakers": 64819790.0,
+    "language_name": "Bangla",
+    "bcp_47": "bn",
+    "speakers": 267193288,
     "scores": [
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.3190660116366235
+        "bleu": 0.30570858536443696
       }
     ],
-    "bleu": 0.3190660116366235,
-    "commonvoice_hours": 362.0
+    "bleu": 0.30570858536443696,
+    "commonvoice_hours": 49.0
   },
   {
-    "language_name": "Gujarati",
-    "language_code": "guj",
-    "speakers": 56400000.0,
+    "language_name": "Portuguese",
+    "bcp_47": "pt",
+    "speakers": 237496885,
     "scores": [
       {
         "model": "openai/gpt-4o-mini",
-        "bleu": 0.25754571533357745
+        "bleu": 0.4122096638493346
       },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.24145756515188838
+        "bleu": 0.39250552075952033
       },
       {
         "model": "mistralai/mistral-small-24b-instruct-2501",
-        "bleu": 0.20092063514315023
+        "bleu": 0.22643923104785263
       },
       {
         "model": "google/gemini-2.0-flash-001",
-        "bleu": 0.3664134239402827
+        "bleu": 0.42197093736929103
       },
       {
         "model": "deepseek/deepseek-chat",
-        "bleu": 0.2908883229704476
+        "bleu": 0.42783260235353093
       },
       {
         "model": "microsoft/phi-4",
-        "bleu": 0.19669824113063106
+        "bleu": 0.38611444119797594
       }
     ],
-    "bleu": 0.2589873172783296,
-    "commonvoice_hours": "N/A"
-  },
-  {
-    "language_name": "Iranian Persian",
-    "language_code": "pes",
-    "speakers": 52800000.0,
-    "scores": [
-      {
-        "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.28359916806993934
-      }
-    ],
-    "bleu": 0.28359916806993934,
-    "commonvoice_hours": "N/A"
+    "bleu": 0.3778453994295843,
+    "commonvoice_hours": 176.0
   },
   {
-    "language_name": "Bhojpuri",
-    "language_code": "bho",
-    "speakers": 52200000.0,
+    "language_name": "Punjabi",
+    "bcp_47": "pa",
+    "speakers": 203571210,
     "scores": [
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
-        "bleu": 0.24311504988281543
+        "bleu": 0.34311946995454473
       }
     ],
-    "bleu": 0.24311504988281543,
-    "commonvoice_hours": "N/A"
-  },
-  {
-    "language_name": "Hakka Chinese",
-    "language_code": "hak",
-    "speakers": 48200000.0,
-    "scores": [],
-    "bleu": null,
-    "commonvoice_hours": "N/A"
+    "bleu": 0.34311946995454473,
+    "commonvoice_hours": 2.3
   }
 ]

uv.lock

@@ -898,6 +898,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ea/8b/d7497df4a1cae9367adf21665dd1f896c2a7aeb8769ad77b662c5e2bcce7/kiwisolver-1.4.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:eb542fe7933aa09d8d8f9d9097ef37532a7df6497819d16efe4359890a2f417a", size = 55715 },
 ]
 
+[[package]]
+name = "langcodes"
+version = "3.5.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "language-data" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3a/7a/5a97e327063409a5caa21541e6d08ae4a0f2da328447e9f2c7b39e179226/langcodes-3.5.0.tar.gz", hash = "sha256:1eef8168d07e51e131a2497ffecad4b663f6208e7c3ae3b8dc15c51734a6f801", size = 191030 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c3/6b/068c2ea7a712bf805c62445bd9e9c06d7340358ef2824150eceac027444b/langcodes-3.5.0-py3-none-any.whl", hash = "sha256:853c69d1a35e0e13da2f427bb68fb2fa4a8f4fb899e0c62ad8df8d073dcfed33", size = 182974 },
+]
+
+[[package]]
+name = "language-data"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "marisa-trie" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/dd/ce/3f144716a9f2cbf42aa86ebc8b085a184be25c80aa453eea17c294d239c1/language_data-1.3.0.tar.gz", hash = "sha256:7600ef8aa39555145d06c89f0c324bf7dab834ea0b0a439d8243762e3ebad7ec", size = 5129310 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5d/e9/5a5ffd9b286db82be70d677d0a91e4d58f7912bb8dd026ddeeb4abe70679/language_data-1.3.0-py3-none-any.whl", hash = "sha256:e2ee943551b5ae5f89cd0e801d1fc3835bb0ef5b7e9c3a4e8e17b2b214548fbf", size = 5385760 },
+]
+
 [[package]]
 name = "languagebench"
 version = "0.1.0"
@@ -914,6 +938,7 @@ dev = [
     { name = "bert-score" },
     { name = "evaluate" },
     { name = "joblib" },
+    { name = "langcodes" },
     { name = "openai" },
     { name = "protobuf" },
     { name = "python-dotenv" },
@@ -937,6 +962,7 @@ dev = [
     { name = "bert-score", specifier = ">=0.3.13" },
     { name = "evaluate", specifier = "==0.4.0" },
     { name = "joblib", specifier = ">=1.4.2" },
+    { name = "langcodes", specifier = ">=3.5.0" },
     { name = "openai", specifier = ">=1.52.2" },
     { name = "protobuf", specifier = ">=5.28.3" },
     { name = "python-dotenv", specifier = ">=1.0.1" },
@@ -1029,6 +1055,61 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ba/b2/6a22fb5c0885da3b00e116aee81f0b829ec9ac8f736cd414b4a09413fc7d/lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba", size = 3487557 },
 ]
 
+[[package]]
+name = "marisa-trie"
+version = "1.2.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "setuptools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/31/15/9d9743897e4450b2de199ee673b50cb018980c4ced477d41cf91304a85e3/marisa_trie-1.2.1.tar.gz", hash = "sha256:3a27c408e2aefc03e0f1d25b2ff2afb85aac3568f6fa2ae2a53b57a2e87ce29d", size = 416124 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e4/83/ccf5b33f2123f3110705c608f8e0caa82002626511aafafc58f82e50d322/marisa_trie-1.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a2eb41d2f9114d8b7bd66772c237111e00d2bae2260824560eaa0a1e291ce9e8", size = 362200 },
+    { url = "https://files.pythonhosted.org/packages/9d/74/f7ce1fc2ee480c7f8ceadd9b992caceaba442a97e5e99d6aea00d3635a0b/marisa_trie-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9e956e6a46f604b17d570901e66f5214fb6f658c21e5e7665deace236793cef6", size = 192309 },
+    { url = "https://files.pythonhosted.org/packages/e4/52/5dbbc13e57ce54c2ef0d04962d7d8f66edc69ed34310c734a2913199a581/marisa_trie-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd45142501300e7538b2e544905580918b67b1c82abed1275fe4c682c95635fa", size = 174713 },
+    { url = "https://files.pythonhosted.org/packages/57/49/2580372f3f980aea95c23d05b2c1d3bbb9ee1ab8cfd441545153e44f1be7/marisa_trie-1.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8443d116c612cfd1961fbf76769faf0561a46d8e317315dd13f9d9639ad500c", size = 1314808 },
+    { url = "https://files.pythonhosted.org/packages/5a/ba/e12a4d450f265414cc68df6a116a78beece72b95f774f04d29cd48e08d19/marisa_trie-1.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:875a6248e60fbb48d947b574ffa4170f34981f9e579bde960d0f9a49ea393ecc", size = 1346678 },
+    { url = "https://files.pythonhosted.org/packages/b2/81/8e130cb1eea741fd17694d821096f7ec9841f0e3d3c69b740257f5eeafa8/marisa_trie-1.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:746a7c60a17fccd3cfcfd4326926f02ea4fcdfc25d513411a0c4fc8e4a1ca51f", size = 1307254 },
+    { url = "https://files.pythonhosted.org/packages/d7/d0/3deb5ea2bf7e4d845339875dbb31f3c3f66c8d6568723db1d137fb08a91c/marisa_trie-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e70869737cc0e5bd903f620667da6c330d6737048d1f44db792a6af68a1d35be", size = 2194712 },
+    { url = "https://files.pythonhosted.org/packages/9c/5f/b38d728dd30954816497b53425cfaddaf7b93ac0912db5911888f191b07a/marisa_trie-1.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06b099dd743676dbcd8abd8465ceac8f6d97d8bfaabe2c83b965495523b4cef2", size = 2355625 },
+    { url = "https://files.pythonhosted.org/packages/7e/4f/61c0faa9ae9e53600a1b7a0c367bc9db1a4fdc625402ec232c755a05e094/marisa_trie-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d2a82eb21afdaf22b50d9b996472305c05ca67fc4ff5a026a220320c9c961db6", size = 2290290 },
+    { url = "https://files.pythonhosted.org/packages/7c/7d/713b970fb3043248881ed776dbf4d54918398aa5dde843a38711d0d62c8f/marisa_trie-1.2.1-cp310-cp310-win32.whl", hash = "sha256:8951e7ce5d3167fbd085703b4cbb3f47948ed66826bef9a2173c379508776cf5", size = 130743 },
+    { url = "https://files.pythonhosted.org/packages/cc/94/3d619cc82c30daeacd18a88674f4e6540ebfb7b4b7752ca0552793be80cf/marisa_trie-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:5685a14b3099b1422c4f59fa38b0bf4b5342ee6cc38ae57df9666a0b28eeaad3", size = 151891 },
+    { url = "https://files.pythonhosted.org/packages/4a/93/ffb01dfa22b6eee918e798e0bc3487427036c608aa4c065725f31aaf4104/marisa_trie-1.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ed3fb4ed7f2084597e862bcd56c56c5529e773729a426c083238682dba540e98", size = 362823 },
+    { url = "https://files.pythonhosted.org/packages/6d/1d/5c36500ac350c278c9bdfd88e17fa846fa4136d75597c167141ed973cdf2/marisa_trie-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0fe69fb9ffb2767746181f7b3b29bbd3454d1d24717b5958e030494f3d3cddf3", size = 192741 },
+    { url = "https://files.pythonhosted.org/packages/e8/04/87dd0840f3f720e511eba56193c02bf64d7d96df1ca9f6d19994f55154be/marisa_trie-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4728ed3ae372d1ea2cdbd5eaa27b8f20a10e415d1f9d153314831e67d963f281", size = 174995 },
+    { url = "https://files.pythonhosted.org/packages/c9/51/9e903a7e13b7593e2e675d0ec4c390ca076dc5df1c1a0d5e85a513b886a3/marisa_trie-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cf4f25cf895692b232f49aa5397af6aba78bb679fb917a05fce8d3cb1ee446d", size = 1384728 },
+    { url = "https://files.pythonhosted.org/packages/e8/3f/7362a5ac60c2b0aad0f52cd57e7bd0c708f20d2660d8df85360f3d8f1c4b/marisa_trie-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cca7f96236ffdbf49be4b2e42c132e3df05968ac424544034767650913524de", size = 1412620 },
+    { url = "https://files.pythonhosted.org/packages/1f/bc/aaa3eaf6875f78a204a8da9692d56e3a36f89997dad2c388628385614576/marisa_trie-1.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7eb20bf0e8b55a58d2a9b518aabc4c18278787bdba476c551dd1c1ed109e509", size = 1361555 },
+    { url = "https://files.pythonhosted.org/packages/18/98/e11b5a6206c5d110f32adab37fa84a85410d684e9c731acdd5c9250e2ce4/marisa_trie-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b1ec93f0d1ee6d7ab680a6d8ea1a08bf264636358e92692072170032dda652ba", size = 2257717 },
+    { url = "https://files.pythonhosted.org/packages/d2/9d/6b4a40867875e738a67c5b29f83e2e490a66bd9067ace3dd9a5c497e2b7f/marisa_trie-1.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e2699255d7ac610dee26d4ae7bda5951d05c7d9123a22e1f7c6a6f1964e0a4e4", size = 2417044 },
+    { url = "https://files.pythonhosted.org/packages/fe/61/e25613c72f2931757334b8bcf6b501569ef713f5ee9c6c7688ec460bd720/marisa_trie-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c484410911182457a8a1a0249d0c09c01e2071b78a0a8538cd5f7fa45589b13a", size = 2351960 },
+    { url = "https://files.pythonhosted.org/packages/19/0a/a90ccaf3eb476d13ec261f80c6c52defaf10ebc7f35eb2bcd7dfb533aef7/marisa_trie-1.2.1-cp311-cp311-win32.whl", hash = "sha256:ad548117744b2bcf0e3d97374608be0a92d18c2af13d98b728d37cd06248e571", size = 130446 },
+    { url = "https://files.pythonhosted.org/packages/fc/98/574b4e143e0a2f5f71af8716b6c4a8a46220f75a6e0847ce7d11ee0ba4aa/marisa_trie-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:436f62d27714970b9cdd3b3c41bdad046f260e62ebb0daa38125ef70536fc73b", size = 152037 },
+    { url = "https://files.pythonhosted.org/packages/4e/bf/8bd4ac8436b33fd46c9e1ffe3c2a131cd9744cc1649dbbe13308f744ef2b/marisa_trie-1.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:638506eacf20ca503fff72221a7e66a6eadbf28d6a4a6f949fcf5b1701bb05ec", size = 360041 },
+    { url = "https://files.pythonhosted.org/packages/ab/dd/4d3151e302e66ae387885f6ec265bd189e096b0c43c1379bfd9a3b9d2543/marisa_trie-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de1665eaafefa48a308e4753786519888021740501a15461c77bdfd57638e6b4", size = 190520 },
+    { url = "https://files.pythonhosted.org/packages/00/28/ae5991c74fb90b173167a366a634c83445f948ad044d37287b478d6b457e/marisa_trie-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f713af9b8aa66a34cd3a78c7d150a560a75734713abe818a69021fd269e927fa", size = 174175 },
+    { url = "https://files.pythonhosted.org/packages/5a/6a/fbfa89a8680eaabc6847a6c421e65427c43182db0c4bdb60e1516c81c822/marisa_trie-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2a7d00f53f4945320b551bccb826b3fb26948bde1a10d50bb9802fabb611b10", size = 1354995 },
+    { url = "https://files.pythonhosted.org/packages/9e/4c/2ba0b385e5f64ca4ddb0c10ec52ddf881bc4521f135948786fc339d1d6c8/marisa_trie-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98042040d1d6085792e8d0f74004fc0f5f9ca6091c298f593dd81a22a4643854", size = 1390989 },
+    { url = "https://files.pythonhosted.org/packages/6b/22/0791ed3045c91d0938345a86be472fc7c188b894f16c5dfad2ef31e7f882/marisa_trie-1.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6532615111eec2c79e711965ece0bc95adac1ff547a7fff5ffca525463116deb", size = 1328810 },
+    { url = "https://files.pythonhosted.org/packages/9d/7d/3f566e563abae6efce7fc311c63282a447c611739b3cd66c0e36077c86f8/marisa_trie-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20948e40ab2038e62b7000ca6b4a913bc16c91a2c2e6da501bd1f917eeb28d51", size = 2230222 },
+    { url = "https://files.pythonhosted.org/packages/a5/0b/38fbb4611b5d1030242ddc2aa62e524438c8076e26f87395dbbf222dc62d/marisa_trie-1.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66b23e5b35dd547f85bf98db7c749bc0ffc57916ade2534a6bbc32db9a4abc44", size = 2383620 },
+    { url = "https://files.pythonhosted.org/packages/ae/17/4553c63de29904d5d2521a24cad817bc7883cfa90506ab702ec4dae59a7b/marisa_trie-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6704adf0247d2dda42e876b793be40775dff46624309ad99bc7537098bee106d", size = 2329202 },
+    { url = "https://files.pythonhosted.org/packages/45/08/6307a630e63cd763fe77ac56516faa67fa9cd342060691e40fabc84be6b0/marisa_trie-1.2.1-cp312-cp312-win32.whl", hash = "sha256:3ad356442c2fea4c2a6f514738ddf213d23930f942299a2b2c05df464a00848a", size = 129652 },
+    { url = "https://files.pythonhosted.org/packages/a1/fe/67c357bfd92710d95a16b86e1453c663d565415d7f7838781c79ff7e1a7e/marisa_trie-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:f2806f75817392cedcacb24ac5d80b0350dde8d3861d67d045c1d9b109764114", size = 150845 },
+    { url = "https://files.pythonhosted.org/packages/2a/a4/a110cd9952f0e72da7bafea1f0084b18b9e03952110d9083bfda52279f5c/marisa_trie-1.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b5ea16e69bfda0ac028c921b58de1a4aaf83d43934892977368579cd3c0a2554", size = 354439 },
+    { url = "https://files.pythonhosted.org/packages/3c/a5/a6099eb1c3fd8d7e93408c45501e1d08536ac57dfef02ec331f78e1ace18/marisa_trie-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9f627f4e41be710b6cb6ed54b0128b229ac9d50e2054d9cde3af0fef277c23cf", size = 188187 },
+    { url = "https://files.pythonhosted.org/packages/7c/cc/f637127e2beffa920d21f7fc45b4029575bcd1b28a90c0d90cb2b08c2205/marisa_trie-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5e649f3dc8ab5476732094f2828cc90cac3be7c79bc0c8318b6fda0c1d248db4", size = 171484 },
+    { url = "https://files.pythonhosted.org/packages/6d/0f/29f2ad7260b956570f69f25a542efa51ba76eb76ecd53c63ee9d21987c3d/marisa_trie-1.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46e528ee71808c961baf8c3ce1c46a8337ec7a96cc55389d11baafe5b632f8e9", size = 1319770 },
+    { url = "https://files.pythonhosted.org/packages/f2/12/0b69ed61fba59551a5f3d569af367afae614db7214ce1da12946ba9a433a/marisa_trie-1.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36aa4401a1180615f74d575571a6550081d84fc6461e9aefc0bb7b2427af098e", size = 1356488 },
+    { url = "https://files.pythonhosted.org/packages/33/23/483b110db7ffe8729d6ebea2bf74258aef51f10fef5775f99e4bac7aef69/marisa_trie-1.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce59bcd2cda9bb52b0e90cc7f36413cd86c3d0ce7224143447424aafb9f4aa48", size = 1302334 },
+    { url = "https://files.pythonhosted.org/packages/1c/6f/46c2be99ce925985127fdf78900f1673bce8cb72debfebee6dccd11032c6/marisa_trie-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f4cd800704a5fc57e53c39c3a6b0c9b1519ebdbcb644ede3ee67a06eb542697d", size = 2202624 },
+    { url = "https://files.pythonhosted.org/packages/fd/b6/ef642327dbd4ec35be55d5682520b8f70fca98a54024f441ef2732f6b305/marisa_trie-1.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2428b495003c189695fb91ceeb499f9fcced3a2dce853e17fa475519433c67ff", size = 2364206 },
+    { url = "https://files.pythonhosted.org/packages/69/04/ef8197a79d0ab5043b781cc9b457bd11b81d4204fe78adf7625a67f48c21/marisa_trie-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:735c363d9aaac82eaf516a28f7c6b95084c2e176d8231c87328dc80e112a9afa", size = 2304801 },
+    { url = "https://files.pythonhosted.org/packages/03/72/f87564d653daf31d8f33d9bf0121e99ccc21f18f5c485fb404ba06abc10e/marisa_trie-1.2.1-cp313-cp313-win32.whl", hash = "sha256:eba6ca45500ca1a042466a0684aacc9838e7f20fe2605521ee19f2853062798f", size = 128799 },
+    { url = "https://files.pythonhosted.org/packages/27/40/5f9eb8b73030cc4b0d6817176e66079a62a2ddd9d5530da54f8011473428/marisa_trie-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:aa7cd17e1c690ce96c538b2f4aae003d9a498e65067dd433c52dd069009951d4", size = 149035 },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"