Update nat_datasets.py

nat_datasets.py

@@ -14,7 +14,7 @@ data = load_dataset("Ibrahemqasim/categories_en2ar", split="train")
 
 nationalities = load_dataset("Ibrahemqasim/nationalities", split="train")
 nationalities_pattern = r'\b(' + '|'.join(map(re.escape, [n["nat_en"].lower() for n in sorted(nationalities, key=lambda x: -x["nat_en"].count(' '))])) + r')\b'
-nationalities_pattern_ar = r'\b(' + '|'.join(map(re.escape, [n["men"].lower() for n in sorted(nationalities, key=lambda x: -x["men"].count(' '))])) + r')\b'
+nationalities_pattern_ar = r'(' + '|'.join(map(re.escape, [n["man"].lower() for n in sorted(nationalities, key=lambda x: -x["man"].count(' '))])) + r')'
 
 # print(nationalities_pattern)
 
@@ -39,23 +39,48 @@ data_lists = {
 
 YEAR_PATTERN = "{YEAR}"
 NAT = "{NAT}"
-AR_NAT_MEN = "{NAT_MEN}"
-AR_NAT_WOMENS = "{NAT_WOMENS}"
 EN_NAT_PATTERN = "{EN_NAT}"
 
 COUNTRY_PATTERN = "{COUNTRY}"
 
 # data = [{"en": "Category:1970s yemeni peoples", "ar": "تصنيف: يمنيون في عقد 1970"}]
 match1_done = 0
+
+
+def new_func(value, ar_tab):
+    # ---
+    ar_country = ar_tab.get("men", "")
+    # ---
+    if ar_country and ar_country in value:
+        return ar_country, "{NAT_MEN}"
+    # ---
+    ar_country2 = ar_tab.get("womens", "")
+    # ---
+    if ar_country2 and ar_country2 in value:
+        return ar_country2, "{NAT_WOMENS}"
+    # ---
+    ar_country3 = ar_tab.get("women", "")
+    # ---
+    if ar_country3 and ar_country2 in value:
+        return ar_country3, "{NAT_WOMEN}"
+    # ---
+    ar_country4 = ar_tab.get("man", "")
+    # ---
+    if ar_country4 and ar_country2 in value:
+        return ar_country4, "{NAT_MAN}"
+    # ---
+    return "", ""
+
+
 for tab in tqdm.tqdm(data):
     # ---
     key = tab["en"]
     value = tab["ar"]
     # ---
-    match1 = re.search(nationalities_pattern, key, re.IGNORECASE)
-    match2 = re.search(nationalities_pattern_ar, value, re.IGNORECASE)
+    match_en = re.search(nationalities_pattern, key, re.IGNORECASE)
+    match_ar = re.search(nationalities_pattern_ar, value, re.IGNORECASE)
     # ----
-    if match1 or match2:
+    if match_en or match_ar:
         # ---
         match1_done += 1
         # ---
@@ -64,25 +89,16 @@ for tab in tqdm.tqdm(data):
         else:
             data_lists["categories_with_nationalities"][key] = {"ar": value, "count": 1}
     # ---
-    if not match1:
+    if not match_en:
         continue
     # ---
-    en_country = match1.group(1)
+    en_country = match_en.group(1)
     ar_tab = nationalities_dict.get(en_country.lower(), {})
     # ---
     if not ar_tab:
         continue
     # ---
-    ar_country = ar_tab.get("men", "")
-    # ---
-    NAT_PATTERN = ""
-    # ---
-    if ar_country and ar_country in value:
-        NAT_PATTERN = AR_NAT_MEN
-    else:
-        ar_country = ar_tab.get("womens", "")
-        if ar_country and ar_country in value:
-            NAT_PATTERN = AR_NAT_WOMENS
+    ar_country, NAT_PATTERN = new_func(value, ar_tab)
     # ---
     if not NAT_PATTERN:
         continue