Update nat_datasets.py

nat_datasets.py

@@ -13,82 +13,113 @@ data = load_dataset("Ibrahemqasim/categories_en2ar", split="train")
 # nationalities.keys() "nat_en","man","men","women","womens","country_en","country_ar",
 
 nationalities = load_dataset("Ibrahemqasim/nationalities", split="train")
-nationalities_pattern = r'\b(' + '|'.join(map(re.escape, [n["nat_en"] for n in sorted(nationalities, key=lambda x: -x["nat_en"].count(' '))])) + r')\b'
+nationalities_pattern = r'\b(' + '|'.join(map(re.escape, [n["nat_en"].lower() for n in sorted(nationalities, key=lambda x: -x["nat_en"].count(' '))])) + r')\b'
+nationalities_pattern_ar = r'\b(' + '|'.join(map(re.escape, [n["men"].lower() for n in sorted(nationalities, key=lambda x: -x["men"].count(' '))])) + r')\b'
+
+# print(nationalities_pattern)
 
 countries = load_dataset("Ibrahemqasim/countries", split="train")
 countries_pattern = r'\b(' + '|'.join(map(re.escape, [n["en"] for n in sorted(countries, key=lambda x: -x["en"].count(' '))])) + r')\b'
 
 # ---
 countries_dict = {cc["en"]: cc for cc in countries}
-nationalities_dict = {cc["nat_en"]: cc for cc in nationalities}
+nationalities_dict = {cc["nat_en"].lower(): cc for cc in nationalities}
 # ---
 to_work = [
     "categories_with_nationalities",
-    # "categories_with_years",
-    # "categories_with_YEAR_COUNTRY_pattern",
-    # "categories_with_YEAR_pattern",
+    "categories_with_NAT_pattern",
+    "categories_with_YEAR_NAT_pattern",
 ]
 
 data_lists = {
     "categories_with_nationalities" : {},
-    "categories_with_years" : {},
-    "categories_with_YEAR_COUNTRY_pattern" : {},
-    "categories_with_YEAR_pattern" : {},
+    "categories_with_NAT_pattern" : {},
+    "categories_with_YEAR_NAT_pattern" : {},
 }
 
 YEAR_PATTERN = "{YEAR}"
 NAT = "{NAT}"
 AR_NAT_MEN = "{NAT_MEN}"
+AR_NAT_WOMENS = "{NAT_WOMENS}"
 EN_NAT_PATTERN = "{EN_NAT}"
 
 COUNTRY_PATTERN = "{COUNTRY}"
 
+# data = [{"en": "Category:1970s yemeni peoples", "ar": "تصنيف: يمنيون في عقد 1970"}]
+match1_done = 0
 for tab in tqdm.tqdm(data):
     # ---
     key = tab["en"]
     value = tab["ar"]
     # ---
-    # Add if key and value has 4 digits and they are the same
-    reg_year = r"(\d+[–-]\d+|\d{4})"
-    # ---
-    key_digits = re.search(reg_year, key, re.IGNORECASE)
-    value_digits = re.search(reg_year, value, re.IGNORECASE)
-    # ----
     match1 = re.search(nationalities_pattern, key, re.IGNORECASE)
+    match2 = re.search(nationalities_pattern_ar, value, re.IGNORECASE)
     # ----
-    if match1:
-        en_country = match1.group(1)
-        ar_country = nationalities_dict.get(en_country, {}).get("men", "")
+    if match1 or match2:
+        # ---
+        match1_done += 1
         # ---
+        if key in data_lists["categories_with_nationalities"]:
+            data_lists["categories_with_nationalities"][key]["count"] += 1
+        else:
+            data_lists["categories_with_nationalities"][key] = {"ar": value, "count": 1}
+    # ---
+    if not match1:
+        continue
+    # ---
+    en_country = match1.group(1)
+    ar_tab = nationalities_dict.get(en_country.lower(), {})
+    # ---
+    if not ar_tab:
+        continue
+    # ---
+    ar_country = ar_tab.get("men", "")
+    # ---
+    NAT_PATTERN = ""
+    # ---
+    if ar_country and ar_country in value:
+        NAT_PATTERN = AR_NAT_MEN
+    else:
+        ar_country = ar_tab.get("womens", "")
         if ar_country and ar_country in value:
-            key1 = re.sub(rf'\b{re.escape(en_country)}\b', EN_NAT_PATTERN, key)
-            value1 = re.sub(rf'\b{re.escape(ar_country)}\b', AR_NAT_MEN, value)
-            # ---
-            if EN_NAT_PATTERN in key1 and AR_NAT_MEN in value1:
-                # ---
-                if key1 in data_lists["categories_with_nationalities"]:
-                    data_lists["categories_with_nationalities"][key1]["count"] += 1
-                else:
-                    data_lists["categories_with_nationalities"][key1] = {"ar": value1, "count": 1}
+            NAT_PATTERN = AR_NAT_WOMENS
+    # ---
+    if not NAT_PATTERN:
+        continue
+    # ---
+    key1 = re.sub(rf'\b{re.escape(en_country)}\b', EN_NAT_PATTERN, key, re.IGNORECASE)
+    value1 = re.sub(rf'\b{re.escape(ar_country)}\b', NAT_PATTERN, value, re.IGNORECASE)
+    # ---
+    # if EN_NAT_PATTERN in key1 and NAT_PATTERN in value1:
     # ---
-    continue
+    if key1 in data_lists["categories_with_NAT_pattern"]:
+        data_lists["categories_with_NAT_pattern"][key1]["count"] += 1
+    else:
+        data_lists["categories_with_NAT_pattern"][key1] = {"ar": value1, "count": 1}
+    # ---
+    # continue
+    # ---
+    # Add if key and value has 4 digits and they are the same
+    reg_year = r"(\d+[–-]\d+|\d{4})"
     # ---
+    key_digits = re.search(reg_year, key, re.IGNORECASE)
+    value_digits = re.search(reg_year, value1, re.IGNORECASE)
+    # ----
     if key_digits and value_digits and key_digits.group() == value_digits.group():
-        # data_lists["categories_with_years"].append({"en": key, "ar": value, "count": 1})
-        if key in data_lists["categories_with_years"]:
-            data_lists["categories_with_years"][key]["count"] += 1
-        else:
-            data_lists["categories_with_years"][key] = {"ar": value, "count": 1}
-        # ---
-        key2 = key.replace(key_digits.group(), YEAR_PATTERN)
-        value2 = value.replace(value_digits.group(), YEAR_PATTERN)
+        # if key1 in data_lists["categories_with_years"]:
+        #     data_lists["categories_with_years"][key1]["count"] += 1
+        # else:
+        #     data_lists["categories_with_years"][key1] = {"ar": value1, "count": 1}
         # ---
-        # data_lists["categories_with_YEAR_pattern"].append({"en": key2, "ar": value2, "count": 1})
+        key2 = key1.replace(key_digits.group(), YEAR_PATTERN)
+        value2 = value1.replace(value_digits.group(), YEAR_PATTERN)
         # ---
-        if key2 in data_lists["categories_with_YEAR_pattern"]:
-            data_lists["categories_with_YEAR_pattern"][key2]["count"] += 1
+        if key2 in data_lists["categories_with_YEAR_NAT_pattern"]:
+            data_lists["categories_with_YEAR_NAT_pattern"][key2]["count"] += 1
         else:
-            data_lists["categories_with_YEAR_pattern"][key2] = {"ar": value2, "count": 1}
+            data_lists["categories_with_YEAR_NAT_pattern"][key2] = {"ar": value2, "count": 1}
+        # ----
+        continue
         # ----
         # البحث عن اسم الدولة في key2
         match = re.search(countries_pattern, key2, re.IGNORECASE)
@@ -98,8 +129,8 @@ for tab in tqdm.tqdm(data):
             ar_country = countries.get(en_country)
             # ---
             if ar_country and ar_country in value2:
-                key3 = re.sub(rf'\b{re.escape(en_country)}\b', COUNTRY_PATTERN, key2)
-                value3 = re.sub(rf'\b{re.escape(ar_country)}\b', COUNTRY_PATTERN, value2)
+                key3 = re.sub(rf'\b{re.escape(en_country)}\b', COUNTRY_PATTERN, key2, re.IGNORECASE)
+                value3 = re.sub(rf'\b{re.escape(ar_country)}\b', COUNTRY_PATTERN, value2, re.IGNORECASE)
                 # ---
                 if COUNTRY_PATTERN in key3 and COUNTRY_PATTERN in value3:
                     # ---
@@ -107,12 +138,8 @@ for tab in tqdm.tqdm(data):
                         data_lists["categories_with_YEAR_COUNTRY_pattern"][key3]["count"] += 1
                     else:
                         data_lists["categories_with_YEAR_COUNTRY_pattern"][key3] = {"ar": value3, "count": 1}
-        # ----
 # ----
-print(f"{len(data_lists['categories_with_YEAR_COUNTRY_pattern'])=}")
-print(f"{len(data_lists['categories_with_YEAR_pattern'])=}")
-
-print(f"all data len: {len(data):,}.")
+print(f"{match1_done=}")
 
 # for x, data_list in data_lists.items():
 for x in to_work: