Update nat_datasets.py

nat_datasets.py

@@ -11,6 +11,7 @@ login(userdata.get('HF_TOKEN'))
 data = load_dataset("Ibrahemqasim/categories_en2ar", split="train")
 
 # nationalities.keys() "nat_en","man","men","women","womens","country_en","country_ar",
+
 nationalities = load_dataset("Ibrahemqasim/nationalities", split="train")
 nationalities_pattern = r'\b(' + '|'.join(map(re.escape, [n["nat_en"] for n in sorted(nationalities, key=lambda x: -x["nat_en"].count(' '))])) + r')\b'
 
@@ -38,6 +39,8 @@ data_lists = {
 YEAR_PATTERN = "{YEAR}"
 NAT = "{NAT}"
 AR_NAT_MEN = "{NAT_MEN}"
+EN_NAT_PATTERN = "{EN_NAT}"
+
 COUNTRY_PATTERN = "{COUNTRY}"
 
 for tab in tqdm.tqdm(data):
@@ -48,20 +51,20 @@ for tab in tqdm.tqdm(data):
     # Add if key and value has 4 digits and they are the same
     reg_year = r"(\d+[–-]\d+|\d{4})"
     # ---
-    key_digits = re.search(reg_year, key)
-    value_digits = re.search(reg_year, value)
+    key_digits = re.search(reg_year, key, re.IGNORECASE)
+    value_digits = re.search(reg_year, value, re.IGNORECASE)
     # ----
-    match1 = re.search(nationalities_pattern, key)
+    match1 = re.search(nationalities_pattern, key, re.IGNORECASE)
     # ----
     if match1:
         en_country = match1.group(1)
         ar_country = nationalities_dict.get(en_country, {}).get("men", "")
         # ---
         if ar_country and ar_country in value:
-            key1 = re.sub(rf'\b{re.escape(en_country)}\b', COUNTRY_PATTERN, key)
+            key1 = re.sub(rf'\b{re.escape(en_country)}\b', EN_NAT_PATTERN, key)
             value1 = re.sub(rf'\b{re.escape(ar_country)}\b', AR_NAT_MEN, value)
             # ---
-            if COUNTRY_PATTERN in key1 and AR_NAT_MEN in value1:
+            if EN_NAT_PATTERN in key1 and AR_NAT_MEN in value1:
                 # ---
                 if key1 in data_lists["categories_with_nationalities"]:
                     data_lists["categories_with_nationalities"][key1]["count"] += 1
@@ -88,7 +91,7 @@ for tab in tqdm.tqdm(data):
             data_lists["categories_with_YEAR_pattern"][key2] = {"ar": value2, "count": 1}
         # ----
         # البحث عن اسم الدولة في key2
-        match = re.search(countries_pattern, key2)
+        match = re.search(countries_pattern, key2, re.IGNORECASE)
         # ----
         if match:
             en_country = match.group(1)
@@ -127,6 +130,9 @@ for x in to_work:
     # ---
     print("____________________________")
     # ---
+    if len(data_list) == 0:
+        continue
+    # ---
     # إنشاء Dataset
     dataset = Dataset.from_list(data_list)