import tqdm import re import requests from huggingface_hub import login from datasets import Dataset from datasets import load_dataset from google.colab import userdata login(userdata.get('HF_TOKEN')) data = load_dataset("Ibrahemqasim/categories_en2ar", split="train") # nationalities.keys() "nat_en","man","men","women","womens","country_en","country_ar", nationalities = load_dataset("Ibrahemqasim/nationalities", split="train") nationalities_pattern = r'\b(' + '|'.join(map(re.escape, [n["nat_en"].lower() for n in sorted(nationalities, key=lambda x: -x["nat_en"].count(' '))])) + r')\b' nationalities_pattern_ar = r'(' + '|'.join(map(re.escape, [n["man"].lower() for n in sorted(nationalities, key=lambda x: -x["man"].count(' '))])) + r')' # print(nationalities_pattern) countries = load_dataset("Ibrahemqasim/countries", split="train") countries_pattern = r'\b(' + '|'.join(map(re.escape, [n["en"] for n in sorted(countries, key=lambda x: -x["en"].count(' '))])) + r')\b' # --- countries_dict = {cc["en"]: cc for cc in countries} nationalities_dict = {cc["nat_en"].lower(): cc for cc in nationalities} # --- to_work = [ "categories_with_nationalities", "categories_with_NAT_pattern", "categories_with_YEAR_NAT_pattern", ] data_lists = { "categories_with_nationalities" : {}, "categories_with_NAT_pattern" : {}, "categories_with_YEAR_NAT_pattern" : {}, } YEAR_PATTERN = "{YEAR}" NAT = "{NAT}" EN_NAT_PATTERN = "{EN_NAT}" COUNTRY_PATTERN = "{COUNTRY}" # data = [{"en": "Category:1970s yemeni peoples", "ar": "تصنيف: يمنيون في عقد 1970"}] match1_done = 0 def new_func(value, ar_tab): # List of possible keys and their corresponding tags patterns = [ ("men", "{NAT_MEN}"), ("womens", "{NAT_WOMENS}"), ("women", "{NAT_WOMEN}"), ("man", "{NAT_MAN}"), ] # Iterate through the patterns for key, tag in patterns: country = ar_tab.get(key, "") if not country: continue # --- country2 = f"ال{country}".replace(" ", " ال") # --- if country2 in value: return country2, tag.replace("}", "_AL}") elif country in value: return country, tag return "", "" for tab in tqdm.tqdm(data): # --- key = tab["en"] value = tab["ar"] # --- match_en = re.search(nationalities_pattern, key, re.IGNORECASE) match_ar = re.search(nationalities_pattern_ar, value, re.IGNORECASE) # ---- if match_en or match_ar: # --- match1_done += 1 # --- if key in data_lists["categories_with_nationalities"]: data_lists["categories_with_nationalities"][key]["count"] += 1 else: data_lists["categories_with_nationalities"][key] = {"ar": value, "count": 1} # --- if not match_en: continue # --- en_country = match_en.group(1) ar_tab = nationalities_dict.get(en_country.lower(), {}) # --- if not ar_tab: continue # --- ar_country, NAT_PATTERN = new_func(value, ar_tab) # --- if not NAT_PATTERN: continue # --- key1 = re.sub(rf'\b{re.escape(en_country)}\b', EN_NAT_PATTERN, f" {key} ", re.IGNORECASE) key1 = key1.strip() # --- if EN_NAT_PATTERN in key1 and key1 in data_lists["categories_with_NAT_pattern"]: data_lists["categories_with_NAT_pattern"][key1]["count"] += 1 # --- value1 = re.sub(rf'\b{re.escape(ar_country)}\b', NAT_PATTERN, f" {value} ", re.IGNORECASE) value1 = value1.strip() # --- if EN_NAT_PATTERN in key1 and NAT_PATTERN in value1: # --- if key1 not in data_lists["categories_with_NAT_pattern"]: data_lists["categories_with_NAT_pattern"][key1] = {"ar": value1, "count": 1} # --- # continue # --- # Add if key and value has 4 digits and they are the same reg_year = r"(\d+[–-]\d+|\d{4})" # --- key_digits = re.search(reg_year, key, re.IGNORECASE) value_digits = re.search(reg_year, value1, re.IGNORECASE) # ---- if not key_digits: continue # ---- key2 = key1.replace(key_digits.group(), YEAR_PATTERN) # --- if key2 in data_lists["categories_with_YEAR_NAT_pattern"]: data_lists["categories_with_YEAR_NAT_pattern"][key2]["count"] += 1 # --- if not value_digits: continue # ---- value2 = value1.replace(value_digits.group(), YEAR_PATTERN) # ---- if key_digits.group() == value_digits.group(): # --- if key2 not in data_lists["categories_with_YEAR_NAT_pattern"]: data_lists["categories_with_YEAR_NAT_pattern"][key2] = {"ar": value2, "count": 1} # ---- continue # ---- # البحث عن اسم الدولة في key2 match = re.search(countries_pattern, key2, re.IGNORECASE) # ---- if match: en_country = match.group(1) ar_country = countries.get(en_country) # --- if ar_country and ar_country in value2: key3 = re.sub(rf'\b{re.escape(en_country)}\b', COUNTRY_PATTERN, key2, re.IGNORECASE) value3 = re.sub(rf'\b{re.escape(ar_country)}\b', COUNTRY_PATTERN, value2, re.IGNORECASE) # --- if COUNTRY_PATTERN in key3 and COUNTRY_PATTERN in value3: # --- if key3 in data_lists["categories_with_YEAR_COUNTRY_pattern"]: data_lists["categories_with_YEAR_COUNTRY_pattern"][key3]["count"] += 1 else: data_lists["categories_with_YEAR_COUNTRY_pattern"][key3] = {"ar": value3, "count": 1} # ---- print(f"{match1_done=}") # for x, data_list in data_lists.items(): for x in to_work: data_list = data_lists.get(x) # --- if x == "countries": data_list = [{"en": key, "ar": value} for key, value in data_list.items()] else: data_list = [{"en": key, "ar": value["ar"], "count": value["count"]} for key, value in data_list.items()] # sort data_list by count data_list = sorted(data_list, key=lambda x: x["count"], reverse=True) # --- print("______________") print(f"len of {x} : {len(data_list)}.") # --- print("____________________________") # --- if len(data_list) == 0: continue # --- # إنشاء Dataset dataset = Dataset.from_list(data_list) # رفع Dataset إلى Hugging Face dataset.push_to_hub(f"Ibrahemqasim/{x}") # --- print(f"dataset: Ibrahemqasim/{x} push_to_hub successfully!")