nat_datasets.py

import tqdm
import re
import requests
from huggingface_hub import login
from datasets import Dataset
from datasets import load_dataset
from google.colab import userdata

login(userdata.get('HF_TOKEN'))

data = load_dataset("Ibrahemqasim/categories_en2ar", split="train")

# nationalities.keys() "nat_en","man","men","women","womens","country_en","country_ar",

nationalities = load_dataset("Ibrahemqasim/nationalities", split="train")
nationalities_pattern = r'\b(' + '|'.join(map(re.escape, [n["nat_en"].lower() for n in sorted(nationalities, key=lambda x: -x["nat_en"].count(' '))])) + r')\b'
nationalities_pattern_ar = r'(' + '|'.join(map(re.escape, [n["man"].lower() for n in sorted(nationalities, key=lambda x: -x["man"].count(' '))])) + r')'

# print(nationalities_pattern)

countries = load_dataset("Ibrahemqasim/countries", split="train")
countries_pattern = r'\b(' + '|'.join(map(re.escape, [n["en"] for n in sorted(countries, key=lambda x: -x["en"].count(' '))])) + r')\b'

# ---
countries_dict = {cc["en"]: cc for cc in countries}
nationalities_dict = {cc["nat_en"].lower(): cc for cc in nationalities}
# ---
to_work = [
    "categories_with_nationalities",
    "categories_with_NAT_pattern",
    "categories_with_YEAR_NAT_pattern",
]

data_lists = {
    "categories_with_nationalities" : {},
    "categories_with_NAT_pattern" : {},
    "categories_with_YEAR_NAT_pattern" : {},
}

YEAR_PATTERN = "{YEAR}"
NAT = "{NAT}"
EN_NAT_PATTERN = "{EN_NAT}"

COUNTRY_PATTERN = "{COUNTRY}"

# data = [{"en": "Category:1970s yemeni peoples", "ar": "تصنيف: يمنيون في عقد 1970"}]
match1_done = 0


def new_func(value, ar_tab):
    # List of possible keys and their corresponding tags
    patterns = [
        ("men", "{NAT_MEN}"),
        ("womens", "{NAT_WOMENS}"),
        ("women", "{NAT_WOMEN}"),
        ("man", "{NAT_MAN}"),
    ]

    # Iterate through the patterns
    for key, tag in patterns:
        country = ar_tab.get(key, "")
        if not country:
            continue
        # ---
        country2 = f"ال{country}".replace(" ", " ال")
        # ---
        if country2 in value:
            return country2, tag.replace("}", "_AL}")
        elif country in value:
            return country, tag

    return "", ""


for tab in tqdm.tqdm(data):
    # ---
    key = tab["en"]
    value = tab["ar"]
    # ---
    match_en = re.search(nationalities_pattern, key, re.IGNORECASE)
    match_ar = re.search(nationalities_pattern_ar, value, re.IGNORECASE)
    # ----
    if match_en or match_ar:
        # ---
        match1_done += 1
        # ---
        if key in data_lists["categories_with_nationalities"]:
            data_lists["categories_with_nationalities"][key]["count"] += 1
        else:
            data_lists["categories_with_nationalities"][key] = {"ar": value, "count": 1}
    # ---
    if not match_en:
        continue
    # ---
    en_country = match_en.group(1)
    ar_tab = nationalities_dict.get(en_country.lower(), {})
    # ---
    if not ar_tab:
        continue
    # ---
    ar_country, NAT_PATTERN = new_func(value, ar_tab)
    # ---
    if not NAT_PATTERN:
        continue
    # ---
    key1 = re.sub(rf'\b{re.escape(en_country)}\b', EN_NAT_PATTERN, f" {key} ", re.IGNORECASE)
    key1 = key1.strip()
    # ---
    if EN_NAT_PATTERN in key1 and key1 in data_lists["categories_with_NAT_pattern"]:
        data_lists["categories_with_NAT_pattern"][key1]["count"] += 1
    # ---
    value1 = re.sub(rf'\b{re.escape(ar_country)}\b', NAT_PATTERN, f" {value} ", re.IGNORECASE)
    value1 = value1.strip()
    # ---
    if EN_NAT_PATTERN in key1 and NAT_PATTERN in value1:
        # ---
        if key1 not in data_lists["categories_with_NAT_pattern"]:
            data_lists["categories_with_NAT_pattern"][key1] = {"ar": value1, "count": 1}
        # ---
        # continue
        # ---
        # Add if key and value has 4 digits and they are the same
        reg_year = r"(\d+[–-]\d+|\d{4})"
        # ---
        key_digits = re.search(reg_year, key, re.IGNORECASE)
        value_digits = re.search(reg_year, value1, re.IGNORECASE)
        # ----
        if not key_digits:
            continue
        # ----
        key2 = key1.replace(key_digits.group(), YEAR_PATTERN)
        # ---
        if key2 in data_lists["categories_with_YEAR_NAT_pattern"]:
            data_lists["categories_with_YEAR_NAT_pattern"][key2]["count"] += 1
        # ---
        if not value_digits:
            continue
        # ----
        value2 = value1.replace(value_digits.group(), YEAR_PATTERN)
        # ----
        if key_digits.group() == value_digits.group():
            # ---
            if key2 not in data_lists["categories_with_YEAR_NAT_pattern"]:
                data_lists["categories_with_YEAR_NAT_pattern"][key2] = {"ar": value2, "count": 1}
            # ----
            continue
            # ----
            # البحث عن اسم الدولة في key2
            match = re.search(countries_pattern, key2, re.IGNORECASE)
            # ----
            if match:
                en_country = match.group(1)
                ar_country = countries.get(en_country)
                # ---
                if ar_country and ar_country in value2:
                    key3 = re.sub(rf'\b{re.escape(en_country)}\b', COUNTRY_PATTERN, key2, re.IGNORECASE)
                    value3 = re.sub(rf'\b{re.escape(ar_country)}\b', COUNTRY_PATTERN, value2, re.IGNORECASE)
                    # ---
                    if COUNTRY_PATTERN in key3 and COUNTRY_PATTERN in value3:
                        # ---
                        if key3 in data_lists["categories_with_YEAR_COUNTRY_pattern"]:
                            data_lists["categories_with_YEAR_COUNTRY_pattern"][key3]["count"] += 1
                        else:
                            data_lists["categories_with_YEAR_COUNTRY_pattern"][key3] = {"ar": value3, "count": 1}
# ----
print(f"{match1_done=}")

# for x, data_list in data_lists.items():
for x in to_work:
    data_list = data_lists.get(x)
    # ---
    if x == "countries":
        data_list = [{"en": key, "ar": value} for key, value in data_list.items()]
    else:
        data_list = [{"en": key, "ar": value["ar"], "count": value["count"]} for key, value in data_list.items()]
        # sort data_list by count
        data_list = sorted(data_list, key=lambda x: x["count"], reverse=True)
    # ---
    print("______________")
    print(f"len of {x} : {len(data_list)}.")
    # ---
    print("____________________________")
    # ---
    if len(data_list) == 0:
        continue
    # ---
    # إنشاء Dataset
    dataset = Dataset.from_list(data_list)

    # رفع Dataset إلى Hugging Face
    dataset.push_to_hub(f"Ibrahemqasim/{x}")
    # ---
    print(f"dataset: Ibrahemqasim/{x} push_to_hub successfully!")