latest

README.md

@@ -0,0 +1,10 @@
+---
+title: Phonemize in Hebrew
+emoji: 🐢
+colorFrom: red
+colorTo: green
+sdk: gradio
+sdk_version: "4.44.0"
+app_file: app.py
+pinned: false
+---

app.py

@@ -0,0 +1,45 @@
+"""
+uv sync
+uv pip install "gradio>=5.15.0"
+uv run gradio examples/editor.py
+"""
+
+from mishkal import phonemize, normalize
+import gradio as gr
+
+default_text = """
+כָּל עֶ֫רֶב יָאִ֫יר (הַשֵּׁ֫ם הַמָּלֵ֫א וּמֽק֫וֹם הָעֲבוֹדָ֫ה שֶׁלּ֫וֹ שְׁמוּרִ֫ים בַּמַּעֲרֶ֫כֶת) רָץ 20 קִילוֹמֶ֫טֶר. הוּא מֽסַפֵּ֫ר לִי שֶׁזֶּ֫ה מֽנַקֶּ֫ה לוֹ אֶת הָרֹ֫אשׁ אַחֲרֵ֫י הָעֲבוֹדָ֫ה, "שָׁעָ֫ה וָחֵ֫צִי בְּלִ֫י עֲבוֹדָ֫ה, אִשָּׁ֫ה וִילָדִ֫ים" כְּמ֫וֹ שֶׁה֫וּא מַגְדִּ֫יר זֹאת. אֲבָ֫ל אַחֲרֵ֫י הַמִּקְלַ֫חַת הוּא מַתְחִ֫יל בּֽמָ֫ה שֶׁנִּתָּ֫ן לֽכַנּ֫וֹת הָעֲבוֹדָ֫ה הַשְּׁנִיָּ֫ה שֶׁלּ֫וֹ: לִמְצֹ֫א לוֹ קוֹלֵ֫גוֹת חֲדָשׁ֫וֹת לָעֲבוֹדָ֫ה, כִּי יָאִ֫יר הוּא כַּנִּרְאֶ֫ה הַמֶּ֫לֶךְ שֶׁל "חָבֵ֫ר מֵבִ֫יא חָבֵ֫ר" בּֽיִשְׂרָאֵ֫ל.
+דֻּגְמָא מַגְנִיבָה: [אנציקלופדיה](/ʔantsikloˈpedja/)
+"""
+
+theme = gr.themes.Soft(font=[gr.themes.GoogleFont("Roboto")])
+
+
+def on_submit_debug(text: str, predict_stress) -> str:
+    phonemes = phonemize(text, preserve_punctuation=True, predict_stress=predict_stress)
+    normalized_text = normalize(text)
+    return phonemes + "\n\nNormalized:\n" + normalized_text
+
+
+def on_submit(text: str, predict_stress) -> str:
+    return phonemize(text, preserve_punctuation=False, predict_stress=predict_stress)
+
+
+with gr.Blocks(theme=theme) as demo:
+    text_input = gr.Textbox(
+        value=default_text, label="Text", rtl=True, elem_classes=["input"]
+    )
+    debug_checkbox = gr.Checkbox(value=False, label="Enable Debug Mode")
+    predict_stress_checkbox = gr.Checkbox(value=False, label="Predict Stress")
+    phonemes_output = gr.Textbox(label="Phonemes")
+    submit_button = gr.Button("Create")
+
+    submit_button.click(
+        fn=lambda text, debug, stress: on_submit_debug(text, stress) if debug else on_submit(text, stress),
+        inputs=[text_input, debug_checkbox, predict_stress_checkbox],
+        outputs=[phonemes_output],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

mishkal/__init__.py

@@ -0,0 +1,32 @@
+"""
+High level phonemize functions
+"""
+
+from .phonemize import Phonemizer
+from .utils import normalize  # noqa: F401
+from typing import Callable
+
+phonemizer = Phonemizer()
+
+
+def phonemize(
+    text: str,
+    preserve_punctuation=True,
+    preserve_stress=True,
+    use_expander=True,
+    use_post_normalize=True,  # For TTS
+    predict_stress=True,
+    predict_shva_nah=True,
+    fallback: Callable[[str], str] = None,
+) -> str:
+    phonemes = phonemizer.phonemize(
+        text,
+        preserve_punctuation=preserve_punctuation,
+        preserve_stress=preserve_stress,
+        fallback=fallback,
+        use_expander=use_expander,
+        use_post_normalize=use_post_normalize,
+        predict_stress=predict_stress,
+        predict_shva_nah=predict_shva_nah,
+    )
+    return phonemes

mishkal/data/kamatz_katan.json

@@ -0,0 +1,5 @@
+{
+    "כל": "ˈkol",
+    "רחבי": "roxˈbi",
+    "אמנות": "omaˈnut"
+}

mishkal/data/rashej_tevot.json

@@ -0,0 +1,3 @@
+{
+    "צה״ל": "ˈtsahal"
+}

mishkal/data/special.json

@@ -0,0 +1,4 @@
+{
+    "וַאלְלָה": "wala",
+    "וַסַבִּי": "wasabi"
+}

mishkal/data/symbols.json

@@ -0,0 +1,4 @@
+{
+    "₪": "ʃeˈkel",
+    "$": "doˈlar"
+}

mishkal/expander/__init__.py

@@ -0,0 +1,32 @@
+"""
+Expand dates and numbers into words with niqqud
+This happens before phonemization
+"""
+
+from .numbers import num_to_word
+from .dates import date_to_word
+from .time_to_word import time_to_word
+from .dictionary import Dictionary
+from mishkal.log import log
+
+
+class Expander:
+    def __init__(self):
+        self.dictionary = Dictionary()
+
+    def expand_text(self, text: str):
+        text = self.dictionary.expand_text(text)
+
+        words = []
+        for source_word in text.split():
+            try:
+                word = date_to_word(source_word)
+                if word == source_word:
+                    word = time_to_word(word)
+                if word == source_word:
+                    word = num_to_word(word)
+                words.append(word)
+            except Exception as e:
+                log.error(f"Failed to expand {word} with error: {e}")
+                words.append(source_word)
+        return " ".join(words)

mishkal/expander/dates.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from .numbers import num_to_word
+
+# Mapping of month names in Hebrew with diacritics (Gregorian months)
+MONTHS = {
+    1: "יָנוּאָר",
+    2: "פֶבְרוּאָר",
+    3: "מֵרְץ",
+    4: "אֵפְרִיל",
+    5: "מַאי",
+    6: "יוּנִי",
+    7: "יוּלִי",
+    8: "אוֹגֻסְט",
+    9: "סֶפְּטֶמְבֶּר",
+    10: "אוֹקְטוֹבֶּר",
+    11: "נוֹבֶמְבֶּר",
+    12: "דֶּצֶמְבֶּר",
+}
+
+# Mapping of day names in Hebrew with diacritics
+DAYS = {
+    0: "יוֹם רִאשׁוֹן",
+    1: "יוֹם שֵׁנִי",
+    2: "יוֹם שְׁלִישִׁי",
+    3: "יוֹם רֵבִיעִי",
+    4: "יוֹם חֲמִישִׁי",
+    5: "יוֹם שִׁישִׁי",
+    6: "יוֹם שַׁבָּת",
+}
+
+
+def date_to_word(word: str, include_day_name=False) -> str:
+    """
+    Converts a given date string in formats (YYYY-MM-DD, YYYY.MM.DD, YYYY/MM/DD) to Hebrew date format with diacritics.
+    Returns the original word if it's not a valid date.
+    """
+    separators = ["-", ".", "/"]
+    orders = [("%Y", "%m", "%d"), ("%d", "%m", "%Y")]
+    date_formats = [sep.join(order) for order in orders for sep in separators]
+
+    for date_format in date_formats:
+        try:
+            # Try parsing the word with each date format
+            date_obj = datetime.strptime(word, date_format)
+
+            # Get the Hebrew day name with diacritics
+            day_name = DAYS[date_obj.weekday()]
+
+            # Convert month to Hebrew name with diacritics
+            month_name = MONTHS[date_obj.month]
+            day = num_to_word(str(date_obj.day))
+            year = num_to_word(str(date_obj.year))
+
+            text = f"{day} בֵּ{month_name} {year}"
+            if include_day_name:
+                text = f"{day_name}, {text}"
+            return text
+        except ValueError:
+            continue
+    return word

mishkal/expander/dictionary.py

@@ -0,0 +1,78 @@
+"""
+Dictionaries are tab separated key value words
+"""
+
+from pathlib import Path
+import json
+import re
+from mishkal.utils import remove_niqqud
+from mishkal.utils import normalize
+import unicodedata
+
+files = Path(__file__).parent.joinpath("../data").glob("*.json")
+# Sort in reverse order to prioritize the most recent and best
+order = {"bronze": 1, "silver": 2, "gold": 3}
+files = sorted(
+    files, key=lambda f: order.get(next((x for x in order if x in f.stem), ""), 0)
+)
+
+
+class Dictionary:
+    def __init__(self):
+        self.dict = {}
+        self.load_dictionaries()
+
+    def load_dictionaries(self):
+        for file in files:
+            with open(file, "r", encoding="utf-8") as f:
+                dictionary: dict = json.load(f)
+                normalized_dictionary = {}
+
+                # normalize niqqud keys
+                for k, v in dictionary.items():
+                    k = normalize(k)
+                    # Ensure not empty
+                    if k and v:
+                        normalized_dictionary[k] = v
+                self.dict.update(normalized_dictionary)
+
+    def replace_hebrew_only_callback(self, match: re.Match[str]) -> str:
+        source: str = match.group(0)
+        # decomposite
+        source = unicodedata.normalize("NFD", source)
+        raw_lookup = self.dict.get(source)
+
+        without_niqqud_lookup = self.dict.get(remove_niqqud(source))
+        with_niqqud_lookup = self.dict.get(normalize(source))
+        # Compare without niqqud ONLY if source has no niqqud
+        if raw_lookup:
+            return raw_lookup
+        if without_niqqud_lookup:
+            return without_niqqud_lookup
+        elif with_niqqud_lookup:
+            return with_niqqud_lookup
+        return source
+
+    def replace_non_whitespace_callback(self, match: re.Match[str]) -> str:
+        raw_source: str = match.group(0)
+        if raw_source.isnumeric():
+            return raw_source
+
+        raw_lookup = self.dict.get(raw_source)
+
+        # Compare without niqqud ONLY if source has no niqqud
+        if raw_lookup:
+            return raw_lookup
+        # search by only ', space, regular niqqud, alphabet
+        raw_source = re.sub(
+            r"[\u05B0-\u05EB ']+", self.replace_hebrew_only_callback, raw_source
+        )
+        return raw_source
+
+    def expand_text(self, text: str) -> str:
+        """
+        TODO: if key doesn't have diacritics expand even diacritized words
+        """
+        text = re.sub(r"\S+", self.replace_non_whitespace_callback, text)
+
+        return text

mishkal/expander/number_names.py

@@ -0,0 +1,193 @@
+"""
+See https://github.com/savoirfairelinux/num2words/blob/master/num2words/lang_HE.py
+"""
+
+# TODO: add niqqud hints
+
+ZERO = {"אפס": "אֶפֶס"}
+
+
+ONES = {
+    "אחת": "אַחַת",
+    "אחד": "אֶחָד",
+    "ראשונה": "רִאשׁוֹנָה",
+    "ראשון": "רִאשׁוֹן",
+    "ראשונות": "רִאשׁוֹנוֹת",
+    "ראשונים": "רִאשׁוֹנִים",
+    "שתיים": "שְׁתַּיִם",
+    "שניים": "שְׁנַיִם",
+    "שתי": "שְׁתֵּי",
+    "שני": "שְׁנֵי",
+    "שנייה": "שְׁנִיָּה",
+    "שניות": "שְׁנִיּוֹת",
+    "שלוש": "שָׁלוֹשׁ",
+    "שלושה": "שְׁלוֹשָׁה",
+    "שלושת": "שְׁלוֹשֶׁת",
+    "שלישית": "שְׁלִישִׁית",
+    "שלישי": "שְׁלִישִׁי",
+    "שלישיות": "שְׁלִישִׁיּוֹת",
+    "שלישיים": "שְׁלִישִׁיִּים",
+    "ארבע": "אַרְבַּע",
+    "ארבעה": "אַרְבַּעָה",
+    "ארבעת": "אַרְבַּעַת",
+    "רביעית": "רֵבִיעִית",
+    "רביעי": "רֵבִיעִי",
+    "רביעיות": "רֵבִיעִיוֹת",
+    "רביעיים": "רֵבִיעִיִּים",
+    "חמש": "חָמֵשׁ",
+    "חמישה": "חֲמִשָּׁה",
+    "חמשת": "חֲמֵשֶׁת",
+    "חמישית": "חֲמִישִּׁית",
+    "חמישי": "חֲמִישִּׁי",
+    "חמישיות": "חֲמִישִּׁיוֹת",
+    "חמישיים": "חֲמִישִּׁיִּים",
+    "שש": "שֵׁשׁ",
+    "שישה": "שִׁשָּׁה",
+    "ששת": "שֵׁשֶׁת",
+    "שישית": "שִׁשִּׁית",
+    "שישי": "שִׁשִּׁי",
+    "שישיות": "שִׁשִּׁיוֹת",
+    "שישיים": "שִׁשִּׁיִּים",
+    "שבע": "שֶׁבַע",
+    "שבעה": "שִׁבְעָה",
+    "שבעת": "שִׁבְעַת",
+    "שביעית": "שְׁבִיעִית",
+    "שביעי": "שְׁבִיעִי",
+    "שביעיות": "שְׁבִיעִיוֹת",
+    "שביעיים": "שְׁבִיעִיִּים",
+    "שמונה": "שְׁמוֹנֶה",
+    "שמונת": "שְׁמוֹנַת",
+    "שמינית": "שְׁמִינִית",
+    "שמיני": "שְׁמִינִי",
+    "שמיניות": "שְׁמִינִיוֹת",
+    "שמיניים": "שְׁמִינִיִּים",
+    "תשע": "תֵּשַׁע",
+    "תשעה": "תִּשְׁעָה",
+    "תשעת": "תִּשְׁעַת",
+    "תשיעית": "תְּשִׁיעִית",
+    "תשיעי": "תְּשִׁיעִי",
+    "תשיעיות": "תְּשִׁיעִיּוֹת",
+    "תשיעיים": "תְּשִׁיעִיִּים",
+}
+
+
+TENS = {
+    "עשר": "עֶשֶׂר",
+    "עשרה": "עֲשָׁרָה",
+    "עשרת": "עֲשֶׁרֶת",
+    "עשירית": "עֲשִׁירִית",
+    "עשירי": "עֲשִׁירִי",
+    "עשיריות": "עֲשִׁירִיוֹת",
+    "עשיריים": "עֲשִׁירִיִּים",
+    "שתים עשרה": "שְׁתֵּים עֶשְׂרֵה",
+    "שנים עשר": "שְׁנֵים עָשָׂר",
+}
+
+
+TWENTIES = {
+    "עשרים": "עֶשְׂרִ֫ים",
+    "שלושים": "שְׁלוֹשִׁים",
+    "ארבעים": "אַרְבָּעִים",
+    "חמישים": "חֲמִשִּׁים",
+    "שישים": "שִׁשִּׁים",
+    "שבעים": "שִׁבְעִים",
+    "שמונים": "שְׁמוֹנִים",
+    "תשעים": "תִּשְׁעִים",
+}
+
+
+HUNDREDS = {
+    "מאה": "מֵאָה",
+    "מאת": "מֵאַת",
+    "מאתיים": "מָאתַיִם",
+    "מאות": "מֵאוֹת",
+}
+
+THOUSANDS = {
+    "אלף": "אֶלֶף",
+    "אלפיים": "אַלְפַּיִם",
+    "אלפים": "אֲלָפִים",
+    "אלפי": "אַלְפִּי",
+}
+
+
+LARGE = {
+    "מיליון": "מִילְיוֹן",
+    "מיליוני": "מִילְיוֹנִי",
+    "מיליארד": "מִילְיַארְד",
+    "מיליארדי": "מִילְיַארְדִּי",
+    "טריליון": "טְרִילְיוֹן",
+    "טריליוני": "טְרִילְיוֹנִי",
+    "קוודריליון": "קוֹוַדְרִילְיוֹן",
+    "קוודריליוני": "קוֹוַדְרִילְיוֹנִי",
+    "קווינטיליון": "קוִוִּנְטִילְיוֹן",
+    "קווינטיליוני": "קוִוִּנְטִילְיוֹנִי",
+    "סקסטיליון": "סְקֶסְטִילְיוֹן",
+    "סקסטיליוני": "סְקֶסְטִילְיוֹנִי",
+    "ספטיליון": "סְפֶּטִילְיוֹן",
+    "ספטיליוני": "סְפֶּטִילְיוֹנִי",
+    "אוקטיליון": "אוֹקְטִילְיוֹן",
+    "אוקטיליוני": "אוֹקְטִילְיוֹנִי",
+    "נוניליון": "נוּנִילְיוֹן",
+    "נוניליוני": "נוּנִילְיוֹנִי",
+    "דסיליון": "דֶּסִילְיוֹן",
+    "דסיליוני": "דֶּסִילְיוֹנִי",
+    "אונדסיליון": "אוּנְדְסִילְיוֹ��",
+    "אונדסיליוני": "אוּנְדְסִילְיוֹנִי",
+    "דואודסיליון": "דוּאודְסִילְיוֹן",
+    "דואודסיליוני": "דוּאודְסִילְיוֹנִי",
+    "טרדסיליון": "טֶרְדְסִילְיוֹן",
+    "טרדסיליוני": "טֶרְדְסִילְיוֹנִי",
+    "קווטואורדסיליון": "קוּוטְוָאורְדְסִילְיוֹן",
+    "קווטואורדסיליוני": "קוּוטְוָאורְדְסִילְיוֹנִי",
+    "קווינדסיליון": "קוִוִּנְדְסִילְיוֹן",
+    "קווינדסיליוני": "קוִוִּנְדְסִילְיוֹנִי",
+    "סקסדסיליון": "סֶקְסְדְסִילְיוֹן",
+    "סקסדסיליוני": "סֶקְסְדְסִילְיוֹנִי",
+    "ספטנדסיליון": "סְפֶּטַנְדְסִילְיוֹן",
+    "ספטנדסיליוני": "סְפֶּטַנְדְסִילְיוֹנִי",
+    "אוקטודסיליון": "אוֹקְטוֹדְסִילְיוֹן",
+    "אוקטודסיליוני": "אוֹקְטוֹדְסִילְיוֹנִי",
+    "נובמדסיליון": "נוֹבְמַדְסִילְיוֹן",
+    "נובמדסיליוני": "נוֹבְמַדְסִילְיוֹנִי",
+    "ויגינטיליון": "וִיגִּינְטִילְיוֹן",
+    "ויגינטיליוני": "וִיגִּינְטִילְיוֹנִי",
+}
+
+
+LETTERS = {
+    "ו": "וֵ",
+    "ה": "הַ",
+}
+
+
+CURRENCY = {
+    "שקל": "שֵׁקֶל",
+    "שקלים": "שְׁקָלִים",
+    "אגורה": "אֲגוֹרָה",
+    "אגורות": "אֲגוֹרוֹת",
+    "אירו": "אֵירוֹ",
+    "סנט": "סֵנְט",
+    "סנטים": "סֵנְטִים",
+    "דולר": "דוֹלָר",
+    "דולרים": "דוֹלָרִים",
+}
+
+
+POINTS = {
+    "מינוס": "מִינּוּס",
+    "נקודה": "נְקֻדָּה",
+}
+
+NUMBER_NAMES = {
+    **CURRENCY,
+    **HUNDREDS,
+    **LARGE,
+    **LETTERS,
+    **ONES,
+    **POINTS,
+    **TENS,
+    **THOUSANDS,
+    **TWENTIES,
+    **ZERO,
+}

mishkal/expander/numbers.py

@@ -0,0 +1,28 @@
+import num2words
+from .number_names import NUMBER_NAMES
+import re
+
+
+def add_diacritics(words: str):
+    new_words = []
+    for word in words.split():
+        if NUMBER_NAMES.get(word):
+            new_words.append(NUMBER_NAMES[word])
+        elif NUMBER_NAMES.get(word[1:]):
+            # With Vav or Bet
+            new_words.append(NUMBER_NAMES[word[0]] + NUMBER_NAMES[word[1:]])
+        else:
+            new_words.append(word)
+    return " ".join(new_words)
+
+
+def num_to_word(maybe_number: str) -> str:
+    def replace_number(match):
+        num = match.group()
+        words = num2words.num2words(num, lang="he", ordinal=False)
+        return add_diacritics(words)
+
+    # Replace all whole numbers in the string
+    result = re.sub(r"\d+", replace_number, maybe_number)
+
+    return result

mishkal/expander/time_to_word.py

@@ -0,0 +1,104 @@
+"""
+Convert time to words
+TODO: fix zeros eg. 22:00
+"""
+
+import re
+
+PATTERNS = [
+    r"(\d{1,2})([apm]{2})",  # AM/PM format
+    r"(\d{1,2}):(\d{2})",  # HH:MM format
+]
+
+
+def extract_time(match):
+    """
+    Extract hour and minute from a string in HH:MM or AM/PM format
+    and return as integers.
+    """
+    time_str = match.group(0).lower().strip()
+
+    # Check for HH:MM format
+    match = re.match(r"(\d{1,2}):(\d{2})", time_str)
+    if match:
+        h = int(match.group(1))
+        m = int(match.group(2))
+        return f"{convert_to_word(h, m)}"
+
+    # Check for AM/PM format
+    match = re.match(r"(\d{1,2})([apm]{2})", time_str)
+    if match:
+        h = int(match.group(1))
+        period = match.group(2)
+
+        # Normalize to 24-hour format
+        if period == "am" and h == 12:
+            h = 0
+        elif period == "pm" and h != 12:
+            h += 12
+        return f"{convert_to_word(h, 0)}"  # Defaulting to 0 minutes when only hour is provided
+
+    return match.group(0)  # Return original text if the format is not recognized
+
+
+def convert_to_word(h, m):
+    hours = [
+        "אֶפֶס",
+        "אַחַת",
+        "שְׁנַיִם",  # Will be replaced with "שֵׁנִי" when needed
+        "שָׁלוֹשׁ",
+        "אַרְבַּע",
+        "חָמֵשׁ",
+        "שֵׁשׁ",
+        "שֶׁבַע",
+        "שְׁמוֹנֵה",
+        "תֵּשַׁע",
+        "עֵשֵׂר",
+        "אַחַת עֶשְׂרֵה",
+        "שְׁתֵּים עֶשְׂרֵה",
+    ]
+
+    tens = ["", "עֵשֵׂר", "עֶשְׂרִים", "שְׁלוֹשִׁים", "אַרְבָּעִים", "חֲמִשִּׁים"]
+
+    ten_to_twenty = [
+        "עֵשֵׂר",
+        "אַחַת עֶשְׂרֵה",
+        "שְׁתֵּים עֶשְׂרֵה",
+        "שְׁלוֹשׁ עֶשְׂרֵה",
+        "אַרְבַּע עֶשְׂרֵה",
+        "חֲמֵשׁ עֶשְׂרֵה",
+        "שֵׁשׁ עֶשְׂרֵה",
+        "שְׁבַע עֶשְׂרֵה",
+        "שְׁמוֹנֶה עֶשְׂרֵה",
+        "תְּשַׁע עֶשְׂרֵה",
+    ]
+
+    vocab = {"minutes": "דַּקּוֹת", "and": "וֵ", "shtey": "שְׁתֵּי"}
+
+    # Convert 0 hours to 12 (midnight)
+    if h == 0:
+        h = 12
+
+    elif h > 12:
+        h -= 12
+
+    if m == 0:
+        return f"{hours[h]}"
+
+    elif 1 <= m <= 9:
+        minute_word = (
+            vocab["shtey"] if m == 2 else hours[m]
+        )  # Replace "שניים" with "שני"
+        return f"{hours[h]} {vocab['and']}{minute_word} {vocab['minutes']}"
+
+    elif 10 <= m <= 19:
+        return f"{hours[h]} {vocab['and']}{ten_to_twenty[m - 10]} {vocab['minutes']}"
+
+    else:
+        tens_part = f"{vocab['and']}{tens[m // 10]}"
+        units_part = f"{vocab['and']}{hours[m % 10]}" if m % 10 != 0 else ""
+        return f"{hours[h]} {tens_part} {units_part} {vocab['minutes']}".strip()
+
+
+def time_to_word(text: str):
+    return re.sub("|".join(PATTERNS), extract_time, text)

mishkal/lexicon.py

@@ -0,0 +1,189 @@
+"""
+ASCII IPA transcription of Hebrew consonants and vowels.
+"""
+
+# https://en.wikipedia.org/wiki/Unicode_and_HTML_for_the_Hebrew_alphabet#Compact_table
+HE_CHARS_PATTERN = (
+    r"\b[\u05B0-\u05EA\u05F3\u0027]+\b"  # Chars including niqqud, geresh and en_geresh
+)
+HE_NIQQUD_PATTERN = r"[\u05B0-\u05C7]"
+PUNCTUATION = r".,!? "
+
+# Special
+GIMEL_OR_ZAIN_WITH_DAGESH = "dʒ"
+TSADIK_WITH_DAGESH = "tʃ"
+SHIN_WITH_POINT = "ʃ"
+SIN_WITH_POINT = "s"
+STRESS = "\u02c8"  # visually looks like '
+HET_GNUVA = "ax"
+W_AS_WALLA = "w"
+
+GERESH_LETTERS = {"ג": "dʒ", "ז": "ʒ", "ת": "ta", "צ": "tʃ", "ץ": "tʃ"}
+
+LETTERS_NAMES_PHONEMES = {
+    "א": "alef",  # Alef, glottal stop
+    "ב": "bet",  # Bet
+    "ג": "gimel",  # Gimel
+    "ד": "dalet",  # Dalet
+    "ה": "hej",  # He
+    "ו": "vav",  # Vav
+    "ז": "zajin",  # Zayin
+    "ח": "xet",  # Het
+    "ט": "tet",  # Tet
+    "י": "jud",  # Yod
+    "ך": "xaf sofit",  # Haf sofit
+    "כ": "xaf",  # Haf
+    "ל": "lamed",  # Lamed
+    "ם": "mem sofit",  # Mem Sofit
+    "מ": "mem",  # Mem
+    "ן": "nun sofit",  # Nun Sofit
+    "נ": "nun",  # Nun
+    "ס": "samex",  # Samekh
+    "ע": "ajin",  # Ayin, glottal stop
+    "פ": "fey",  # Fey
+    "ף": "fey sofit",  # Fey Sofit
+    "ץ": "tsadik sofit",  # Tsadik sofit
+    "צ": "tsadik",  # Tsadik
+    "ק": "kuf",  # Kuf
+    "ר": "rejiʃ",  # Resh
+    "ש": "ʃin",  # Shin
+    "ת": "taf",  # Taf
+}
+
+# Consonants
+LETTERS_PHONEMES = {
+    "א": "ʔ",  # Alef
+    "ב": "v",  # Bet
+    "ג": "g",  # Gimel
+    "ד": "d",  # Dalet
+    "ה": "h",  # He
+    "ו": "v",  # Vav
+    "ז": "z",  # Zayin
+    "ח": "x",  # Het
+    "ט": "t",  # Tet
+    "י": "j",  # Yod
+    "ך": "x",  # Haf sofit
+    "כ": "x",  # Haf
+    "ל": "l",  # Lamed
+    "ם": "m",  # Mem Sofit
+    "מ": "m",  # Mem
+    "ן": "n",  # Nun Sofit
+    "נ": "n",  # Nun
+    "ס": "s",  # Samekh
+    "ע": "ʔ",  # Ayin, only voweled
+    "פ": "f",  # Fey
+    "ף": "f",  # Fey Sofit
+    "ץ": "ts",  # Tsadik sofit
+    "צ": "ts",  # Tsadik
+    "ק": "k",  # Kuf
+    "ר": "r",  # Resh
+    "ש": "ʃ",  # Shin
+    "ת": "t",  # Taf
+    # Beged Kefet
+    "בּ": "b",
+    "כּ": "k",
+    "פּ": "p",
+    "שׁ": "ʃ",
+    "שׂ": "s",
+    "'": "",
+}
+
+# Vowels
+VOWEL_A = "a"
+VOWEL_E = "e"
+VOWEL_I = "i"
+VOWEL_O = "o"
+VOWEL_U = "u"
+
+SHVA_NA_DIACRITIC = "\u05bd"
+ATAMAHA_DIACRITIC = "\u05ab"
+
+NIQQUD_PHONEMES = {
+    "\u05b4": "i",  # Hiriq
+    "\u05b5": "e",  # Tsere
+    "\u05b7": "a",  # Patah
+    "\u05b9": "o",  # Holam
+    "\u05ba": "o",  # Holam haser for vav
+    "\u05bb": "u",  # Qubuts
+
+    "\u05b3": 'o', # Hataf qamats
+    "\u05b8": "a", # Kamataz
+    
+    ATAMAHA_DIACRITIC: "ˈ",  # Stress (Atmaha)
+    SHVA_NA_DIACRITIC: "e",  # Shva na
+}
+
+SET_LETTER_SYMBOLS = {
+    "\u05b0",  # Shva
+    "\u05b4",  # Hiriq
+    "\u05b5",  # Tsere
+    "\u05b7",  # Patah
+    "\u05b9",  # Holam
+    "\u05ba",  # Holam haser for vav
+    "\u05bb",  # Qubuts
+    "\u05bc",  # Dagesh
+    "\u05c1",  # Shin dot
+    "\u05b3", # Hataf qamats
+    "\u05b8", # Kamataz
+    "\u05c2",  # Sin dot
+    "'",  # Geresh
+}
+
+"""
+We're left with the following niqqud (10):  
+Shva, Hiriq, Tsere, Patah, Holam, Qubuts, Dagesh, 
+Holam haser for vav, Shin dot, Sin dot
+"""
+NIQQUD_DEDUPLICATE = {
+    "\u05b1": "\u05b5",  # Hataf Segol -> Tsere
+    "\u05b2": "\u05b7",  # Hataf Patah -> Patah
+    # "\u05b3": "\u05b9",  # Hataf Qamats -> Holam
+    "\u05b6": "\u05b5",  # Segol -> Tsere
+    # Kamatz -> Patah
+    # "\u05b8": "\u05b7",  # Qamats -> Patah
+    "\u05c7": "\u05b9",  # Qamats Qatan -> Holam
+    "\u05f3": "'",  # Hebrew geresh to regular geresh
+}
+
+
+SET_OUTPUT_CHARACTERS = set(
+    [
+        *GIMEL_OR_ZAIN_WITH_DAGESH,
+        TSADIK_WITH_DAGESH,
+        SHIN_WITH_POINT,
+        SIN_WITH_POINT,
+        W_AS_WALLA,
+    ]
+    + [STRESS]
+    + list(LETTERS_PHONEMES.values())
+    + list(NIQQUD_PHONEMES.values())
+    + [VOWEL_A, VOWEL_E, VOWEL_I, VOWEL_O, VOWEL_U]
+    + list(PUNCTUATION)
+)
+
+SET_NIQQUD = {
+    # Shva, Hiriq, Tsere, Patah, Holam, Holam haser for vav, Qubuts, Dagesh, Shin dot, Sin dot
+    "\u05b0", # Shva
+    "\u05b4", # Hiriq
+    "\u05b5", # Tsere
+    "\u05b7", # Patah
+    "\u05b9", # Holam
+    "\u05ba", # Holam for vav
+    "\u05bb", # Qubuts
+    "\u05bc", # Dagesh
+    "\u05c1", # Shin
+    "\u05c2", # Sin
+    "\u05b3", # Hataf qamats
+    "\u05b8", # Kamataz
+    # shva na and atmaha
+    "\u05bd",  # shva na
+    "\u05ab",  # atmaha
+}
+SET_LETTERS = set(LETTERS_PHONEMES.keys())
+SET_PUNCTUATION = set(PUNCTUATION)
+
+
+# Set for fast lookup
+SET_INPUT_CHARACTERS = set(
+    list(LETTERS_PHONEMES.keys()) + list(SET_NIQQUD) + list(PUNCTUATION) + ["'"]
+)

mishkal/log.py

@@ -0,0 +1,35 @@
+import logging
+import os
+import colorlog
+
+
+def _create_logger():
+    """
+    Create a logger with colorized output
+    Usage: LOG_LEVEL=DEBUG python <script.py>
+    """
+
+    handler = colorlog.StreamHandler()
+    fmt = "%(log_color)s%(levelname)-8s%(reset)s [%(filename)s:%(lineno)d] %(message)s"
+    handler.setFormatter(
+        colorlog.ColoredFormatter(
+            fmt=fmt,
+            log_colors={
+                "DEBUG": "blue",
+                "INFO": "green",
+                "WARNING": "yellow",
+                "ERROR": "red",
+                "CRITICAL": "red",
+            },
+        )
+    )
+    # Get log level from LOG_LEVEL environment variable
+    log_level = os.getenv("LOG_LEVEL", "WARNING").upper()
+    logger = colorlog.getLogger(__package__)
+    logger.setLevel(level=getattr(logging, log_level, logging.WARNING))
+    # Setup logging to stdout
+    logger.addHandler(handler)
+    return logger
+
+
+log = _create_logger()

mishkal/phonemize.py

@@ -0,0 +1,310 @@
+"""
+The actual letters phonemization happens here.
+Phonemes generated based on rules.
+
+Early rules:
+1. Niqqud malle vowels
+2. Dagesh (custom beged kefet)
+3. Final letter without niqqud
+4. Final Het gnuva
+5. Geresh (Gimel, Ttadik, Zain)
+6. Shva na
+Revised rules:
+1. Consonants
+2. Niqqud
+
+Reference:
+- https://hebrew-academy.org.il/2020/08/11/איך-הוגים-את-השווא-הנע
+- https://hebrew-academy.org.il/2010/03/24/צהרים-נעמי-הגיית-קמץ-לפני-חט/
+- https://en.wikipedia.org/wiki/Unicode_and_HTML_for_the_Hebrew_alphabet#Compact_table
+- https://en.wikipedia.org/wiki/Help:IPA/Hebrew
+- https://he.wikipedia.org/wiki/הברה
+"""
+
+from mishkal import lexicon
+from .expander import Expander
+from mishkal.utils import normalize, post_normalize, has_vowel, has_constant, remove_niqqud
+from typing import Callable
+import regex as re
+from mishkal.variants import Letter, Syllable
+
+class Phonemizer:
+    def __init__(self):
+        self.expander = Expander()
+
+    def phonemize(
+        self,
+        text: str,
+        preserve_punctuation=True,
+        preserve_stress=True,
+        use_expander=False,
+        use_post_normalize=False,  # For TTS
+        predict_stress=False,
+        predict_shva_nah=False,
+        fallback: Callable[[str], str] = None,
+    ) -> str:
+        # normalize
+        text = normalize(text)
+        
+        
+        # TODO: is that enough? what if there's punctuation around? other chars?
+        he_pattern = r"[\u05b0-\u05ea\u05ab\u05bd']+"
+        fallback_pattern = r"[a-zA-Z]+"
+
+        def fallback_replace_callback(match: re.Match):
+            word = match.group(0)
+
+            if self.expander.dictionary.dict.get(word):
+                # skip
+                # TODO: better API
+                return word
+            phonemes = fallback(word).strip()
+            # TODO: check that it has only IPA?!
+            for c in phonemes:
+                lexicon.SET_OUTPUT_CHARACTERS.add(c)
+            return phonemes
+
+        if fallback is not None:
+            text = re.sub(fallback_pattern, fallback_replace_callback, text)
+        if use_expander:
+            text = self.expander.expand_text(text)
+        self.fallback = fallback
+
+        def heb_replace_callback(match: re.Match):
+            word = match.group(0)
+
+            word = normalize(word)
+            word = "".join(
+                i for i in word if i in lexicon.SET_LETTERS or i in lexicon.SET_NIQQUD
+            )
+            letters: list[tuple[str, str]] = re.findall(r"(\p{L})([\p{M}']*)", word)  # with en_geresh
+            letters: list[Letter] = [Letter(i[0], i[1]) for i in letters]
+            syllables: list[Syllable] = self.phonemize_hebrew(letters, predict_shva_na=predict_shva_nah)
+            phonemes = "".join(syllable.phones for syllable in syllables)
+            if use_post_normalize:
+                phonemes = post_normalize(phonemes)
+
+
+            if predict_stress and lexicon.STRESS not in phonemes:
+                stressed = []
+
+                is_milra = True
+
+                milhel_patterns = ['יים', 'וע', 'טו']
+                if syllables and any(remove_niqqud(syllables[-1].chars).endswith(i) for i in milhel_patterns):
+                    is_milra = False
+
+                # Iterate through each syllable
+                for idx, syllable in enumerate(syllables):
+                    # If it's the last syllable, add stress
+                    if not is_milra and idx == len(syllables) - 2:
+                        stressed.append(f'ˈ{syllable.phones}')
+                    elif is_milra and idx == len(syllables) - 1:
+                        stressed.append(f'ˈ{syllable.phones}')
+                    else:
+                        stressed.append(syllable.phones)
+                phonemes = "".join(stressed)
+                phonemes = post_normalize(phonemes)
+            
+            return phonemes
+
+        
+        text = re.sub(he_pattern, heb_replace_callback, text)
+
+        if not preserve_punctuation:
+            text = "".join(i for i in text if i not in lexicon.PUNCTUATION or i == " ")
+        if not preserve_stress:
+            text = "".join(
+                i for i in text if i not in [lexicon.STRESS]
+            )
+
+        def expand_hyper_phonemes(text: str):
+            """
+            Expand hyper phonemes into normal phonemes
+            eg. [hello](/hɛˈloʊ/) -> hɛˈloʊ
+            """
+            def hyper_phonemes_callback(match: re.Match):
+                matched_phonemes = match.group(2)
+                for c in matched_phonemes:
+                    lexicon.SET_OUTPUT_CHARACTERS.add(c)
+                return matched_phonemes  # The phoneme is in the second group
+
+            text = re.sub(r"\[(.+?)\]\(\/(.+?)\/\)", hyper_phonemes_callback, text)
+            return text
+
+        text = expand_hyper_phonemes(text)
+        text = "".join(i for i in text if i in lexicon.SET_OUTPUT_CHARACTERS)
+
+        return text
+
+    def phonemize_hebrew(self, letters: list[Letter], predict_shva_na: bool) -> list[Syllable]:
+        phonemes = []
+        i = 0
+
+        
+        syllables = []
+        cur_syllable = Syllable('', '')
+        while i < len(letters):
+            
+            cur = letters[i]
+            prev = letters[i - 1] if i > 0 else None
+            next = letters[i + 1] if i < len(letters) - 1 else None
+            cur_phonemes = []
+            skip_diacritics = False
+            skip_constants = False
+            skip_offset = 0
+            # revised rules
+
+            # יַאלְלָה
+            if cur.char == "ל" and cur.diac == "\u05b0" and next and next.char == "ל":
+                skip_diacritics = True
+                skip_constants = True
+
+            if (
+                cur.char == "ו"
+                and not prev
+                and next
+                and not next.diac
+                and cur.char + cur.diac == "וַא"
+            ):
+                skip_offset += 1
+                cur_phonemes.append("wa")
+
+            if cur.char == "א" and not cur.diac and prev:
+                if next and next.char != 'ו':
+                    skip_constants = True
+
+            # TODO ?
+            if cur.char == "י" and next and not cur.diac and prev.char + prev.diac != 'אֵ':
+                skip_constants = True
+
+            if cur.char == "ש" and "\u05c2" in cur.diac:
+                cur_phonemes.append("s")
+                skip_constants = True
+
+            # shin without niqqud after sin = sin
+            if cur.char == "ש" and not cur.diac and prev and "\u05c2" in prev.diac:
+                cur_phonemes.append("s")
+                skip_constants = True
+
+            if not next and cur.char == "ח" and '\u05b7' in cur.diac:
+                # Final Het gnuva
+                cur_phonemes.append("ax")
+                skip_diacritics = True
+                skip_constants = True
+
+            if cur and "'" in cur.diac and cur.char in lexicon.GERESH_LETTERS:
+                if cur.char == "ת":
+                    cur_phonemes.append(lexicon.GERESH_LETTERS.get(cur.char, ""))
+                    skip_diacritics = True
+                    skip_constants = True
+                else:
+                    # Geresh
+                    cur_phonemes.append(lexicon.GERESH_LETTERS.get(cur.char, ""))
+                    skip_constants = True
+
+            elif (
+                "\u05bc" in cur.diac and cur.char + "\u05bc" in lexicon.LETTERS_PHONEMES
+            ):  # dagesh
+                cur_phonemes.append(lexicon.LETTERS_PHONEMES.get(cur.char + "\u05bc", ""))
+                skip_constants = True
+            elif cur.char == "ו":
+                skip_constants = True
+                if next and next.char == "ו" and next.diac == cur.diac:
+                    # patah and next.diac empty
+                    if cur.diac in ["\u05b7", "\u05b8"] and not next.diac:
+                        cur_phonemes.append("w")
+                        skip_diacritics = True
+                        skip_offset += 1
+                    elif cur.diac == next.diac:
+                        # double vav
+                        cur_phonemes.append("wo")
+                        skip_diacritics = True
+                        skip_offset += 1
+                    else:
+                        # TODO ?
+                        # skip_consonants = False
+                        skip_diacritics = False
+                else:
+                    # Single vav
+
+                    # Vav with Patah
+                    if re.search("[\u05b7-\u05b8]", cur.diac):
+                        cur_phonemes.append("va")
+
+                    # Holam haser
+                    elif "\u05b9" in cur.diac:
+                        cur_phonemes.append("o")
+                    # Shuruk / Kubutz
+                    elif "\u05bb" in cur.diac or "\u05bc" in cur.diac:
+                        cur_phonemes.append("u")
+                    # Vav with Shva in start
+                    elif "\u05b0" in cur.diac and not prev:
+                        cur_phonemes.append("ve")
+                    # Hirik
+                    elif "\u05b4" in cur.diac:
+                        cur_phonemes.append("vi")
+                    # Tsere
+                    elif "\u05b5" in cur.diac:
+                        cur_phonemes.append("ve")
+                    
+                    else:
+                        cur_phonemes.append("v")
+                    skip_diacritics = True
+
+            if not skip_constants:
+                cur_phonemes.append(lexicon.LETTERS_PHONEMES.get(cur.char, ""))
+            
+            if predict_shva_na and '\u05b0' in cur.diac and not skip_diacritics and lexicon.SHVA_NA_DIACRITIC not in cur.diac:
+                # shva na prediction
+                if not prev:
+                    if cur.char in 'למנרי' or cur.char in 'אהע' or cur.char in 'וכלב':
+                        cur_phonemes.append("e")
+                        skip_diacritics = True 
+                else:
+                    if next and next.char == cur.char:
+                        cur_phonemes.append("e")
+                        skip_diacritics = True
+                    elif prev and '\u05b0' in prev.diac and phonemes[-1] != 'e':
+                        cur_phonemes.append("e")
+                        skip_diacritics = True
+
+            if '\u05b8' in cur.diac and next and '\u05b3' in next.diac:
+                cur_phonemes.append('o')
+                skip_diacritics = True
+
+            niqqud_phonemes = (
+                [lexicon.NIQQUD_PHONEMES.get(niqqud, "") for niqqud in cur.diac]
+                if not skip_diacritics
+                else []
+            )            
+            
+            cur_phonemes.extend(niqqud_phonemes)
+            # Ensure the stress is at the beginning of the syllable
+            cur_phonemes.sort(key=lambda x: x != 'ˈ')
+            phonemes.extend(cur_phonemes)
+            
+
+            if not next:
+                cur_syllable.chars += cur.char + cur.diac
+                cur_syllable.phones += ''.join(cur_phonemes)
+                syllables.append(cur_syllable)
+            elif not prev:
+                cur_syllable = Syllable(cur.char + cur.diac, ''.join(cur_phonemes))
+
+            elif len(re.findall('[א-ת]', cur_syllable.chars)) >= 2 and has_vowel(cur_syllable.phones) and cur.diac:
+                syllables.append(cur_syllable)
+                cur_syllable = Syllable(cur.char + cur.diac, ''.join(cur_phonemes))
+
+            elif not has_vowel(cur_phonemes):                
+                cur_syllable.chars += cur.char + cur.diac
+                cur_syllable.phones += ''.join(cur_phonemes)
+
+            elif not has_vowel(cur_syllable.phones):
+                cur_syllable.chars += cur.char + cur.diac
+                cur_syllable.phones += ''.join(cur_phonemes)
+            else:
+                syllables.append(cur_syllable)
+                cur_syllable = Syllable(cur.char + cur.diac, ''.join(cur_phonemes))
+            i += skip_offset + 1
+        return syllables

mishkal/utils.py

@@ -0,0 +1,68 @@
+from mishkal import lexicon
+import unicodedata
+import regex as re
+
+
+def sort_diacritics(match):
+    letter = match.group(1)
+    diacritics = "".join(sorted(match.group(2)))  # Sort diacritics
+    return letter + diacritics
+
+
+NORMALIZE_PATTERNS = {
+    # Alphabet followed by 1/2 symbols then dagesh. make dagesh first
+    r"(\p{L})(\p{M}+)": sort_diacritics,
+    "״": '"',
+    "׳": "'",
+}
+
+
+def remove_niqqud(text: str):
+    return re.sub(lexicon.HE_NIQQUD_PATTERN, "", text)
+
+
+def has_niqqud(text: str):
+    return re.search(lexicon.HE_NIQQUD_PATTERN, text) is not None
+
+
+def normalize(text: str) -> str:
+    """
+    Normalize unicode (decomposite)
+    Deduplicate niqqud (eg. only Patah instead of Kamatz)
+    Keep only Hebrew characters / punctuation / IPA
+    Sort diacritics
+    """
+
+    # Decompose text
+    text = unicodedata.normalize("NFD", text)
+    for k, v in NORMALIZE_PATTERNS.items():
+        text = re.sub(k, v, text)
+    # Normalize niqqud, remove duplicate phonetics 'sounds' (eg. only Patah)
+    for k, v in lexicon.NIQQUD_DEDUPLICATE.items():
+        text = text.replace(k, v)
+    return text
+
+
+def post_normalize(phonemes: str):
+    new_phonemes = []
+    for word in phonemes.split(" "):
+        # remove glottal stop from end
+        word = re.sub(r"ʔ$", "", word)
+        # remove h from end
+        word = re.sub(r"h$", "", word)
+        word = re.sub(r"ˈh$", "", word)
+        # remove j followed by a i
+        word = re.sub(r"ij", "i", word)
+        new_phonemes.append(word)
+    return " ".join(new_phonemes)
+
+
+def get_unicode_names(text: str):
+    return [unicodedata.name(c, "?") for c in text]
+
+def has_vowel(s: iter):
+    return any(i in s for i in 'aeiou')
+
+def has_constant(s: iter):
+    return any(i not in 'aeiou' for i in s)
+

mishkal/variants.py

@@ -0,0 +1,15 @@
+class Letter:
+    def __init__(self, char: str, diac: list[str]):
+        self.char = char
+        self.diac = diac
+
+    def __repr__(self):
+        return f"{self.char}{''.join(self.diac)}"
+    
+class Syllable:
+    def __init__(self, chars, phones):
+        self.chars = chars
+        self.phones = phones
+
+    def __repr__(self):
+        return f'{self.chars}: {self.phones}'

requirements.txt

@@ -0,0 +1,4 @@
+gradio>=5.15.0
+num2words
+colorlog
+regex