Handle numbers properly

app.py

@@ -18,13 +18,15 @@ from torch.cuda import is_available
 
 class VoiceOption(Enum):
     Sevil = "Севіль (жіночий) 👩"
-    #Arslan = "Арслан (чоловічий) 👨"
+    # Arslan = "Арслан (чоловічий) 👨"
     Eskander = "Ескандер (чоловічий) 👨"
     # Abibulla = "Абібулла (чоловічий) 👨"
 
 
 def check_thread(logging_queue: Queue):
-    logging_callback = log_data(hf_token=getenv("HF_API_TOKEN"), dataset_name="crh-tts-output", private=False)
+    logging_callback = log_data(
+        hf_token=getenv("HF_API_TOKEN"), dataset_name="crh-tts-output", private=False
+    )
     while True:
         sleep(60)
         batch = []
@@ -35,10 +37,13 @@ def check_thread(logging_queue: Queue):
             try:
                 logging_callback(batch)
             except:
-                print("Error happened while pushing data to HF. Puttting items back in queue...")
+                print(
+                    "Error happened while pushing data to HF. Puttting items back in queue..."
+                )
                 for item in batch:
                     logging_queue.put(item)
 
+
 if getenv("HF_API_TOKEN") is not None:
     log_queue = Queue()
     t = Thread(target=check_thread, args=(log_queue,))
@@ -62,9 +67,9 @@ def tts(text: str, voice: str):
 
     voice_mapping = {
         VoiceOption.Sevil.value: Voices.Sevil.value,
-        #VoiceOption.Arslan.value: Voices.Arslan.value,
+        # VoiceOption.Arslan.value: Voices.Arslan.value,
         VoiceOption.Eskander.value: Voices.Eskander.value,
-        #VoiceOption.Abibulla.value: Voices.Abibulla.value,
+        # VoiceOption.Abibulla.value: Voices.Abibulla.value,
     }
 
     speaker_name = voice_mapping[voice]
@@ -114,6 +119,7 @@ iface = gr.Interface(
         ],
         ["Селям! Ишлер насыл?", VoiceOption.Sevil.value],
         ["Selâm! 123456789", VoiceOption.Eskander.value],
+        ["Selâm! 1,2,3,4,5,6,789", VoiceOption.Eskander.value],
     ],
 )
 iface.launch()

crh_preprocessor/preprocessor.py

@@ -11,44 +11,44 @@ mapping = {
     "s\u0327": "\u015f",
     "a\u0302": "\xe2",
     "w": "v",
-    "x": "ks"
+    "x": "ks",
 }
 
 zero = {
-    0: 'sıfır',
+    0: "sıfır",
 }
 
 numbers_map = {
-    1: 'bir',
-    2: 'eki',
-    3: 'üç',
-    4: 'dört',
-    5: 'beş',
-    6: 'altı',
-    7: 'yedi',
-    8: 'sekiz',
-    9: 'doquz',
-    10: 'on',
-    20: 'yigirmi',
-    30: 'otuz',
-    40: 'qırq',
-    50: 'elli',
-    60: 'altmış',
-    70: 'yetmiş',
-    80: 'seksen',
-    90: 'doqsan',
-    100: 'yüz',
-    1000: 'biñ',
-    1_000_000: 'million',
-    1_000_000_000: 'milliard'
+    1: "bir",
+    2: "eki",
+    3: "üç",
+    4: "dört",
+    5: "beş",
+    6: "altı",
+    7: "yedi",
+    8: "sekiz",
+    9: "doquz",
+    10: "on",
+    20: "yigirmi",
+    30: "otuz",
+    40: "qırq",
+    50: "elli",
+    60: "altmış",
+    70: "yetmiş",
+    80: "seksen",
+    90: "doqsan",
+    100: "yüz",
+    1000: "biñ",
+    1_000_000: "million",
+    1_000_000_000: "milliard",
 }
 
 
 def spell_numbers(numbers: str) -> str:
-    numbers_map_with_zero = {**numbers_map,**zero}
+    numbers_map_with_zero = {**numbers_map, **zero}
     for i in range(0, 10):
-        numbers = numbers.replace(str(i), numbers_map_with_zero[i] + ' ')
-    return numbers.strip()  
+        numbers = numbers.replace(str(i), numbers_map_with_zero[i] + " ")
+    return numbers.strip()
 
 
 def num2word(n):
@@ -58,24 +58,32 @@ def num2word(n):
         tens = (n // 10) * 10
         units = n % 10
         if units == 0:
-            return ''
-        return (numbers_map[tens] + ' ' + numbers_map[units]).strip()
+            return ""
+        return (numbers_map[tens] + " " + numbers_map[units]).strip()
     elif n < 1000:
         hundreds = n // 100
         rest = n % 100
-        return (num2word(hundreds) + ' ' + numbers_map[100] + ' ' + num2word(rest)).strip()
+        return (
+            num2word(hundreds) + " " + numbers_map[100] + " " + num2word(rest)
+        ).strip()
     elif n < 1_000_000:
         thousands = n // 1_000
         rest = n % 1_000
-        return (num2word(thousands) + ' ' + numbers_map[1_000] + ' ' + num2word(rest)).strip()
+        return (
+            num2word(thousands) + " " + numbers_map[1_000] + " " + num2word(rest)
+        ).strip()
     elif n < 1_000_000_000:
         millions = n // 1_000_000
         rest = n % 1_000_000
-        return (num2word(millions) + ' ' + numbers_map[1_000_000] + ' ' + num2word(rest)).strip()
+        return (
+            num2word(millions) + " " + numbers_map[1_000_000] + " " + num2word(rest)
+        ).strip()
     elif n < 1_000_000_000_000:
         billions = n // 1_000_000_000
         rest = n % 1_000_000_000
-        return (num2word(billions) + ' ' + numbers_map[1_000_000_000] + ' ' + num2word(rest)).strip()
+        return (
+            num2word(billions) + " " + numbers_map[1_000_000_000] + " " + num2word(rest)
+        ).strip()
     else:
         return spell_numbers(str(n))
 
@@ -87,20 +95,29 @@ def preprocess(text):
     for symbol in mapping.keys():
         text = re.sub(symbol, mapping[symbol], text)
 
-    separators = "?!" # TODO: add proper symbols to tts
+    separators = "?!"  # TODO: add proper symbols to tts
     for symbol in separators:
         text = text.replace(symbol, ".")
 
     while True:
-        number_match = re.search("-?\d+(\.|,)?(\d+)?", text)
-        
+        groups_match = re.search("((\d,)+){2,}", text)
+        if groups_match is not None:
+            text = text.replace(
+                groups_match.string[groups_match.start() : groups_match.end()],
+                " ".join(
+                    groups_match.string[
+                        groups_match.start() : groups_match.end()
+                    ].split(",")
+                ),
+            )
+            continue
+
+        number_match = re.search("(\-|\+)?(\d)+((\.|,)?\d+)?", text)
         if number_match is None:
             break
-        
-        print(number_match.string, number_match.start(), number_match.end())
-        
-        number = number_match.string.strip()
 
+        number = number_match.string[number_match.start() : number_match.end()]
+        number_to_replace = number
         prefix = ""
 
         if number.startswith("-"):
@@ -112,21 +129,33 @@ def preprocess(text):
 
         if "." in number:
             number = number.split(".")
-            number = prefix + " noqta ".join((num2word(int(number[0])) if int(number[0]) != 0 else spell_numbers(number[0]), spell_numbers(number[1])))
-            text = text.replace(number_match.string.strip(), number, 1)
+            number = prefix + " noqta ".join(
+                (
+                    num2word(int(number[0]))
+                    if int(number[0]) != 0
+                    else spell_numbers(number[0]),
+                    spell_numbers(number[1]),
+                )
+            )
+            text = text.replace(number_to_replace, number, 1)
             continue
         elif "," in number:
             number = number.split(",")
-            number = prefix + " virgül ".join((num2word(int(number[0])) if int(number[0]) != 0 else spell_numbers(number[0]), spell_numbers(number[1])))
-            text = text.replace(number_match.string.strip(), number, 1)
+            number = prefix + " virgül ".join(
+                (
+                    num2word(int(number[0]))
+                    if int(number[0]) != 0
+                    else spell_numbers(number[0]),
+                    spell_numbers(number[1]),
+                )
+            )
+            text = text.replace(number_to_replace, number, 1)
             continue
 
         if number.startswith("0"):
-            text = text.replace(number_match.string.strip(), prefix + spell_numbers(number), 1)
+            text = text.replace(number_to_replace, prefix + spell_numbers(number), 1)
             continue
-        
-        text = text.replace(number_match.string.strip(), prefix + num2word(int(number)), 1)
-        
+
+        text = text.replace(number_to_replace, prefix + num2word(int(number)), 1)
 
     return text.strip()
-    

crh_tts/tts.py

@@ -10,10 +10,10 @@ from torch import no_grad
 class Voices(Enum):
     """List of available voices for the model."""
 
-    #Arslan = "arslan"
+    # Arslan = "arslan"
     Sevil = "sevil"
     Eskander = "eskander"
-    #Abibulla = "abibulla"
+    # Abibulla = "abibulla"
 
 
 class TTS:

data_logger.py

@@ -3,21 +3,22 @@ import os
 import csv
 import huggingface_hub
 
+
 def log_data(hf_token: str, dataset_name: str, private=True):
     path_to_dataset_repo = huggingface_hub.create_repo(
-            repo_id=dataset_name,
-            token=hf_token,
-            private=private,
-            repo_type="dataset",
-            exist_ok=True,
-        )
+        repo_id=dataset_name,
+        token=hf_token,
+        private=private,
+        repo_type="dataset",
+        exist_ok=True,
+    )
     flagging_dir = "flagged"
     dataset_dir = os.path.join(flagging_dir, dataset_name)
     repo = huggingface_hub.Repository(
-            local_dir=dataset_dir,
-            clone_from=path_to_dataset_repo,
-            use_auth_token=hf_token,
-        )
+        local_dir=dataset_dir,
+        clone_from=path_to_dataset_repo,
+        use_auth_token=hf_token,
+    )
     repo.git_pull(lfs=True)
     log_file = os.path.join(dataset_dir, "data.csv")
 
@@ -38,4 +39,3 @@ def log_data(hf_token: str, dataset_name: str, private=True):
         return line_count
 
     return log_function
-

tests/test_preprocessor.py

@@ -2,69 +2,44 @@ from crh_preprocessor.preprocessor import preprocess, num2word
 
 
 def test_num2word():
+    assert num2word(16) == "on altı"
     assert (
-        num2word(16) == "on altı"
-    )
-    assert (
-        num2word(1324759813) == "bir milliard üç yüz yigirmi dört million yedi yüz elli doquz biñ sekiz yüz on üç"
-    )
-    assert (
-        num2word(1_000_000) == "million"
+        num2word(1324759813)
+        == "bir milliard üç yüz yigirmi dört million yedi yüz elli doquz biñ sekiz yüz on üç"
     )
+    assert num2word(1_000_000) == "million"
 
 
 def test_preprocessor():
     assert (
         preprocess("İşanç Alla-Taalâğa.") == "işan\u04ab alla-taalâğa."
     )  # first i is two symbols (i without dot and dot)
+    assert preprocess("1000000") == "million"
     assert (
-        preprocess("1000000") == "million"
-    )
-    assert (
-        preprocess("1324700000") == "bir milliard üç yüz yigirmi dört million yedi yüz biñ"
-    )
-    assert (
-        preprocess("1000002") == "bir million eki"
-    )
-    assert (
-        preprocess("16") == "on altı"
-    )
-    assert (
-        preprocess("001") == "sıfır sıfır bir"
-    )
-    assert (
-        preprocess("00") == "sıfır sıfır"
-    )
-    assert (
-        preprocess("10.02") == "on noqta sıfır eki"
-    )
-    assert (
-        preprocess("0.01") == "sıfır noqta sıfır bir"
-    )
-    assert (
-        preprocess("0,01") == "sıfır virgül sıfır bir"
-    )
-    assert (
-        preprocess("00,01") == "sıfır sıfır virgül sıfır bir"
-    )
-    assert (
-        preprocess("-10") == "minus on"
-    )
-    assert (
-        preprocess("+10") == "plüs on"
-    )
-    assert (
-        preprocess("+10.1400") == "plüs on noqta bir dört sıfır sıfır"
-    )
-    assert (
-        preprocess("-10.14156") == "minus on noqta bir dört bir beş altı"
+        preprocess("1324700000")
+        == "bir milliard üç yüz yigirmi dört million yedi yüz biñ"
     )
+    assert preprocess("1000002") == "bir million eki"
+    assert preprocess("16") == "on altı"
+    assert preprocess("001") == "sıfır sıfır bir"
+    assert preprocess("00") == "sıfır sıfır"
+    assert preprocess("10.02") == "on noqta sıfır eki"
+    assert preprocess("0.01") == "sıfır noqta sıfır bir"
+    assert preprocess("0,01") == "sıfır virgül sıfır bir"
+    assert preprocess("00,01") == "sıfır sıfır virgül sıfır bir"
+    assert preprocess("-10") == "minus on"
+    assert preprocess("+10") == "plüs on"
+    assert preprocess("+10.1400") == "plüs on noqta bir dört sıfır sıfır"
+    assert preprocess("-10.14156") == "minus on noqta bir dört bir beş altı"
+    assert preprocess("10,14156") == "on virgül bir dört bir beş altı"
     assert (
-        preprocess("10,14156") == "on virgül bir dört bir beş altı"
+        preprocess("1, 2, 3, 4, 5, 6,7") == "bir, eki, üç, dört, beş, altı virgül yedi"
     )
     assert (
-        preprocess("1, 2, 3, 4, 5, 6,7") == "bir virgül eki virgül"
+        preprocess("1. 2. 3. 4. 5. 6.7") == "bir. eki. üç. dört. beş. altı noqta yedi"
     )
+    assert preprocess("1,2,3,4,5,6,7") == "bir eki üç dört beş altı yedi"
     assert (
-        preprocess("1,2,3,4,5,6,7") == "on altı"
+        preprocess("1,2,3,4,5,6,74.3")
+        == "bir eki üç dört beş altı yetmiş dört noqta üç"
     )