try to figure out how ZeroGPU works

InferenceInterfaces/ToucanTTSInterface.py

@@ -107,7 +107,7 @@ class ToucanTTSInterface(torch.nn.Module):
         self.set_accent_language(lang_id=lang_id)
 
     def set_phonemizer_language(self, lang_id):
-        self.text2phone = ArticulatoryCombinedTextFrontend(language=lang_id, add_silence_to_end=True)
+        self.text2phone.change_lang(language=lang_id, add_silence_to_end=True)
 
     def set_accent_language(self, lang_id):
         if lang_id in ['ajp', 'ajt', 'lak', 'lno', 'nul', 'pii', 'plj', 'slq', 'smd', 'snb', 'tpw', 'wya', 'zua', 'en-us', 'en-sc', 'fr-be', 'fr-sw', 'pt-br', 'spa-lat', 'vi-ctr', 'vi-so']:

Preprocessing/TextFrontend.py

@@ -567,6 +567,543 @@ class ArticulatoryCombinedTextFrontend:
         self.id_to_phone = {v: k for k, v in self.phone_to_id.items()}
         self.text_vector_to_phone_cache = dict()
 
+    def change_lang(self, language,
+                    use_explicit_eos=True,
+                    use_lexical_stress=True,
+                    silent=True,
+                    add_silence_to_end=True,
+                    use_word_boundaries=True):
+        self.language = language
+        self.use_explicit_eos = use_explicit_eos
+        self.use_stress = use_lexical_stress
+        self.add_silence_to_end = add_silence_to_end
+        self.use_word_boundaries = use_word_boundaries
+        from transphone.g2p import read_g2p
+
+        register_to_height = {
+            "˥": 5,
+            "˦": 4,
+            "˧": 3,
+            "˨": 2,
+            "˩": 1
+        }
+        self.rising_perms = list()
+        self.falling_perms = list()
+        self.peaking_perms = list()
+        self.dipping_perms = list()
+
+        for first_tone in ["˥", "˦", "˧", "˨", "˩"]:
+            for second_tone in ["˥", "˦", "˧", "˨", "˩"]:
+                if register_to_height[first_tone] > register_to_height[second_tone]:
+                    self.falling_perms.append(first_tone + second_tone)
+                else:
+                    self.rising_perms.append(first_tone + second_tone)
+                for third_tone in ["˥", "˦", "˧", "˨", "˩"]:
+                    if register_to_height[first_tone] > register_to_height[second_tone] < register_to_height[third_tone]:
+                        self.dipping_perms.append(first_tone + second_tone + third_tone)
+                    elif register_to_height[first_tone] < register_to_height[second_tone] > register_to_height[third_tone]:
+                        self.peaking_perms.append(first_tone + second_tone + third_tone)
+
+        if language == "eng":
+            self.g2p_lang = "en-us"  # English as spoken in USA
+            self.expand_abbreviations = english_text_expansion
+            self.phonemizer = "espeak"
+
+        elif language == "deu":
+            self.g2p_lang = "de"  # German
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ell":
+            self.g2p_lang = "el"  # Greek
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "spa":
+            self.g2p_lang = "es"  # Spanish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "fin":
+            self.g2p_lang = "fi"  # Finnish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "rus":
+            self.g2p_lang = "ru"  # Russian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "hun":
+            self.g2p_lang = "hu"  # Hungarian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "nld":
+            self.g2p_lang = "nl"  # Dutch
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "fra":
+            self.g2p_lang = "fr-fr"  # French
+            self.expand_abbreviations = remove_french_spacing
+            self.phonemizer = "espeak"
+
+        elif language == "ita":
+            self.g2p_lang = "it"  # Italian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "por":
+            self.g2p_lang = "pt"  # Portuguese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "pol":
+            self.g2p_lang = "pl"  # Polish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "cmn":
+            self.g2p_lang = "cmn"  # Mandarin
+            self.expand_abbreviations = convert_kanji_to_pinyin_mandarin
+            self.phonemizer = "dragonmapper"
+
+        elif language == "vie":
+            self.g2p_lang = "vi"  # Northern Vietnamese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ukr":
+            self.g2p_lang = "uk"  # Ukrainian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "pes":
+            self.g2p_lang = "fa"  # Western Farsi
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "afr":
+            self.g2p_lang = "af"  # Afrikaans
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "aln":
+            self.g2p_lang = "sq"  # Albanian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "amh":
+            self.g2p_lang = "am"  # Amharic
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "arb":
+            self.g2p_lang = "ar"  # Arabic
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "arg":
+            self.g2p_lang = "an"  # Aragonese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "hye":
+            self.g2p_lang = "hy"  # East Armenian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "hyw":
+            self.g2p_lang = "hyw"  # West Armenian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "azj":
+            self.g2p_lang = "az"  # Azerbaijani
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "bak":
+            self.g2p_lang = "ba"  # Bashkir
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "eus":
+            self.g2p_lang = "eu"  # Basque
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "bel":
+            self.g2p_lang = "be"  # Belarusian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ben":
+            self.g2p_lang = "bn"  # Bengali
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "bpy":
+            self.g2p_lang = "bpy"  # Bishnupriya Manipuri
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "bos":
+            self.g2p_lang = "bs"  # Bosnian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "bul":
+            self.g2p_lang = "bg"  # Bulgarian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "mya":
+            self.g2p_lang = "my"  # Burmese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "chr":
+            self.g2p_lang = "chr"  # Cherokee
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "yue":
+            self.g2p_lang = "yue"  # Chinese	Cantonese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "hak":
+            self.g2p_lang = "hak"  # Chinese	Hakka
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "haw":
+            self.g2p_lang = "haw"  # Hawaiian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "hrv":
+            self.g2p_lang = "hr"  # Croatian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ces":
+            self.g2p_lang = "cs"  # Czech
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "dan":
+            self.g2p_lang = "da"  # Danish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ekk":
+            self.g2p_lang = "et"  # Estonian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "gle":
+            self.g2p_lang = "ga"  # Gaelic	Irish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "gla":
+            self.g2p_lang = "gd"  # Gaelic	Scottish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "kat":
+            self.g2p_lang = "ka"  # Georgian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "kal":
+            self.g2p_lang = "kl"  # Greenlandic
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "guj":
+            self.g2p_lang = "gu"  # Gujarati
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "heb":
+            self.g2p_lang = "he"  # Hebrew
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "hin":
+            self.g2p_lang = "hi"  # Hindi
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "isl":
+            self.g2p_lang = "is"  # Icelandic
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ind":
+            self.g2p_lang = "id"  # Indonesian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "jpn":
+            self.g2p_lang = "ja"  # Japanese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "kan":
+            self.g2p_lang = "kn"  # Kannada
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "knn":
+            self.g2p_lang = "kok"  # Konkani
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "kor":
+            self.g2p_lang = "ko"  # Korean
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ckb":
+            self.g2p_lang = "ku"  # Kurdish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "kaz":
+            self.g2p_lang = "kk"  # Kazakh
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "kir":
+            self.g2p_lang = "ky"  # Kyrgyz
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "lat":
+            self.g2p_lang = "la"  # Latin
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ltz":
+            self.g2p_lang = "lb"  # Luxembourgish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "lvs":
+            self.g2p_lang = "lv"  # Latvian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "lit":
+            self.g2p_lang = "lt"  # Lithuanian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "mri":
+            self.g2p_lang = "mi"  # Māori
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "mkd":
+            self.g2p_lang = "mk"  # Macedonian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "zlm":
+            self.g2p_lang = "ms"  # Malay
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "mal":
+            self.g2p_lang = "ml"  # Malayalam
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "mlt":
+            self.g2p_lang = "mt"  # Maltese
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "mar":
+            self.g2p_lang = "mr"  # Marathi
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "nci":
+            self.g2p_lang = "nci"  # Nahuatl
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "npi":
+            self.g2p_lang = "ne"  # Nepali
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "nob":
+            self.g2p_lang = "nb"  # Norwegian Bokmål
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "nog":
+            self.g2p_lang = "nog"  # Nogai
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ory":
+            self.g2p_lang = "or"  # Oriya
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "gaz":
+            self.g2p_lang = "om"  # Oromo
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "pap":
+            self.g2p_lang = "pap"  # Papiamento
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "pan":
+            self.g2p_lang = "pa"  # Punjabi
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "ron":
+            self.g2p_lang = "ro"  # Romanian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "lav":
+            self.g2p_lang = "ru-lv"  # Russian	Latvia
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "srp":
+            self.g2p_lang = "sr"  # Serbian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tsn":
+            self.g2p_lang = "tn"  # Setswana
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "snd":
+            self.g2p_lang = "sd"  # Sindhi
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "slk":
+            self.g2p_lang = "sk"  # Slovak
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "slv":
+            self.g2p_lang = "sl"  # Slovenian
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "smj":
+            self.g2p_lang = "smj"  # Lule Saami
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "swh":
+            self.g2p_lang = "sw"  # Swahili
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "swe":
+            self.g2p_lang = "sv"  # Swedish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tam":
+            self.g2p_lang = "ta"  # Tamil
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tha":
+            self.g2p_lang = "th"  # Thai
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tuk":
+            self.g2p_lang = "tk"  # Turkmen
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tat":
+            self.g2p_lang = "tt"  # Tatar
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tel":
+            self.g2p_lang = "te"  # Telugu
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "tur":
+            self.g2p_lang = "tr"  # Turkish
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "uig":
+            self.g2p_lang = "ug"  # Uyghur
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "urd":
+            self.g2p_lang = "ur"  # Urdu
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "uzn":
+            self.g2p_lang = "uz"  # Uzbek
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        elif language == "cym":
+            self.g2p_lang = "cy"  # Welsh
+            self.expand_abbreviations = lambda x: x
+            self.phonemizer = "espeak"
+
+        else:
+            # blanket solution for the rest
+            self.g2p_lang = language
+            self.phonemizer = "transphone"
+            self.expand_abbreviations = lambda x: x
+            self.transphone = read_g2p()
+
+        # remember to also update get_language_id() below when adding something here, as well as the get_example_sentence function
+
+        if self.phonemizer == "espeak":
+            try:
+                self.phonemizer_backend = EspeakBackend(language=self.g2p_lang,
+                                                        punctuation_marks=';:,.!?¡¿—…"«»“”~/。【】、‥،؟“”؛',
+                                                        preserve_punctuation=True,
+                                                        language_switch='remove-flags',
+                                                        with_stress=self.use_stress)
+            except RuntimeError:
+                print("Error in loading espeak! \n"
+                      "Maybe espeak is not installed on your system? \n"
+                      "Falling back to transphone.")
+                from transphone.g2p import read_g2p
+                self.g2p_lang = self.language
+                self.phonemizer = "transphone"
+                self.expand_abbreviations = lambda x: x
+                self.transphone = read_g2p()
+        self.phone_to_vector = generate_feature_table()
+        self.phone_to_id = get_phone_to_id()
+        self.id_to_phone = {v: k for k, v in self.phone_to_id.items()}
+        self.text_vector_to_phone_cache = dict()
+
     @staticmethod
     def get_example_sentence(lang):
         if lang == "eng":