docstr: Generic IPA G2P module. This module converts words from graphemes to International Phonetic Alphabet
  representations. Optionally, it can ignore heteronyms, ambiguous words, or words marked as unchangeable
  by `word_tokenize_func` (see code for details). Ignored words are left unchanged or passed through
  `apply_to_oov_word` for handling.
  
  Args:
      phoneme_dict (str, Path, or Dict): Path to file in CMUdict format or an IPA dict object with CMUdict-like
          entries. For example,
          a dictionary file: scripts/tts_dataset_files/ipa_cmudict-0.7b_nv22.06.txt;
          a dictionary object: {..., "Wire": [["ˈ", "w", "a", "ɪ", "ɚ"], ["ˈ", "w", "a", "ɪ", "ɹ"]], ...}.
      locale (str): Locale used to determine a locale-specific tokenization logic. Currently, it supports "en-US",
          "de-DE", and "es-ES". Defaults to "en-US". Specify None if implementing custom logic for a new locale.
      apply_to_oov_word (Callable): Function that deals with the out-of-vocabulary (OOV) words that do not exist
          in the `phoneme_dict`.
      ignore_ambiguous_words (bool): Whether to handle word via phoneme_dict with ambiguous phoneme sequences.
          Defaults to True.
      heteronyms (str, Path, List[str]): Path to file that includes heteronyms (one word entry per line), or a
          list of words.
      use_chars (bool): Whether to include chars/graphemes in the token list. It is True if `phoneme_probability`
          is not None or if `apply_to_oov_word` function ever returns graphemes.
      phoneme_probability (Optional[float]): The probability (0.0 <= ε <= 1.0) that is used to balance the action
          that a word in a sentence is whether transliterated into a sequence of phonemes, or kept as a sequence
          of graphemes. If a random number for a word is greater than ε, then the word is kept as graphemes;
          otherwise, the word is transliterated as phonemes. Defaults to None which is equivalent to setting it
          to 1.0, meaning always transliterating the word into phonemes. Note that this code path is only run if
          the word can be transliterated into phonemes, otherwise, if a word does not have an entry in the g2p
          dict, it will be kept as graphemes. If a word has multiple pronunciations as shown in the g2p dict and
          `ignore_ambiguous_words` is True, it will be kept as graphemes as well.
      use_stresses (Optional[bool]): Whether to include the stress symbols (ˈ and ˌ).
      grapheme_case (Optional[str]): Trigger converting all graphemes to uppercase, lowercase, or keeping them as
          original mix-cases. You may want to use this feature to distinguish the grapheme set from the phoneme
          set if there is an overlap in between. Defaults to `upper` because phoneme set only uses lowercase
          symbols. You could explicitly prepend `grapheme_prefix` to distinguish them.
      grapheme_prefix (Optional[str]): Prepend a special symbol to any graphemes in order to distinguish graphemes
          from phonemes because there may be overlaps between the two set. It is suggested to choose a prefix that
          is not used or preserved somewhere else. "#" could be a good candidate. Default to "".
      TODO @borisfom: add docstring for newly added `mapping_file` argument.
docstr_tok: ["Generic","IPA","G2P","module",".","This","module","converts","words","from","graphemes","to","International","Phonetic","Alphabet","representations",".","Optionally",",","it","can","ignore","heteronyms",",","ambiguous","words",",","or","words","marked","as","unchangeable","by","`","word_tokenize_func","`","(","see","code","for","details",")",".","Ignored","words","are","left","unchanged","or","passed","through","`","apply_to_oov_word","`","for","handling",".","Args",":","phoneme_dict","(","str",",","Path",",","or","Dict",")",":","Path","to","file","in","CMUdict","format","or","an","IPA","dict","object","with","CMUdict-like","entries",".","For","example",",","a","dictionary","file",":","scripts\/tts_dataset_files\/ipa_cmudict-0.7b_nv22.06.txt",";","a","dictionary","object",":","{","...",",","``","Wire","''",":","[","[","``","\u02c8","''",",","``","w","''",",","``","a","''",",","``","\u026a","''",",","``","\u025a","''","]",",","[","``","\u02c8","''",",","``","w","''",",","``","a","''",",","``","\u026a","''",",","``","\u0279","''","]","]",",","...","}",".","locale","(","str",")",":","Locale","used","to","determine","a","locale-specific","tokenization","logic",".","Currently",",","it","supports","``","en-US","''",",","``","de-DE","''",",","and","``","es-ES","''",".","Defaults","to","``","en-US","''",".","Specify","None","if","implementing","custom","logic","for","a","new","locale",".","apply_to_oov_word","(","Callable",")",":","Function","that","deals","with","the","out-of-vocabulary","(","OOV",")","words","that","do","not","exist","in","the","`","phoneme_dict","`",".","ignore_ambiguous_words","(","bool",")",":","Whether","to","handle","word","via","phoneme_dict","with","ambiguous","phoneme","sequences",".","Defaults","to","True",".","heteronyms","(","str",",","Path",",","List","[","str","]",")",":","Path","to","file","that","includes","heteronyms","(","one","word","entry","per","line",")",",","or","a","list","of","words",".","use_chars","(","bool",")",":","Whether","to","include","chars\/graphemes","in","the","token","list",".","It","is","True","if","`","phoneme_probability","`","is","not","None","or","if","`","apply_to_oov_word","`","function","ever","returns","graphemes",".","phoneme_probability","(","Optional","[","float","]",")",":","The","probability","(","0.0","<","=","\u03b5","<","=","1.0",")","that","is","used","to","balance","the","action","that","a","word","in","a","sentence","is","whether","transliterated","into","a","sequence","of","phonemes",",","or","kept","as","a","sequence","of","graphemes",".","If","a","random","number","for","a","word","is","greater","than","\u03b5",",","then","the","word","is","kept","as","graphemes",";","otherwise",",","the","word","is","transliterated","as","phonemes",".","Defaults","to","None","which","is","equivalent","to","setting","it","to","1.0",",","meaning","always","transliterating","the","word","into","phonemes",".","Note","that","this","code","path","is","only","run","if","the","word","can","be","transliterated","into","phonemes",",","otherwise",",","if","a","word","does","not","have","an","entry","in","the","g2p","dict",",","it","will","be","kept","as","graphemes",".","If","a","word","has","multiple","pronunciations","as","shown","in","the","g2p","dict","and","`","ignore_ambiguous_words","`","is","True",",","it","will","be","kept","as","graphemes","as","well",".","use_stresses","(","Optional","[","bool","]",")",":","Whether","to","include","the","stress","symbols","(","\u02c8","and","\u02cc",")",".","grapheme_case","(","Optional","[","str","]",")",":","Trigger","converting","all","graphemes","to","uppercase",",","lowercase",",","or","keeping","them","as","original","mix-cases",".","You","may","want","to","use","this","feature","to","distinguish","the","grapheme","set","from","the","phoneme","set","if","there","is","an","overlap","in","between",".","Defaults","to","`","upper","`","because","phoneme","set","only","uses","lowercase","symbols",".","You","could","explicitly","prepend","`","grapheme_prefix","`","to","distinguish","them",".","grapheme_prefix","(","Optional","[","str","]",")",":","Prepend","a","special","symbol","to","any","graphemes","in","order","to","distinguish","graphemes","from","phonemes","because","there","may","be","overlaps","between","the","two","set",".","It","is","suggested","to","choose","a","prefix","that","is","not","used","or","preserved","somewhere","else",".","``","#","''","could","be","a","good","candidate",".","Default","to","``","''",".","TODO","@","borisfom",":","add","docstring","for","newly","added","`","mapping_file","`","argument","."]
code: def __init__(
      self,
      phoneme_dict: Union[
          str, pathlib.Path, Dict[str, List[List[str]]]
      ],
      locale: str = "en-US",
      apply_to_oov_word: Optional[Callable[[str], str]] = None,
      ignore_ambiguous_words: bool = True,
      heteronyms: Optional[Union[str, pathlib.Path, List[str]]] = None,
      use_chars: bool = False,
      phoneme_probability: Optional[float] = None,
      use_stresses: Optional[bool] = True,
      grapheme_case: Optional[str] = GRAPHEME_CASE_UPPER,
      grapheme_prefix: Optional[str] = "",
      mapping_file: Optional[str] = None,
  ) -> None:
      """
      Generic IPA G2P module. This module converts words from graphemes to International Phonetic Alphabet
      representations. Optionally, it can ignore heteronyms, ambiguous words, or words marked as unchangeable
      by `word_tokenize_func` (see code for details). Ignored words are left unchanged or passed through
      `apply_to_oov_word` for handling.
  
      Args:
          phoneme_dict (str, Path, or Dict): Path to file in CMUdict format or an IPA dict object with CMUdict-like
              entries. For example,
              a dictionary file: scripts/tts_dataset_files/ipa_cmudict-0.7b_nv22.06.txt;
              a dictionary object: {..., "Wire": [["ˈ", "w", "a", "ɪ", "ɚ"], ["ˈ", "w", "a", "ɪ", "ɹ"]], ...}.
          locale (str): Locale used to determine a locale-specific tokenization logic. Currently, it supports "en-US",
              "de-DE", and "es-ES". Defaults to "en-US". Specify None if implementing custom logic for a new locale.
          apply_to_oov_word (Callable): Function that deals with the out-of-vocabulary (OOV) words that do not exist
              in the `phoneme_dict`.
          ignore_ambiguous_words (bool): Whether to handle word via phoneme_dict with ambiguous phoneme sequences.
              Defaults to True.
          heteronyms (str, Path, List[str]): Path to file that includes heteronyms (one word entry per line), or a
              list of words.
          use_chars (bool): Whether to include chars/graphemes in the token list. It is True if `phoneme_probability`
              is not None or if `apply_to_oov_word` function ever returns graphemes.
          phoneme_probability (Optional[float]): The probability (0.0 <= ε <= 1.0) that is used to balance the action
              that a word in a sentence is whether transliterated into a sequence of phonemes, or kept as a sequence
              of graphemes. If a random number for a word is greater than ε, then the word is kept as graphemes;
              otherwise, the word is transliterated as phonemes. Defaults to None which is equivalent to setting it
              to 1.0, meaning always transliterating the word into phonemes. Note that this code path is only run if
              the word can be transliterated into phonemes, otherwise, if a word does not have an entry in the g2p
              dict, it will be kept as graphemes. If a word has multiple pronunciations as shown in the g2p dict and
              `ignore_ambiguous_words` is True, it will be kept as graphemes as well.
          use_stresses (Optional[bool]): Whether to include the stress symbols (ˈ and ˌ).
          grapheme_case (Optional[str]): Trigger converting all graphemes to uppercase, lowercase, or keeping them as
              original mix-cases. You may want to use this feature to distinguish the grapheme set from the phoneme
              set if there is an overlap in between. Defaults to `upper` because phoneme set only uses lowercase
              symbols. You could explicitly prepend `grapheme_prefix` to distinguish them.
          grapheme_prefix (Optional[str]): Prepend a special symbol to any graphemes in order to distinguish graphemes
              from phonemes because there may be overlaps between the two set. It is suggested to choose a prefix that
              is not used or preserved somewhere else. "#" could be a good candidate. Default to "".
          TODO @borisfom: add docstring for newly added `mapping_file` argument.
      """
      self.use_stresses = use_stresses
      self.grapheme_case = grapheme_case
      self.grapheme_prefix = grapheme_prefix
      self.phoneme_probability = phoneme_probability
      self.locale = locale
      self._rng = random.Random()
  
      if locale is not None:
          validate_locale(locale)
  
      if not use_chars and self.phoneme_probability is not None:
          self.use_chars = True
          logging.warning(
              "phoneme_probability was not None, characters will be enabled even though "
              "use_chars was set to False."
          )
      else:
          self.use_chars = use_chars
  
      phoneme_dict_obj = self._parse_phoneme_dict(phoneme_dict)
  
      # verify if phoneme dict obj is empty
      if phoneme_dict_obj:
          _phoneme_dict, self.symbols = self._normalize_dict(
              phoneme_dict_obj
          )
      else:
          raise ValueError(f"{phoneme_dict} contains no entries!")
  
      if apply_to_oov_word is None:
          logging.warning(
              "apply_to_oov_word=None, This means that some of words will remain unchanged "
              "if they are not handled by any of the rules in self.parse_one_word(). "
              "This may be intended if phonemes and chars are both valid inputs, otherwise, "
              "you may see unexpected deletions in your input."
          )
  
      # word_tokenize_func returns a List[Tuple[List[str], bool]] where every tuple denotes
      # a word representation (a list tokens) and a flag indicating whether to process the word or
      # leave it unchanged.
      if locale == "en-US":
          word_tokenize_func = english_word_tokenize
      else:
          word_tokenize_func = any_locale_word_tokenize
  
      super().__init__(
          phoneme_dict=_phoneme_dict,
          word_tokenize_func=word_tokenize_func,
          apply_to_oov_word=apply_to_oov_word,
          mapping_file=mapping_file,
      )
  
      self.ignore_ambiguous_words = ignore_ambiguous_words
      if isinstance(heteronyms, str) or isinstance(
          heteronyms, pathlib.Path
      ):
          self.heteronyms = set(self._parse_file_by_lines(heteronyms))
      elif isinstance(heteronyms, list) and all(
          isinstance(het, str) for het in heteronyms
      ):
          self.heteronyms = set(heteronyms)
      else:
          self.heteronyms = None
  
      if self.heteronyms:
          self.heteronyms = {
              set_grapheme_case(het, case=self.grapheme_case)
              for het in self.heteronyms
          }
  
imports: import pathlib
  import random
  import re
  from collections import defaultdict
  from typing import Callable, Dict, List, Optional, Set, Tuple, Union
  from nemo.collections.common.tokenizers.text_to_speech.ipa_lexicon import validate_locale
  from nemo.collections.common.tokenizers.text_to_speech.tokenizer_utils import LATIN_CHARS_ALL, any_locale_word_tokenize, english_word_tokenize, normalize_unicode_text
  from nemo.collections.tts.g2p.models.base import BaseG2p
  from nemo.collections.tts.g2p.utils import GRAPHEME_CASE_MIXED, GRAPHEME_CASE_UPPER, set_grapheme_case
  from nemo.utils import logging
  from nemo.utils.decorators import experimental
label_desc: Use image node_id 1 to create a new IpaG2p object from inherited base classes: BaseG2p with example: obj = IpaG2p(phoneme_dict, locale, apply_to_oov_word, ignore_ambiguous_words, heteronyms, use_chars, phoneme_probability, use_stresses, grapheme_case, grapheme_prefix, mapping_file)