docstr: General-purpose IPA-based tokenizer.
  Args:
      g2p: Grapheme to phoneme module, should be IpaG2p or some subclass thereof.
      locale: Locale used to determine default text processing logic and punctuation.
          Supports ["en-US", "de-DE", "es-ES", "fr-FR"]. Defaults to "en-US".
          Specify None if implementing custom logic for a new locale.
      punct: Whether to reserve grapheme for basic punctuation or not.
      non_default_punct_list: List of punctuation marks which will be used instead default, if any.
      fixed_vocab: List of valid grapheme/phoneme tokens for the model.
          Set only if overriding the default vocab generation process (reading from G2P dict).
          If set, any dataset entries that have unincluded graphemes will be filtered out, and any words whose
          pronunciations have unincluded phonemes will be treated as OOV.
          Please make sure that the grapheme prefixes and cases are consistent with the G2P module's settings.
          Defaults to None, which means default vocab generation is used.
      space: Space token as string.
      silence: Silence token as string (will be disabled if it is None).
      apostrophe: Whether to use apostrophe or not.
      oov: OOV token as string.
      sep: Separation token as string.
      add_blank_at: Add blank to labels in the specified order ("last") or after tokens (any non None),
          if None then no blank in labels.
      pad_with_space: Whether to pad text with spaces at the beginning and at the end or not.
docstr_tok: ["General-purpose","IPA-based","tokenizer",".","Args",":","g2p",":","Grapheme","to","phoneme","module",",","should","be","IpaG2p","or","some","subclass","thereof",".","locale",":","Locale","used","to","determine","default","text","processing","logic","and","punctuation",".","Supports","[","``","en-US","''",",","``","de-DE","''",",","``","es-ES","''",",","``","fr-FR","''","]",".","Defaults","to","``","en-US","''",".","Specify","None","if","implementing","custom","logic","for","a","new","locale",".","punct",":","Whether","to","reserve","grapheme","for","basic","punctuation","or","not",".","non_default_punct_list",":","List","of","punctuation","marks","which","will","be","used","instead","default",",","if","any",".","fixed_vocab",":","List","of","valid","grapheme\/phoneme","tokens","for","the","model",".","Set","only","if","overriding","the","default","vocab","generation","process","(","reading","from","G2P","dict",")",".","If","set",",","any","dataset","entries","that","have","unincluded","graphemes","will","be","filtered","out",",","and","any","words","whose","pronunciations","have","unincluded","phonemes","will","be","treated","as","OOV",".","Please","make","sure","that","the","grapheme","prefixes","and","cases","are","consistent","with","the","G2P","module","'s","settings",".","Defaults","to","None",",","which","means","default","vocab","generation","is","used",".","space",":","Space","token","as","string",".","silence",":","Silence","token","as","string","(","will","be","disabled","if","it","is","None",")",".","apostrophe",":","Whether","to","use","apostrophe","or","not",".","oov",":","OOV","token","as","string",".","sep",":","Separation","token","as","string",".","add_blank_at",":","Add","blank","to","labels","in","the","specified","order","(","``","last","''",")","or","after","tokens","(","any","non","None",")",",","if","None","then","no","blank","in","labels",".","pad_with_space",":","Whether","to","pad","text","with","spaces","at","the","beginning","and","at","the","end","or","not","."]
code: def __init__(
      self,
      g2p,
      locale="en-US",
      punct=True,
      non_default_punct_list=None,
      fixed_vocab=None,
      *,
      space=" ",
      silence=None,
      apostrophe=False,
      oov=BaseTokenizer.OOV,
      sep="|",  # To be able to distinguish between symbols
      add_blank_at=None,
      pad_with_space=False,
  ):
      """General-purpose IPA-based tokenizer.
      Args:
          g2p: Grapheme to phoneme module, should be IpaG2p or some subclass thereof.
          locale: Locale used to determine default text processing logic and punctuation.
              Supports ["en-US", "de-DE", "es-ES", "fr-FR"]. Defaults to "en-US".
              Specify None if implementing custom logic for a new locale.
          punct: Whether to reserve grapheme for basic punctuation or not.
          non_default_punct_list: List of punctuation marks which will be used instead default, if any.
          fixed_vocab: List of valid grapheme/phoneme tokens for the model.
              Set only if overriding the default vocab generation process (reading from G2P dict).
              If set, any dataset entries that have unincluded graphemes will be filtered out, and any words whose
              pronunciations have unincluded phonemes will be treated as OOV.
              Please make sure that the grapheme prefixes and cases are consistent with the G2P module's settings.
              Defaults to None, which means default vocab generation is used.
          space: Space token as string.
          silence: Silence token as string (will be disabled if it is None).
          apostrophe: Whether to use apostrophe or not.
          oov: OOV token as string.
          sep: Separation token as string.
          add_blank_at: Add blank to labels in the specified order ("last") or after tokens (any non None),
              if None then no blank in labels.
          pad_with_space: Whether to pad text with spaces at the beginning and at the end or not.
      """
      if not hasattr(g2p, "symbols"):
          logging.error(
              f"Please make sure the G2P module passed into the IPATokenizer has a `symbols` attribute. "
              f"This is required in order to build the tokenizer vocabulary.\n"
              f"Expected e.g. IpaG2p, found {type(g2p)}"
          )
          raise ValueError(
              "G2P modules passed into the IPATokenizer must have `symbols` defined."
          )
  
      if locale is not None:
          validate_locale(locale)
  
      self.phoneme_probability = None
      if hasattr(g2p, "phoneme_probability"):
          self.phoneme_probability = g2p.phoneme_probability
  
      if locale == "en-US":
          self.text_preprocessing_func = (
              lambda text: english_text_preprocessing(text, lower=False)
          )
      else:
          self.text_preprocessing_func = any_locale_text_preprocessing
  
      # Build tokens list if fixed_vocab isn't set
      if fixed_vocab:
          tokens = {
              self.text_preprocessing_func(c) for c in fixed_vocab
          }
          self.set_fixed_vocab = True  # Used to check whether dataset entries need filtering
  
          if g2p.symbols == tokens:
              logging.info(
                  "Did not replace G2P valid symbol set since the given set is equivalent to the existing one."
              )
              self.set_fixed_vocab = False
          else:
              g2p.replace_symbols(tokens)
      else:
          tokens = set(g2p.symbols)
          self.set_fixed_vocab = False
  
      if apostrophe:
          tokens.add("'")
  
      if punct:
          if non_default_punct_list is not None:
              self.punct_list = non_default_punct_list
          else:
              self.punct_list = get_ipa_punctuation_list(locale)
  
          tokens.update(self.punct_list)
  
      # Sort to ensure that vocab is in the same order every time
      tokens = sorted(list(tokens))
  
      if space in g2p.symbols:
          self.space = tokens.index(space)
      else:
          self.space, tokens = len(tokens), tokens + [space]
  
      if silence is not None:
          self.silence, tokens = len(tokens), tokens + [silence]
  
      super().__init__(
          tokens, oov=oov, sep=sep, add_blank_at=add_blank_at
      )
  
      self.tokens_set = set(
          self.tokens
      )  # To save some repeated work when filtering entries
  
      self.punct = punct
      self.pad_with_space = pad_with_space
  
      self.g2p = g2p
  
imports: import itertools
  import string
  from abc import ABC, abstractmethod
  from contextlib import contextmanager
  from typing import List, Optional
  from nemo.collections.common.tokenizers.text_to_speech.ipa_lexicon import get_grapheme_character_set, get_ipa_punctuation_list, validate_locale
  from nemo.collections.common.tokenizers.text_to_speech.tokenizer_utils import any_locale_text_preprocessing, chinese_text_preprocessing, english_text_preprocessing, french_text_preprocessing, italian_text_preprocessing, spanish_text_preprocessing
  from nemo.utils import logging
  from nemo.utils.decorators import experimental
label_desc: Use image node_id 1 to create a new IPATokenizer object from inherited base classes: BaseTokenizer with example: obj = IPATokenizer(g2p, locale, punct, non_default_punct_list, fixed_vocab)