docstr: Chinese phoneme-based tokenizer.
  Note: This tokenizer for now covers Chinese phonemes/tones and English letters because our dataset contains
        both Chinese and English graphemes.
  Args:
      g2p: Grapheme to phoneme module.
      punct: Whether to reserve grapheme for basic punctuation or not.
      non_default_punct_list: List of punctuation marks which will be used instead default.
      space: Space token as string.
      silence: Silence token as string (will be disabled if it is None).
      apostrophe: Whether to use apostrophe or not.
      sep: Separation token as string.
      add_blank_at: Add blank to labels in the specified order ("last") or after tokens (any non None),
       if None then no blank in labels.
      pad_with_space: Whether to pad text with spaces at the beginning and at the end or not.
      text_preprocessing_func: Text preprocessing function for correct execution of the tokenizer.
       Basically, it replaces all non-unicode characters with unicode ones.
       Note that lower() function shouldn't be applied here, in case the text contains phonemes (it will be handled by g2p).
docstr_tok: ["Chinese","phoneme-based","tokenizer",".","Note",":","This","tokenizer","for","now","covers","Chinese","phonemes\/tones","and","English","letters","because","our","dataset","contains","both","Chinese","and","English","graphemes",".","Args",":","g2p",":","Grapheme","to","phoneme","module",".","punct",":","Whether","to","reserve","grapheme","for","basic","punctuation","or","not",".","non_default_punct_list",":","List","of","punctuation","marks","which","will","be","used","instead","default",".","space",":","Space","token","as","string",".","silence",":","Silence","token","as","string","(","will","be","disabled","if","it","is","None",")",".","apostrophe",":","Whether","to","use","apostrophe","or","not",".","sep",":","Separation","token","as","string",".","add_blank_at",":","Add","blank","to","labels","in","the","specified","order","(","``","last","''",")","or","after","tokens","(","any","non","None",")",",","if","None","then","no","blank","in","labels",".","pad_with_space",":","Whether","to","pad","text","with","spaces","at","the","beginning","and","at","the","end","or","not",".","text_preprocessing_func",":","Text","preprocessing","function","for","correct","execution","of","the","tokenizer",".","Basically",",","it","replaces","all","non-unicode","characters","with","unicode","ones",".","Note","that","lower","(",")","function","should","n't","be","applied","here",",","in","case","the","text","contains","phonemes","(","it","will","be","handled","by","g2p",")","."]
code: def __init__(
      self,
      g2p,
      punct=True,
      non_default_punct_list=None,
      *,
      space=" ",
      silence=None,
      apostrophe=True,
      sep="|",  # To be able to distinguish between 2/3 letters codes.
      add_blank_at=None,
      pad_with_space=False,
      text_preprocessing_func=chinese_text_preprocessing,
  ):
      """Chinese phoneme-based tokenizer.
      Note: This tokenizer for now covers Chinese phonemes/tones and English letters because our dataset contains
            both Chinese and English graphemes.
      Args:
          g2p: Grapheme to phoneme module.
          punct: Whether to reserve grapheme for basic punctuation or not.
          non_default_punct_list: List of punctuation marks which will be used instead default.
          space: Space token as string.
          silence: Silence token as string (will be disabled if it is None).
          apostrophe: Whether to use apostrophe or not.
          sep: Separation token as string.
          add_blank_at: Add blank to labels in the specified order ("last") or after tokens (any non None),
           if None then no blank in labels.
          pad_with_space: Whether to pad text with spaces at the beginning and at the end or not.
          text_preprocessing_func: Text preprocessing function for correct execution of the tokenizer.
           Basically, it replaces all non-unicode characters with unicode ones.
           Note that lower() function shouldn't be applied here, in case the text contains phonemes (it will be handled by g2p).
      """
      tokens = []
      self.space, tokens = len(tokens), tokens + [space]  # Space
  
      if silence is not None:
          self.silence, tokens = len(tokens), tokens + [
              silence
          ]  # Silence
  
      self.phoneme_list = g2p.phoneme_list
      self.tone_list = g2p.tone_list
      self.ascii_letter_list = g2p.ascii_letter_list
  
      tokens.extend(self.phoneme_list)
      tokens.extend(self.tone_list)
      tokens.extend(self.ascii_letter_list)
  
      self.text_preprocessing_func = text_preprocessing_func
  
      if apostrophe:
          tokens.append("'")  # Apostrophe
  
      if punct:
          if non_default_punct_list is not None:
              self.PUNCT_LIST = non_default_punct_list
          else:
              self.PUNCT_LIST = list(self.ZH_PUNCT_LIST)
          tokens.extend(self.PUNCT_LIST)
  
      super().__init__(tokens, sep=sep, add_blank_at=add_blank_at)
  
      self.punct = punct
      self.pad_with_space = pad_with_space
      self.g2p = g2p
  
imports: import itertools
  import string
  from abc import ABC, abstractmethod
  from contextlib import contextmanager
  from typing import List, Optional
  from nemo.collections.common.tokenizers.text_to_speech.ipa_lexicon import get_grapheme_character_set, get_ipa_punctuation_list, validate_locale
  from nemo.collections.common.tokenizers.text_to_speech.tokenizer_utils import any_locale_text_preprocessing, chinese_text_preprocessing, english_text_preprocessing, french_text_preprocessing, italian_text_preprocessing, spanish_text_preprocessing
  from nemo.utils import logging
  from nemo.utils.decorators import experimental
label_desc: Use image node_id 1 to create a new ChinesePhonemesTokenizer object from inherited base classes: BaseTokenizer with example: obj = ChinesePhonemesTokenizer(g2p, punct, non_default_punct_list)