docstr: Abstract class for creating an arbitrary tokenizer to convert string to list of int tokens.
  Args:
      tokens: List of tokens.
      pad: Pad token as string.
      blank: Blank token as string.
      oov: OOV token as string.
      sep: Separation token as string.
      add_blank_at: Add blank to labels in the specified order ("last") or after tokens (any non None),
       if None then no blank in labels.
docstr_tok: ["Abstract","class","for","creating","an","arbitrary","tokenizer","to","convert","string","to","list","of","int","tokens",".","Args",":","tokens",":","List","of","tokens",".","pad",":","Pad","token","as","string",".","blank",":","Blank","token","as","string",".","oov",":","OOV","token","as","string",".","sep",":","Separation","token","as","string",".","add_blank_at",":","Add","blank","to","labels","in","the","specified","order","(","``","last","''",")","or","after","tokens","(","any","non","None",")",",","if","None","then","no","blank","in","labels","."]
code: def __init__(
      self,
      tokens,
      *,
      pad=PAD,
      blank=BLANK,
      oov=OOV,
      sep="",
      add_blank_at=None,
  ):
      """Abstract class for creating an arbitrary tokenizer to convert string to list of int tokens.
      Args:
          tokens: List of tokens.
          pad: Pad token as string.
          blank: Blank token as string.
          oov: OOV token as string.
          sep: Separation token as string.
          add_blank_at: Add blank to labels in the specified order ("last") or after tokens (any non None),
           if None then no blank in labels.
      """
      super().__init__()
  
      tokens = list(tokens)
      # TODO @xueyang: in general, IDs of pad, sil, blank, and oov are preserved ahead instead of dynamically
      #  assigned according to the number of tokens. The downside of using dynamical assignment leads to different IDs
      #  for each.
      self.pad, tokens = len(tokens), tokens + [pad]  # Padding
  
      if add_blank_at is not None:
          self.blank, tokens = len(tokens), tokens + [
              blank
          ]  # Reserved for blank from asr-model
      else:
          # use add_blank_at=None only for ASR where blank is added automatically, disable blank here
          self.blank = None
  
      self.oov, tokens = len(tokens), tokens + [
          oov
      ]  # Out Of Vocabulary
  
      if add_blank_at == "last":
          tokens[-1], tokens[-2] = tokens[-2], tokens[-1]
          self.oov, self.blank = self.blank, self.oov
  
      self.tokens = tokens
      self.sep = sep
  
      self._util_ids = {self.pad, self.blank, self.oov}
      self._token2id = {l: i for i, l in enumerate(tokens)}
      self._id2token = tokens
  
imports: import itertools
  import string
  from abc import ABC, abstractmethod
  from contextlib import contextmanager
  from typing import List, Optional
  from nemo.collections.common.tokenizers.text_to_speech.ipa_lexicon import get_grapheme_character_set, get_ipa_punctuation_list, validate_locale
  from nemo.collections.common.tokenizers.text_to_speech.tokenizer_utils import any_locale_text_preprocessing, chinese_text_preprocessing, english_text_preprocessing, french_text_preprocessing, italian_text_preprocessing, spanish_text_preprocessing
  from nemo.utils import logging
  from nemo.utils.decorators import experimental
label_desc: Use image node_id 1 to create a new BaseTokenizer object from inherited base classes: ABC with example: obj = BaseTokenizer(tokens)