class_docstr: Base class for all slow tokenizers.
  
  Inherits from :class:`~transformers.tokenization_utils_base.PreTrainedTokenizerBase`.
  
  Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
  pretrained tokenizers as well as adding tokens to the vocabulary.
  
  This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","slow","tokenizers",".","Inherits","from",":","class",":","`","~transformers.tokenization_utils_base.PreTrainedTokenizerBase","`",".","Handle","all","the","shared","methods","for","tokenization","and","special","tokens","as","well","as","methods","downloading\/caching\/loading","pretrained","tokenizers","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contain","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
docstr: Converts a string in a sequence of tokens, using the tokenizer.
  
  Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
  (BPE/SentencePieces/WordPieces). Takes care of added tokens.
  
  Args:
      text (:obj:`str`):
          The sequence to be encoded.
      **kwargs (additional keyword arguments):
          Passed along to the model-specific ``prepare_for_tokenization`` preprocessing method.
  
  Returns:
      :obj:`List[str]`: The list of tokens.
docstr_tok: ["Converts","a","string","in","a","sequence","of","tokens",",","using","the","tokenizer",".","Split","in","words","for","word-based","vocabulary","or","sub-words","for","sub-word-based","vocabularies","(","BPE\/SentencePieces\/WordPieces",")",".","Takes","care","of","added","tokens",".","Args",":","text","(",":","obj",":","`","str","`",")",":","The","sequence","to","be","encoded",".","*","*","kwargs","(","additional","keyword","arguments",")",":","Passed","along","to","the","model-specific","``","prepare_for_tokenization","``","preprocessing","method",".","Returns",":",":","obj",":","`","List","[","str","]","`",":","The","list","of","tokens","."]
code: def tokenize(self, text: TextInput, **kwargs) -> List[str]:
      """
      Converts a string in a sequence of tokens, using the tokenizer.
  
      Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
      (BPE/SentencePieces/WordPieces). Takes care of added tokens.
  
      Args:
          text (:obj:`str`):
              The sequence to be encoded.
          **kwargs (additional keyword arguments):
              Passed along to the model-specific ``prepare_for_tokenization`` preprocessing method.
  
      Returns:
          :obj:`List[str]`: The list of tokens.
      """
      # Simple mapping string => AddedToken for special tokens with specific tokenization behaviors
      all_special_tokens_extended = dict(
          (str(t), t)
          for t in self.all_special_tokens_extended
          if isinstance(t, AddedToken)
      )
  
      text, kwargs = self.prepare_for_tokenization(text, **kwargs)
  
      if kwargs:
          logger.warning(f"Keyword arguments {kwargs} not recognized.")
  
      # TODO: should this be in the base class?
      if hasattr(self, "do_lower_case") and self.do_lower_case:
          # convert non-special tokens to lowercase
          escaped_special_toks = [
              re.escape(s_tok)
              for s_tok in (
                  self.unique_no_split_tokens + self.all_special_tokens
              )
          ]
          pattern = (
              r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
          )
          text = re.sub(
              pattern,
              lambda m: m.groups()[0] or m.groups()[1].lower(),
              text,
          )
  
      no_split_token = set(self.unique_no_split_tokens)
      tokens = self.tokens_trie.split(text)
      # ["This is something", "<special_token_1>", "  else"]
      for i, token in enumerate(tokens):
          if token in no_split_token:
              tok_extended = all_special_tokens_extended.get(
                  token, None
              )
              left = tokens[i - 1] if i > 0 else None
              right = tokens[i + 1] if i < len(tokens) - 1 else None
              if isinstance(tok_extended, AddedToken):
                  if tok_extended.rstrip and right:
                      # A bit counter-intuitive but we strip the left of the string
                      # since tok_extended.rstrip means the special token is eating all white spaces on its right
                      tokens[i + 1] = right.lstrip()
                  # Strip white spaces on the left
                  if tok_extended.lstrip and left:
                      tokens[i - 1] = left.rstrip()  # Opposite here
              else:
                  # We strip left and right by default
                  if right:
                      tokens[i + 1] = right.lstrip()
                  if left:
                      tokens[i - 1] = left.rstrip()
      # ["This is something", "<special_token_1>", "else"]
      tokenized_text = []
      for token in tokens:
          # Need to skip eventual empty (fully stripped) tokens
          if not token:
              continue
          if token in no_split_token:
              tokenized_text.append(token)
          else:
              tokenized_text.extend(self._tokenize(token))
      # ["This", " is", " something", "<special_token_1>", "else"]
      return tokenized_text
  
imports: import bisect
  import itertools
  import re
  import unicodedata
  from collections import OrderedDict
  from typing import Any, Dict, List, Optional, Tuple, Union, overload
  from .file_utils import PaddingStrategy, TensorType, add_end_docstrings
  from .tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING, INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, EncodedInputPair, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, TextInput, TextInputPair, TruncationStrategy
  from fastNLP.core.log import logger
label_desc: Use image node_id 9 for calling the PreTrainedTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: tokenized_text