class_docstr: Base class for all slow tokenizers.
  
  Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].
  
  Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
  pretrained tokenizers as well as adding tokens to the vocabulary.
  
  This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","slow","tokenizers",".","Inherits","from","[","`","~tokenization_utils_base.PreTrainedTokenizerBase","`","]",".","Handle","all","the","shared","methods","for","tokenization","and","special","tokens","as","well","as","methods","downloading\/caching\/loading","pretrained","tokenizers","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contain","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
docstr: Converts a string into a sequence of tokens, using the tokenizer.
  
  Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
  (BPE/SentencePieces/WordPieces). Takes care of added tokens.
  
  Args:
      text (`str`):
          The sequence to be encoded.
      **kwargs (additional keyword arguments):
          Passed along to the model-specific `prepare_for_tokenization` preprocessing method.
  
  Returns:
      `List[str]`: The list of tokens.
docstr_tok: ["Converts","a","string","into","a","sequence","of","tokens",",","using","the","tokenizer",".","Split","in","words","for","word-based","vocabulary","or","sub-words","for","sub-word-based","vocabularies","(","BPE\/SentencePieces\/WordPieces",")",".","Takes","care","of","added","tokens",".","Args",":","text","(","`","str","`",")",":","The","sequence","to","be","encoded",".","*","*","kwargs","(","additional","keyword","arguments",")",":","Passed","along","to","the","model-specific","`","prepare_for_tokenization","`","preprocessing","method",".","Returns",":","`","List","[","str","]","`",":","The","list","of","tokens","."]
code: def tokenize(self, text: TextInput, **kwargs) -> List[str]:
      """
      Converts a string into a sequence of tokens, using the tokenizer.
  
      Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
      (BPE/SentencePieces/WordPieces). Takes care of added tokens.
  
      Args:
          text (`str`):
              The sequence to be encoded.
          **kwargs (additional keyword arguments):
              Passed along to the model-specific `prepare_for_tokenization` preprocessing method.
  
      Returns:
          `List[str]`: The list of tokens.
      """
      split_special_tokens = kwargs.pop(
          "split_special_tokens", self.split_special_tokens
      )
  
      text, kwargs = self.prepare_for_tokenization(text, **kwargs)
  
      if kwargs:
          logger.warning(f"Keyword arguments {kwargs} not recognized.")
  
      if hasattr(self, "do_lower_case") and self.do_lower_case:
          # convert non-special tokens to lowercase. Might be super slow as well?
          escaped_special_toks = [
              re.escape(s_tok) for s_tok in (self.all_special_tokens)
          ]
          escaped_special_toks += [
              re.escape(s_tok.content)
              for s_tok in (self._added_tokens_decoder.values())
              if not s_tok.special and s_tok.normalized
          ]
          pattern = (
              r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
          )
          text = re.sub(
              pattern,
              lambda m: m.groups()[0] or m.groups()[1].lower(),
              text,
          )
  
      if split_special_tokens:
          no_split_token = []
          tokens = [text]
      else:
          no_split_token = (
              self._added_tokens_encoder.keys()
          )  # don't split on any of the added tokens
          # "This is something<special_token_1>  else"
          tokens = self.tokens_trie.split(text)
  
      # ["This is something", "<special_token_1>", "  else"]
      for i, token in enumerate(tokens):
          if token in no_split_token:
              tok_extended = self._added_tokens_decoder.get(
                  self._added_tokens_encoder[token], None
              )
              left = tokens[i - 1] if i > 0 else None
              right = tokens[i + 1] if i < len(tokens) - 1 else None
              if isinstance(tok_extended, AddedToken):
                  if tok_extended.rstrip and right:
                      # A bit counter-intuitive but we strip the left of the string
                      # since tok_extended.rstrip means the special token is eating all white spaces on its right
                      tokens[i + 1] = right.lstrip()
                  # Strip white spaces on the left
                  if tok_extended.lstrip and left:
                      tokens[i - 1] = left.rstrip()  # Opposite here
                  if (
                      tok_extended.single_word
                      and left
                      and left[-1] != " "
                  ):
                      tokens[i - 1] += token
                      tokens[i] = ""
                  elif (
                      tok_extended.single_word
                      and right
                      and right[0] != " "
                  ):
                      tokens[i + 1] = token + tokens[i + 1]
                      tokens[i] = ""
              else:
                  raise ValueError(
                      f"{tok_extended} cannot be tokenized because it was not properly added"
                      f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"
                  )
      # ["This is something", "<special_token_1>", "else"]
      tokenized_text = []
      for token in tokens:
          # Need to skip eventual empty (fully stripped) tokens
          if not token:
              continue
          if token in no_split_token:
              tokenized_text.append(token)
          else:
              tokenized_text.extend(self._tokenize(token))
      # ["This", " is", " something", "<special_token_1>", "else"]
      return tokenized_text
  
imports: import bisect
  import itertools
  import re
  import unicodedata
  from collections import OrderedDict
  from typing import Any, Dict, List, Optional, Tuple, Union, overload
  from .tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING, INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, EncodedInputPair, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, TextInput, TextInputPair, TruncationStrategy
  from .utils import PaddingStrategy, TensorType, add_end_docstrings, logging
label_desc: Use image node_id 12 for calling the PreTrainedTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: tokenized_text