class_docstr: Base class for all slow tokenizers.
  
  Inherits from :class:`~transformers.tokenization_utils_base.PreTrainedTokenizerBase`.
  
  Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
  pretrained tokenizers as well as adding tokens to the vocabulary.
  
  This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","slow","tokenizers",".","Inherits","from",":","class",":","`","~transformers.tokenization_utils_base.PreTrainedTokenizerBase","`",".","Handle","all","the","shared","methods","for","tokenization","and","special","tokens","as","well","as","methods","downloading\/caching\/loading","pretrained","tokenizers","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contain","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
docstr: Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
  vocabulary.
  
  Args:
      tokens (:obj:`str` or :obj:`List[str]`): One or several token(s) to convert to token id(s).
  
  Returns:
      :obj:`int` or :obj:`List[int]`: The token id or list of token ids.
docstr_tok: ["Converts","a","token","string","(","or","a","sequence","of","tokens",")","in","a","single","integer","id","(","or","a","sequence","of","ids",")",",","using","the","vocabulary",".","Args",":","tokens","(",":","obj",":","`","str","`","or",":","obj",":","`","List","[","str","]","`",")",":","One","or","several","token","(","s",")","to","convert","to","token","id","(","s",")",".","Returns",":",":","obj",":","`","int","`","or",":","obj",":","`","List","[","int","]","`",":","The","token","id","or","list","of","token","ids","."]
code: def convert_tokens_to_ids(
      self, tokens: Union[str, List[str]]
  ) -> Union[int, List[int]]:
      """
      Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
      vocabulary.
  
      Args:
          tokens (:obj:`str` or :obj:`List[str]`): One or several token(s) to convert to token id(s).
  
      Returns:
          :obj:`int` or :obj:`List[int]`: The token id or list of token ids.
      """
      if tokens is None:
          return None
  
      if isinstance(tokens, str):
          return self._convert_token_to_id_with_added_voc(tokens)
  
      ids = []
      for token in tokens:
          ids.append(self._convert_token_to_id_with_added_voc(token))
      return ids
  
imports: import bisect
  import itertools
  import re
  import unicodedata
  from collections import OrderedDict
  from typing import Any, Dict, List, Optional, Tuple, Union, overload
  from .file_utils import PaddingStrategy, TensorType, add_end_docstrings
  from .tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING, INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, EncodedInputPair, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, TextInput, TextInputPair, TruncationStrategy
  from fastNLP.core.log import logger
label_desc: Use image node_id 11 for calling the PreTrainedTokenizer obj's underlying member method code with example usage: obj.convert_tokens_to_ids(tokens) and returns: ids, None, self