class_docstr: Base class for all slow tokenizers.
  
  Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].
  
  Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
  pretrained tokenizers as well as adding tokens to the vocabulary.
  
  This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","slow","tokenizers",".","Inherits","from","[","`","~tokenization_utils_base.PreTrainedTokenizerBase","`","]",".","Handle","all","the","shared","methods","for","tokenization","and","special","tokens","as","well","as","methods","downloading\/caching\/loading","pretrained","tokenizers","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contain","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
docstr: Returns the number of added tokens when encoding a sequence with special tokens.
  
  <Tip>
  
  This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
  this inside your training loop.
  
  </Tip>
  
  Args:
      pair (`bool`, *optional*, defaults to `False`):
          Whether the number of added tokens should be computed in the case of a sequence pair or a single
          sequence.
  
  Returns:
      `int`: Number of special tokens added to sequences.
docstr_tok: ["Returns","the","number","of","added","tokens","when","encoding","a","sequence","with","special","tokens",".","<","Tip",">","This","encodes","a","dummy","input","and","checks","the","number","of","added","tokens",",","and","is","therefore","not","efficient",".","Do","not","put","this","inside","your","training","loop",".","<","\/Tip",">","Args",":","pair","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","the","number","of","added","tokens","should","be","computed","in","the","case","of","a","sequence","pair","or","a","single","sequence",".","Returns",":","`","int","`",":","Number","of","special","tokens","added","to","sequences","."]
code: def num_special_tokens_to_add(self, pair: bool = False) -> int:
      """
      Returns the number of added tokens when encoding a sequence with special tokens.
  
      <Tip>
  
      This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
      this inside your training loop.
  
      </Tip>
  
      Args:
          pair (`bool`, *optional*, defaults to `False`):
              Whether the number of added tokens should be computed in the case of a sequence pair or a single
              sequence.
  
      Returns:
          `int`: Number of special tokens added to sequences.
      """
      token_ids_0 = []
      token_ids_1 = []
      return len(
          self.build_inputs_with_special_tokens(
              token_ids_0, token_ids_1 if pair else None
          )
      )
  
imports: import bisect
  import itertools
  import re
  import unicodedata
  from collections import OrderedDict
  from typing import Any, Dict, List, Optional, Tuple, Union, overload
  from .tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING, INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, EncodedInputPair, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, TextInput, TextInputPair, TruncationStrategy
  from .utils import PaddingStrategy, TensorType, add_end_docstrings, logging
label_desc: Use image node_id 8 for calling the PreTrainedTokenizer obj's underlying member method code with example usage: obj.num_special_tokens_to_add(pair) and returns: len