class_docstr: Construct a Perceiver tokenizer. The Perceiver simply uses raw bytes utf-8 encoding.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods.
  
  Args:
      pad_token (`str`, *optional*, defaults to `"[PAD]"`):
          The token used for padding, for example when batching sequences of different lengths.
      bos_token (`str`, *optional*, defaults to `"[BOS]"`):
          The BOS token (reserved in the vocab, but not actually used).
      eos_token (`str`, *optional*, defaults to `"[EOS]"`):
          The end of sequence token (reserved in the vocab, but not actually used).
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      mask_token (`str`, *optional*, defaults to `"[MASK]"`):
          The MASK token, useful for masked language modeling.
      cls_token (`str`, *optional*, defaults to `"[CLS]"`):
          The CLS token (reserved in the vocab, but not actually used).
      sep_token (`str`, *optional*, defaults to `"[SEP]"`):
          The separator token, which is used when building a sequence from two sequences.
class_docstr_tok: ["Construct","a","Perceiver","tokenizer",".","The","Perceiver","simply","uses","raw","bytes","utf-8","encoding",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","PAD","]","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","BOS","]","''","`",")",":","The","BOS","token","(","reserved","in","the","vocab",",","but","not","actually","used",")",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","EOS","]","''","`",")",":","The","end","of","sequence","token","(","reserved","in","the","vocab",",","but","not","actually","used",")",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","mask_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","MASK","]","''","`",")",":","The","MASK","token",",","useful","for","masked","language","modeling",".","cls_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","CLS","]","''","`",")",":","The","CLS","token","(","reserved","in","the","vocab",",","but","not","actually","used",")",".","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","SEP","]","''","`",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","two","sequences","."]
code: def vocab_size(self):
      return self._utf_vocab_size + self._num_special_tokens
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/perceiver/tokenization_perceiver.py
imports: from typing import Dict, List, Optional, Tuple
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 3 for calling the PerceiverTokenizer obj's underlying member method code with example usage: obj.vocab_size() and returns: unknown