class_docstr: Construct a Perceiver tokenizer. The Perceiver simply uses raw bytes utf-8 encoding.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods.
  
  Args:
      pad_token (`str`, *optional*, defaults to `"[PAD]"`):
          The token used for padding, for example when batching sequences of different lengths.
      bos_token (`str`, *optional*, defaults to `"[BOS]"`):
          The BOS token (reserved in the vocab, but not actually used).
      eos_token (`str`, *optional*, defaults to `"[EOS]"`):
          The end of sequence token (reserved in the vocab, but not actually used).
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      mask_token (`str`, *optional*, defaults to `"[MASK]"`):
          The MASK token, useful for masked language modeling.
      cls_token (`str`, *optional*, defaults to `"[CLS]"`):
          The CLS token (reserved in the vocab, but not actually used).
      sep_token (`str`, *optional*, defaults to `"[SEP]"`):
          The separator token, which is used when building a sequence from two sequences.
class_docstr_tok: ["Construct","a","Perceiver","tokenizer",".","The","Perceiver","simply","uses","raw","bytes","utf-8","encoding",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","PAD","]","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","BOS","]","''","`",")",":","The","BOS","token","(","reserved","in","the","vocab",",","but","not","actually","used",")",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","EOS","]","''","`",")",":","The","end","of","sequence","token","(","reserved","in","the","vocab",",","but","not","actually","used",")",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","mask_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","MASK","]","''","`",")",":","The","MASK","token",",","useful","for","masked","language","modeling",".","cls_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","CLS","]","''","`",")",":","The","CLS","token","(","reserved","in","the","vocab",",","but","not","actually","used",")",".","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","SEP","]","''","`",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","two","sequences","."]
docstr: Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
  special tokens using the tokenizer `prepare_for_model` method.
  
  Args:
      token_ids_0 (`List[int]`):
          List of IDs.
      token_ids_1 (`List[int]`, *optional*):
          Optional second list of IDs for sequence pairs.
      already_has_special_tokens (`bool`, *optional*, defaults to `False`):
          Whether or not the token list is already formatted with special tokens for the model.
  
  Returns:
      `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
docstr_tok: ["Retrieve","sequence","ids","from","a","token","list","that","has","no","special","tokens","added",".","This","method","is","called","when","adding","special","tokens","using","the","tokenizer","`","prepare_for_model","`","method",".","Args",":","token_ids_0","(","`","List","[","int","]","`",")",":","List","of","IDs",".","token_ids_1","(","`","List","[","int","]","`",",","*","optional","*",")",":","Optional","second","list","of","IDs","for","sequence","pairs",".","already_has_special_tokens","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","token","list","is","already","formatted","with","special","tokens","for","the","model",".","Returns",":","`","List","[","int","]","`",":","A","list","of","integers","in","the","range","[","0",",","1","]",":","1","for","a","special","token",",","0","for","a","sequence","token","."]
code: def get_special_tokens_mask(
      self,
      token_ids_0: List[int],
      token_ids_1: Optional[List[int]] = None,
      already_has_special_tokens: bool = False,
  ) -> List[int]:
      """
      Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
      special tokens using the tokenizer `prepare_for_model` method.
  
      Args:
          token_ids_0 (`List[int]`):
              List of IDs.
          token_ids_1 (`List[int]`, *optional*):
              Optional second list of IDs for sequence pairs.
          already_has_special_tokens (`bool`, *optional*, defaults to `False`):
              Whether or not the token list is already formatted with special tokens for the model.
  
      Returns:
          `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
      """
      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_0=token_ids_0,
              token_ids_1=token_ids_1,
              already_has_special_tokens=True,
          )
  
      # normal case: some special tokens
      if token_ids_1 is None:
          return [1] + [0] * len(token_ids_0) + [1]
      return (
          [1]
          + ([0] * len(token_ids_0))
          + [1]
          + ([0] * len(token_ids_1))
          + [1]
      )
  
file_path: transformers/src/transformers/models/perceiver/tokenization_perceiver.py
imports: from typing import Dict, List, Optional, Tuple
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 4 for calling the PerceiverTokenizer obj's underlying member method code with example usage: obj.get_special_tokens_mask(token_ids_0, token_ids_1, already_has_special_tokens) and returns: unknown, super, unknown