class_docstr: Construct an Whisper tokenizer.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
  the superclass for more information regarding such methods.
  
   Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
      normalizer_file (`str`, *optional*, defaults to `None`):
          Path to the normalizer_file file.
      errors (`str`, *optional*, defaults to `"replace"`):
          Paradigm to follow when decoding bytes to UTF-8. See
          [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
      unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
          The beginning of sequence token.
      eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
          The end of sequence token.
      add_prefix_space (`bool`, *optional*, defaults to `False`):
          Whether or not to add an initial space to the input. This allows to treat the leading word just as any
          other word.
      add_bos_token (`bool`, *optional*, defaults to `False`):
          Whether or not to add an initial <|endoftext|> to the input. This allows to treat the leading word just as
          any other word.
class_docstr_tok: ["Construct","an","Whisper","tokenizer",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","some","of","the","main","methods",".","Users","should","refer","to","the","superclass","for","more","information","regarding","such","methods",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","normalizer_file","(","`","str","`",",","*","optional","*",",","defaults","to","`","None","`",")",":","Path","to","the","normalizer_file","file",".","errors","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","replace","''","`",")",":","Paradigm","to","follow","when","decoding","bytes","to","UTF-8",".","See","[","bytes.decode","]","(","https",":","\/\/docs.python.org\/3\/library\/stdtypes.html","#","bytes.decode",")","for","more","information",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|endoftext|",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|endoftext|",">","''","`",")",":","The","beginning","of","sequence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|endoftext|",">","''","`",")",":","The","end","of","sequence","token",".","add_prefix_space","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","add","an","initial","space","to","the","input",".","This","allows","to","treat","the","leading","word","just","as","any","other","word",".","add_bos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","add","an","initial","<","|endoftext|",">","to","the","input",".","This","allows","to","treat","the","leading","word","just","as","any","other","word","."]
docstr: Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
  special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
  
  Args:
      token_ids_0 (`List[int]`):
          List of IDs.
      token_ids_1 (`List[int]`, *optional*):
          Optional second list of IDs for sequence pairs.
      already_has_special_tokens (`bool`, *optional*, defaults to `False`):
          Whether or not the token list is already formatted with special tokens for the model.
  
  Returns:
      `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
docstr_tok: ["Retrieves","sequence","ids","from","a","token","list","that","has","no","special","tokens","added",".","This","method","is","called","when","adding","special","tokens","using","the","tokenizer","`","prepare_for_model","`","or","`","encode_plus","`","methods",".","Args",":","token_ids_0","(","`","List","[","int","]","`",")",":","List","of","IDs",".","token_ids_1","(","`","List","[","int","]","`",",","*","optional","*",")",":","Optional","second","list","of","IDs","for","sequence","pairs",".","already_has_special_tokens","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","token","list","is","already","formatted","with","special","tokens","for","the","model",".","Returns",":","`","List","[","int","]","`",":","A","list","of","integers","in","the","range","[","0",",","1","]",":","1","for","a","special","token",",","0","for","a","sequence","token","."]
code: def get_special_tokens_mask(
      self,
      token_ids_0: List[int],
      token_ids_1: Optional[List[int]] = None,
      already_has_special_tokens: bool = False,
  ) -> List[int]:
      """
      Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
      special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
  
      Args:
          token_ids_0 (`List[int]`):
              List of IDs.
          token_ids_1 (`List[int]`, *optional*):
              Optional second list of IDs for sequence pairs.
          already_has_special_tokens (`bool`, *optional*, defaults to `False`):
              Whether or not the token list is already formatted with special tokens for the model.
  
      Returns:
          `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
      """
      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_0=token_ids_0,
              token_ids_1=token_ids_1,
              already_has_special_tokens=True,
          )
  
      if not self.add_bos_token:
          return super().get_special_tokens_mask(
              token_ids_0=token_ids_0,
              token_ids_1=token_ids_1,
              already_has_special_tokens=False,
          )
  
      if token_ids_1 is None:
          return [1] + ([0] * len(token_ids_0))
      return (
          [1]
          + ([0] * len(token_ids_0))
          + [1]
          + ([0] * len(token_ids_1))
      )
  
imports: import json
  import os
  from typing import List, Optional, Tuple, Union
  import regex
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  from .english_normalizer import EnglishTextNormalizer
label_desc: Use image node_id 6 for calling the WhisperTokenizer obj's underlying member method code with example usage: obj.get_special_tokens_mask(token_ids_0, token_ids_1, already_has_special_tokens) and returns: unknown, super, super, unknown