class_docstr: Construct a "fast" T5 tokenizer (backed by HuggingFace's *tokenizers* library). Based on
  [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
  
  This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
  refer to this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`):
          [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
          contains the vocabulary necessary to instantiate a tokenizer.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
      extra_ids (`int`, *optional*, defaults to 100):
          Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
          accessible as "<extra_id_{%d}>" where "{%d}" is a number between 0 and extra_ids-1. Extra tokens are
          indexed from the end of the vocabulary up to beginning ("<extra_id_0>" is the last token in the vocabulary
          like in T5 preprocessing see
          [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
      additional_special_tokens (`List[str]`, *optional*):
          Additional special tokens used by the tokenizer.
class_docstr_tok: ["Construct","a","``","fast","''","T5","tokenizer","(","backed","by","HuggingFace","'s","*","tokenizers","*","library",")",".","Based","on","[","Unigram","]","(","https",":","\/\/huggingface.co\/docs\/tokenizers\/python\/latest\/components.html","?","highlight=unigram","#","models",")",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizerFast","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",")",":","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")","file","(","generally","has","a","*",".spm","*","extension",")","that","contains","the","vocabulary","necessary","to","instantiate","a","tokenizer",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","extra_ids","(","`","int","`",",","*","optional","*",",","defaults","to","100",")",":","Add","a","number","of","extra","ids","added","to","the","end","of","the","vocabulary","for","use","as","sentinels",".","These","tokens","are","accessible","as","``","<","extra_id_","{","%","d","}",">","''","where","``","{","%","d","}","''","is","a","number","between","0","and","extra_ids-1",".","Extra","tokens","are","indexed","from","the","end","of","the","vocabulary","up","to","beginning","(","``","<","extra_id_0",">","''","is","the","last","token","in","the","vocabulary","like","in","T5","preprocessing","see","[","here","]","(","https",":","\/\/github.com\/google-research\/text-to-text-transfer-transformer\/blob\/9fd7b14a769417be33bc6c850f9598764913c833\/t5\/data\/preprocessors.py","#","L2117",")",")",".","additional_special_tokens","(","`","List","[","str","]","`",",","*","optional","*",")",":","Additional","special","tokens","used","by","the","tokenizer","."]
code: def _eventually_correct_t5_max_length(
      pretrained_model_name_or_path,
      max_model_length,
      init_max_model_length,
  ):
      if (
          pretrained_model_name_or_path
          in T5TokenizerFast.max_model_input_sizes
      ):
          deprecated_max_model_length = (
              T5TokenizerFast.max_model_input_sizes[
                  pretrained_model_name_or_path
              ]
          )
          if (
              init_max_model_length is not None
              and init_max_model_length != max_model_length
          ):
              return init_max_model_length
          elif init_max_model_length is None:
              warnings.warn(
                  "This tokenizer was incorrectly instantiated with a model max length of"
                  f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
                  " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
                  " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
                  f" {pretrained_model_name_or_path} automatically truncating your input to"
                  f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
                  f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
                  " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
                  " instantiate this tokenizer with `model_max_length` set to your preferred value.",
                  FutureWarning,
              )
  
      return max_model_length
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/t5/tokenization_t5_fast.py
imports: import os
  import warnings
  from shutil import copyfile
  from typing import List, Optional, Tuple
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
label_desc: Use image node_id 2 for calling the T5TokenizerFast obj's underlying member method code with example usage: obj._eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length) and returns: max_model_length, init_max_model_length