class_docstr: Construct a "fast" T5 tokenizer (backed by HuggingFace's *tokenizers* library). Based on
  [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
  
  This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
  refer to this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`):
          [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
          contains the vocabulary necessary to instantiate a tokenizer.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
      extra_ids (`int`, *optional*, defaults to 100):
          Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
          accessible as "<extra_id_{%d}>" where "{%d}" is a number between 0 and extra_ids-1. Extra tokens are
          indexed from the end of the vocabulary up to beginning ("<extra_id_0>" is the last token in the vocabulary
          like in T5 preprocessing see
          [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
      additional_special_tokens (`List[str]`, *optional*):
          Additional special tokens used by the tokenizer.
class_docstr_tok: ["Construct","a","``","fast","''","T5","tokenizer","(","backed","by","HuggingFace","'s","*","tokenizers","*","library",")",".","Based","on","[","Unigram","]","(","https",":","\/\/huggingface.co\/docs\/tokenizers\/python\/latest\/components.html","?","highlight=unigram","#","models",")",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizerFast","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",")",":","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")","file","(","generally","has","a","*",".spm","*","extension",")","that","contains","the","vocabulary","necessary","to","instantiate","a","tokenizer",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","extra_ids","(","`","int","`",",","*","optional","*",",","defaults","to","100",")",":","Add","a","number","of","extra","ids","added","to","the","end","of","the","vocabulary","for","use","as","sentinels",".","These","tokens","are","accessible","as","``","<","extra_id_","{","%","d","}",">","''","where","``","{","%","d","}","''","is","a","number","between","0","and","extra_ids-1",".","Extra","tokens","are","indexed","from","the","end","of","the","vocabulary","up","to","beginning","(","``","<","extra_id_0",">","''","is","the","last","token","in","the","vocabulary","like","in","T5","preprocessing","see","[","here","]","(","https",":","\/\/github.com\/google-research\/text-to-text-transfer-transformer\/blob\/9fd7b14a769417be33bc6c850f9598764913c833\/t5\/data\/preprocessors.py","#","L2117",")",")",".","additional_special_tokens","(","`","List","[","str","]","`",",","*","optional","*",")",":","Additional","special","tokens","used","by","the","tokenizer","."]
code: def __init__(
      self,
      vocab_file=None,
      tokenizer_file=None,
      eos_token="</s>",
      unk_token="<unk>",
      pad_token="<pad>",
      extra_ids=100,
      additional_special_tokens=None,
      **kwargs,
  ):
      # Add extra_ids to the special token list
      if extra_ids > 0 and additional_special_tokens is None:
          additional_special_tokens = [
              f"<extra_id_{i}>" for i in range(extra_ids)
          ]
      elif extra_ids > 0 and additional_special_tokens is not None:
          # Check that we have the right number of extra special tokens
          extra_tokens = len(
              set(
                  filter(
                      lambda x: bool("extra_id_" in str(x)),
                      additional_special_tokens,
                  )
              )
          )
          if extra_tokens != extra_ids:
              raise ValueError(
                  f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                  " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                  " tokens"
              )
  
      super().__init__(
          vocab_file,
          tokenizer_file=tokenizer_file,
          eos_token=eos_token,
          unk_token=unk_token,
          pad_token=pad_token,
          extra_ids=extra_ids,
          additional_special_tokens=additional_special_tokens,
          **kwargs,
      )
  
      self.vocab_file = vocab_file
      self.can_save_slow_tokenizer = (
          False if not self.vocab_file else True
      )
      self._extra_ids = extra_ids
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/t5/tokenization_t5_fast.py
imports: import os
  import warnings
  from shutil import copyfile
  from typing import List, Optional, Tuple
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
label_desc: Use image node_id 1 to create a new T5TokenizerFast object from inherited base classes: PreTrainedTokenizerFast with example: obj = T5TokenizerFast(vocab_file, tokenizer_file, eos_token, unk_token, pad_token, extra_ids, additional_special_tokens)