class_docstr: Construct a "fast" XGLM tokenizer (backed by HuggingFace's *tokenizers* library). Adapted from [`RobertaTokenizer`]
  and [`XLNetTokenizer`]. Based on
  [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
  
  This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
  refer to this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the beginning of
          sequence. The token used is the `cls_token`.
  
          </Tip>
  
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      sep_token (`str`, *optional*, defaults to `"</s>"`):
          The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
          sequence classification or for a text and a question for question answering. It is also used as the last
          token of a sequence built with special tokens.
      cls_token (`str`, *optional*, defaults to `"<s>"`):
          The classifier token which is used when doing sequence classification (classification of the whole sequence
          instead of per-token classification). It is the first token of the sequence when built with special tokens.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
      additional_special_tokens (`List[str]`, *optional*, defaults to `["<s>NOTUSED", "</s>NOTUSED"]`):
          Additional special tokens used by the tokenizer.
class_docstr_tok: ["Construct","a","``","fast","''","XGLM","tokenizer","(","backed","by","HuggingFace","'s","*","tokenizers","*","library",")",".","Adapted","from","[","`","RobertaTokenizer","`","]","and","[","`","XLNetTokenizer","`","]",".","Based","on","[","BPE","]","(","https",":","\/\/huggingface.co\/docs\/tokenizers\/python\/latest\/components.html","?","highlight=BPE","#","models",")",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizerFast","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sequence","token","that","was","used","during","pretraining",".","Can","be","used","a","sequence","classifier","token",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","beginning","of","sequence",".","The","token","used","is","the","`","cls_token","`",".","<","\/Tip",">","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","multiple","sequences",",","e.g",".","two","sequences","for","sequence","classification","or","for","a","text","and","a","question","for","question","answering",".","It","is","also","used","as","the","last","token","of","a","sequence","built","with","special","tokens",".","cls_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","classifier","token","which","is","used","when","doing","sequence","classification","(","classification","of","the","whole","sequence","instead","of","per-token","classification",")",".","It","is","the","first","token","of","the","sequence","when","built","with","special","tokens",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","additional_special_tokens","(","`","List","[","str","]","`",",","*","optional","*",",","defaults","to","`","[","``","<","s",">","NOTUSED","''",",","``","<","\/s",">","NOTUSED","''","]","`",")",":","Additional","special","tokens","used","by","the","tokenizer","."]
code: def __init__(
      self,
      vocab_file=None,
      tokenizer_file=None,
      bos_token="<s>",
      eos_token="</s>",
      sep_token="</s>",
      cls_token="<s>",
      unk_token="<unk>",
      pad_token="<pad>",
      **kwargs,
  ):
      # Compatibility with the original tokenizer
      self.num_madeup_words = 7
      madeup_words = [
          f"<madeupword{i}>" for i in range(self.num_madeup_words)
      ]
  
      kwargs["additional_special_tokens"] = kwargs.get(
          "additional_special_tokens", []
      )
      kwargs["additional_special_tokens"] += [
          word
          for word in madeup_words
          if word not in kwargs["additional_special_tokens"]
      ]
  
      super().__init__(
          vocab_file,
          tokenizer_file=tokenizer_file,
          bos_token=bos_token,
          eos_token=eos_token,
          sep_token=sep_token,
          cls_token=cls_token,
          unk_token=unk_token,
          pad_token=pad_token,
          **kwargs,
      )
  
      self.vocab_file = vocab_file
      self.can_save_slow_tokenizer = (
          False if not self.vocab_file else True
      )
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/xglm/tokenization_xglm_fast.py
imports: import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
label_desc: Use image node_id 1 to create a new XGLMTokenizerFast object from inherited base classes: PreTrainedTokenizerFast with example: obj = XGLMTokenizerFast(vocab_file, tokenizer_file, bos_token, eos_token, sep_token, cls_token, unk_token, pad_token)