class_docstr: Construct a "Fast" BPE tokenizer for HerBERT (backed by HuggingFace's *tokenizers* library).
  
  Peculiarities:
  
  - uses BERT's pre-tokenizer: BertPreTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of
    a punctuation character will be treated separately.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the methods. Users should refer to the
  superclass for more information regarding methods.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
class_docstr_tok: ["Construct","a","``","Fast","''","BPE","tokenizer","for","HerBERT","(","backed","by","HuggingFace","'s","*","tokenizers","*","library",")",".","Peculiarities",":","-","uses","BERT","'s","pre-tokenizer",":","BertPreTokenizer","splits","tokens","on","spaces",",","and","also","on","punctuation",".","Each","occurrence","of","a","punctuation","character","will","be","treated","separately",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","methods",".","Users","should","refer","to","the","superclass","for","more","information","regarding","methods",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file","."]
docstr: Create a mask from the two sequences passed to be used in a sequence-pair classification task. HerBERT, like
  BERT sequence pair mask has the following format:
  
  ```
  0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
  | first sequence    | second sequence |
  ```
  
  Args:
      token_ids_0 (`List[int]`):
          List of IDs.
      token_ids_1 (`List[int]`, *optional*):
          Optional second list of IDs for sequence pairs.
  
  Returns:
      `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
docstr_tok: ["Create","a","mask","from","the","two","sequences","passed","to","be","used","in","a","sequence-pair","classification","task",".","HerBERT",",","like","BERT","sequence","pair","mask","has","the","following","format",":","``","`","0","0","0","0","0","0","0","0","0","0","0","1","1","1","1","1","1","1","1","1","|","first","sequence","|","second","sequence","|","``","`","Args",":","token_ids_0","(","`","List","[","int","]","`",")",":","List","of","IDs",".","token_ids_1","(","`","List","[","int","]","`",",","*","optional","*",")",":","Optional","second","list","of","IDs","for","sequence","pairs",".","Returns",":","`","List","[","int","]","`",":","List","of","[","token","type","IDs","]","(","..","\/glossary","#","token-type-ids",")","according","to","the","given","sequence","(","s",")","."]
code: def create_token_type_ids_from_sequences(
      self,
      token_ids_0: List[int],
      token_ids_1: Optional[List[int]] = None,
  ) -> List[int]:
      """
      Create a mask from the two sequences passed to be used in a sequence-pair classification task. HerBERT, like
      BERT sequence pair mask has the following format:
  
      ```
      0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
      | first sequence    | second sequence |
      ```
  
      Args:
          token_ids_0 (`List[int]`):
              List of IDs.
          token_ids_1 (`List[int]`, *optional*):
              Optional second list of IDs for sequence pairs.
  
      Returns:
          `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
      """
      sep = [self.sep_token_id]
      cls = [self.cls_token_id]
  
      if token_ids_1 is None:
          return len(cls + token_ids_0 + sep) * [0]
      return len(cls + token_ids_0 + sep) * [0] + len(
          token_ids_1 + sep
      ) * [1]
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/herbert/tokenization_herbert_fast.py
imports: from typing import List, Optional, Tuple
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_herbert import HerbertTokenizer
label_desc: Use image node_id 4 for calling the HerbertTokenizerFast obj's underlying member method code with example usage: obj.create_token_type_ids_from_sequences(token_ids_0, token_ids_1) and returns: unknown, unknown