class_docstr: Constructs a Speech2Text2Tokenizer.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
  the superclass for more information regarding such methods.
  
  Args:
      vocab_file (`str`):
          File containing the vocabulary.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The beginning of sentence token.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sentence token.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
  
      **kwargs
          Additional keyword arguments passed along to [`PreTrainedTokenizer`]
class_docstr_tok: ["Constructs","a","Speech2Text2Tokenizer",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","some","of","the","main","methods",".","Users","should","refer","to","the","superclass","for","more","information","regarding","such","methods",".","Args",":","vocab_file","(","`","str","`",")",":","File","containing","the","vocabulary",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sentence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sentence","token",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","*","*","kwargs","Additional","keyword","arguments","passed","along","to","[","`","PreTrainedTokenizer","`","]"]
docstr: Tokenize a string.
docstr_tok: ["Tokenize","a","string","."]
code: def _tokenize(self, text):
      """Tokenize a string."""
  
      if self.bpe_ranks is None:
          raise ValueError(
              "This tokenizer was instantiated without a `merges.txt` file, so"
              " that it can only be used for decoding, not for encoding."
              "Make sure to provide `merges.txt` file at instantiation to enable "
              "encoding."
          )
  
      if self.do_lower_case:
          text = text.lower()
  
      text = text.split()
  
      split_tokens = []
      for token in text:
          if token:
              split_tokens.extend(
                  [t for t in self.bpe(token).split(" ")]
              )
  
      return split_tokens
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py
imports: import json
  import os
  from typing import Dict, List, Optional, Tuple
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 5 for calling the Speech2Text2Tokenizer obj's underlying member method code with example usage: obj._tokenize(text) and returns: split_tokens