class_docstr: Constructs a Speech2Text2Tokenizer.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
  the superclass for more information regarding such methods.
  
  Args:
      vocab_file (`str`):
          File containing the vocabulary.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The beginning of sentence token.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sentence token.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
  
      **kwargs
          Additional keyword arguments passed along to [`PreTrainedTokenizer`]
class_docstr_tok: ["Constructs","a","Speech2Text2Tokenizer",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","some","of","the","main","methods",".","Users","should","refer","to","the","superclass","for","more","information","regarding","such","methods",".","Args",":","vocab_file","(","`","str","`",")",":","File","containing","the","vocabulary",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sentence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sentence","token",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","*","*","kwargs","Additional","keyword","arguments","passed","along","to","[","`","PreTrainedTokenizer","`","]"]
code: def bpe(self, token):
      word = tuple(token[:-1]) + (token[-1] + BPE_TOKEN_MERGES,)
      if token in self.cache:
          return self.cache[token]
      pairs = get_pairs(word)
  
      if not pairs:
          return token
  
      while True:
          bigram = min(
              pairs,
              key=lambda pair: self.bpe_ranks.get(pair, float("inf")),
          )
          if bigram not in self.bpe_ranks:
              break
          first, second = bigram
          new_word = []
          i = 0
          while i < len(word):
              try:
                  j = word.index(first, i)
              except ValueError:
                  new_word.extend(word[i:])
                  break
              else:
                  new_word.extend(word[i:j])
                  i = j
  
              if (
                  word[i] == first
                  and i < len(word) - 1
                  and word[i + 1] == second
              ):
                  new_word.append(first + second)
                  i += 2
              else:
                  new_word.append(word[i])
                  i += 1
          new_word = tuple(new_word)
          word = new_word
          if len(word) == 1:
              break
          else:
              pairs = get_pairs(word)
      word = " ".join(word)
      if word == "\n  " + BPE_TOKEN_MERGES:
          word = "\n" + BPE_TOKEN_MERGES
  
      if word.endswith(BPE_TOKEN_MERGES):
          word = word.replace(BPE_TOKEN_MERGES, "")
  
      word = word.replace(" ", BPE_TOKEN_VOCAB)
      self.cache[token] = word
      return word
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py
imports: import json
  import os
  from typing import Dict, List, Optional, Tuple
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 4 for calling the Speech2Text2Tokenizer obj's underlying member method code with example usage: obj.bpe(token) and returns: word, self, token