class_docstr: A modified wrapper for a public python BPE tokenizer. The modifications allow encoding directly into the formats
  required for finetune. Particularly with respect to formatting with multiple inputs.
class_docstr_tok: ["A","modified","wrapper","for","a","public","python","BPE","tokenizer",".","The","modifications","allow","encoding","directly","into","the","formats","required","for","finetune",".","Particularly","with","respect","to","formatting","with","multiple","inputs","."]
code: def bpe(self, token):
      word = tuple(token[:-1]) + (token[-1] + "</w>",)
      if token in self.cache:
          return self.cache[token]
      pairs = get_pairs(word)
  
      if not pairs:
          return token + "</w>"
  
      while True:
          bigram = min(
              pairs,
              key=lambda pair: self.bpe_ranks.get(pair, float("inf")),
          )
          if bigram not in self.bpe_ranks:
              break
          first, second = bigram
          new_word = []
          i = 0
          while i < len(word):
              try:
                  j = word.index(first, i)
                  new_word.extend(word[i:j])
                  i = j
              except:
                  new_word.extend(word[i:])
                  break
  
              if (
                  word[i] == first
                  and i < len(word) - 1
                  and word[i + 1] == second
              ):
                  new_word.append(first + second)
                  i += 2
              else:
                  new_word.append(word[i])
                  i += 1
          new_word = tuple(new_word)
          word = new_word
          if len(word) == 1:
              break
          else:
              pairs = get_pairs(word)
      word = " ".join(word)
      if word == "\n  </w>":
          word = "\n</w>"
      self.cache[token] = word
      return word
  
file_path: finetune/finetune/base_models/gpt/encoder.py
imports: import re
  import json
  import os
  import warnings
  import codecs
  import numpy
  from ftfy.fixes import uncurl_quotes
  import finetune
  from finetune.encoding.input_encoder import get_spacy, EncodedOutput, BaseEncoder, get_pairs
label_desc: Use image node_id 3 for calling the GPTEncoder obj's underlying member method code with example usage: obj.bpe(token) and returns: word, self, unknown