class_docstr: A modified wrapper for a public python BPE tokenizer. The modifications allow encoding directly into the formats
  required for finetune. Particularly with respect to formatting with multiple inputs.
class_docstr_tok: ["A","modified","wrapper","for","a","public","python","BPE","tokenizer",".","The","modifications","allow","encoding","directly","into","the","formats","required","for","finetune",".","Particularly","with","respect","to","formatting","with","multiple","inputs","."]
code: def __init__(self, encoder_path=ENCODER_PATH, vocab_path=VOCAB_PATH):
      super().__init__(encoder_path=encoder_path, vocab_path=vocab_path)
  
      with open(self.encoder_path, "r") as f:
          self.encoder = json.load(f)
  
      with codecs.open(self.vocab_path, encoding="utf8"):
          merges = (
              codecs.open(self.vocab_path, encoding="utf8")
              .read()
              .split("\n")[1:-1]
          )
          merges = [tuple(merge.split()) for merge in merges]
          self.bpe_ranks = dict(zip(merges, range(len(merges))))
  
      self.decoder = {v: k for k, v in self.encoder.items()}
  
      self.special_tokens = ["_start_", "_delimiter_", "_classify_"]
      for token in self.special_tokens:
          self.encoder[token] = len(self.encoder)
  
      self.decoder = {v: k for k, v in self.encoder.items()}
      self.start_token = self.encoder["_start_"]
      self.delimiter_token = self.encoder["_delimiter_"]
      self.end_token = self.encoder["_classify_"]
  
file_path: finetune/finetune/base_models/gpt/encoder.py
imports: import re
  import json
  import os
  import warnings
  import codecs
  import numpy
  from ftfy.fixes import uncurl_quotes
  import finetune
  from finetune.encoding.input_encoder import get_spacy, EncodedOutput, BaseEncoder, get_pairs
label_desc: Use image node_id 1 to create a new GPTEncoder object from inherited base classes: BaseEncoder with example: obj = GPTEncoder(encoder_path, vocab_path)