class_docstr: A modified wrapper for a public python BPE tokenizer. The modifications allow encoding directly into the formats
  required for finetune. Particularly with respect to formatting with multiple inputs.
class_docstr_tok: ["A","modified","wrapper","for","a","public","python","BPE","tokenizer",".","The","modifications","allow","encoding","directly","into","the","formats","required","for","finetune",".","Particularly","with","respect","to","formatting","with","multiple","inputs","."]
code: def __init__(self, encoder_path=ENCODER_PATH, vocab_path=VOCAB_PATH):
      super().__init__(encoder_path=encoder_path, vocab_path=vocab_path)
  
      # Load encoder
      with open(self.encoder_path, "r") as f:
          self.encoder = json.load(f)
  
      if self.offset != 0:
          self.encoder.update(
              (token, idx + self.offset)
              for token, idx in self.encoder.items()
          )
  
      # Load BPE
      with open(self.vocab_path, "r", encoding="utf-8") as f:
          bpe_data = f.read()
      bpe_merges = [
          tuple(merge_str.split())
          for merge_str in bpe_data.split("\n")[1:-1]
      ]
      self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
  
      self._add_extra_toks()
  
      self.decoder = {v: k for k, v in self.encoder.items()}
      self.errors = "replace"
      self.init_encoders()
  
file_path: finetune/finetune/base_models/gpt2/encoder.py
imports: import os
  import json
  import traceback
  import regex
  from functools import lru_cache
  import numpy
  import finetune
  from finetune.encoding.input_encoder import BaseEncoder, EncodedOutput, get_pairs
label_desc: Use image node_id 1 to create a new GPT2Encoder object from inherited base classes: BaseEncoder with example: obj = GPT2Encoder(encoder_path, vocab_path)