code: def _load(self):
      if self._token_str_to_idx is not None:
          return
  
      self._token_str_to_idx = json.loads(
          read_file(self.full_encoder_path).decode()
      )
      bpe_data = read_file(self.full_bpe_path).decode()
      bpe_merges = [
          tuple(merge_str.split())
          for merge_str in bpe_data.split("\n")[1:-1]
      ]
  
      assert self.eot_token == self._token_str_to_idx["<|endoftext|>"]
  
      # Add an <|end_of_prefix|> token
      if self.eoprefix_token is not None:
          assert not self._token_str_to_idx.get("<|endofprefix|>")
          self._token_str_to_idx[
              "<|endofprefix|>"
          ] = self.eoprefix_token
  
      # Add denoise sentinel tokens like <|dn_1|> <|dn_2|> etc.
      # These tokens are added to the end of the vocabulary range
      for denoise_sentinel_idx in range(self.n_denoise_sentinels):
          str_repr = f"<|dn_{denoise_sentinel_idx}|>"
          assert not self._token_str_to_idx.get(str_repr)
  
          n_non_sentinel_tokens = (
              self.n_vocab - self.n_denoise_sentinels
          )
          sentinel_token = n_non_sentinel_tokens + denoise_sentinel_idx
  
          self._token_str_to_idx[str_repr] = sentinel_token
  
      assert len(self._token_str_to_idx) == self.n_vocab
  
      self._token_idx_to_str = {
          v: k for k, v in self._token_str_to_idx.items()
      }
      self.byte_encoder = bytes_to_unicode()
      self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
      self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
  
imports: import json
  import os.path
  from dataclasses import dataclass
  from functools import lru_cache
  from typing import ClassVar
  import blobfile
  import regex
  import torch
label_desc: Use image node_id 2 for calling the BPEEncoding obj's underlying member method code with example usage: obj._load() without return types