code: def data_gym_to_mergeable_bpe_ranks(
      vocab_bpe_file: str, encoder_json_file: str
  ) -> dict[bytes, int]:
      # NB: do not add caching to this function
      rank_to_intbyte = [
          b
          for b in range(2**8)
          if chr(b).isprintable() and chr(b) != " "
      ]
  
      data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte}
      n = 0
      for b in range(2**8):
          if b not in rank_to_intbyte:
              rank_to_intbyte.append(b)
              data_gym_byte_to_byte[chr(2**8 + n)] = b
              n += 1
      assert len(rank_to_intbyte) == 2**8
  
      # vocab_bpe contains the merges along with associated ranks
      vocab_bpe_contents = read_file_cached(vocab_bpe_file).decode()
      bpe_merges = [
          tuple(merge_str.split())
          for merge_str in vocab_bpe_contents.split("\n")[1:-1]
      ]
  
      def decode_data_gym(value: str) -> bytes:
          return bytes(data_gym_byte_to_byte[b] for b in value)
  
      # add the single byte tokens
      bpe_ranks = {bytes([b]): i for i, b in enumerate(rank_to_intbyte)}
      # add the merged tokens
      n = len(bpe_ranks)
      for first, second in bpe_merges:
          bpe_ranks[
              decode_data_gym(first) + decode_data_gym(second)
          ] = n
          n += 1
  
      # check that the encoder file matches the merges file
      # this sanity check is important since tiktoken assumes that ranks are ordered the same
      # as merge priority
      encoder_json = json.loads(read_file_cached(encoder_json_file))
      encoder_json_loaded = {
          decode_data_gym(k): v for k, v in encoder_json.items()
      }
      # drop these two special tokens if present, since they're not mergeable bpe tokens
      encoder_json_loaded.pop(b"<|endoftext|>", None)
      encoder_json_loaded.pop(b"<|startoftext|>", None)
      assert bpe_ranks == encoder_json_loaded
  
      return bpe_ranks
  
file_path: tiktoken/tiktoken/load.py
imports: from __future__ import annotations
  import base64
  import hashlib
  import json
  import os
  import tempfile
  import uuid
  import requests
label_desc: Use image node_id 3 for calling a global function with example usage: data_gym_to_mergeable_bpe_ranks(vocab_bpe_file, encoder_json_file) and returns: bpe_ranks, bytes