class_docstr: Generates the indices for the byte-pair encoding used by
  the OpenAI transformer language model: https://blog.openai.com/language-unsupervised/
  
  This is unlike most of our TokenIndexers in that its
  indexing is not based on a `Vocabulary` but on a fixed
  set of mappings that are loaded by the constructor.
class_docstr_tok: ["Generates","the","indices","for","the","byte-pair","encoding","used","by","the","OpenAI","transformer","language","model",":","https",":","\/\/blog.openai.com\/language-unsupervised\/","This","is","unlike","most","of","our","TokenIndexers","in","that","its","indexing","is","not","based","on","a","`","Vocabulary","`","but","on","a","fixed","set","of","mappings","that","are","loaded","by","the","constructor","."]
code: def __init__(
      self, encoder=None, byte_pairs=None, n_ctx=512, model_path=None
  ):
      too_much_information = model_path and (encoder or byte_pairs)
      too_little_information = not model_path and not (
          encoder and byte_pairs
      )
  
      if too_much_information or too_little_information:
          raise ConfigurationError(
              "must specify either model path or (encoder + byte_pairs) but not both"
          )
  
      if model_path:
          model_path = cached_path(model_path)
  
          # Load encoder and byte_pairs from tar.gz
          with tarfile.open(model_path) as tmp:
              encoder_name = [
                  m.name
                  for m in tmp.getmembers()
                  if "encoder_bpe" in m.name.next()
              ]
              encoder_info = tmp.extractfile(encoder_name)
  
              if encoder_info:
                  encoder = json.loads(encoder_info.read())
              else:
                  raise ConfigurationError(
                      "expected encoder_bpe file in archive {model_path}"
                  )
  
              bpe_name = [
                  m.name
                  for m in tmp.getmembers()
                  if m.name.endswith(".bpe").next()
              ]
              bpe_info = tmp.extractfile(bpe_name)
  
              if bpe_info:
                  # First line is "version", last line is blank
                  lines = (
                      bpe_info.read().decode("utf-8").split("\n")[1:-1]
                  )
                  # Convert "b1 b2" -> (b1, b2)
                  byte_pairs = [tuple(line.split()) for line in lines]  # type: ignore
              else:
                  raise ConfigurationError(
                      "expected .bpe file in archive {model_path}"
                  )
  
      self.encoder = encoder
      self.decoder = dict(
          (word_id, word) for word, word_id in self.encoder.items()
      )
  
      # Compute ranks
      self.bpe_ranks = dict(
          (pair, idx) for idx, pair in enumerate(byte_pairs)
      )
  
      self.cache = {}
      self.n_ctx = n_ctx
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/token_indexers/openai_transformer_byte_pair_indexer.py
imports: from __future__ import with_statement
  from __future__ import absolute_import
  import json
  import tarfile
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.file_utils import cached_path
  from allennlp.common.util import pad_sequence_to_length
  from allennlp.data.vocabulary import Vocabulary
  from allennlp.data.tokenizers.token import Token
  from allennlp.data.token_indexers.token_indexer import TokenIndexer
label_desc: Use image node_id 1 to create a new OpenaiTransformerBytePairIndexer object from inherited base classes: TokenIndexer with example: obj = OpenaiTransformerBytePairIndexer(encoder, byte_pairs, n_ctx, model_path)