class_docstr: Generates the indices for the byte-pair encoding used by
  the OpenAI transformer language model: https://blog.openai.com/language-unsupervised/
  
  This is unlike most of our TokenIndexers in that its
  indexing is not based on a `Vocabulary` but on a fixed
  set of mappings that are loaded by the constructor.
class_docstr_tok: ["Generates","the","indices","for","the","byte-pair","encoding","used","by","the","OpenAI","transformer","language","model",":","https",":","\/\/blog.openai.com\/language-unsupervised\/","This","is","unlike","most","of","our","TokenIndexers","in","that","its","indexing","is","not","based","on","a","`","Vocabulary","`","but","on","a","fixed","set","of","mappings","that","are","loaded","by","the","constructor","."]
code: def tokens_to_indices(self, tokens, _vocabulary, index_name):
      text_tokens = []
      offsets = []
      offset = -1
  
      for token in tokens:
          bpe_tokens = [
              self.encoder.get(t, 0)
              for t in self.byte_pair_encode(token)
          ]
          offset += len(bpe_tokens)
          offsets.append(offset)
          text_tokens.extend(bpe_tokens)
  
      num_tokens = len(text_tokens)
  
      # If there's too many tokens, that's going to cause problems.
      if num_tokens > self.n_ctx:
          raise RuntimeError(
              "The transformer model has a maximum sequence length of {self.n_ctx} "
              "but your byte pair encoded sequence has length {num_tokens}. "
              "The offending text input is {tokens}."
          )
  
      # If there's too few tokens, just pad with zeros.
      text_tokens.extend(0 for _ in range(self.n_ctx - num_tokens))
  
      return {
          index_name: text_tokens,
          "{index_name}-offsets": offsets,
          # add mask here according to the original tokens,
          # because calling util.get_text_field_mask on the
          # "byte pair" tokens will produce the wrong shape
          "mask": [1 for _ in offsets],
      }
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/token_indexers/openai_transformer_byte_pair_indexer.py
imports: from __future__ import with_statement
  from __future__ import absolute_import
  import json
  import tarfile
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.file_utils import cached_path
  from allennlp.common.util import pad_sequence_to_length
  from allennlp.data.vocabulary import Vocabulary
  from allennlp.data.tokenizers.token import Token
  from allennlp.data.token_indexers.token_indexer import TokenIndexer
label_desc: Use image node_id 4 for calling the OpenaiTransformerBytePairIndexer obj's underlying member method code with example usage: obj.tokens_to_indices(tokens, _vocabulary, index_name) and returns: dict