class_docstr: Generates the indices for the byte-pair encoding used by
  the OpenAI transformer language model: https://blog.openai.com/language-unsupervised/
  
  This is unlike most of our TokenIndexers in that its
  indexing is not based on a `Vocabulary` but on a fixed
  set of mappings that are loaded by the constructor.
class_docstr_tok: ["Generates","the","indices","for","the","byte-pair","encoding","used","by","the","OpenAI","transformer","language","model",":","https",":","\/\/blog.openai.com\/language-unsupervised\/","This","is","unlike","most","of","our","TokenIndexers","in","that","its","indexing","is","not","based","on","a","`","Vocabulary","`","but","on","a","fixed","set","of","mappings","that","are","loaded","by","the","constructor","."]
code: def byte_pair_encode(self, token, lowercase=True):
      if lowercase:
          text = token.text.lower()
      else:
          text = token.text
  
      if text in self.cache:
          return self.cache[text]
  
      # Split into letters, but add a `</w>` to the last
      word = [c for c in text[:-1]]
      word.append(text[-1] + "</w>")
  
      # Get unique pairs (prev_symbol, next_symbol)
      pairs = set(
          (prev_symbol, next_symbol)
          for prev_symbol, next_symbol in izip(word, word[1:])
      )
  
      if not pairs:
          return [text + "</w>"]
  
      while True:
          # Find the highest ranked pair
          bigram = min(
              pairs,
              key=lambda pair: self.bpe_ranks.get(pair, float("inf")),
          )
  
          # If that pair is not actually ranked, stop.
          if bigram not in self.bpe_ranks:
              break
  
          # Split up the pair
          first, second = bigram
  
          # and make a helper for a new word
          new_word = []
          i = 0
  
          # Iterate over the letters of the word
          while i < len(word):
              try:
                  # Find first occurrence of `first` after i,
                  j = word.index(first, i)
                  # add all the characters preceding it,
                  new_word.extend(word[i:j])
                  # and update i to j
                  i = j
              except ValueError:
                  # `first` didn't occur, so just add the rest
                  new_word.extend(word[i:])
                  break  # out of while i < len(word)
  
              # At this point we know word[i] == first
              if i < len(word) - 1 and word[i + 1] == second:
                  new_word.append(first + second)
                  i += 2
              else:
                  new_word.append(word[i])
                  i += 1
  
          word = new_word
          if len(word) == 1:
              break  # out of while True
          else:
              pairs = set(
                  (prev_symbol, next_symbol)
                  for prev_symbol, next_symbol in izip(word, word[1:])
              )
  
      if " ".join(word) == "\n  </w>":
          word = ["\n</w>"]
  
      self.cache[text] = word
      return word
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/token_indexers/openai_transformer_byte_pair_indexer.py
imports: from __future__ import with_statement
  from __future__ import absolute_import
  import json
  import tarfile
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.file_utils import cached_path
  from allennlp.common.util import pad_sequence_to_length
  from allennlp.data.vocabulary import Vocabulary
  from allennlp.data.tokenizers.token import Token
  from allennlp.data.token_indexers.token_indexer import TokenIndexer
label_desc: Use image node_id 3 for calling the OpenaiTransformerBytePairIndexer obj's underlying member method code with example usage: obj.byte_pair_encode(token, lowercase) and returns: word, self, list