code: def test_bpe(self):
      # [e, w, o, e</w>] -> best pair (e, w)
      # [ew, o, e</w>] -> best pair (o, e</w>)
      # [ew, oe</w>] -> done
      token = Token("ewoe")
      assert self.indexer.byte_pair_encode(token) == ["ew", "oe</w>"]
  
      # Prefer "ew" to "we"
      token = Token("ewe")
      assert self.indexer.byte_pair_encode(token) == ["ew", "e</w>"]
  
      # Prefer ending a word
      token = Token("eee")
      assert self.indexer.byte_pair_encode(token) == ["e", "ee</w>"]
  
      # Encodes up to a single symbol when appropriate
      token = Token("woe")
      assert self.indexer.byte_pair_encode(token) == ["woe</w>"]
  
file_path: magnitude/pymagnitude/third_party/allennlp/tests/data/token_indexers/openai_transformer_byte_pair_indexer_test.py
imports: from __future__ import with_statement
  from __future__ import division
  from __future__ import absolute_import
  import json
  import tarfile
  import pytest
  from allennlp.common.testing import AllenNlpTestCase
  from allennlp.data import Token
  from allennlp.data.token_indexers import OpenaiTransformerBytePairIndexer
  from io import open
label_desc: Use image node_id 2 for calling the TestOpenaiTransformerBytePairIndexer obj's underlying member method code with example usage: obj.test_bpe() without return types