code: def test_tokens_to_indices(self):
      tokens = [Token("ewoe"), Token("woe"), Token("ewe"), Token("ee")]
  
      indices = self.indexer.tokens_to_indices(tokens, None, "test")
  
      assert set(indices.keys()) == set(
          ["test", "test-offsets", "mask"]
      )
  
      text_tokens = indices["test"]
      offsets = indices["test-offsets"]
  
      assert text_tokens[:6] == [
          self.indexer.encoder.get(symbol, 0)
          for symbol in ["ew", "oe</w>"]
          + ["woe</w>"]
          + ["ew", "e</w>"]
          + ["ee</w>"]
      ]
  
      assert offsets == [
          1,  # end of first word
          2,  # end of second word
          4,  # end of third word
          5,  # end of last word
      ]
  
file_path: magnitude/pymagnitude/third_party/allennlp/tests/data/token_indexers/openai_transformer_byte_pair_indexer_test.py
imports: from __future__ import with_statement
  from __future__ import division
  from __future__ import absolute_import
  import json
  import tarfile
  import pytest
  from allennlp.common.testing import AllenNlpTestCase
  from allennlp.data import Token
  from allennlp.data.token_indexers import OpenaiTransformerBytePairIndexer
  from io import open
label_desc: Use image node_id 3 for calling the TestOpenaiTransformerBytePairIndexer obj's underlying member method code with example usage: obj.test_tokens_to_indices() without return types