code: def test_tokenized_single(self):
      tokens = [
          ["hello", "world", "!"],
          [
              "pine",
              "##apple",
              "is",
              "not",
              "in",
              "the",
              "bert",
              "vocabulary",
              ".",
          ],
      ]
      encoded_input = model_utils.batch_encode_pretokenized(
          self.tokenizer, tokens
      )
      # We don't care about the raw IDs, but let's detokenize and make sure we get
      # something reasonable.
      recovered_tokens = [
          self.tokenizer.convert_ids_to_tokens(ids[:ntok])
          for ids, ntok in zip(
              encoded_input["input_ids"].numpy(),
              encoded_input["attention_mask"].numpy().sum(axis=1),
          )
      ]
      expected_tokens = [["[CLS]"] + ts + ["[SEP]"] for ts in tokens]
      self.assertEqual(recovered_tokens, expected_tokens)
  
file_path: lit/lit_nlp/examples/models/model_utils_test.py
imports: import os
  from absl.testing import absltest
  from lit_nlp.examples.models import model_utils
  import numpy
  import transformers
label_desc: Use image node_id 3 for calling the BatchEncodePretokenizedTest obj's underlying member method code with example usage: obj.test_tokenized_single() without return types