code: def test_full_tokenizer(self):
      tokenizer = self.tokenizer_class(
          vocab_file=self.vocab_file, merges_file=self.merges_file
      )
  
      text = "lower"
      bpe_tokens = ["low", "er</w>"]
      tokens = tokenizer.tokenize(text)
      self.assertListEqual(tokens, bpe_tokens)
  
      input_tokens = tokens + ["<unk>"]
      input_bpe_tokens = [16, 17, 23]
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(input_tokens),
          input_bpe_tokens,
      )
  
file_path: transformers/tests/models/herbert/test_tokenization_herbert.py
imports: import json
  import os
  import unittest
  from transformers import HerbertTokenizer, HerbertTokenizerFast
  from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
  from transformers.testing_utils import get_tests_dir, require_tokenizers, slow
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 3 for calling the HerbertTokenizationTest obj's underlying member method code with example usage: obj.test_full_tokenizer() without return types