code: def test_full_blenderbot_small_tokenizer(self):
      tokenizer = BlenderbotSmallTokenizer(
          self.vocab_file, self.merges_file, **self.special_tokens_map
      )
      text = "adapt act apte"
      bpe_tokens = ["adapt", "act", "ap@@", "te"]
      tokens = tokenizer.tokenize(text)
      self.assertListEqual(tokens, bpe_tokens)
  
      input_tokens = (
          [tokenizer.bos_token] + tokens + [tokenizer.eos_token]
      )
  
      input_bpe_tokens = [0, 1, 2, 3, 4, 5]
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(input_tokens),
          input_bpe_tokens,
      )
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/tests/models/blenderbot_small/test_tokenization_blenderbot_small.py
imports: import json
  import os
  import unittest
  from transformers.models.blenderbot_small.tokenization_blenderbot_small import VOCAB_FILES_NAMES, BlenderbotSmallTokenizer
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 4 for calling the BlenderbotSmallTokenizerTest obj's underlying member method code with example usage: obj.test_full_blenderbot_small_tokenizer() without return types