docstr: Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
docstr_tok: ["Adapted","from","Sennrich","et","al",".","2015","and","https",":","\/\/github.com\/rsennrich\/subword-nmt"]
code: def test_full_tokenizer(self):
      """Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt"""
      tokenizer = FSMTTokenizer(
          self.langs,
          self.src_vocab_file,
          self.tgt_vocab_file,
          self.merges_file,
      )
  
      text = "lower"
      bpe_tokens = ["low", "er</w>"]
      tokens = tokenizer.tokenize(text)
      self.assertListEqual(tokens, bpe_tokens)
  
      input_tokens = tokens + ["<unk>"]
      input_bpe_tokens = [14, 15, 20]
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(input_tokens),
          input_bpe_tokens,
      )
  
file_path: transformers/tests/models/fsmt/test_tokenization_fsmt.py
imports: import json
  import os
  import unittest
  from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES, FSMTTokenizer
  from transformers.testing_utils import slow
  from transformers.utils import cached_property
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 5 for calling the FSMTTokenizationTest obj's underlying member method code with example usage: obj.test_full_tokenizer() without return types