code: def test_subword_nmt_tokenizer():
      with tempfile.TemporaryDirectory() as dir_path:
          model_path = os.path.join(dir_path, "subword_nmt.model")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/subword-nmt/test_ende-d189ff.model",
              path=model_path,
          )
          vocab_path = os.path.join(dir_path, "subword_nmt.vocab")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/subword-nmt/test_ende_vocab-900f81.json",
              path=vocab_path,
          )
  
          # Case 1
          tokenizer = SubwordNMTTokenizer(model_path, vocab_path)
          gt_tokenized = [
              [
                  "Hel",
                  "lo",
                  ",</w>",
                  "y",
                  "'",
                  "all",
                  "!</w>",
                  "How</w>",
                  "are</w>",
                  "you</w>",
                  "Ⅷ</w>",
                  "😁</w>",
                  "😁</w>",
                  "😁</w>",
                  "?</w>",
              ],
              [
                  "Gl",
                  "u",
                  "on",
                  "N",
                  "L",
                  "P</w>",
                  "is</w>",
                  "great",
                  "！",
                  "！",
                  "！",
                  "!!",
                  "!</w>",
              ],
              [
                  "Gl",
                  "u",
                  "on",
                  "N",
                  "L",
                  "P",
                  "-",
                  "Amaz",
                  "on-",
                  "H",
                  "ai",
                  "b",
                  "in-",
                  "Le",
                  "on",
                  "ard",
                  "-",
                  "Sh",
                  "eng",
                  "-",
                  "Sh",
                  "u",
                  "ai",
                  "-",
                  "X",
                  "ing",
                  "ji",
                  "an",
                  "..",
                  "...",
                  "/",
                  ":",
                  "!",
                  "@",
                  "#</w>",
                  "'",
                  "ab",
                  "c",
                  "'</w>",
              ],
          ]
          gt_offsets = [
              [
                  (0, 3),
                  (3, 5),
                  (5, 6),
                  (7, 8),
                  (8, 9),
                  (9, 12),
                  (12, 13),
                  (14, 17),
                  (18, 21),
                  (22, 25),
                  (26, 27),
                  (28, 29),
                  (30, 31),
                  (32, 33),
                  (34, 35),
              ],
              [
                  (0, 2),
                  (2, 3),
                  (3, 5),
                  (5, 6),
                  (6, 7),
                  (7, 8),
                  (9, 11),
                  (12, 17),
                  (17, 18),
                  (18, 19),
                  (19, 20),
                  (20, 22),
                  (22, 23),
              ],
              [
                  (0, 2),
                  (2, 3),
                  (3, 5),
                  (5, 6),
                  (6, 7),
                  (7, 8),
                  (8, 9),
                  (9, 13),
                  (13, 16),
                  (16, 17),
                  (17, 19),
                  (19, 20),
                  (20, 23),
                  (23, 25),
                  (25, 27),
                  (27, 30),
                  (30, 31),
                  (31, 33),
                  (33, 36),
                  (36, 37),
                  (37, 39),
                  (39, 40),
                  (40, 42),
                  (42, 43),
                  (43, 44),
                  (44, 47),
                  (47, 49),
                  (49, 51),
                  (51, 53),
                  (53, 56),
                  (56, 57),
                  (57, 58),
                  (58, 59),
                  (59, 60),
                  (60, 61),
                  (62, 63),
                  (63, 65),
                  (65, 66),
                  (66, 67),
              ],
          ]
          gt_int_decode = [
              "Hello, y'all! How are you Ⅷ 😁 😁 😁 ?",
              "GluonNLP is great！！！!!!",
              "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'",
          ]
          gt_str_decode = SUBWORD_TEST_SAMPLES
          verify_encode_token(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized
          )
          verify_pickleble(tokenizer, SubwordNMTTokenizer)
          verify_encode_token_with_offsets(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets
          )
          verify_decode_subword_nmt(
              tokenizer,
              SUBWORD_TEST_SAMPLES,
              gt_int_decode,
              gt_str_decode,
          )
  
          # Case 2, bpe_dropout
          # We use str decode here because we may not perfectly recover the original sentence with int decode.
          tokenizer = SubwordNMTTokenizer(
              model_path, vocab_path, bpe_dropout=0.5
          )
          verify_decode(tokenizer, SUBWORD_TEST_SAMPLES, out_type=str)
  
          os.remove(model_path)
          os.remove(vocab_path)
  
imports: import pytest
  import random
  import collections
  import pickle
  from uuid import uuid4
  import os
  import unicodedata
  import tempfile
  from pkg_resources import parse_version
  import gluonnlp
  from gluonnlp.data.tokenizers import WhitespaceTokenizer, MosesTokenizer, JiebaTokenizer, SpacyTokenizer, SubwordNMTTokenizer, YTTMTokenizer, SentencepieceTokenizer, HuggingFaceBPETokenizer, HuggingFaceByteBPETokenizer, HuggingFaceWordPieceTokenizer, HuggingFaceTokenizer
  from gluonnlp.base import get_repo_url
  from gluonnlp.data import Vocab, load_vocab
  from gluonnlp.utils.misc import download
  from gluonnlp.models.t5 import T5Tokenizer
label_desc: Use image node_id 18 for calling a global function with example usage: test_subword_nmt_tokenizer() without return types