code: def test_sentencepiece_tokenizer():
      with tempfile.TemporaryDirectory() as dir_path:
          model_path = os.path.join(dir_path, "spm.model")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/sentencepiece/case1/test_ende-a9bee4.model",
              path=model_path,
          )
          # Case1
          tokenizer = SentencepieceTokenizer(model_path)
          gt_tokenized = [
              [
                  "▁Hel",
                  "lo",
                  ",",
                  "▁y",
                  "'",
                  "all",
                  "!",
                  "▁How",
                  "▁are",
                  "▁you",
                  "▁",
                  "VI",
                  "II",
                  "▁",
                  "😁",
                  "▁",
                  "😁",
                  "▁",
                  "😁",
                  "▁?",
              ],
              [
                  "▁G",
                  "lu",
                  "on",
                  "N",
                  "L",
                  "P",
                  "▁is",
                  "▁great",
                  "!",
                  "!",
                  "!",
                  "!",
                  "!",
                  "!",
              ],
              [
                  "▁G",
                  "lu",
                  "on",
                  "N",
                  "L",
                  "P",
                  "-",
                  "A",
                  "ma",
                  "zo",
                  "n",
                  "-",
                  "H",
                  "ai",
                  "bin",
                  "-",
                  "L",
                  "e",
                  "on",
                  "ard",
                  "-",
                  "S",
                  "hen",
                  "g",
                  "-",
                  "S",
                  "hu",
                  "ai",
                  "-",
                  "X",
                  "ing",
                  "j",
                  "ian",
                  ".",
                  ".",
                  ".",
                  ".",
                  ".",
                  "/",
                  ":",
                  "!",
                  "@",
                  "#",
                  "▁",
                  "'",
                  "ab",
                  "c",
                  "'",
              ],
          ]
          gt_offsets = [
              [
                  (0, 3),
                  (3, 5),
                  (5, 6),
                  (6, 8),
                  (8, 9),
                  (9, 12),
                  (12, 13),
                  (13, 17),
                  (17, 21),
                  (21, 25),
                  (25, 26),
                  (26, 26),
                  (26, 27),
                  (27, 28),
                  (28, 29),
                  (29, 30),
                  (30, 31),
                  (31, 32),
                  (32, 33),
                  (33, 35),
              ],
              [
                  (0, 1),
                  (1, 3),
                  (3, 5),
                  (5, 6),
                  (6, 7),
                  (7, 8),
                  (8, 11),
                  (11, 17),
                  (17, 18),
                  (18, 19),
                  (19, 20),
                  (20, 21),
                  (21, 22),
                  (22, 23),
              ],
              [
                  (0, 1),
                  (1, 3),
                  (3, 5),
                  (5, 6),
                  (6, 7),
                  (7, 8),
                  (8, 9),
                  (9, 10),
                  (10, 12),
                  (12, 14),
                  (14, 15),
                  (15, 16),
                  (16, 17),
                  (17, 19),
                  (19, 22),
                  (22, 23),
                  (23, 24),
                  (24, 25),
                  (25, 27),
                  (27, 30),
                  (30, 31),
                  (31, 32),
                  (32, 35),
                  (35, 36),
                  (36, 37),
                  (37, 38),
                  (38, 40),
                  (40, 42),
                  (42, 43),
                  (43, 44),
                  (44, 47),
                  (47, 48),
                  (48, 51),
                  (51, 52),
                  (52, 53),
                  (53, 54),
                  (54, 55),
                  (55, 56),
                  (56, 57),
                  (57, 58),
                  (58, 59),
                  (59, 60),
                  (60, 61),
                  (61, 62),
                  (62, 63),
                  (63, 65),
                  (65, 66),
                  (66, 67),
              ],
          ]
          gt_int_decode = [
              "Hello, y ⁇ all! How are you VIII  ⁇   ⁇   ⁇  ?",
              "GluonNLP is great!!!!!!",
              "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:! ⁇ #  ⁇ abc ⁇ ",
          ]
          verify_encode_token(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized
          )
          verify_pickleble(tokenizer, SentencepieceTokenizer)
          verify_encode_token_with_offsets(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets
          )
          verify_decode_spm(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_int_decode
          )
  
          # Case2, lower_case
          gt_lower_case_int_decode = [
              "hello, y ⁇ all! how are you viii  ⁇   ⁇   ⁇  ?",
              "gluonnlp is great!!!!!!",
              "gluonnlp-amazon-haibin-leonard-sheng-shuai-xingjian...../:! ⁇ #  ⁇ abc ⁇ ",
          ]
          tokenizer = SentencepieceTokenizer(model_path, lowercase=True)
          verify_decode_spm(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_lower_case_int_decode
          )
  
          # Case3, Use the sentencepiece regularization commands, we test whether we can obtain different encoding results
          tokenizer = SentencepieceTokenizer(
              model_path, lowercase=True, nbest=-1, alpha=1.0
          )
          has_different_encode_out = False
          encode_out = None
          for _ in range(10):
              if encode_out is None:
                  encode_out = tokenizer.encode(SUBWORD_TEST_SAMPLES[0])
              else:
                  ele_out = tokenizer.encode(SUBWORD_TEST_SAMPLES[0])
                  if ele_out != encode_out:
                      has_different_encode_out = True
                      break
          assert has_different_encode_out
          os.remove(model_path)
  
      # Case of T5 Tokenizer
      with tempfile.TemporaryDirectory() as dir_path:
          vocab_path = os.path.join(dir_path, "t5_spm.model")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/sentencepiece/case_t5/test_t5spm-5f05e7.model",
              path=vocab_path,
          )
          extra_ids = 100
          tokenizer = T5Tokenizer(vocab_path, extra_ids)
          gt_tokenized = [
              [
                  "▁Hello",
                  ",",
                  "▁",
                  "y",
                  "'",
                  "all",
                  "!",
                  "▁How",
                  "▁are",
                  "▁you",
                  "▁VIII",
                  "▁",
                  "😁",
                  "▁",
                  "😁",
                  "▁",
                  "😁",
                  "▁",
                  "?",
              ],
              [
                  "▁",
                  "Glu",
                  "on",
                  "N",
                  "LP",
                  "▁is",
                  "▁great",
                  "!",
                  "!!!!!",
              ],
              [
                  "▁",
                  "Glu",
                  "on",
                  "N",
                  "LP",
                  "-",
                  "Am",
                  "a",
                  "zon",
                  "-",
                  "H",
                  "a",
                  "i",
                  "bin",
                  "-",
                  "Le",
                  "on",
                  "ard",
                  "-",
                  "She",
                  "ng",
                  "-",
                  "Sh",
                  "u",
                  "a",
                  "i",
                  "-",
                  "X",
                  "ing",
                  "j",
                  "i",
                  "an",
                  ".....",
                  "/",
                  ":",
                  "!",
                  "@",
                  "#",
                  "▁",
                  "'",
                  "a",
                  "b",
                  "c",
                  "'",
              ],
          ]
          gt_offsets = [
              [
                  (0, 5),
                  (5, 6),
                  (6, 7),
                  (7, 8),
                  (8, 9),
                  (9, 12),
                  (12, 13),
                  (13, 17),
                  (17, 21),
                  (21, 25),
                  (25, 27),
                  (27, 28),
                  (28, 29),
                  (29, 30),
                  (30, 31),
                  (31, 32),
                  (32, 33),
                  (33, 34),
                  (34, 35),
              ],
              [
                  (0, 0),
                  (0, 3),
                  (3, 5),
                  (5, 6),
                  (6, 8),
                  (8, 11),
                  (11, 17),
                  (17, 18),
                  (18, 23),
              ],
              [
                  (0, 0),
                  (0, 3),
                  (3, 5),
                  (5, 6),
                  (6, 8),
                  (8, 9),
                  (9, 11),
                  (11, 12),
                  (12, 15),
                  (15, 16),
                  (16, 17),
                  (17, 18),
                  (18, 19),
                  (19, 22),
                  (22, 23),
                  (23, 25),
                  (25, 27),
                  (27, 30),
                  (30, 31),
                  (31, 34),
                  (34, 36),
                  (36, 37),
                  (37, 39),
                  (39, 40),
                  (40, 41),
                  (41, 42),
                  (42, 43),
                  (43, 44),
                  (44, 47),
                  (47, 48),
                  (48, 49),
                  (49, 51),
                  (51, 56),
                  (56, 57),
                  (57, 58),
                  (58, 59),
                  (59, 60),
                  (60, 61),
                  (61, 62),
                  (62, 63),
                  (63, 64),
                  (64, 65),
                  (65, 66),
                  (66, 67),
              ],
          ]
          gt_int_decode = [
              "Hello, y'all! How are you VIII  ⁇   ⁇   ⁇  ?",
              "GluonNLP is great!!!!!!",
              "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'",
          ]
          inserted_special_tokens = list(
              "<extra_id_{}>".format(i)
              for i in range(extra_ids - 1, -1, -1)
          )
          assert (
              list(
                  tokenizer.vocab.to_tokens(i)
                  for i in range(
                      len(tokenizer._sp_model), len(tokenizer._vocab)
                  )
              )
              == inserted_special_tokens
          ), "Some <extra_id> tokens are not properly inserted."
          verify_encode_token(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized
          )
          verify_pickleble(tokenizer, SentencepieceTokenizer)
          verify_encode_token_with_offsets(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets
          )
          verify_decode_spm(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_int_decode
          )
          os.remove(vocab_path)
  
imports: import pytest
  import random
  import collections
  import pickle
  from uuid import uuid4
  import os
  import unicodedata
  import tempfile
  from pkg_resources import parse_version
  import gluonnlp
  from gluonnlp.data.tokenizers import WhitespaceTokenizer, MosesTokenizer, JiebaTokenizer, SpacyTokenizer, SubwordNMTTokenizer, YTTMTokenizer, SentencepieceTokenizer, HuggingFaceBPETokenizer, HuggingFaceByteBPETokenizer, HuggingFaceWordPieceTokenizer, HuggingFaceTokenizer
  from gluonnlp.base import get_repo_url
  from gluonnlp.data import Vocab, load_vocab
  from gluonnlp.utils.misc import download
  from gluonnlp.models.t5 import T5Tokenizer
label_desc: Use image node_id 17 for calling a global function with example usage: test_sentencepiece_tokenizer() without return types