code: def test_huggingface_bpe_tokenizer():
      with tempfile.TemporaryDirectory() as dir_path:
          model_path = os.path.join(dir_path, "test_hf_bpe.model")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/hf_bpe/test_hf_bpe.model",
              path=model_path,
          )
          vocab_path = os.path.join(dir_path, "test_hf_bpe.vocab")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/hf_bpe/test_hf_bpe.vocab",
              path=vocab_path,
          )
          hf_vocab_path = os.path.join(dir_path, "test_hf_bpe.hf_vocab")
          download(
              url=get_repo_url()
              + "tokenizer_test_models/hf_bpe/test_hf_bpe.hf_vocab",
              path=hf_vocab_path,
          )
  
          # Case 1, default lowercase=False
          tokenizer = HuggingFaceBPETokenizer(model_path, vocab_path)
          gt_tokenized = [
              [
                  "Hello</w>",
                  ",</w>",
                  "y</w>",
                  "'</w>",
                  "all</w>",
                  "!</w>",
                  "How</w>",
                  "are</w>",
                  "you</w>",
                  "<unk>",
                  "<unk>",
                  "<unk>",
                  "<unk>",
                  "?</w>",
              ],
              [
                  "Gl",
                  "u",
                  "on",
                  "N",
                  "LP</w>",
                  "is</w>",
                  "great</w>",
                  "！</w>",
                  "！</w>",
                  "！</w>",
                  "!</w>",
                  "!</w>",
                  "!</w>",
              ],
              [
                  "Gl",
                  "u",
                  "on",
                  "N",
                  "LP</w>",
                  "-</w>",
                  "Amazon</w>",
                  "-</w>",
                  "H",
                  "ai",
                  "bin</w>",
                  "-</w>",
                  "Leonard</w>",
                  "-</w>",
                  "Sh",
                  "en",
                  "g</w>",
                  "-</w>",
                  "Sh",
                  "u",
                  "ai</w>",
                  "-</w>",
                  "X",
                  "ing",
                  "j",
                  "ian</w>",
                  ".</w>",
                  ".</w>",
                  ".</w>",
                  ".</w>",
                  ".</w>",
                  "/</w>",
                  ":</w>",
                  "!</w>",
                  "@</w>",
                  "#</w>",
                  "'</w>",
                  "ab",
                  "c</w>",
                  "'</w>",
              ],
          ]
          gt_offsets = [
              [
                  (0, 5),
                  (5, 6),
                  (7, 8),
                  (8, 9),
                  (9, 12),
                  (12, 13),
                  (14, 17),
                  (18, 21),
                  (22, 25),
                  (26, 27),
                  (28, 29),
                  (30, 31),
                  (32, 33),
                  (34, 35),
              ],
              [
                  (0, 2),
                  (2, 3),
                  (3, 5),
                  (5, 6),
                  (6, 8),
                  (9, 11),
                  (12, 17),
                  (17, 18),
                  (18, 19),
                  (19, 20),
                  (20, 21),
                  (21, 22),
                  (22, 23),
              ],
              [
                  (0, 2),
                  (2, 3),
                  (3, 5),
                  (5, 6),
                  (6, 8),
                  (8, 9),
                  (9, 15),
                  (15, 16),
                  (16, 17),
                  (17, 19),
                  (19, 22),
                  (22, 23),
                  (23, 30),
                  (30, 31),
                  (31, 33),
                  (33, 35),
                  (35, 36),
                  (36, 37),
                  (37, 39),
                  (39, 40),
                  (40, 42),
                  (42, 43),
                  (43, 44),
                  (44, 47),
                  (47, 48),
                  (48, 51),
                  (51, 52),
                  (52, 53),
                  (53, 54),
                  (54, 55),
                  (55, 56),
                  (56, 57),
                  (57, 58),
                  (58, 59),
                  (59, 60),
                  (60, 61),
                  (62, 63),
                  (63, 65),
                  (65, 66),
                  (66, 67),
              ],
          ]
          # gt_int_decode = gt_str_decode for hf
          # hf removed the unk tokens in decode result
          gt_decode = [
              "Hello , y ' all ! How are you ?",
              "GluonNLP is great ！ ！ ！ ! ! !",
              "GluonNLP - Amazon - Haibin - Leonard - Sheng - Shuai - Xingjian . . . . . / : ! @ # ' abc '",
          ]
          verify_encode_token(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized
          )
          verify_pickleble(tokenizer, HuggingFaceBPETokenizer)
          verify_encode_token_with_offsets(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets
          )
          verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)
  
          # Case 2, lowercase=True
          gt_lowercase_decode = [
              "hello , y ' all ! how are you ?",
              "gluonnlp is great ！ ！ ！ ! ! !",
              "gluonnlp - amazon - haibin - leonard - sheng - shuai - xingjian . . . . . / : ! @ # ' abc '",
          ]
          tokenizer = HuggingFaceBPETokenizer(
              model_path, vocab_path, lowercase=True
          )
          verify_decode_hf(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_lowercase_decode
          )
  
          # Case 3, using original hf vocab
          tokenizer = HuggingFaceBPETokenizer(model_path, hf_vocab_path)
          verify_encode_token(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized
          )
          verify_pickleble(tokenizer, HuggingFaceBPETokenizer)
          verify_encode_token_with_offsets(
              tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets
          )
          verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)
  
          os.remove(model_path)
          os.remove(vocab_path)
          os.remove(hf_vocab_path)
  
imports: import pytest
  import random
  import collections
  import pickle
  from uuid import uuid4
  import os
  import unicodedata
  import tempfile
  from pkg_resources import parse_version
  import gluonnlp
  from gluonnlp.data.tokenizers import WhitespaceTokenizer, MosesTokenizer, JiebaTokenizer, SpacyTokenizer, SubwordNMTTokenizer, YTTMTokenizer, SentencepieceTokenizer, HuggingFaceBPETokenizer, HuggingFaceByteBPETokenizer, HuggingFaceWordPieceTokenizer, HuggingFaceTokenizer
  from gluonnlp.base import get_repo_url
  from gluonnlp.data import Vocab, load_vocab
  from gluonnlp.utils.misc import download
  from gluonnlp.models.t5 import T5Tokenizer
label_desc: Use image node_id 19 for calling a global function with example usage: test_huggingface_bpe_tokenizer() without return types