code: def verify_sentencepiece_tokenizer_with_offsets(
      tokenizer, all_sentences
  ):
      for sentences in [all_sentences[0], all_sentences]:
          enc_tokens = tokenizer.encode(sentences, str)
          tokens, offsets = tokenizer.encode_with_offsets(
              sentences, str
          )
          if isinstance(sentences, list):
              for (
                  ele_tokens,
                  ele_enc_tokens,
                  ele_offsets,
                  ele_sentence,
              ) in zip(tokens, enc_tokens, offsets, sentences):
                  for i, (tok, offset, enc_tok) in enumerate(
                      zip(ele_tokens, ele_offsets, ele_enc_tokens)
                  ):
                      assert tok == enc_tok
                      ele_sel_tok = unicodedata.normalize(
                          "NFKC", ele_sentence[offset[0] : offset[1]]
                      ).strip()
                      if tokenizer.is_first_subword(tok):
                          real_tok = tok[1:]
                      else:
                          real_tok = tok
                      assert (
                          ele_sel_tok == real_tok
                      ), "ele_sel_tok={}, real_tok={}".format(
                          ele_sel_tok, real_tok
                      )
  
imports: import pytest
  import random
  import collections
  import pickle
  from uuid import uuid4
  import os
  import unicodedata
  import tempfile
  from pkg_resources import parse_version
  import gluonnlp
  from gluonnlp.data.tokenizers import WhitespaceTokenizer, MosesTokenizer, JiebaTokenizer, SpacyTokenizer, SubwordNMTTokenizer, YTTMTokenizer, SentencepieceTokenizer, HuggingFaceBPETokenizer, HuggingFaceByteBPETokenizer, HuggingFaceWordPieceTokenizer, HuggingFaceTokenizer
  from gluonnlp.base import get_repo_url
  from gluonnlp.data import Vocab, load_vocab
  from gluonnlp.utils.misc import download
  from gluonnlp.models.t5 import T5Tokenizer
label_desc: Use image node_id 3 for calling a global function with example usage: verify_sentencepiece_tokenizer_with_offsets(tokenizer, all_sentences) without return types