code: def verify_encode_token_with_offsets(
      tokenizer, all_sentences, gt_offsets=None
  ):
      if gt_offsets is None:
          for sentences in [all_sentences[0], all_sentences]:
              enc_tokens = tokenizer.encode(sentences, str)
              tokens, offsets = tokenizer.encode_with_offsets(
                  sentences, str
              )
              if isinstance(sentences, list):
                  for (
                      ele_tokens,
                      ele_enc_tokens,
                      ele_offsets,
                      ele_sentence,
                  ) in zip(tokens, enc_tokens, offsets, sentences):
                      for tok, offset, enc_tok in zip(
                          ele_tokens, ele_offsets, ele_enc_tokens
                      ):
                          assert (
                              ele_sentence[offset[0] : offset[1]] == tok
                          )
                          assert tok == enc_tok
              else:
                  for tok, offset, enc_tok in zip(
                      tokens, offsets, enc_tokens
                  ):
                      assert sentences[offset[0] : offset[1]] == tok
                      assert tok == enc_tok
      else:
          for sentences, ele_gt_offsets in [
              (all_sentences[0], gt_offsets[0]),
              (all_sentences, gt_offsets),
          ]:
              enc_tokens = tokenizer.encode(sentences, str)
              tokens, offsets = tokenizer.encode_with_offsets(
                  sentences, str
              )
              assert ele_gt_offsets == offsets
              assert enc_tokens == tokens
  
imports: import pytest
  import random
  import collections
  import pickle
  from uuid import uuid4
  import os
  import unicodedata
  import tempfile
  from pkg_resources import parse_version
  import gluonnlp
  from gluonnlp.data.tokenizers import WhitespaceTokenizer, MosesTokenizer, JiebaTokenizer, SpacyTokenizer, SubwordNMTTokenizer, YTTMTokenizer, SentencepieceTokenizer, HuggingFaceBPETokenizer, HuggingFaceByteBPETokenizer, HuggingFaceWordPieceTokenizer, HuggingFaceTokenizer
  from gluonnlp.base import get_repo_url
  from gluonnlp.data import Vocab, load_vocab
  from gluonnlp.utils.misc import download
  from gluonnlp.models.t5 import T5Tokenizer
label_desc: Use image node_id 2 for calling a global function with example usage: verify_encode_token_with_offsets(tokenizer, all_sentences, gt_offsets) without return types