code: def test_basic_offsets():
      enc = tiktoken.get_encoding("cl100k_base")
  
      prompt = "hello world"
      p, o = enc.decode_with_offsets(enc.encode(prompt))
      assert p == prompt
      assert o == [0, 5]
  
      prompt = "hello world<|endoftext|> green cow"
      p, o = enc.decode_with_offsets(
          enc.encode(prompt, allowed_special="all")
      )
      assert p == prompt
      assert o == [0, 5, 11, 24, 30]
  
      prompt = "我非常渴望与人工智能一起工作"
      p, o = enc.decode_with_offsets(enc.encode(prompt))
      assert p == prompt
      assert o == [
          0,
          1,
          2,
          3,
          3,
          4,
          4,
          5,
          6,
          7,
          8,
          8,
          9,
          10,
          11,
          12,
          13,
      ]
  
      # contains the interesting tokens b'\xe0\xae\xbf\xe0\xae' and b'\xe0\xaf\x8d\xe0\xae'
      # in which \xe0 is the start of a 3-byte UTF-8 character
      prompt = "நடிகர் சூர்யா"
      p, o = enc.decode_with_offsets(enc.encode(prompt))
      assert p == prompt
      assert o == [
          0,
          0,
          1,
          1,
          2,
          3,
          4,
          4,
          5,
          6,
          7,
          8,
          8,
          9,
          9,
          10,
          11,
          12,
          12,
      ]
  
      # contains the interesting token b'\xa0\xe9\x99\xa4'
      # in which \xe9 is the start of a 3-byte UTF-8 character and \xa0 is a continuation byte
      prompt = " Ġ除"
      p, o = enc.decode_with_offsets(enc.encode(prompt))
      assert p == prompt
      assert o == [0, 1]
  
file_path: tiktoken/tests/test_offsets.py
imports: from typing import Callable
  import hypothesis
  import pytest
  from hypothesis import strategies
  import tiktoken
  from .test_helpers import MAX_EXAMPLES, SOME_ENCODING_FACTORIES
label_desc: Use image node_id 4 for calling a global function with example usage: test_basic_offsets() without return types