code: def test_llama_cpp_tokenization():
      llama = llama_cpp.Llama(
          model_path=MODEL, vocab_only=True, verbose=False
      )
  
      assert llama
      assert llama._ctx.ctx is not None
  
      text = b"Hello World"
  
      tokens = llama.tokenize(text)
      assert tokens[0] == llama.token_bos()
      assert tokens == [1, 15043, 2787]
      detokenized = llama.detokenize(tokens)
      assert detokenized == text
  
      tokens = llama.tokenize(text, add_bos=False)
      assert tokens[0] != llama.token_bos()
      assert tokens == [15043, 2787]
  
      detokenized = llama.detokenize(tokens)
      assert detokenized != text
  
      text = b"Hello World</s>"
      tokens = llama.tokenize(text)
      assert tokens[-1] != llama.token_eos()
      assert tokens == [1, 15043, 2787, 829, 29879, 29958]
  
      tokens = llama.tokenize(text, special=True)
      assert tokens[-1] == llama.token_eos()
      assert tokens == [1, 15043, 2787, 2]
  
      text = b""
      tokens = llama.tokenize(text, add_bos=True, special=True)
      assert tokens[-1] != llama.token_eos()
      assert tokens == [llama.token_bos()]
      assert text == llama.detokenize(tokens)
  
file_path: llama-cpp-python/tests/test_llama.py
imports: import ctypes
  import numpy
  import pytest
  from scipy.special import log_softmax
  import llama_cpp
label_desc: Use image node_id 1 for calling a global function with example usage: test_llama_cpp_tokenization() without return types