code: def tokenize_text(
      client: AutoClient,
      tokenizer: str,
      output_path: str = "synthetic_efficiency_instances",
      base_path: str = "prod_env",
  ) -> Tuple[Dict[str, List[TokenizationToken]], Dict[str, List[str]]]:
      """Tokenizes each book using the requested tokenizer service."""
      sources = {
          "alice": "https://www.gutenberg.org/files/11/11-0.txt",
          "pride_and_prejudice": "https://www.gutenberg.org/files/1342/1342-0.txt",
          "sherlock_holmes": "https://www.gutenberg.org/files/1661/1661-0.txt",
          "monte_cristo": "https://www.gutenberg.org/cache/epub/1184/pg1184.txt",
          "crime_and_punishment": "https://www.gutenberg.org/files/2554/2554-0.txt",
      }
  
      tokens: Dict[str, List[TokenizationToken]] = {}
      text_chunks: Dict[str, List[str]] = {}
  
      tokenizer_organization: str = tokenizer.split("/")[0]
      ai21_tokenizer: bool = tokenizer_organization == "ai21"
  
      # Extract tokens from book sources
      seen_tokens = set()
      for book, source_url in sources.items():
          data_path = os.path.join(output_path, f"{book}.txt")
          ensure_file_downloaded(
              source_url=source_url,
              target_path=data_path,
              unpack=False,
          )
          with open(data_path, "r") as f:
              raw_text = f.read()
          batch_size = 2048
          # Skip intro text
          text = raw_text.split(" ")[batch_size:]
          i = 0
          tokens[book] = []
          text_chunks[book] = []
          while i * batch_size < len(text):
              batch = " ".join(
                  text[i * batch_size : (i + 1) * batch_size]
              )
              while True:
                  request: TokenizationRequest = TokenizationRequest(
                      text=batch,
                      tokenizer=tokenizer,
                      encode=(not ai21_tokenizer),
                  )
                  result: TokenizationRequestResult = client.tokenize(
                      request
                  )
                  tokens_ = frozenset(
                      [token.value for token in result.tokens]
                  )
                  if tokens_ not in seen_tokens:
                      seen_tokens.add(tokens_)
                      break
              tokens[book] += result.tokens
              if ai21_tokenizer:
                  text_chunks[book] += [
                      result.text[token.text_range.start : token.text_range.end]  # type: ignore
                      for token in result.tokens
                  ]
              i += 1
      return tokens, text_chunks
  
file_path: H2O/h2o_hf/helm/scripts/efficiency/generate_instances.py
imports: import os
  from typing import Dict, List, Tuple
  from helm.common.general import ensure_directory_exists, ensure_file_downloaded, parse_hocon, write
  from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult, DecodeRequest, DecodeRequestResult, TokenizationToken
  from helm.proxy.clients.client import Client
  from helm.proxy.clients.auto_client import AutoClient
  from helm.proxy.services.service import CREDENTIALS_FILE, CACHE_DIR
  from helm.benchmark.scenarios.synthetic_efficiency_scenario import NUM_INPUT_TOKENS
label_desc: Use image node_id 3 for calling a global function with example usage: tokenize_text(client, tokenizer, output_path, base_path) and returns: tokens,  text_chunks