docstr: Encode the text using Aleph Alpha's tokenizer library:
  https://aleph-alpha-client.readthedocs.io/en/latest/aleph_alpha_client.html#aleph_alpha_client.Client.tokenizer
docstr_tok: ["Encode","the","text","using","Aleph","Alpha","'s","tokenizer","library",":","https",":","\/\/aleph-alpha-client.readthedocs.io\/en\/latest\/aleph_alpha_client.html","#","aleph_alpha_client.Client.tokenizer"]
code: def tokenize(
      self, request: TokenizationRequest
  ) -> TokenizationRequestResult:
      """
      Encode the text using Aleph Alpha's tokenizer library:
      https://aleph-alpha-client.readthedocs.io/en/latest/aleph_alpha_client.html#aleph_alpha_client.Client.tokenizer
      """
      try:
  
          def do_it():
              tokenizer: Tokenizer = self._get_tokenizer(
                  request.tokenizer_name
              )
              result: Encoding = tokenizer.encode(
                  request.text, add_special_tokens=False
              )
              return {"token_ids": result.ids, "tokens": result.tokens}
  
          cache_key = {
              "model": request.tokenizer_name,
              "prompt": request.text,
              "tokens": True,
              "token_ids": True,
          }
          response, cached = self.cache.get(
              cache_key, wrap_request_time(do_it)
          )
      except RuntimeError as e:
          error: str = f"AlephAlphaClient tokenize error: {e}"
          return TokenizationRequestResult(
              error=error,
              success=False,
              cached=False,
              text="",
              tokens=[],
          )
  
      tokens = response["token_ids" if request.encode else "tokens"]
      if request.truncation:
          tokens = tokens[: request.max_length]
  
      return TokenizationRequestResult(
          success=True,
          cached=cached,
          tokens=[TokenizationToken(value) for value in tokens],
          text=request.text,
          request_time=response["request_time"],
      )
  
file_path: H2O/h2o_hf/helm/src/helm/proxy/clients/aleph_alpha_client.py
imports: import json
  import requests
  from typing import Any, Dict, List
  from aleph_alpha_client import Client
  from tokenizers import Tokenizer, Encoding
  from helm.common.cache import Cache, CacheConfig
  from helm.common.hierarchical_logger import hlog
  from helm.common.request import Request, RequestResult, Sequence, Token
  from helm.common.tokenization_request import DecodeRequest, DecodeRequestResult, TokenizationRequest, TokenizationRequestResult, TokenizationToken
  from .client import Client, wrap_request_time, truncate_sequence
label_desc: Use image node_id 5 for calling the AlephAlphaClient obj's underlying member method code with example usage: obj.tokenize(request) and returns: TokenizationRequestResult, dict, TokenizationRequestResult