class_docstr: AI21 Labs provides Jurassic models.
  https://studio.ai21.com/docs/api/
docstr: Tokenizes the text by using the AI21 endpoint: https://api.ai21.com/studio/v1/tokenize.
docstr_tok: ["Tokenizes","the","text","by","using","the","AI21","endpoint",":","https",":","\/\/api.ai21.com\/studio\/v1\/tokenize","."]
code: def tokenize(
      self, request: TokenizationRequest
  ) -> TokenizationRequestResult:
      """
      Tokenizes the text by using the AI21 endpoint: https://api.ai21.com/studio/v1/tokenize.
      """
      raw_request: Dict[str, str] = {"text": request.text}
  
      def do_it():
          response = requests.post(
              "https://api.ai21.com/studio/v1/tokenize",
              headers={"Authorization": f"Bearer {self.api_key}"},
              json=raw_request,
          ).json()
  
          # If 'tokens' is not present in the response, assume request failed.
          if "tokens" not in response:
              AI21Client.handle_failed_request(
                  api_type="tokenizer", response=response
              )
  
          return response
  
      try:
          response, cached = self.cache.get(raw_request, do_it)
      except AI21RequestError:
          return TokenizationRequestResult(
              success=False, cached=False, text="", tokens=[]
          )
  
      # Each token is represented like this in the response:
      # {'token': '▁Hello', 'textRange': {'start': 0, 'end': 5}}
      tokens: List[TokenizationToken] = []
      for token_dict in response["tokens"]:
          tokens.append(
              TokenizationToken(
                  value=token_dict["token"],
                  text_range=from_dict(
                      TextRange, token_dict["textRange"]
                  ),
              )
          )
      text: str = response["text"]
      return TokenizationRequestResult(
          success=True, cached=cached, tokens=tokens, text=text
      )
  
file_path: H2O/h2o_hf/helm/src/helm/proxy/clients/ai21_client.py
imports: from typing import Dict, List
  import requests
  from dacite import from_dict
  from helm.common.cache import Cache, CacheConfig
  from helm.common.request import EMBEDDING_UNAVAILABLE_REQUEST_RESULT, Request, RequestResult, Sequence, Token
  from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult, TokenizationToken, TextRange, DecodeRequest, DecodeRequestResult
  from .client import Client, wrap_request_time, truncate_sequence
label_desc: Use image node_id 4 for calling the AI21Client obj's underlying member method code with example usage: obj.tokenize(request) and returns: TokenizationRequestResult, response, TokenizationRequestResult