class_docstr: Tokenizes by making a request to the proxy server with REST endpoint: `/api/tokenize`.
docstr: Truncates the text using the AI21 Jurassic tokenizer.
  First, ensures the text is shorter than `AI21Tokenizer.MAX_CHARACTER_LENGTH` long.
  Tokenizes, then truncates the list of tokens to fit within the context window minus the
  expected completion length (defaults to 0), then uses the start of the text range of the first
  token and the end of the text range of the last token of the truncated list of tokens to
  build the truncated text.
docstr_tok: ["Truncates","the","text","using","the","AI21","Jurassic","tokenizer",".","First",",","ensures","the","text","is","shorter","than","`","AI21Tokenizer.MAX_CHARACTER_LENGTH","`","long",".","Tokenizes",",","then","truncates","the","list","of","tokens","to","fit","within","the","context","window","minus","the","expected","completion","length","(","defaults","to","0",")",",","then","uses","the","start","of","the","text","range","of","the","first","token","and","the","end","of","the","text","range","of","the","last","token","of","the","truncated","list","of","tokens","to","build","the","truncated","text","."]
code: def truncate_from_right(
      self, text: str, expected_completion_token_length: int = 0
  ) -> str:
      """
      Truncates the text using the AI21 Jurassic tokenizer.
      First, ensures the text is shorter than `AI21Tokenizer.MAX_CHARACTER_LENGTH` long.
      Tokenizes, then truncates the list of tokens to fit within the context window minus the
      expected completion length (defaults to 0), then uses the start of the text range of the first
      token and the end of the text range of the last token of the truncated list of tokens to
      build the truncated text.
      """
      text = text[: AI21WindowService.MAX_CHARACTER_LENGTH]
      response: TokenizationRequestResult = (
          self._make_tokenization_request(text)
      )
  
      # Only look at the first `self.max_request_length` - `expected_completion_token_length`
      # number of tokens to the fit the text within the context window.
      # Each token is represented like this: {'text': '▁Hello', 'textRange': {'start': 0, 'end': 5}}
      max_length: int = (
          self.max_request_length - expected_completion_token_length
      )
      tokens: List[TokenizationToken] = response.tokens[:max_length]
  
      # If there is no tokens, just return the original text
      if len(tokens) == 0:
          return text
  
      # AI21 uses "_" to represent a single space in their tokens, so we have to build the new text from the
      # original text after truncation using the text ranges of tokens generated from the original text.
      assert tokens[0].text_range is not None
      first_text_range: TextRange = tokens[0].text_range
      assert tokens[-1].text_range is not None
      last_text_range: TextRange = tokens[-1].text_range
      start: int = first_text_range.start
      end: int = last_text_range.end
      truncated_text: str = text[start:end]
  
      # HACK: For the vast majority of cases, the above logic works, but there are a few where the
      # token count exceeds `max_length` by 1. This might be a bug with the AI21 tokenizer API.
      # We handle those by removing characters one by one until it fits within the context window.
      while not self.fits_within_context_window(
          truncated_text, expected_completion_token_length
      ):
          end -= 1
          truncated_text = text[start:end]
      return truncated_text
  
imports: import re
  from typing import List, Optional, Tuple
  from urllib.parse import unquote
  from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult, TokenizationToken, TextRange
  from .window_service import WindowService, EncodeResult
  from .tokenizer_service import TokenizerService
  from .gpt2_window_service import GPT2WindowService
label_desc: Use image node_id 12 for calling the AI21WindowService obj's underlying member method code with example usage: obj.truncate_from_right(text, expected_completion_token_length) and returns: truncated_text, text