class_docstr: Tokenizes by making a request to the proxy server with REST endpoint: `/api/tokenize`.
docstr: If the text is too long  (longer than 11,000 tokens when tokenized by the GPT-2 tokenizer),
  the AI21 server will close the connection. Therefore, we need to split the text into smaller
  chunks, tokenize each chunk, and re-assemble the tokenization results.
docstr_tok: ["If","the","text","is","too","long","(","longer","than","11,000","tokens","when","tokenized","by","the","GPT-2","tokenizer",")",",","the","AI21","server","will","close","the","connection",".","Therefore",",","we","need","to","split","the","text","into","smaller","chunks",",","tokenize","each","chunk",",","and","re-assemble","the","tokenization","results","."]
code: def _make_long_tokenization_request(
      self, text: str
  ) -> Tuple[List[TokenizationToken], str]:
      """If the text is too long  (longer than 11,000 tokens when tokenized by the GPT-2 tokenizer),
      the AI21 server will close the connection. Therefore, we need to split the text into smaller
      chunks, tokenize each chunk, and re-assemble the tokenization results.
      """
      # Uses the number of gpt2-style tokens as a measure of text length.
      gpt2_tokens: List[
          TokenizationToken
      ] = self.gpt2_window_service.encode(text).tokens
  
      # If the text is short, just makes one request and returns the result.
      if (
          len(gpt2_tokens)
          < AI21WindowService.MAX_TOKENIZATION_REQUEST_LENGTH
      ):
          result: TokenizationRequestResult = (
              self._make_tokenization_request(text)
          )
          return result.tokens, result.text
      # Otherwise, splits the text to chunks, tokenizes each chunk, and re-assembles them.
      else:
          all_tokens: List[TokenizationToken] = []
          normalized_text_chunks: List[str] = []
          # The number of gpt2-style tokens we have tokenized with the AI21 tokenizer.
          num_processed_tokens: int = 0
          # The length of the (normalized) text string we have tokenized with the AI21 tokenizer.
          num_processed_positions: int = 0
          while num_processed_tokens < len(gpt2_tokens):
              token_chunk_size: int = min(
                  len(gpt2_tokens) - num_processed_tokens,
                  AI21WindowService.MAX_TOKENIZATION_REQUEST_LENGTH,
              )
              token_chunk: List[TokenizationToken] = gpt2_tokens[
                  num_processed_tokens : num_processed_tokens
                  + token_chunk_size
              ]
              text_chunk: str = self.gpt2_window_service.decode(
                  token_chunk
              )
              # We need to avoid generating byte tokens when splitting the text
              while text_chunk.endswith("\ufffd"):
                  token_chunk_size -= 1
                  token_chunk = gpt2_tokens[
                      num_processed_tokens : num_processed_tokens
                      + token_chunk_size
                  ]
                  text_chunk = self.gpt2_window_service.decode(
                      token_chunk
                  )
              chunk_result: TokenizationRequestResult = (
                  self._make_tokenization_request(text_chunk)
              )
              chunk_tokens: List[TokenizationToken]
              normalized_text_chunk: str
              chunk_tokens, normalized_text_chunk = (
                  chunk_result.tokens,
                  chunk_result.text,
              )
              # Removes the empty tokens introduced by the split.
              assert chunk_tokens[0].text_range is not None
              if (
                  num_processed_tokens != 0
                  and chunk_tokens[0].text_range.start
                  == chunk_tokens[0].text_range.end
              ):
                  chunk_tokens = chunk_tokens[1:]
              else:
                  chunk_tokens = chunk_tokens[:]
  
              # Shifts the start and end index of each token
              shifted_tokens: List[TokenizationToken] = []
              for token in chunk_tokens:
                  assert token.text_range is not None
                  shifted_tokens.append(
                      TokenizationToken(
                          value=token.value,
                          text_range=TextRange(
                              start=token.text_range.start
                              + num_processed_positions,
                              end=token.text_range.end
                              + num_processed_positions,
                          ),
                      )
                  )
              all_tokens.extend(shifted_tokens)
              normalized_text_chunks.append(normalized_text_chunk)
              num_processed_tokens += token_chunk_size
              num_processed_positions += len(normalized_text_chunk)
  
          return all_tokens, "".join(normalized_text_chunks)
  
imports: import re
  from typing import List, Optional, Tuple
  from urllib.parse import unquote
  from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult, TokenizationToken, TextRange
  from .window_service import WindowService, EncodeResult
  from .tokenizer_service import TokenizerService
  from .gpt2_window_service import GPT2WindowService
label_desc: Use image node_id 14 for calling the AI21WindowService obj's underlying member method code with example usage: obj._make_long_tokenization_request(text) and returns: result,  result, all_tokens,  str