class_docstr: Tokenizes by making a request to the proxy server with REST endpoint: `/api/tokenize`.
docstr: Given the model and a list of tokens, outputs the corresponding text.
  
  For models using the GPT-2 tokenizer, the tokens are integers; for AI21
  models, the tokens are `TokenizationToken`s.
  
  Some tokenizers (e.g. AI21) normalize the text before encoding it and
  thus require the `normalized_text` for decoding.
docstr_tok: ["Given","the","model","and","a","list","of","tokens",",","outputs","the","corresponding","text",".","For","models","using","the","GPT-2","tokenizer",",","the","tokens","are","integers",";","for","AI21","models",",","the","tokens","are","`","TokenizationToken","`","s",".","Some","tokenizers","(","e.g",".","AI21",")","normalize","the","text","before","encoding","it","and","thus","require","the","`","normalized_text","`","for","decoding","."]
code: def decode(
      self,
      tokens: List[TokenizationToken],
      normalized_text: Optional[str] = None,
  ) -> str:
      """
      Given the model and a list of tokens, outputs the corresponding text.
  
      For models using the GPT-2 tokenizer, the tokens are integers; for AI21
      models, the tokens are `TokenizationToken`s.
  
      Some tokenizers (e.g. AI21) normalize the text before encoding it and
      thus require the `normalized_text` for decoding.
      """
      if not tokens:
          return ""
  
      # `normalized_text` is necessary for decoding AI21 tokens.
      assert (
          normalized_text
      ), "The AI21 tokenizer needs `normalized_text` for decoding"
      for j in range(len(tokens) - 1):
          first_text_range = tokens[j].text_range
          second_text_range = tokens[j + 1].text_range
          assert first_text_range is not None
          assert second_text_range is not None
          assert (
              first_text_range.end == second_text_range.start
          ), "The tokens to be decoded must form a substring of `normalized_text`."
  
      token_texts: List[str] = []
      # The format of AI21 byte token representations. e.g. <0xE8>
      byte_pattern = "<0x[0-9A-F]{2}>"
      i: int = 0
      while i < len(tokens):
          # If there are byte tokens, aggregates them to a string
          token_value = tokens[i].value
          assert isinstance(token_value, str)
          if re.match(byte_pattern, token_value):
              bytestring = ""
              while i < len(tokens) and re.match(
                  byte_pattern, token_value
              ):
                  # e.g. <0xE8> -> \xE8
                  bytestring += "\\" + token_value[2:-1]
                  i += 1
              # Convert to encoded URI (e.g., %e2%80%99) and decode
              token_text = unquote(bytestring.replace("\\x", "%"))
          # Not a byte token: retrieves the token text based on text_range.
          else:
              token: TokenizationToken = tokens[i]
              assert token.text_range is not None
              token_text = normalized_text[
                  token.text_range.start : token.text_range.end
              ]
              i += 1
          token_texts.append(token_text)
      return "".join(token_texts)
  
imports: import re
  from typing import List, Optional, Tuple
  from urllib.parse import unquote
  from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult, TokenizationToken, TextRange
  from .window_service import WindowService, EncodeResult
  from .tokenizer_service import TokenizerService
  from .gpt2_window_service import GPT2WindowService
label_desc: Use image node_id 8 for calling the AI21WindowService obj's underlying member method code with example usage: obj.decode(tokens, normalized_text) and returns: str, str