code: def detokenize_incrementally(
      tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
      all_input_ids: List[int],
      prev_tokens: Optional[List[str]],
      prefix_offset: int = 0,
      read_offset: int = 0,
      skip_special_tokens: bool = False,
      spaces_between_special_tokens: bool = True,
  ) -> Tuple[List[str], str, int, int]:
      new_token_id = all_input_ids[-1]
      # This is the first iteration for this sequence
      if prev_tokens is None:
          new_tokens = tokenizer.convert_ids_to_tokens(
              all_input_ids, skip_special_tokens=skip_special_tokens
          )
          output_tokens = new_tokens
          # 5 is an arbitrary value that should work for all
          # tokenizers (bigger = more conservative).
          # Subtract 1 extra to account for the generated token.
          prefix_offset = max(len(output_tokens) - 6, 0)
          # If the first new token is a special token, we can't skip 1 extra token
          if (
              skip_special_tokens
              and new_token_id in tokenizer.all_special_ids
          ):
              read_offset = max(len(output_tokens), 0)
          else:
              read_offset = max(len(output_tokens) - 1, 0)
      else:
          # Put new_token_id in a list so skip_special_tokens is respected
          new_tokens = tokenizer.convert_ids_to_tokens(
              [new_token_id], skip_special_tokens=skip_special_tokens
          )
          output_tokens = prev_tokens + new_tokens
  
      # The prefix text is necessary only to defeat cleanup algorithms in
      # the decode which decide to add a space or not depending on the
      # surrounding ids.
      if tokenizer.is_fast or not tokenizer.get_added_vocab():
          prefix_text = tokenizer.convert_tokens_to_string(
              output_tokens[prefix_offset:read_offset]
          )
          new_text = tokenizer.convert_tokens_to_string(
              output_tokens[prefix_offset:]
          )
      else:
          prefix_text = _convert_tokens_to_string_with_added_encoders(
              tokenizer,
              output_tokens[prefix_offset:read_offset],
              skip_special_tokens=skip_special_tokens,
              spaces_between_special_tokens=spaces_between_special_tokens,
          )
          new_text = _convert_tokens_to_string_with_added_encoders(
              tokenizer,
              output_tokens[prefix_offset:],
              skip_special_tokens=skip_special_tokens,
              spaces_between_special_tokens=spaces_between_special_tokens,
          )
  
      if len(new_text) > len(prefix_text) and not new_text.endswith(
          "�"
      ):
          # utf-8 char at the end means it's a potential unfinished byte sequence
          # from byte fallback tokenization.
          # If it's in the middle, it's probably a real invalid id generated
          # by the model
          new_text = new_text[len(prefix_text) :]
          return new_tokens, new_text, read_offset, len(output_tokens)
      else:
          return new_tokens, "", prefix_offset, read_offset
  
imports: from typing import List, Optional, Tuple, Union
  from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
  from bigdl.llm.vllm.logger import init_logger
  from bigdl.llm.utils.common import invalidInputError
label_desc: Use image node_id 3 for calling a global function with example usage: detokenize_incrementally(tokenizer, all_input_ids, prev_tokens, prefix_offset, read_offset, skip_special_tokens, spaces_between_special_tokens) and returns: new_tokens,  new_text,  read_offset,  len, new_tokens,  str,  prefix_offset,  read_offset