code: def chatglm3_stream_chat(model, tokenizer):
      chat_history = []
      past_key_values = None
      current_length = 0
      # https://github.com/THUDM/ChatGLM3/issues/274#issuecomment-1810160305
      stopping_criteria = StoppingCriteriaList(
          [
              StopSequenceCriteria(
                  ["<|user|>", "<|observation|>"], tokenizer
              )
          ]
      )
      max_past_length = 2048
  
      while True:
          user_input = input(Fore.GREEN + "\nHuman: " + Fore.RESET)
          # let's stop the conversation when user input "stop"
          if user_input == "stop":
              break
          print(Fore.BLUE + "BigDL-LLM: " + Fore.RESET, end="")
          # https://github.com/THUDM/ChatGLM3/blob/main/PROMPT_en.md
          prompt = f"""
              <|system|>
              You are an intelligent AI assistant, named ChatGLM3. Follow the user's instructions carefully.
              <|user|>
              {user_input}
              <|assistant|>
          """
          for (
              response,
              chat_history,
              past_key_values,
          ) in model.stream_chat(
              tokenizer,
              prompt,
              history=chat_history,
              stopping_criteria=stopping_criteria,
              past_key_values=past_key_values,
              return_past_key_values=True,
          ):
              print(response[current_length:], end="", flush=True)
              current_length = len(response)
              if past_key_values[0][0].shape[0] > max_past_length:
                  # To avoid out of memory, only keep recent key_values
                  new_values_list = []
                  for i in range(len(past_key_values)):
                      new_value = []
                      for val in past_key_values[i]:
                          new_v = val[-max_past_length:]
                          new_value.append(new_v)
                      new_values_list.append(tuple(new_value))
                  past_key_values = tuple(new_values_list)
  
file_path: BigDL/python/llm/portable-zip/chat.py
imports: import torch
  import argparse
  import sys
  from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, AutoConfig
  from transformers import TextIteratorStreamer
  from transformers.tools.agents import StopSequenceCriteria
  from transformers.generation.stopping_criteria import StoppingCriteriaList
  from colorama import Fore
  from bigdl.llm import optimize_model
  from kv_cache import StartRecentKVCache
label_desc: Use image node_id 4 for calling a global function with example usage: chatglm3_stream_chat(model, tokenizer) without return types