Update app.py

app.py

@@ -42,18 +42,22 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
     messages.append({"role": "user", "content": message})
     
     try:
-        response = llm.create_chat_completion(
+        # Stream response from LLM
+        stream = llm.create_chat_completion(
             messages=messages,
             max_tokens=max_tokens,
             temperature=temperature,
-            top_p=top_p
+            top_p=top_p,
+            stream=True  # Enable streaming
         )
-        return response["choices"][0]["message"]["content"]
+        response_content = ""
+        for chunk in stream:
+            response_content += chunk["choices"][0]["delta"]["content"]
+            yield response_content  # Stream each chunk back to the frontend
     except Exception as e:
-        return f"Error: {e}"
+        yield f"Error: {e}"
 
 def clear_context():
-    # Initialize chat history with a greeting from the assistant
     greeting_message = get_random_greeting()
     return [("", greeting_message)], ""
 
@@ -74,10 +78,13 @@ with gr.Blocks() as demo:
         temperature = gr.Slider(0.1, 4.0, value=0.7, step=0.1, label="Temperature")
         top_p = gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top-p")
     
-    # Automatically handle submission on Enter key press
+    # Automatically handle submission on Enter key press with streaming
     def handle_submit(message, history, system_message, max_tokens, temperature, top_p):
-        response = respond(message, history, system_message, max_tokens, temperature, top_p)
-        history.append((message, response))
+        history.append((message, None))  # Append user's message first
+        chatbot.update(history)  # Display user's message before response
+        for response in respond(message, history, system_message, max_tokens, temperature, top_p):
+            history[-1] = (message, response)  # Update the last response with streaming content
+            chatbot.update(history)
         return history, ""
 
     msg.submit(