Update app.py

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 from openai import OpenAI, APIError
 import os
 import tenacity
+import asyncio
 
 ACCESS_TOKEN = os.getenv("HF_TOKEN")
 
@@ -11,7 +12,7 @@ client = OpenAI(
 )
 
 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10))
-def respond(
+async def respond(
     message,
     history,
     system_message,
@@ -30,14 +31,18 @@ def respond(
 
         messages.append({"role": "user", "content": message})
 
-        response = client.chat.completions.create(
+        response = ""
+        async for message in client.chat.completions.create(
             model="NousResearch/Hermes-3-Llama-3.1-8B",
             max_tokens=max_tokens,
+            stream=True,
             temperature=temperature,
             top_p=top_p,
             messages=messages,
-        )
-        return response.choices[0].delta.content
+        ):
+            token = message.choices[0].text
+            response += token
+            yield response
     except APIError as e:
         error_details = e.body
         error_type = error_details.get("type")
@@ -51,10 +56,10 @@ def respond(
             error_str = "An error occurred during streaming"
 
         print(f"Error: {error_str}")
-        return error_str
+        yield error_str
     except Exception as e:
         print(f"Error: {e}")
-        return "Error occurred. Please try again."
+        yield "Error occurred. Please try again."
 
 def launch_app():
     try:
@@ -74,7 +79,7 @@ def launch_app():
                     label="Top-P",
                 ),
             ],
-            "text",
+            "queue",
             title="Chatbot",
             description="A chatbot that responds to user input",
         )