feat: stream

app.py

@@ -4,10 +4,8 @@ import json
 import os
 from dotenv import load_dotenv
 
-# 加载.env文件中的环境变量
 load_dotenv()
 
-# 从环境变量中读取配置
 API_URL = os.getenv("API_URL")
 API_TOKEN = os.getenv("API_TOKEN")
 
@@ -24,7 +22,7 @@ For more information on `huggingface_hub` Inference API support, please check th
 
 def respond(
     message,
-    history: list[dict],  # 修改为新的消息格式
+    history: list[dict],
     system_message,
     max_tokens,
     temperature,
@@ -32,10 +30,8 @@ def respond(
 ):
     messages = [{"role": "system", "content": system_message}]
 
-    # 添加历史消息
     messages.extend(history)
     
-    # 添加当前用户消息
     messages.append({"role": "user", "content": message})
 
     headers = {
@@ -45,7 +41,7 @@ def respond(
 
     data = {
         "model": "/data/DMind-1",
-        "stream": False,
+        "stream": True,
         "messages": messages,
         "temperature": temperature,
         "top_p": top_p,
@@ -58,29 +54,29 @@ def respond(
     print(f"[INFO] userMsg: {message}")
 
     try:
-        with requests.post(API_URL, headers=headers, json=data) as r:
-            # print(f"[INFO] response status: {r.status_code}")
+        with requests.post(API_URL, headers=headers, json=data, stream=True) as r:
             if r.status_code == 200:
-                json_response = r.json()
-                # print(f"[INFO] response json: {json_response}")
-                if 'choices' in json_response and len(json_response['choices']) > 0:
-                    content = json_response['choices'][0].get('message', {}).get('content', '')
-                    # print(f"[INFO] response content: {content}")
-                    if content:
-                        if '<think>' in content and '</think>' in content:
-                            content = content.split('</think>')[-1].strip()
-                        if '**Final Answer**' in content:
-                            content = content.split('**Final Answer**')[-1].strip()
-                        print(f"[INFO] final response: {content}")
-                        return content
-                else:
-                    print(f"[ERROR] No choices in response: {json_response}")
+                for line in r.iter_lines():
+                    if line:
+                        line = line.decode('utf-8')
+                        if line.startswith('data: '):
+                            try:
+                                json_response = json.loads(line[6:])
+                                if 'choices' in json_response and len(json_response['choices']) > 0:
+                                    delta = json_response['choices'][0].get('delta', {})
+                                    if 'content' in delta:
+                                        content = delta['content']
+                                        if content:
+                                            yield content
+                            except json.JSONDecodeError as e:
+                                print(f"[ERROR] JSON decode error: {e}")
+                                continue
             else:
                 print(f"[ERROR] Bad status code: {r.status_code}, response: {r.text}")
-            return "Service temporarily unavailable"
+                yield "Service temporarily unavailable"
     except Exception as e:
         print(f"[ERROR] Request error: {e}")
-        return "Service error occurred"
+        yield "Service error occurred"
 
 
 """