Update app.py

app.py

@@ -10,9 +10,12 @@ from pydantic import BaseModel
 
 class GenModel(BaseModel):
     question: str
-    system: str = "You are a helpful medical assistant."
+    system: str = "You are a helpful medical AI assistant. Help as much as you can. Remember, response in English."
     temperature: float = 0.8
-    seed: int = 101
+    seed: int = 101,
+    mirostat_mode=2,
+    mirostat_tau=4.0,
+    mirostat_eta=1.1
     
 llm_chat = llama_cpp.Llama.from_pretrained(
     repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
@@ -30,6 +33,9 @@ llm_generate = llama_cpp.Llama.from_pretrained(
     verbose=False,
      n_ctx=4096,
      n_gpu_layers=0,
+    mirostat_mode=2,
+    mirostat_tau=4.0,
+    mirostat_eta=1.1
     #chat_format="llama-2"
 )
 # Logger setup
@@ -40,7 +46,13 @@ app = fastapi.FastAPI(
     title="OpenGenAI",
     description="Your Excellect AI Physician")
 
-
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins = ["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"]
+)
 @app.get("/")
 def index():
     return fastapi.responses.RedirectResponse(url="/docs")
@@ -66,17 +78,7 @@ async def chat(gen:GenModel):
         )
         messages.append({"role": "user", "content": gen.question},)
         print(output)
-        """
-        for chunk in output:
-            
-            delta = chunk['choices'][0]['delta']
-            if 'role' in delta:
-                print(delta['role'], end=': ')
-            elif 'content' in delta:
-                print(delta['content'], end='')
-            
-            print(chunk)
-        """
+        
         et = time()
         output["time"] = et - st
         messages.append({'role': "assistant", "content": output['choices'][0]['message']})
@@ -96,16 +98,17 @@ async def generate(gen:GenModel):
     gen.seed: int = 42
     try:
         st = time()
-        output = llm_generate.create_chat_completion(
+        output = llm_generate.create_completion(
             messages=[
                 {"role": "system", "content": gen.system},
                 {"role": "user", "content": gen.question},
             ],
             temperature = gen.temperature,
             seed= gen.seed,
-            stream=True
+            #stream=True,
+            #echo=True
         )
-        
+        """
         for chunk in output:
             delta = chunk['choices'][0]['delta']
             if 'role' in delta:
@@ -113,7 +116,7 @@ async def generate(gen:GenModel):
             elif 'content' in delta:
                 print(delta['content'], end='')
             #print(chunk)
-    
+        """
         et = time()
         #output["time"] = et - st
         return output