stream output

app.py

@@ -1,11 +1,5 @@
 import gradio as gr
-
-import copy
-import random
-import os
-import requests
-import time
-import sys
+import json
 
 from huggingface_hub import snapshot_download
 from llama_cpp import Llama
@@ -31,29 +25,31 @@ def generate(
     top_k=40,
     max_tokens=512,
 ):
+    
     prompt = template.format(input)
-    output = model(prompt, 
+    output = ""
+    for chunk in model.create_completion(prompt, 
                    temperature = temperature,
                    top_k = top_k,
                    top_p = top_p,
                    max_tokens = max_tokens,
                    stop=["<|im_end|>"], 
-                   echo=True)
-    output = output["choices"][0]['text']
-    return output.split("assistant\n")[1]
+                   echo = False,
+                   stream = True):
+        output +=chunk["choices"][0]["text"]
+        yield output
+    return output
 
 g = gr.Interface(
     fn=generate,
     inputs=[
         gr.components.Textbox(
-            lines=2, label="Prompt", value = "What is huggingface?"
+            lines=2, label="Prompt", value = "What is Huggingface?"
         ),
         gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature"),
         gr.components.Slider(minimum=0, maximum=1, value=1, label="Top p"),
         gr.components.Slider(minimum=0, maximum=100, step=1, value=50, label="Top k"),
-        gr.components.Slider(
-            minimum=1, maximum=1024, step=1, value=128, label="Max tokens"
-        ),
+        gr.components.Slider(minimum=1, maximum=1024, step=1, value=256, label="Max tokens"),  
     ],
     outputs=[
         gr.Textbox(