Update app.py

app.py

@@ -3,27 +3,18 @@ from llama_cpp import Llama
 
 
 llm = Llama.from_pretrained(
-    repo_id="bartowski/Phi-3.5-mini-instruct-GGUF",
-    filename="Phi-3.5-mini-instruct-Q4_K_M.gguf",
+    repo_id="maddes8cht/smallcloudai-Refact-1_6B-fim-gguf",
+    filename="smallcloudai-Refact-1_6B-fim-Q4_K_M.gguf",
     numa=True,
-    use_mmap=False,
-    use_mlock=True,
     seed=-1,
-    # flash_attn=True,
-    # n_gpu_layers=-1,
     n_batch=1024,
     n_ctx=4095,
 )
 
 def respond(prompt: str):
-    stream = llm.create_chat_completion(stream=True, messages=[{"role": "user", "content": prompt}])
+    print(llm(prompt, max_tokens=64))
+    return prompt
 
-    response = ""
-    for chunk in stream:
-        if "content" in chunk["choices"][0]["delta"]:
-            response += chunk["choices"][0]["delta"]["content"]
-            yield response
-
-
-demo = gr.Interface(fn=respond, inputs=[gr.TextArea("What is the capital of France?")], outputs=[gr.TextArea()])
+prompt = "<fim_prefix>import socket\n\ndef ping_exponential_backoff():\n    <fim_suffix>\n\nif __name__ == \"main\":\n   ping_exponential_backoff()<fim_middle>"
+demo = gr.Interface(fn=respond, inputs=[gr.TextArea(prompt)], outputs=[gr.TextArea()])
 demo.launch(server_name="0.0.0.0", server_port=7860)