Update app.py

app.py

@@ -50,10 +50,12 @@ def respond(
     llm = Llama(
         model_path=f"models/{model}",
         flash_attn=True,
+        n_threads=4,
         n_gpu_layers=81,
+        n_batch=1024,
         n_ctx=8192,
     )
-    provider = LlamaCppPythonProvider(llm, temperature, top_k, top_p, max_tokens, repeat_penalty)
+    provider = LlamaCppPythonProvider(llm)
 
     agent = LlamaCppAgent(
         provider,
@@ -63,7 +65,11 @@ def respond(
     )
     
     settings = provider.get_provider_default_settings()
+    settings.temperature = temperature
+    settings.top_k = top_k
+    settings.top_p = top_p
     settings.max_tokens = max_tokens
+    settings.repeat_penalty = repeat_penalty
     settings.stream = True
 
     messages = BasicChatHistory()