Update llm_handler.py

llm_handler.py

@@ -2,12 +2,45 @@ import requests
 import json
 from openai import OpenAI
 from params import OPENAI_MODEL, OPENAI_API_KEY
+from llama_cpp import Llama
+from llama_cpp_agent import LlamaCppAgent
+from llama_cpp_agent import MessagesFormatterType
+from llama_cpp_agent.providers import LlamaCppPythonProvider
+
+# Add this at the top of the file
+local_model_base_url = "http://localhost:11434/v1"
+anything_llm_workspace = "<input-workspace-name-here>"
+
+def set_local_model_base_url(url):
+    global local_model_base_url
+    local_model_base_url = url
+
+def set_anything_llm_workspace(workspace):
+    global anything_llm_workspace
+    anything_llm_workspace = workspace
 
 # Create an instance of the OpenAI class for the local model
-client = OpenAI(api_key="local-model", base_url="http://localhost:11434/v1")
+client = OpenAI(api_key="local-model", base_url=local_model_base_url)
+
+# Initialize LlamaCpp model and agent
+llama_model = Llama("/Users/anima/.cache/lm-studio/models/arcee-ai/Arcee-Spark-GGUF/Arcee-Spark-Q4_K_M.gguf", n_batch=1024, n_threads=24, n_gpu_layers=33, n_ctx=4098, verbose=False)
+provider = LlamaCppPythonProvider(llama_model)
+llama_agent = LlamaCppAgent(
+    provider,
+    system_prompt="You are a helpful assistant.",
+    predefined_messages_formatter_type=MessagesFormatterType.MISTRAL,
+    debug_output=True
+)
+
+# Configure provider settings
+settings = provider.get_provider_default_settings()
+settings.max_tokens = 2000
+settings.stream = True
 
 def send_to_chatgpt(msg_list):
     try:
+        # Update the send_to_chatgpt function to use the dynamic base_url
+        client = OpenAI(api_key="local-model", base_url=local_model_base_url)
         completion = client.chat.completions.create(
             model=OPENAI_MODEL,
             temperature=0.6,
@@ -21,7 +54,7 @@ def send_to_chatgpt(msg_list):
         return f"Error: {str(e)}", None
 
 def send_to_anything_llm(msg_list):
-    url = 'http://localhost:3001/api/v1/workspace/mycoworks/chat'
+    url = f'http://localhost:3001/api/v1/workspace/{anything_llm_workspace}/chat'
     headers = {
         'accept': 'application/json',
         'Authorization': 'Bearer 0MACR41-7804XQB-MGC1GS0-FGSKB44',
@@ -44,10 +77,24 @@ def send_to_anything_llm(msg_list):
         print(f"Error in send_to_anything_llm: {str(e)}")
         return f"Error: {str(e)}", None
 
+def send_to_llamacpp(msg_list):
+    try:
+        # Convert the message list to the format expected by LlamaCppAgent
+        formatted_messages = [{"role": msg["role"], "content": msg["content"]} for msg in msg_list]
+        response = llama_agent(formatted_messages, settings=settings)
+        chatgpt_response = response.message.content
+        chatgpt_usage = {"prompt_tokens": response.usage.prompt_tokens, "completion_tokens": response.usage.completion_tokens, "total_tokens": response.usage.total_tokens}
+        return chatgpt_response, chatgpt_usage
+    except Exception as e:
+        print(f"Error in send_to_llamacpp: {str(e)}")
+        return f"Error: {str(e)}", None
+
 def send_to_llm(provider, msg_list):
     if provider == "local-model":
         return send_to_chatgpt(msg_list)
     elif provider == "anything-llm":
         return send_to_anything_llm(msg_list)
+    elif provider == "llamacpp":
+        return send_to_llamacpp(msg_list)
     else:
         raise ValueError(f"Unknown provider: {provider}")