|
import spaces |
|
import json |
|
import subprocess |
|
import gradio as gr |
|
from huggingface_hub import hf_hub_download |
|
|
|
subprocess.run('pip install llama-cpp-python==0.2.75 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124', shell=True) |
|
subprocess.run('pip install llama-cpp-agent==0.2.10', shell=True) |
|
|
|
hf_hub_download( |
|
repo_id="bartowski/Meta-Llama-3-70B-Instruct-GGUF", |
|
filename="Meta-Llama-3-70B-Instruct-Q3_K_M.gguf", |
|
local_dir = "./models" |
|
) |
|
hf_hub_download( |
|
repo_id="bartowski/Llama-3-8B-Synthia-v3.5-GGUF", |
|
filename="Llama-3-8B-Synthia-v3.5-f16.gguf", |
|
local_dir = "./models" |
|
) |
|
hf_hub_download( |
|
repo_id="bartowski/Mistral-7B-Instruct-v0.3-GGUF", |
|
filename="Mistral-7B-Instruct-v0.3-f32.gguf", |
|
local_dir = "./models" |
|
) |
|
|
|
css = """ |
|
.message-row { |
|
justify-content: space-evenly !important; |
|
} |
|
.message-bubble-border { |
|
border-radius: 6px !important; |
|
} |
|
.dark.message-bubble-border { |
|
border-color: #343140 !important; |
|
} |
|
.dark.user { |
|
background: #1e1c26 !important; |
|
} |
|
.dark.assistant.dark, .dark.pending.dark { |
|
background: #16141c !important; |
|
} |
|
""" |
|
|
|
def get_messages_formatter_type(model_name): |
|
from llama_cpp_agent import MessagesFormatterType |
|
if "Llama" in model_name: |
|
return MessagesFormatterType.LLAMA_3 |
|
elif "Mistral" in model_name: |
|
return MessagesFormatterType.MISTRAL |
|
else: |
|
raise ValueError(f"Unsupported model: {model_name}") |
|
|
|
@spaces.GPU(duration=120) |
|
def respond( |
|
message, |
|
history: list[tuple[str, str]], |
|
system_message, |
|
max_tokens, |
|
temperature, |
|
top_p, |
|
top_k, |
|
repeat_penalty, |
|
model, |
|
): |
|
from llama_cpp import Llama |
|
from llama_cpp_agent import LlamaCppAgent |
|
from llama_cpp_agent.providers import LlamaCppPythonProvider |
|
from llama_cpp_agent.chat_history import BasicChatHistory |
|
from llama_cpp_agent.chat_history.messages import Roles |
|
|
|
chat_template = get_messages_formatter_type(model) |
|
|
|
llm = Llama( |
|
model_path=f"models/{model}", |
|
flash_attn=True, |
|
n_threads=40, |
|
n_gpu_layers=81, |
|
n_batch=1024, |
|
n_ctx=8192, |
|
) |
|
provider = LlamaCppPythonProvider(llm) |
|
|
|
agent = LlamaCppAgent( |
|
provider, |
|
system_prompt=f"{system_message}", |
|
predefined_messages_formatter_type=chat_template, |
|
debug_output=True |
|
) |
|
|
|
settings = provider.get_provider_default_settings() |
|
settings.temperature = temperature |
|
settings.top_k = top_k |
|
settings.top_p = top_p |
|
settings.max_tokens = max_tokens |
|
settings.repeat_penalty = repeat_penalty |
|
settings.stream = True |
|
|
|
messages = BasicChatHistory() |
|
|
|
for msn in history: |
|
user = { |
|
'role': Roles.user, |
|
'content': msn[0] |
|
} |
|
assistant = { |
|
'role': Roles.assistant, |
|
'content': msn[1] |
|
} |
|
messages.add_message(user) |
|
messages.add_message(assistant) |
|
|
|
stream = agent.get_chat_response( |
|
message, |
|
llm_sampling_settings=settings, |
|
chat_history=messages, |
|
returns_streaming_generator=True, |
|
print_output=False |
|
) |
|
|
|
outputs = "" |
|
for output in stream: |
|
outputs += output |
|
yield outputs |
|
|
|
PLACEHOLDER = """ |
|
<div class="container" style="max-width: 600px; margin: 0 auto; padding: 30px; background-color: #fff; box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);"> |
|
<h1 style="font-size: 28px; margin-bottom: 15px;">llama-cpp-agent: Simplify LLM Interactions</h1> |
|
<p style="font-size: 16px; line-height: 1.5; margin-bottom: 15px;">The llama-cpp-agent framework simplifies interactions with Large Language Models (LLMs), providing an interface for chatting, executing function calls, generating structured output, performing retrieval augmented generation, and processing text using agentic chains with tools.</p> |
|
<p style="font-size: 16px; line-height: 1.5; margin-bottom: 15px;">The framework uses guided sampling to constrain model output to user-defined structures, enabling models not fine-tuned for function calling and JSON output to do so. It is compatible with llama.cpp server, llama-cpp-python and its server, TGI, and vllm servers.</p> |
|
<h2 style="font-size: 22px; margin-bottom: 10px;">Key Features</h2> |
|
<ul style="list-style-type: none; padding: 0;"> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Simple Chat Interface</strong>: Engage in seamless conversations with LLMs.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Structured Output</strong>: Generate structured output (objects) from LLMs.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Function Calling</strong>: Execute functions using LLMs.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>RAG</strong>: Perform retrieval augmented generation with colbert reranking.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Agent Chains</strong>: Process text using agent chains with tools.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Guided Sampling</strong>: Allows most 7B LLMs to do function calling and structured output.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Multiple Providers</strong>: Works with various servers and providers.</li> |
|
<li style="font-size: 16px; line-height: 1.5; margin-bottom: 8px;"><strong>Compatibility</strong>: Works with python functions, pydantic tools, llama-index tools, and OpenAI tool schemas.</li> |
|
<li style="font-size: 16px; line-height: 1.5;"><strong>Flexibility</strong>: Suitable for various applications, from casual chatting to specific function executions.</li> |
|
</ul> |
|
</div> |
|
""" |
|
|
|
demo = gr.ChatInterface( |
|
respond, |
|
additional_inputs=[ |
|
gr.Textbox(value="You are a helpful assistant.", label="System message"), |
|
gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"), |
|
gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), |
|
gr.Slider( |
|
minimum=0.1, |
|
maximum=1.0, |
|
value=0.95, |
|
step=0.05, |
|
label="Top-p", |
|
), |
|
gr.Slider( |
|
minimum=0, |
|
maximum=100, |
|
value=40, |
|
step=1, |
|
label="Top-k", |
|
), |
|
gr.Slider( |
|
minimum=0.0, |
|
maximum=2.0, |
|
value=1.1, |
|
step=0.1, |
|
label="Repetition penalty", |
|
), |
|
gr.Dropdown([ |
|
'Meta-Llama-3-70B-Instruct-Q3_K_M.gguf', |
|
'Llama-3-8B-Synthia-v3.5-f16.gguf', |
|
'Mistral-7B-Instruct-v0.3-f32.gguf' |
|
], |
|
value="Meta-Llama-3-70B-Instruct-Q3_K_M.gguf", |
|
label="Model" |
|
), |
|
], |
|
theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set( |
|
body_background_fill_dark="#16141c", |
|
block_background_fill_dark="#16141c", |
|
block_border_width="1px", |
|
block_title_background_fill_dark="#1e1c26", |
|
input_background_fill_dark="#292733", |
|
button_secondary_background_fill_dark="#24212b", |
|
border_color_primary_dark="#343140", |
|
background_fill_secondary_dark="#16141c", |
|
color_accent_soft_dark="transparent" |
|
), |
|
css=css, |
|
retry_btn="Retry", |
|
undo_btn="Undo", |
|
clear_btn="Clear", |
|
submit_btn="Send", |
|
description="Llama-cpp-agent: Chat multi llm selection", |
|
chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER) |
|
) |
|
|
|
if __name__ == "__main__": |
|
demo.launch() |
