File size: 2,428 Bytes
fec4cfa 732e00c fec4cfa 732e00c fec4cfa 732e00c fec4cfa 732e00c fec4cfa 732e00c fec4cfa 732e00c fec4cfa 732e00c bd7b4bf 732e00c fec4cfa 732e00c fec4cfa 732e00c fec4cfa 732e00c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
import gradio as gr
from huggingface_hub import InferenceClient
import random
# Initialize the model
model = "mistralai/Mixtral-8x7B-Instruct-v0.1"
client = InferenceClient(model)
def chat_response(prompt, history, seed, temp, tokens, top_p, rep_p):
generate_kwargs = {
"temperature": temp,
"max_new_tokens": tokens,
"top_p": top_p,
"repetition_penalty": rep_p,
"do_sample": True,
"seed": seed,
}
# Include the chat history in the prompt
formatted_prompt = "\n".join([f"Q: {user_prompt}\nA: {bot_response}" for user_prompt, bot_response in history]) + f"\nQ: {prompt}\nA:"
output = ""
# Generating text in streaming mode
for response in client.text_generation(formatted_prompt, **generate_kwargs, stream=True):
# Assuming response is directly a string or contains a message
output += response # Using response directly since it's a string
# Yield the updated output for real-time display
yield [(prompt, output)]
# Append the full response to history after completion
history.append((prompt, output))
yield history # Yielding the updated history
def clear_chat():
return [], [] # Returning an empty history
# Gradio interface
with gr.Blocks() as app:
gr.HTML("<center><h1>Chatbot</h1><h3>Ask your questions!</h3></center>")
chat_box = gr.Chatbot(height=500)
inp = gr.Textbox(label="Your Question", lines=5)
btn = gr.Button("Ask")
clear_btn = gr.Button("Clear")
rand_seed = gr.Checkbox(label="Random Seed", value=True)
seed_slider = gr.Slider(label="Seed", minimum=1, maximum=1111111111111111, value=random.randint(1, 1111111111111111))
tokens_slider = gr.Slider(label="Max new tokens", value=3840, minimum=0, maximum=8000)
temp_slider = gr.Slider(label="Temperature", value=0.9, minimum=0.01, maximum=1.0)
top_p_slider = gr.Slider(label="Top-P", value=0.9, minimum=0.01, maximum=1.0)
rep_p_slider = gr.Slider(label="Repetition Penalty", value=1.0, minimum=0.1, maximum=2.0)
# Handle button click to get chat response
btn.click(
lambda prompt: chat_response(prompt, [], seed_slider.value, temp_slider.value, tokens_slider.value, top_p_slider.value, rep_p_slider.value),
inp,
chat_box,
)
clear_btn.click(clear_chat, None, [inp, chat_box])
app.launch(share=True, auth=("admin", "0112358"))
|