File size: 2,309 Bytes
f6c2dc2 86a0c79 f6c2dc2 86a0c79 f6c2dc2 ed4b678 f6c2dc2 86a0c79 f6c2dc2 86a0c79 f6c2dc2 86a0c79 f6c2dc2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
import gradio as gr
import os
os.system('CMAKE_ARGS="-DLLAMA_OPENBLAS=on" FORCE_CMAKE=1 pip install llama-cpp-python')
import wget
from llama_cpp import Llama
import random
url = 'https://huggingface.co/TheBloke/WizardLM-7B-uncensored-GGML/resolve/main/WizardLM-7B-uncensored.ggmlv3.q2_K.bin'
filename = wget.download(url)
llm2 = Llama(model_path=filename, seed=random.randint(1, 2**31))
title = """<h1 align="center">Chat with awesome WizardLM 7b model!</h1><br>"""
with gr.Blocks(theme=gr.themes.clean) as demo:
gr.HTML(title)
gr.HTML("This model is awesome for its size! It is only 20th the size of Chatgpt but is around 90% as good as Chatgpt. However, please don't rely on WizardLM to provide 100% true information as it might be wrong sometimes.")
chatbot = gr.Chatbot()
msg = gr.Textbox()
clear = gr.ClearButton([msg, chatbot])
temperature = gr.Slider(minimum=0.1, maximum=1.0, default=0.72, step=0.01, label="Temperature")
top_p = gr.Slider(minimum=0.1, maximum=1.0, default=0.73, step=0.01, label="Top-p")
top_k = gr.Slider(minimum=1, maximum=100, default=50, step=1, label="Top-k")
repeat_penalty = gr.Slider(minimum=0.1, maximum=2.0, default=1.1, step=0.1, label="Repeat Penalty")
def user(user_message, history):
return gr.update(value="", interactive=True), history + [[user_message, None]]
def bot(history):
user_message = history[-1][0]
tokens3 = llm2.tokenize(user_message.encode())
token4 = llm2.tokenize(b"\n\n### Response:")
tokens = tokens3 + token4
history[-1][1] = ""
count = 0
output = ""
for token in llm2.generate(tokens, top_k=top_k.value, top_p=top_p.value, temp=temperature.value, repeat_penalty=repeat_penalty.value):
text = llm2.detokenize([token])
output += text.decode()
count += 1
if count >= 500 or (token == llm2.token_eos()):
break
history[-1][1] += text.decode()
yield history
response = msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
bot, chatbot, chatbot
)
response.then(lambda: gr.update(interactive=True), None, [msg], queue=False)
gr.HTML("Thanks for checking out this app!")
demo.queue()
demo.launch(debug=True) |