app.py

import gradio as gr
import time
import os
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer,TextIteratorStreamer
from threading import Thread
output_dir_merge = "Elliot4AI/Dugong-Llama2-7b-chinese"
# load base LLM model and tokenizer
model = AutoModelForCausalLM.from_pretrained(
    output_dir_merge,
    low_cpu_mem_usage=True,
    torch_dtype=torch.float16,
    load_in_8bit=True,
)

tokenizer = AutoTokenizer.from_pretrained(output_dir_merge)

def run_generation(user_text, top_p, temperature, top_k, max_new_tokens):
    # Get the model and tokenizer, and tokenize the user text.
    model_inputs = tokenizer([user_text], return_tensors="pt").input_ids.cuda()

    # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
    # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
    generate_kwargs = dict(
        inputs=model_inputs,
        streamer=streamer,
        max_new_tokens=max_new_tokens,
        do_sample=True,
        top_p=top_p,
        temperature=float(temperature),
        top_k=top_k
        # repetition_penalty=2.0
    )
    t = Thread(target=model.generate, kwargs=generate_kwargs)
    t.start()

    # Pull the generated text from the streamer, and update the model output.
    model_output = ""
    for new_text in streamer:
        model_output += new_text
        yield model_output
    return model_output


def reset_textbox():
    return gr.update(value='')
with gr.Blocks() as demo:
    with gr.Tab("PatentQA-Dugong-Llama2-7b-chinese Agent"):
        gr.Markdown(
        "# 🤗 PatentQA_Dugong 🔥PatentQA_Dugong Agent🔥 \n"
        "Dugong是一个用中文微调的Llama2-7b的模型， 微调后中文回答更顺畅 "
        "目前采用流式输出"
        "🤗💛"
    )
        # gr.Markdown("PatentQA_Dugong Agent: Dugong是一个用中文微调的Llama2-7b的模型， 微调后中文回答更顺畅，并且具有丰富英业达专利知识的人工智能助手，可以回答专利的相关信息,目前恢复速度稍慢")
        with gr.Row():
           with gr.Column(scale=4):
            user_text = gr.Textbox(
                placeholder="请输入你的问题",
                label="问题"
            )
            model_output = gr.Textbox(label="回答", lines=10, interactive=False)
            button_submit = gr.Button(value="提交")
            clear = gr.ClearButton([user_text, model_output])

           with gr.Column(scale=1):
            max_new_tokens = gr.Slider(
                minimum=1, maximum=1000, value=250, step=1, interactive=True, label="最大输出token数量",
            )
            top_p = gr.Slider(
                minimum=0.05, maximum=1.0, value=0.95, step=0.05, interactive=True, label="Top-p (nucleus sampling)",
            )
            top_k = gr.Slider(
                minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k",
            )
            temperature = gr.Slider(
                minimum=0.1, maximum=5.0, value=0.8, step=0.1, interactive=True, label="温度",
            )
    
    user_text.submit(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)
    button_submit.click(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)
    
demo.queue(max_size=32)
demo.launch(enable_queue=True)