minhnguyent546 commited on
Commit
d5d675e
·
unverified ·
1 Parent(s): 89c8ae7

feat: update app

Browse files
Files changed (3) hide show
  1. app.py +164 -51
  2. assets/assistant_avavar.png +0 -0
  3. requirements.txt +3 -1
app.py CHANGED
@@ -1,64 +1,177 @@
 
 
 
1
  import gradio as gr
2
- from huggingface_hub import InferenceClient
3
 
4
- """
5
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
6
- """
7
- client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
8
 
9
 
10
- def respond(
11
- message,
12
- history: list[tuple[str, str]],
13
- system_message,
14
- max_tokens,
15
- temperature,
16
- top_p,
17
- ):
18
- messages = [{"role": "system", "content": system_message}]
19
 
20
- for val in history:
21
- if val[0]:
22
- messages.append({"role": "user", "content": val[0]})
23
- if val[1]:
24
- messages.append({"role": "assistant", "content": val[1]})
25
 
26
- messages.append({"role": "user", "content": message})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
- response = ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
- for message in client.chat_completion(
31
- messages,
32
- max_tokens=max_tokens,
 
 
 
 
 
 
 
 
 
 
33
  stream=True,
 
 
34
  temperature=temperature,
35
  top_p=top_p,
36
- ):
37
- token = message.choices[0].delta.content
38
-
39
- response += token
40
- yield response
41
-
42
-
43
- """
44
- For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
45
- """
46
- demo = gr.ChatInterface(
47
- respond,
48
- additional_inputs=[
49
- gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
50
- gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
51
- gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
52
- gr.Slider(
53
- minimum=0.1,
54
- maximum=1.0,
55
- value=0.95,
56
- step=0.05,
57
- label="Top-p (nucleus sampling)",
58
- ),
59
- ],
60
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
 
62
 
63
- if __name__ == "__main__":
64
- demo.launch()
 
1
+ import gc
2
+ import os
3
+
4
  import gradio as gr
 
5
 
6
+ from llama_cpp import Llama
 
 
 
7
 
8
 
9
+ ALPACA_SYSTEM_PROMPT = 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request'
10
+ ALPACA_SYSTEM_PROMPT_NO_INPUT = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'
 
 
 
 
 
 
 
11
 
12
+ DEFAULT_MODEL = 'Med-Alpaca-2-7b-chat.Q4_K_M'
 
 
 
 
13
 
14
+ model_paths = {
15
+ 'Med-Alpaca-2-7b-chat.Q2_K': {
16
+ 'repo_id': 'minhnguyent546/Med-Alpaca-2-7b-chat-GGUF',
17
+ 'filename': 'Med-Alpaca-2-7B-chat.Q2_K.gguf',
18
+ },
19
+ 'Med-Alpaca-2-7b-chat.Q4_K_M': {
20
+ 'repo_id': 'minhnguyent546/Med-Alpaca-2-7b-chat-GGUF',
21
+ 'filename': 'Med-Alpaca-2-7B-chat.Q4_K_M.gguf',
22
+ },
23
+ 'Med-Alpaca-2-7b-chat.Q6_K': {
24
+ 'repo_id': 'minhnguyent546/Med-Alpaca-2-7b-chat-GGUF',
25
+ 'filename': 'Med-Alpaca-2-7B-chat.Q6_K.gguf',
26
+ },
27
+ 'Med-Alpaca-2-7b-chat.Q8_0': {
28
+ 'repo_id': 'minhnguyent546/Med-Alpaca-2-7b-chat-GGUF',
29
+ 'filename': 'Med-Alpaca-2-7B-chat.Q8_0.gguf',
30
+ },
31
+ 'Med-Alpaca-2-7b-chat.F16': {
32
+ 'repo_id': 'minhnguyent546/Med-Alpaca-2-7b-chat-GGUF',
33
+ 'filename': 'Med-Alpaca-2-7B-chat.F16.gguf',
34
+ },
35
+ }
36
+
37
+ model = Llama.from_pretrained(
38
+ **model_paths[DEFAULT_MODEL],
39
+ n_ctx=4096,
40
+ n_threads=4,
41
+ cache_dir='./hf-cache'
42
+ )
43
 
44
+ def generate_alpaca_prompt(
45
+ instruction: str,
46
+ input: str | None = None,
47
+ response: str = '',
48
+ ) -> str:
49
+ prompt = ''
50
+ if input is not None and input and input.strip() != '<noinput>':
51
+ prompt = (
52
+ f'{ALPACA_SYSTEM_PROMPT}\n\n'
53
+ f'### Instruction:\n'
54
+ f'{instruction}\n\n'
55
+ f'### Input:\n'
56
+ f'{input}\n\n'
57
+ f'### Response: '
58
+ f'{response}'
59
+ )
60
+ else:
61
+ prompt = (
62
+ f'{ALPACA_SYSTEM_PROMPT_NO_INPUT}\n\n'
63
+ f'### Instruction:\n'
64
+ f'{instruction}\n\n'
65
+ f'### Response: '
66
+ f'{response}'
67
+ )
68
+ return prompt.strip()
69
 
70
+ def chat_completion(
71
+ message,
72
+ history,
73
+ seed: int,
74
+ max_new_tokens: int,
75
+ temperature: float,
76
+ repeatition_penalty: float,
77
+ top_k: int,
78
+ top_p: float,
79
+ ):
80
+ prompt = generate_alpaca_prompt(instruction=message)
81
+ response_iterator = model(
82
+ prompt,
83
  stream=True,
84
+ seed=seed,
85
+ max_tokens=max_new_tokens,
86
  temperature=temperature,
87
  top_p=top_p,
88
+ top_k=top_k,
89
+ repeat_penalty=repeatition_penalty,
90
+ )
91
+ partial_response = ''
92
+ for token in response_iterator:
93
+ partial_response += token['choices'][0]['text']
94
+ yield partial_response
95
+
96
+ def on_model_changed(model_name: str):
97
+ global model
98
+ if 'model' in globals():
99
+ del model
100
+ gc.collect()
101
+ model = Llama.from_pretrained(
102
+ **model_paths[model_name],
103
+ n_ctx=4096,
104
+ n_threads=4,
105
+ cache_dir='./hf-cache'
106
+ )
107
+
108
+ app_title_mark = gr.Markdown(f"""<center><font size=16>{model_name}</center>""")
109
+ chatbot = gr.Chatbot(
110
+ type='messages',
111
+ height=500,
112
+ placeholder='<strong>Hi, I have a headache, what should I do?</strong>',
113
+ label=model_name,
114
+ avatar_images=[None, './assets/assistant_avavar.png'], # pyright: ignore[reportArgumentType]
115
+ )
116
+ return app_title_mark, chatbot
117
+
118
+ def main() -> None:
119
+ with gr.Blocks(theme=gr.themes.Ocean()) as demo:
120
+ app_title_mark = gr.Markdown(f"""<center><font size=18>{DEFAULT_MODEL}</center>""")
121
+
122
+ model_options = list(model_paths.keys())
123
+
124
+ with gr.Row():
125
+ with gr.Column(scale=2):
126
+ with gr.Row():
127
+ model_radio = gr.Radio(choices=model_options, label='Model', value=DEFAULT_MODEL)
128
+ with gr.Row():
129
+ seed = gr.Number(value=998244353, label='Seed')
130
+ max_new_tokens = gr.Number(value=512, minimum=64, maximum=2048, label='Max new tokens')
131
+
132
+ with gr.Row():
133
+ temperature = gr.Slider(0, 2, step=0.01, label='Temperature', value=0.6, info='Info')
134
+ repeatition_penalty = gr.Slider(0.01, 5, step=0.05, label='Repetition penalty', value=1.1)
135
+
136
+ with gr.Row():
137
+ top_k = gr.Slider(1, 100, step=1, label='Top k', value=40)
138
+ top_p = gr.Slider(0, 1, step=0.01, label='Top p', value=0.9)
139
+
140
+ with gr.Column(scale=5):
141
+ chatbot = gr.Chatbot(
142
+ type='messages',
143
+ height=500,
144
+ placeholder='<strong>Hi, I have a headache, what should I do?</strong>',
145
+ label=DEFAULT_MODEL,
146
+ avatar_images=[None, './assets/assistant_avavar.png'], # pyright: ignore[reportArgumentType]
147
+ )
148
+ textbox = gr.Textbox(
149
+ placeholder='Hi, I have a headache, what should I do?',
150
+ container=False,
151
+ submit_btn=True,
152
+ stop_btn=True,
153
+ )
154
+
155
+ chat_interface = gr.ChatInterface(
156
+ chat_completion,
157
+ type='messages',
158
+ chatbot=chatbot,
159
+ textbox=textbox,
160
+ additional_inputs=[
161
+ seed,
162
+ max_new_tokens,
163
+ temperature,
164
+ repeatition_penalty,
165
+ top_k,
166
+ top_p,
167
+ ],
168
+ )
169
+
170
+ model_radio.change(on_model_changed, inputs=[model_radio], outputs=[app_title_mark, chatbot])
171
+
172
+ demo.queue(api_open=False, default_concurrency_limit=20)
173
+ demo.launch(max_threads=5, share=os.environ.get('GRADIO_SHARE', False))
174
 
175
 
176
+ if __name__ == '__main__':
177
+ main()
assets/assistant_avavar.png ADDED
requirements.txt CHANGED
@@ -1 +1,3 @@
1
- huggingface_hub==0.25.2
 
 
 
1
+ gradio~=5.6.0
2
+ huggingface_hub==0.25.2
3
+ llama-cpp-python~=0.3.2