Spaces:
Runtime error
Runtime error
ffreemt
commited on
Commit
·
3c0a531
1
Parent(s):
0acdcc9
Update deq DequeCallback
Browse files
app.py
CHANGED
@@ -8,6 +8,7 @@ import random
|
|
8 |
import time
|
9 |
from pathlib import Path
|
10 |
from queue import deque
|
|
|
11 |
from typing import Any, Dict, List, Union
|
12 |
|
13 |
# from types import SimpleNamespace
|
@@ -18,9 +19,12 @@ from ctransformers import Config
|
|
18 |
from dl_hf_model import dl_hf_model
|
19 |
from langchain.callbacks.base import BaseCallbackHandler
|
20 |
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
|
|
|
|
|
21 |
|
22 |
# from ctransformers import AutoModelForCausalLM
|
23 |
from langchain.llms import CTransformers
|
|
|
24 |
from langchain.schema import LLMResult
|
25 |
from loguru import logger
|
26 |
|
@@ -124,36 +128,28 @@ prompt_template = """### HUMAN:
|
|
124 |
|
125 |
### RESPONSE:"""
|
126 |
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
_ = psutil.cpu_count(logical=False) - 1
|
133 |
-
cpu_count: int = int(_) if _ else 1
|
134 |
-
logger.debug(f"{cpu_count=}")
|
135 |
|
136 |
-
|
137 |
-
|
|
|
|
|
|
|
138 |
|
139 |
-
|
140 |
-
model_loc, file_size = dl_hf_model(url)
|
141 |
-
except Exception as exc_:
|
142 |
-
logger.error(exc_)
|
143 |
-
raise SystemExit(1) from exc_
|
144 |
|
145 |
-
|
146 |
-
|
|
|
147 |
|
148 |
-
|
149 |
-
|
150 |
-
model=model_loc,
|
151 |
-
model_type="llama",
|
152 |
-
threads=cpu_count,
|
153 |
-
callbacks=[StreamingStdOutCallbackHandler()],
|
154 |
-
)
|
155 |
|
156 |
-
logger.
|
157 |
|
158 |
os.environ["TZ"] = "Asia/Shanghai"
|
159 |
try:
|
@@ -166,7 +162,7 @@ except Exception:
|
|
166 |
class DequeCallbackHandler(BaseCallbackHandler):
|
167 |
"""Mediate gradio and stream output."""
|
168 |
|
169 |
-
def __init__(self, deq: deque):
|
170 |
"""Init deque for FIFO, may need to upgrade to queue.Queue or queue.SimpleQueue."""
|
171 |
self.q = deq
|
172 |
|
@@ -191,6 +187,63 @@ class DequeCallbackHandler(BaseCallbackHandler):
|
|
191 |
self.q.put(sig_end)
|
192 |
|
193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
194 |
def user(user_message, history):
|
195 |
# return user_message, history + [[user_message, None]]
|
196 |
history.append([user_message, None])
|
@@ -223,28 +276,29 @@ def bot(history):
|
|
223 |
|
224 |
logger.debug(f"{user_message=}")
|
225 |
|
226 |
-
|
227 |
-
|
228 |
-
|
229 |
-
then = time.time()
|
230 |
-
|
231 |
-
logger.debug("about to generate")
|
232 |
-
|
233 |
-
config = Config(reset=True)
|
234 |
-
for elm in generate(user_message, config=config):
|
235 |
-
if flag == 1:
|
236 |
-
logger.debug("in the loop")
|
237 |
-
prefix = f"({time.time() - then:.2f}s) "
|
238 |
-
flag = 0
|
239 |
-
print(prefix, end="", flush=True)
|
240 |
-
logger.debug(f"{prefix=}")
|
241 |
-
print(elm, end="", flush=True)
|
242 |
-
# logger.debug(f"{elm}")
|
243 |
-
|
244 |
-
response.append(elm)
|
245 |
-
history[-1][1] = prefix + "".join(response)
|
246 |
-
yield history
|
247 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
248 |
_ = (
|
249 |
f"(time elapsed: {atime.duration_human}, " # type: ignore
|
250 |
f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
|
@@ -258,7 +312,7 @@ def predict_api(prompt):
|
|
258 |
logger.debug(f"{prompt=}")
|
259 |
try:
|
260 |
# user_prompt = prompt
|
261 |
-
|
262 |
temperature=0.2,
|
263 |
top_k=10,
|
264 |
top_p=0.9,
|
@@ -270,12 +324,18 @@ def predict_api(prompt):
|
|
270 |
# threads=cpu_count,
|
271 |
# stop=prompt_prefix[1:2],
|
272 |
)
|
273 |
-
|
274 |
response = generate(
|
275 |
prompt,
|
276 |
config=config,
|
277 |
)
|
278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
279 |
logger.debug(f"api: {response=}")
|
280 |
except Exception as exc:
|
281 |
logger.error(exc)
|
@@ -283,7 +343,7 @@ def predict_api(prompt):
|
|
283 |
# bot = {"inputs": [response]}
|
284 |
# bot = [(prompt, response)]
|
285 |
|
286 |
-
return response
|
287 |
|
288 |
|
289 |
css = """
|
|
|
8 |
import time
|
9 |
from pathlib import Path
|
10 |
from queue import deque
|
11 |
+
from threading import Thread
|
12 |
from typing import Any, Dict, List, Union
|
13 |
|
14 |
# from types import SimpleNamespace
|
|
|
19 |
from dl_hf_model import dl_hf_model
|
20 |
from langchain.callbacks.base import BaseCallbackHandler
|
21 |
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
|
22 |
+
from langchain.chains import ConversationChain
|
23 |
+
from langchain.chains.conversation.memory import ConversationBufferWindowMemory
|
24 |
|
25 |
# from ctransformers import AutoModelForCausalLM
|
26 |
from langchain.llms import CTransformers
|
27 |
+
from langchain.prompts import PromptTemplate
|
28 |
from langchain.schema import LLMResult
|
29 |
from loguru import logger
|
30 |
|
|
|
128 |
|
129 |
### RESPONSE:"""
|
130 |
|
131 |
+
prompt_template = """### HUMAN:
|
132 |
+
You are a helpful assistant. Think step by step.
|
133 |
+
{history}
|
134 |
+
{input}
|
135 |
+
### RESPONSE:"""
|
|
|
|
|
|
|
136 |
|
137 |
+
prompt_template = """You are a helpful assistant.
|
138 |
+
{history}
|
139 |
+
### HUMAN:
|
140 |
+
{input}
|
141 |
+
### RESPONSE:"""
|
142 |
|
143 |
+
# PromptTemplate(input_variables=['history', 'input'], output_parser=None, partial_variables={}, template='The following is afriendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n{history}\nHuman: {input}\nAI:', template_format='f-string', validate_template=True)
|
|
|
|
|
|
|
|
|
144 |
|
145 |
+
human_prefix = "### HUMAN"
|
146 |
+
ai_prefix = "### RESPONSE"
|
147 |
+
stop = [f"{human_prefix}:"]
|
148 |
|
149 |
+
_ = [elm for elm in prompt_template.splitlines() if elm.strip()]
|
150 |
+
stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
|
|
|
|
|
|
|
|
|
|
|
151 |
|
152 |
+
# logger.debug(f"{stop_string=} not used")
|
153 |
|
154 |
os.environ["TZ"] = "Asia/Shanghai"
|
155 |
try:
|
|
|
162 |
class DequeCallbackHandler(BaseCallbackHandler):
|
163 |
"""Mediate gradio and stream output."""
|
164 |
|
165 |
+
def __init__(self, deq: deque = deque()):
|
166 |
"""Init deque for FIFO, may need to upgrade to queue.Queue or queue.SimpleQueue."""
|
167 |
self.q = deq
|
168 |
|
|
|
187 |
self.q.put(sig_end)
|
188 |
|
189 |
|
190 |
+
_ = psutil.cpu_count(logical=False) - 1
|
191 |
+
cpu_count: int = int(_) if _ else 1
|
192 |
+
logger.debug(f"{cpu_count=}")
|
193 |
+
|
194 |
+
LLM = None
|
195 |
+
gc.collect()
|
196 |
+
|
197 |
+
try:
|
198 |
+
model_loc, file_size = dl_hf_model(url)
|
199 |
+
except Exception as exc_:
|
200 |
+
logger.error(exc_)
|
201 |
+
raise SystemExit(1) from exc_
|
202 |
+
|
203 |
+
config = Config()
|
204 |
+
config.stream = True
|
205 |
+
config.stop = stop
|
206 |
+
config.threads=cpu_count
|
207 |
+
|
208 |
+
deqcb = DequeCallbackHandler(deq)
|
209 |
+
|
210 |
+
# LLM = AutoModelForCausalLM.from_pretrained(
|
211 |
+
LLM = CTransformers(
|
212 |
+
model=model_loc,
|
213 |
+
model_type="llama",
|
214 |
+
callbacks=[StreamingStdOutCallbackHandler(), deqcb],
|
215 |
+
# config=config,
|
216 |
+
**vars(config),
|
217 |
+
)
|
218 |
+
|
219 |
+
logger.info(f"done load llm {model_loc=} {file_size=}G")
|
220 |
+
|
221 |
+
|
222 |
+
prompt = PromptTemplate(
|
223 |
+
input_variables=['history', 'input'],
|
224 |
+
output_parser=None,
|
225 |
+
partial_variables={},
|
226 |
+
template=prompt_template,
|
227 |
+
template_format='f-string',
|
228 |
+
validate_template=True
|
229 |
+
)
|
230 |
+
|
231 |
+
memory = ConversationBufferWindowMemory(
|
232 |
+
human_prefix=human_prefix,
|
233 |
+
ai_prefix=ai_prefix,
|
234 |
+
) # default k=5
|
235 |
+
|
236 |
+
conversation = ConversationChain(
|
237 |
+
llm=LLM,
|
238 |
+
prompt=prompt,
|
239 |
+
memory=memory,
|
240 |
+
verbose=True,
|
241 |
+
)
|
242 |
+
logger.debug(f"{conversation.prompt.template=}")
|
243 |
+
|
244 |
+
# conversation.predict(input="Hello, my name is Andrea")
|
245 |
+
|
246 |
+
|
247 |
def user(user_message, history):
|
248 |
# return user_message, history + [[user_message, None]]
|
249 |
history.append([user_message, None])
|
|
|
276 |
|
277 |
logger.debug(f"{user_message=}")
|
278 |
|
279 |
+
# conversation.predict(input="What's my name?")
|
280 |
+
thr = Thread(target=conversation.predict, kwargs={"input": user_message})
|
281 |
+
thr.start()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
282 |
|
283 |
+
# preocess deq
|
284 |
+
response = []
|
285 |
+
flag = 1
|
286 |
+
then = time.time()
|
287 |
+
with about_time() as atime: # type: ignore
|
288 |
+
while True:
|
289 |
+
if deq:
|
290 |
+
if flag:
|
291 |
+
prefix = f"({time.time() - then:.2f}s) "
|
292 |
+
flag = 0
|
293 |
+
_ = deq.popleft()
|
294 |
+
if _ is sig_end:
|
295 |
+
break
|
296 |
+
# print(_, end='')
|
297 |
+
response.append(_)
|
298 |
+
history[-1][1] = prefix + "".join(response).strip()
|
299 |
+
yield history
|
300 |
+
else:
|
301 |
+
time.sleep(0.01)
|
302 |
_ = (
|
303 |
f"(time elapsed: {atime.duration_human}, " # type: ignore
|
304 |
f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
|
|
|
312 |
logger.debug(f"{prompt=}")
|
313 |
try:
|
314 |
# user_prompt = prompt
|
315 |
+
Config(
|
316 |
temperature=0.2,
|
317 |
top_k=10,
|
318 |
top_p=0.9,
|
|
|
324 |
# threads=cpu_count,
|
325 |
# stop=prompt_prefix[1:2],
|
326 |
)
|
327 |
+
_ = """
|
328 |
response = generate(
|
329 |
prompt,
|
330 |
config=config,
|
331 |
)
|
332 |
+
# """
|
333 |
+
conversation = ConversationChain(
|
334 |
+
llm=LLM,
|
335 |
+
prompt=prompt,
|
336 |
+
verbose=True,
|
337 |
+
)
|
338 |
+
response = conversation.predict(prompt)
|
339 |
logger.debug(f"api: {response=}")
|
340 |
except Exception as exc:
|
341 |
logger.error(exc)
|
|
|
343 |
# bot = {"inputs": [response]}
|
344 |
# bot = [(prompt, response)]
|
345 |
|
346 |
+
return response.strip()
|
347 |
|
348 |
|
349 |
css = """
|