|
import gradio as gr |
|
from huggingface_hub import InferenceClient |
|
|
|
|
|
client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.3") |
|
|
|
|
|
def generate_text(messages): |
|
generated = "" |
|
for token in client.chat_completion(messages, max_tokens=50,stream=True): |
|
content = (token.choices[0].delta.content) |
|
generated+=content |
|
print(generated) |
|
|
|
return generated |
|
|
|
def call_generate_text(message, history): |
|
|
|
|
|
|
|
print(message) |
|
print(history) |
|
|
|
user_message = [{"role":"user","content":message}] |
|
messages = history + user_message |
|
try: |
|
text = generate_text(messages) |
|
assistant_message=[{"role":"assistant","content":text}] |
|
messages += assistant_message |
|
return "",messages |
|
except RuntimeError as e: |
|
print(f"An unexpected error occurred: {e}") |
|
|
|
return "",history |
|
|
|
head = ''' |
|
<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.webgpu.min.js" ></script> |
|
<script type="module"> |
|
import { matcha_tts,env } from "https://akjava.github.io/Matcha-TTS-Japanese/js-esm/v001-20240921/matcha_tts_onnx_en.js"; |
|
window.MatchaTTSEn = matcha_tts |
|
function replaceSpecialChars(text) { |
|
const pattern = /[^a-zA-Z0-9,.!?-_']/g; |
|
return text.replace(pattern, ' '); |
|
} |
|
window.replaceSpecialChars = replaceSpecialChars |
|
</script> |
|
''' |
|
|
|
with gr.Blocks(title="LLM with TTS",head=head) as demo: |
|
gr.Markdown("## LLM is unstable:The inference client used in this demo exhibits inconsistent performance. While it can provide responses in milliseconds, it sometimes becomes unresponsive and times out.") |
|
gr.Markdown("## TTS talke a long loading time:Please be patient, the first response may have a delay of up to over 20 seconds while loading.") |
|
gr.Markdown("**Mistral-7B-Instruct-v0.3/LJSpeech**.LLM and TTS models will change without notice.") |
|
|
|
js = """ |
|
function(chatbot){ |
|
text = (chatbot[chatbot.length -1])["content"] |
|
tts_text = window.replaceSpecialChars(text) |
|
console.log(tts_text) |
|
window.MatchaTTSEn(tts_text,"/file=models/ljspeech_sim.onnx") |
|
} |
|
""" |
|
chatbot = gr.Chatbot(type="messages") |
|
chatbot.change(None,[chatbot],[],js=js) |
|
msg = gr.Textbox() |
|
with gr.Row(): |
|
clear = gr.ClearButton([msg, chatbot]) |
|
submit = gr.Button("Submit",variant="primary").click(call_generate_text, inputs=[msg, chatbot], outputs=[msg,chatbot]) |
|
|
|
gr.HTML(""" |
|
<br> |
|
<div id="footer"> |
|
<b>Spaces</b><br> |
|
<a href="https://huggingface.co/spaces/Akjava/matcha-tts_vctk-onnx" style="font-size: 9px" target="link">Match-TTS VCTK-ONNX</a> | |
|
<a href="https://huggingface.co/spaces/Akjava/matcha-tts-onnx-benchmarks" style="font-size: 9px" target="link">Match-TTS ONNX-Benchmark</a> | |
|
<a href="https://huggingface.co/spaces/Akjava/AIChat-matcha-tts-onnx-en" style="font-size: 9px" target="link">AIChat-Matcha-TTS ONNX English</a> | |
|
|
|
<br><br> |
|
<b>Credits</b><br> |
|
<a href="https://github.com/akjava/Matcha-TTS-Japanese" style="font-size: 9px" target="link">Matcha-TTS-Japanese</a> | |
|
<a href = "http://www.udialogue.org/download/cstr-vctk-corpus.html" style="font-size: 9px" target="link">CSTR VCTK Corpus</a> | |
|
<a href = "https://github.com/cmusphinx/cmudict" style="font-size: 9px" target="link">CMUDict</a> | |
|
<a href = "https://huggingface.co/docs/transformers.js/index" style="font-size: 9px" target="link">Transformer.js</a> | |
|
<a href = "https://huggingface.co/cisco-ai/mini-bart-g2p" style="font-size: 9px" target="link">mini-bart-g2p</a> | |
|
<a href = "https://onnxruntime.ai/docs/get-started/with-javascript/web.html" style="font-size: 9px" target="link">ONNXRuntime-Web</a> | |
|
<a href = "https://github.com/akjava/English-To-IPA-Collections" style="font-size: 9px" target="link">English-To-IPA-Collections</a> | |
|
<a href ="https://huggingface.co/papers/2309.03199" style="font-size: 9px" target="link">Matcha-TTS Paper</a> |
|
</div> |
|
""") |
|
|
|
msg.submit(call_generate_text, [msg, chatbot], [msg, chatbot]) |
|
|
|
import os |
|
print(os.listdir("/home/user/app/")) |
|
print(os.listdir("/home/user/app/models")) |
|
demo.launch(allowed_paths=["/home/user/app/models/ljspeech_sim.onnx"]) |
|
|
