app.py

import os
import re
import gradio as gr
import edge_tts
import asyncio
import time
import tempfile
from huggingface_hub import InferenceClient

Fast = """## Fastest Model"""

Complex = """## Best in Complex Question"""

Detail = """## Best for Detailed Generation or Long Answers"""

client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
#client1 = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")

system_instructions1 = "<s>[SYSTEM] You are AI assistant named DorjGPT, Keep conversation very short, clear, friendly and concise. The text provided is a request for a specific type of response from you, the virtual assistant. The request asks you to provide friendly responses as if super interlligent AI assistant. The expectation is that I will avoid introductions and start answering the query directly, elaborating on all aspects of the request. As an AI-powered assistant, your task is to generate responses that appear as if they are created by the user and not an AI-powered assistant. [USER]"

global history
history = []
def format_prompt(message, history):
    prompt = system_instructions1
    for user_prompt, bot_response in history:
        prompt += f"[INST] {user_prompt} [/INST]"
        prompt += f" {bot_response}</s> "
    prompt += f"[INST] {message} [/INST]"
    return prompt

async def generate1(prompt, history=[], b=None):
    generate_kwargs = dict(
        temperature=0.6,
        max_new_tokens=256,
        top_p=0.95,
        repetition_penalty=1,
        do_sample=True,
        seed=42,
    )
    #formatted_prompt = system_instructions1 + prompt + "[JARVIS]"
    formatted_prompt = format_prompt(f"{system_instructions1}, {prompt}", history) + "[JARVIS]"
    stream = client1.text_generation(
        formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True)

    output = ""
    for response in stream:
        output += response.token.text
    output = output.replace("</s>","")
    history.append([prompt, output])

    communicate = edge_tts.Communicate(output)
    with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
        tmp_path = tmp_file.name
        await communicate.save(tmp_path)
    yield tmp_path

with gr.Blocks(theme="gradio/monochrome", title="Dorj Assistant") as demo:
    gr.HTML("""
        <h1 style="text-align: center; style="font-size: 3m;">
        DorjGPT
        </h1>
        """)
    with gr.Column():
        output_audio = gr.Audio(label="DorjGPT", type="filepath",
                  interactive=False,
                  autoplay=True,
                  elem_classes="audio")
        
        user_input = gr.Textbox(label="Prompt", value="What is Mongolia")

    with gr.Tab():
      with gr.Row():
        translate_btn = gr.Button("Response")
        translate_btn.click(fn=generate1, inputs=user_input,
                            outputs=output_audio, api_name="translate")  

if __name__ == "__main__":
    demo.queue(max_size=30).launch()