app.py

import gradio as gr
from openai import OpenAI
import os

# Retrieve the access token from the environment variable
ACCESS_TOKEN = os.getenv("HF_TOKEN")
print("Access token loaded.")

# Initialize the OpenAI client with the Hugging Face Inference API endpoint
client = OpenAI(
    base_url="https://api-inference.huggingface.co/v1/",
    api_key=ACCESS_TOKEN,
)
print("OpenAI client initialized.")

def respond(
    message,
    history: list[tuple[str, str]],
    system_message,
    max_tokens,
    temperature,
    top_p,
    frequency_penalty,
    seed,
    model,
    custom_model
):
    """
    This function handles the chatbot response. It takes in:
    - message: the user's new message
    - history: the list of previous messages, each as a tuple (user_msg, assistant_msg)
    - system_message: the system prompt
    - max_tokens: the maximum number of tokens to generate in the response
    - temperature: sampling temperature
    - top_p: top-p (nucleus) sampling
    - frequency_penalty: penalize repeated tokens in the response
    - seed: a fixed seed for reproducibility; -1 will mean 'random'
    - model: the selected model
    - custom_model: the custom model path
    """

    print(f"Received message: {message}")
    print(f"History: {history}")
    print(f"system message: {system_message}")
    print(f"max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}")
    print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}")
    print(f"Selected Model: {model}")
    print(f"Custom model: {custom_model}")

    # Convert seed to None if -1 (meaning random)
    if seed == -1:
        seed = None

    # Construct the messages array required by the API
    messages = [{"role": "system", "content": system_message}]

    # Add conversation history to the context
    for val in history:
        user_part = val[0]
        assistant_part = val[1]
        if user_part:
            messages.append({"role": "user", "content": user_part})
            print(f"Added user message to context: {user_part}")
        ifassistant_part:
            messages.append({"role": "assistant", "content": assistant_part})
            print(f"Added assistant message to context: {assistant_part}")

    # Append the latest user message
    messages.append({"role": "user", "content": message})

    # Start with an empty string to build the response as tokens stream in
    response = ""
    print("Sending request to OpenAI API.")

    # Make the request to the HF Inference API via openAI-like client
    for message_chunk in client.chat.completions.create(
        model=custom_model if custom_model.strip() != "" else model,
        max_tokens=max_tokens,
        stream=True,  # Stream the response
        temperature=temperature,
        top_p=top_p,
        frequency_penalty=frequency_penalty,  # <--
        seed=seed,  # <--
        messages=messages
    ):
        # Extract the token text from the response chunk
        token_text = message_chunk.choices[0].message.content
        print(f"Received token: {token_text}")
        response += token_text
        yield response

    print("Completed response generation.")

# Create a Chatbot component with a specified height
chatbot = gr.Chatbot(height=600)
print("Chatbot interface created.")

# Define the Gradio interface
with gr.Blocks(theme='Nymbo/Nymbo_Theme') as demo:
    # Tab for basic settings
    with gr.Tab("Basic Settings"):
        with gr.Column(elem_id="prompt-container"):
            with gr.Row():
                # Textbox for user to input the message
                text_prompt = gr.Textbox(label="Prompt", placeholder="Enter a prompt here", lines=3, elem_id="prompt-text-input")
            with gr.Row():
                # Textbox for custom model input
                custom_model = gr.textbox(label="Custom Model", info="HuggingFace model path (optional)", placeholder="meta-llama/Llama-3.3-70B-Instruct", lines=1, elem_id="model-search-input")
            # Accordion for selecting the model
            with gr.Accordion("Featured models", open=True):
                # Textbox for searching models
                model_search = gr.textbox(Label="Filter models", placeholder="Search for a featured model...", lines=1, elem_id="model-search-input")
                # Radio buttons to select the desired model
                model = gr.Radio(label="Select a model below", value="meta-llama/Llama-3.3-70B-Instruct", choices=[
                    "meta-llama/Llama-3.3-70B-Instruct",
                    "anthropic/claude-3",
                    "anthropic/claude-instant-3",
                    "anthropic/claude-2",
                    "anthropic/claude-2",
                    "anthropic/claude-instant-2",
                    "anthropic/claude-1.3",
                    "anthropic/claude-instant-1.3",
                    "anthropic/claude-1",
                    "anthropic/claude-instant-1",
                    "anthropic/claude-0.3",
                    "anthropic/claude-instant-0.3",
                    "anthropic/claude-0.1",
                    "anthropic/claude-instant-0.1",
                    "anthropic/claude-v2",
                    "anthropic/claude-instant-v2",
                    "anthropic/claude-v1",
                    "anthropic/claude-instant-v1",
                    "anthropic/claude-v0.3",
                    "anthropic/claude-instant-v0.3",
                    "anthropic/claude-v0.1",
                    "anthropic/claude-instant-v0.1",
                ], interactive=True, elem_id="model-radio")

                # Filtering models based on search input
                def filter_models(search_term):
                    filtered_models = [m for m in model.choices if search_term.lower() in m.lower()]
                    return gr.update(choices=filtered_models)

                # Update model list when search box is used
                model_search.change(filter_models, inputs=model, outputs=model)

    # Tab for advanced settings
    with gr.Tab("Advanced Settings"):
        with gr.Row():
            # Text box for specifying the system message
            system_message = gr.text box(value="", label="System message")
        with gr.Row():
            # Slider for setting the maximum new tokens
            max_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max new tokens")
        with gr.Row():
            # Slider for setting the temperature
            temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
        with gr.Row():
            #Slider for setting top-p
            top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Top-P")
        with gr.Row():
            #Slider for setting frequency penalty
            frequency_penalty = gr.Slider(minimum=-2.0, maximum=2.0, value=0.0, step=0.1, label="Frequency Penalty")
        with gr.Row():
            #Slider for setting the seed
            seed = gr.SLider(minimum=-1, maximum=65535, value=-1, step=1, label="Seed (-1 for random)")

    # Tab for information
    with gr.tab("Information"):
        with gr.Row():
            # Display a sample prompt
            gr.textbox(label="Sample prompt", value="Enter a prompt | ultra detail, ultra elaboration, ultra quality, perfect.")
        with gr.Accordion("Featured Models (WiP)", open=False):
            gr.html(
                """
            <p><a href="https://huggingface.co/models?inferences=warm&pipeline_tag=text-to-text&sort=trending">View more models</a></p>
            <table style="width:100%; text-align:center; margin:auto;">
                <tr>
                    <th>Model</th>
                    <th>Description</th>
                </tr>
                <tr>
                    <td>meta-llama/Llama-3.3-70B-Instruct</td>
                    <td>High-quality, large-scale language model</td>
                </tr>
                <tr>
                    <td>anthropic/claude-3</td>
                    <td> Advanced conversational AI model</td>
                </tr>
                <tr>
                    <td>anthropic/claude-instant-3</td>
                    <td> Fast and efficient conversational AI model</td>
                </tr>
            </table>
            """
            )
        with gr.Accordion("Parameters Overview", open=False):
            gr.markdown(
            """
            ## System Message
            - **Description**: The system message provides context and instructions to the model.
            - **Default**: ""

            ## Max New Tokens
            - **Description**: The maximum number of tokens to generate in the response.
            - **Default**: 512
            - **Range**: 1 to 4096

            ## Temperature
            - **Description**: Controls the randomness of the output. Lower values make the output more deterministic, higher values make it output more varied.
 - **Default**: 0.7
 - **Range**: 0.1 to 4.0

            ## Top-P
            - **Description**: Controls the diversity of the output. Lower values make the output more focused, higher values make it more varied.
            - **Default**: 0.7
            - **Range**: 0.1 to 1.0

            ## Frequency Penalty
            - **Description**: Penalizes repeated tokens in the response. Higher values makes the output less repetitive.
 - **Default**: 0.0
 - **Range**: -2.0 to 2.0

            ## Seed
            - **Description**: A fixed seed for reproducibility. -1 for random.
            - **Default**: -1
            - **Range**: -1 to 65535

            """
            )
"""

    # Row containing the 'Run' button to trigger the query function
    with gr.Row():
        text_button = gr.Button("Run", variant='primary', elem_id="gen-button")
    # Row for displaying the generated response
    with gr.Row():
        response_output = gr.Textbox(label="Response Output", elem_id="response-output")

    # Set up button to call the respond function
    text_button.click(
        respond,
        inputs=[
            text_prompt, model, custom_model, system_message, max_tokens, temperature, top_p, frequency_penalty, seed
        ],
        outputs=[response_output]
    )

print("Gradio interface initialized.")

if __name__ == "__main__":
    demo.launch(show_api=False, share=False)