import gradio as gr from openai import OpenAI import os # Retrieve the access token from the environment variable ACCESS_TOKEN = os.getenv("HF_TOKEN") print("Access token loaded.") # Initialize the OpenAI client with the Hugging Face Inference API endpoint client = OpenAI( base_url="https://api-inference.huggingface.co/v1/", api_key=ACCESS_TOKEN, ) print("OpenAI client initialized.") def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, frequency_penalty, seed, model, custom_model ): """ This function handles the chatbot response. It takes in: - message: the user's new message - history: the list of previous messages, each as a tuple (user_msg, assistant_msg) - system_message: the system prompt - max_tokens: the maximum number of tokens to generate in the response - temperature: sampling temperature - top_p: top-p (nucleus) sampling - frequency_penalty: penalize repeated tokens in the response - seed: a fixed seed for reproducibility; -1 will mean 'random' - model: the selected model - custom_model: the custom model path """ print(f"Received message: {message}") print(f"History: {history}") print(f"system message: {system_message}") print(f"max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}") print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}") print(f"Selected Model: {model}") print(f"Custom model: {custom_model}") # Convert seed to None if -1 (meaning random) if seed == -1: seed = None # Construct the messages array required by the API messages = [{"role": "system", "content": system_message}] # Add conversation history to the context for val in history: user_part = val[0] assistant_part = val[1] if user_part: messages.append({"role": "user", "content": user_part}) print(f"Added user message to context: {user_part}") ifassistant_part: messages.append({"role": "assistant", "content": assistant_part}) print(f"Added assistant message to context: {assistant_part}") # Append the latest user message messages.append({"role": "user", "content": message}) # Start with an empty string to build the response as tokens stream in response = "" print("Sending request to OpenAI API.") # Make the request to the HF Inference API via openAI-like client for message_chunk in client.chat.completions.create( model=custom_model if custom_model.strip() != "" else model, max_tokens=max_tokens, stream=True, # Stream the response temperature=temperature, top_p=top_p, frequency_penalty=frequency_penalty, # <-- seed=seed, # <-- messages=messages ): # Extract the token text from the response chunk token_text = message_chunk.choices[0].message.content print(f"Received token: {token_text}") response += token_text yield response print("Completed response generation.") # Create a Chatbot component with a specified height chatbot = gr.Chatbot(height=600) print("Chatbot interface created.") # Define the Gradio interface with gr.Blocks(theme='Nymbo/Nymbo_Theme') as demo: # Tab for basic settings with gr.Tab("Basic Settings"): with gr.Column(elem_id="prompt-container"): with gr.Row(): # Textbox for user to input the message text_prompt = gr.Textbox(label="Prompt", placeholder="Enter a prompt here", lines=3, elem_id="prompt-text-input") with gr.Row(): # Textbox for custom model input custom_model = gr.textbox(label="Custom Model", info="HuggingFace model path (optional)", placeholder="meta-llama/Llama-3.3-70B-Instruct", lines=1, elem_id="model-search-input") # Accordion for selecting the model with gr.Accordion("Featured models", open=True): # Textbox for searching models model_search = gr.textbox(Label="Filter models", placeholder="Search for a featured model...", lines=1, elem_id="model-search-input") # Radio buttons to select the desired model model = gr.Radio(label="Select a model below", value="meta-llama/Llama-3.3-70B-Instruct", choices=[ "meta-llama/Llama-3.3-70B-Instruct", "anthropic/claude-3", "anthropic/claude-instant-3", "anthropic/claude-2", "anthropic/claude-2", "anthropic/claude-instant-2", "anthropic/claude-1.3", "anthropic/claude-instant-1.3", "anthropic/claude-1", "anthropic/claude-instant-1", "anthropic/claude-0.3", "anthropic/claude-instant-0.3", "anthropic/claude-0.1", "anthropic/claude-instant-0.1", "anthropic/claude-v2", "anthropic/claude-instant-v2", "anthropic/claude-v1", "anthropic/claude-instant-v1", "anthropic/claude-v0.3", "anthropic/claude-instant-v0.3", "anthropic/claude-v0.1", "anthropic/claude-instant-v0.1", ], interactive=True, elem_id="model-radio") # Filtering models based on search input def filter_models(search_term): filtered_models = [m for m in model.choices if search_term.lower() in m.lower()] return gr.update(choices=filtered_models) # Update model list when search box is used model_search.change(filter_models, inputs=model, outputs=model) # Tab for advanced settings with gr.Tab("Advanced Settings"): with gr.Row(): # Text box for specifying the system message system_message = gr.text box(value="", label="System message") with gr.Row(): # Slider for setting the maximum new tokens max_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max new tokens") with gr.Row(): # Slider for setting the temperature temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature") with gr.Row(): #Slider for setting top-p top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Top-P") with gr.Row(): #Slider for setting frequency penalty frequency_penalty = gr.Slider(minimum=-2.0, maximum=2.0, value=0.0, step=0.1, label="Frequency Penalty") with gr.Row(): #Slider for setting the seed seed = gr.SLider(minimum=-1, maximum=65535, value=-1, step=1, label="Seed (-1 for random)") # Tab for information with gr.tab("Information"): with gr.Row(): # Display a sample prompt gr.textbox(label="Sample prompt", value="Enter a prompt | ultra detail, ultra elaboration, ultra quality, perfect.") with gr.Accordion("Featured Models (WiP)", open=False): gr.html( """

View more models

Model Description
meta-llama/Llama-3.3-70B-Instruct High-quality, large-scale language model
anthropic/claude-3 Advanced conversational AI model
anthropic/claude-instant-3 Fast and efficient conversational AI model
""" ) with gr.Accordion("Parameters Overview", open=False): gr.markdown( """ ## System Message - **Description**: The system message provides context and instructions to the model. - **Default**: "" ## Max New Tokens - **Description**: The maximum number of tokens to generate in the response. - **Default**: 512 - **Range**: 1 to 4096 ## Temperature - **Description**: Controls the randomness of the output. Lower values make the output more deterministic, higher values make it output more varied. - **Default**: 0.7 - **Range**: 0.1 to 4.0 ## Top-P - **Description**: Controls the diversity of the output. Lower values make the output more focused, higher values make it more varied. - **Default**: 0.7 - **Range**: 0.1 to 1.0 ## Frequency Penalty - **Description**: Penalizes repeated tokens in the response. Higher values makes the output less repetitive. - **Default**: 0.0 - **Range**: -2.0 to 2.0 ## Seed - **Description**: A fixed seed for reproducibility. -1 for random. - **Default**: -1 - **Range**: -1 to 65535 """ ) """ # Row containing the 'Run' button to trigger the query function with gr.Row(): text_button = gr.Button("Run", variant='primary', elem_id="gen-button") # Row for displaying the generated response with gr.Row(): response_output = gr.Textbox(label="Response Output", elem_id="response-output") # Set up button to call the respond function text_button.click( respond, inputs=[ text_prompt, model, custom_model, system_message, max_tokens, temperature, top_p, frequency_penalty, seed ], outputs=[response_output] ) print("Gradio interface initialized.") if __name__ == "__main__": demo.launch(show_api=False, share=False)