import gradio as gr from openai import OpenAI import os # Retrieve the access token from the environment variable ACCESS_TOKEN = os.getenv("HF_TOKEN") print("Access token loaded.") # Initialize the OpenAI client with the Hugging Face Inference API endpoint client = OpenAI( base_url="https://api-inference.huggingface.co/v1/", api_key=ACCESS_TOKEN, ) print("OpenAI client initialized.") def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, frequency_penalty, seed, selected_model, ): """ This function handles the chatbot response. It takes in: - message: the user's new message - history: the list of previous messages, each as a tuple (user_msg, assistant_msg) - system_message: the system prompt - max_tokens: the maximum number of tokens to generate in the response - temperature: sampling temperature - top_p: top-p (nucleus) sampling - frequency_penalty: penalize repeated tokens in the output - seed: a fixed seed for reproducibility; -1 will mean 'random' - selected_model: the model to use for generating the response """ print(f"Received message: {message}") print(f"History: {history}") print(f"System message: {system_message}") print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}") print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}") print(f"Selected model: {selected_model}") # Convert seed to None if -1 (meaning random) if seed == -1: seed = None # Construct the messages array required by the API messages = [{"role": "system", "content": system_message}] # Add conversation history to the context for val in history: user_part = val[0] assistant_part = val[1] if user_part: messages.append({"role": "user", "content": user_part}) print(f"Added user message to context: {user_part}") if assistant_part: messages.append({"role": "assistant", "content": assistant_part}) print(f"Added assistant message to context: {assistant_part}") # Append the latest user message messages.append({"role": "user", "content": message}) # Start with an empty string to build the response as tokens stream in response = "" print("Sending request to OpenAI API.") # Make the streaming request to the HF Inference API via openai-like client for message_chunk in client.chat.completions.create( model=selected_model, # Use the selected model max_tokens=max_tokens, stream=True, # Stream the response temperature=temperature, top_p=top_p, frequency_penalty=frequency_penalty, # <-- NEW seed=seed, # <-- NEW messages=messages, ): # Extract the token text from the response chunk token_text = message_chunk.choices[0].delta.content print(f"Received token: {token_text}") response += token_text yield response print("Completed response generation.") # Create a Chatbot component with a specified height chatbot = gr.Chatbot(height=600) print("Chatbot interface created.") # Define the list of featured models featured_models = [ "meta-llama/Llama-3.3-70B-Instruct", "google/flan-t5-xl", "facebook/bart-large-cnn", "EleutherAI/gpt-neo-2.7B", # Add more featured models here ] # Create the Gradio Blocks interface with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo: # Tab for model selection with gr.Tab("Models"): with gr.Row(): with gr.Column(): with gr.Accordion("Featured Models", open=True): model_search = gr.Textbox(label="Filter Models", placeholder="Search for a featured model...", lines=1) model = gr.Dropdown(label="Select a model below", choices=featured_models, value="meta-llama/Llama-3.3-70B-Instruct", interactive=True) def filter_models(search_term): filtered_models = [m for m in featured_models if search_term.lower() in m.lower()] return gr.update(choices=filtered_models) model_search.change(filter_models, inputs=model_search, outputs=model) custom_model = gr.Textbox(label="Custom Model", placeholder="Enter a custom model ID here", interactive=True) # Tab for chat interface with gr.Tab("Chat"): with gr.Row(): with gr.Column(): txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False) # Additional parameters with gr.Row(): with gr.Column(): system_message = gr.Textbox(label="System Message", value="", lines=3) max_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max New Tokens") temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature") top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-P") frequency_penalty = gr.Slider(minimum=-2.0, maximum=2.0, value=0.0, step=0.1, label="Frequency Penalty") seed = gr.Slider(minimum=-1, maximum=65535, value=-1, step=1, label="Seed (-1 for random)") # Chatbot display chatbot = gr.Chatbot(height=600) # Submit button submit_btn = gr.Button("Submit") # Tab for information with gr.Tab("Information"): with gr.Row(): gr.Markdown( """ # Featured Models - **meta-llama/Llama-3.3-70B-Instruct**: A large language model from Meta. - **google/flan-t5-xl**: A pretrained encoder-decoder model from Google. - **facebook/bart-large-cnn**: A pretrained sequence-to-sequence model from Facebook. - **EleutherAI/gpt-neo-2.7B**: A large autoregressive language model from EleutherAI. # Parameters Overview - **System Message**: Sets the behavior and context for the assistant. - **Max New Tokens**: Limits the length of the generated response. - **Temperature**: Controls the randomness of the output. Higher values make output more random. - **Top-P**: Controls the diversity of text by selecting tokens that account for top-p probability mass. - **Frequency Penalty**: Decreases the model's likelihood to repeat the same lines. - **Seed**: Ensures reproducibility of results; set to -1 for random seed. """ ) # Function to handle chat submission def user(user_message, history): return "", history + [[user_message, None]] # Function to process the chat def bot(history, system_message, max_tokens, temperature, top_p, frequency_penalty, seed, selected_model): # Get the last user message user_message = history[-1][0] # Generate response response_iter = respond( user_message, history[:-1], # Exclude the last user message which doesn't have a response yet system_message, max_tokens, temperature, top_p, frequency_penalty, seed, selected_model, ) # Collect the entire response full_response = "" for resp in response_iter: full_response = resp # Update history with the bot's response history[-1][1] = full_response return history # Set up the chat flow txt.submit(user, [txt, chatbot], [txt, chatbot], queue=False).then( bot, [chatbot, system_message, max_tokens, temperature, top_p, frequency_penalty, seed, model], chatbot ) submit_btn.click(user, [txt, chatbot], [txt, chatbot], queue=False).then( bot, [chatbot, system_message, max_tokens, temperature, top_p, frequency_penalty, seed, model], chatbot ) print("Gradio interface initialized.") if __name__ == "__main__": print("Launching the demo application.") demo.launch()