Update app.py

app.py

@@ -2,17 +2,15 @@ import gradio as gr
 from openai import OpenAI
 import os
 
-# Load the Hugging Face access token from environment variables
+# Retrieve the access token from the environment variable
 ACCESS_TOKEN = os.getenv("HF_TOKEN")
-
 print("Access token loaded.")
 
-# Initialize the OpenAI client with Hugging Face's serverless API
+# Initialize the OpenAI client with the Hugging Face Inference API endpoint
 client = OpenAI(
     base_url="https://api-inference.huggingface.co/v1/",
     api_key=ACCESS_TOKEN,
 )
-
 print("OpenAI client initialized.")
 
 def respond(
@@ -23,79 +21,96 @@ def respond(
     temperature,
     top_p,
     frequency_penalty,
-    seed,
+    seed
 ):
+    """
+    This function handles the chatbot response. It takes in:
+    - message: the user's new message
+    - history: the list of previous messages, each as a tuple (user_msg, assistant_msg)
+    - system_message: the system prompt
+    - max_tokens: the maximum number of tokens to generate in the response
+    - temperature: sampling temperature
+    - top_p: top-p (nucleus) sampling
+    - frequency_penalty: penalize repeated tokens in the output
+    - seed: a fixed seed for reproducibility; -1 will mean 'random'
+    """
+
     print(f"Received message: {message}")
     print(f"History: {history}")
     print(f"System message: {system_message}")
     print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}")
-    print(f"Frequency penalty: {frequency_penalty}, Seed: {seed}")
+    print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}")
+
+    # Convert seed to None if -1 (meaning random)
+    if seed == -1:
+        seed = None
 
-    # Construct the messages list for the conversation context
+    # Construct the messages array required by the API
     messages = [{"role": "system", "content": system_message}]
 
+    # Add conversation history to the context
     for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-            print(f"Added user message to context: {val[0]}")
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-            print(f"Added assistant message to context: {val[1]}")
+        user_part = val[0]
+        assistant_part = val[1]
+        if user_part:
+            messages.append({"role": "user", "content": user_part})
+            print(f"Added user message to context: {user_part}")
+        if assistant_part:
+            messages.append({"role": "assistant", "content": assistant_part})
+            print(f"Added assistant message to context: {assistant_part}")
 
+    # Append the latest user message
     messages.append({"role": "user", "content": message})
 
+    # Start with an empty string to build the response as tokens stream in
     response = ""
     print("Sending request to OpenAI API.")
 
-    for message in client.chat.completions.create(
-        model="meta-llama/Llama-3.3-70B-Instruct",
+    # Make the streaming request to the HF Inference API via openai-like client
+    for message_chunk in client.chat.completions.create(
+        model="meta-llama/Llama-3.3-70B-Instruct",   # You can update this to your specific model
         max_tokens=max_tokens,
-        stream=True,
+        stream=True,  # Stream the response
         temperature=temperature,
         top_p=top_p,
-        frequency_penalty=frequency_penalty,
-        seed=seed,
+        frequency_penalty=frequency_penalty,  # <-- NEW
+        seed=seed,                             # <-- NEW
         messages=messages,
     ):
-        token = message.choices[0].delta.content
-        print(f"Received token: {token}")
-        response += token
+        # Extract the token text from the response chunk
+        token_text = message_chunk.choices[0].delta.content
+        print(f"Received token: {token_text}")
+        response += token_text
         yield response
 
     print("Completed response generation.")
 
-# Initialize the chatbot interface
+# Create a Chatbot component with a specified height
 chatbot = gr.Chatbot(height=600)
-
 print("Chatbot interface created.")
 
-# Create the Gradio interface with additional inputs for the new parameters
+# Create the Gradio ChatInterface
+# We add two new sliders for Frequency Penalty and Seed
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
         gr.Textbox(value="", label="System message"),
-        gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-P",
-        ),
+        gr.Slider(minimum=1,   maximum=4096, value=512, step=1,   label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0,  value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=1.0,  value=0.95, step=0.05, label="Top-P"),
         gr.Slider(
             minimum=-2.0,
             maximum=2.0,
             value=0.0,
             step=0.1,
-            label="Frequency Penalty",
+            label="Frequency Penalty"
         ),
         gr.Slider(
             minimum=-1,
-            maximum=2**31 - 1,
+            maximum=65535,  # Arbitrary upper limit for demonstration
             value=-1,
             step=1,
-            label="Seed",
+            label="Seed (-1 for random)"
         ),
     ],
     fill_height=True,