more

app.py

@@ -16,19 +16,25 @@ FastLanguageModel.for_inference(model)  # Enable optimized inference
 
 # Define the response function
 def respond(message, history, system_message, max_tokens, temperature, top_p):
+    # Add the system message and include previous conversation history
     messages = [{"role": "system", "content": system_message}]
-    for exchange in history:
-        messages.append({"role": "user", "content": exchange["user"]})
-        messages.append({"role": "assistant", "content": exchange["assistant"]})
+    if history:
+        for entry in history:
+            messages.append({"role": "user", "content": entry["user"]})
+            messages.append({"role": "assistant", "content": entry["assistant"]})
+
+    # Add the user's new input
     messages.append({"role": "user", "content": message})
-    
+
+    # Tokenize inputs
     inputs = tokenizer.apply_chat_template(
         messages,
         tokenize=True,
         add_generation_prompt=True,
         return_tensors="pt",
     ).to("cuda" if torch.cuda.is_available() else "cpu")
-    
+
+    # Generate the response
     attention_mask = inputs.ne(tokenizer.pad_token_id).long()
     generated_tokens = model.generate(
         input_ids=inputs,
@@ -40,25 +46,37 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
     )
     response = tokenizer.decode(generated_tokens[0], skip_special_tokens=True)
 
+    # Update history
+    if history is None:
+        history = []
     history.append({"user": message, "assistant": response})
-    formatted_history = [
-        {"role": "user", "content": exchange["user"]} if "user" in exchange else
-        {"role": "assistant", "content": exchange["assistant"]}
-        for exchange in history
-    ]
+
+    print("history:")
+    print(history)
+    # Format history for Gradio (strictly enforce the role-content format)
+    formatted_history = []
+    for entry in history:
+        formatted_history.append({"role": "user", "content": entry["user"]})
+        formatted_history.append({"role": "assistant", "content": entry["assistant"]})
+
+    print("formatted_history:")
+    print(formatted_history)
+    # Return formatted history
     return formatted_history
 
+
 # Define the Gradio interface
 demo = gr.ChatInterface(
-    respond,
+    fn=respond,
     additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Textbox(value="You are a helpful assistant.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
     ],
     type="messages",
 )
 
+
 if __name__ == "__main__":
     demo.launch(share=False)  # Use share=False for local testing