small changes

app.py

@@ -20,13 +20,18 @@ FastLanguageModel.for_inference(model)  # Enable optimized inference
 # Define the response function
 def respond(message, history, system_message, max_tokens, temperature, top_p):
     messages = [{"role": "system", "content": system_message}]
+    
+    # Append past conversation history
     for user_msg, assistant_msg in history:
         if user_msg:
             messages.append({"role": "user", "content": user_msg})
         if assistant_msg:
             messages.append({"role": "assistant", "content": assistant_msg})
+    
+    # Add the user's current message
     messages.append({"role": "user", "content": message})
     
+    # Tokenize the input with proper attention mask
     inputs = tokenizer.apply_chat_template(
         messages,
         tokenize=True,
@@ -36,15 +41,23 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
     inputs = inputs.to("cuda" if torch.cuda.is_available() else "cpu")
     attention_mask = inputs.ne(tokenizer.pad_token_id).long()
     
+    # Generate the output using the model
     output = model.generate(
         input_ids=inputs,
+        attention_mask=attention_mask,
         max_new_tokens=max_tokens,
-        use_cache=True,
         temperature=temperature,
         top_p=top_p,
+        pad_token_id=tokenizer.eos_token_id,  # Ensure padding is replaced with EOS
     )
-    response = tokenizer.decode(output[0], skip_special_tokens=True)
-    yield response  # Return full response directly
+    
+    # Decode the generated output
+    response = tokenizer.decode(
+        output[0], skip_special_tokens=True
+    ).strip()  # Remove any extra whitespace or unexpected tokens
+    
+    # Yield the clean response for display
+    yield response
 
 
 # Define the Gradio interface