more

app.py

@@ -26,6 +26,9 @@ def respond(message, max_new_tokens, temperature, system_message=""):
         messages = [{"role": "system", "content": system_message}] if system_message else []
         messages.append({"role": "user", "content": message})
 
+        # Debug: Show messages
+        print("[DEBUG] Messages:", messages)
+
         # Tokenize inputs
         inputs = tokenizer.apply_chat_template(
             messages,
@@ -34,10 +37,13 @@ def respond(message, max_new_tokens, temperature, system_message=""):
             return_tensors="pt",
         ).to("cuda")
 
+        # Debug: Inspect input tensors
+        print("[DEBUG] Tokenized input IDs:", inputs["input_ids"].shape)
+        print("[DEBUG] Attention mask:", inputs["attention_mask"].shape)
+
         # Stream response
-        response = []
         text_streamer = TextStreamer(tokenizer, skip_prompt=True)
-        _ = model.generate(
+        response = model.generate(
             input_ids=inputs["input_ids"],
             attention_mask=inputs["attention_mask"],
             max_new_tokens=max_new_tokens,
@@ -45,8 +51,11 @@ def respond(message, max_new_tokens, temperature, system_message=""):
             use_cache=True,
             streamer=text_streamer,
         )
-        return "".join(response)
+
+        return response
     except Exception as e:
+        # Debug: Log errors
+        print("[ERROR]", str(e))
         return f"Error: {str(e)}"
 
 # Gradio UI