small changes

app.py

@@ -20,11 +20,11 @@ FastLanguageModel.for_inference(model)  # Enable optimized inference
 def respond(message, history, system_message, max_tokens, temperature, top_p):
     # Build the chat message history
     messages = [{"role": "system", "content": system_message}]
-    for val in history:
-        if val[0]:  # User message
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:  # Assistant message
-            messages.append({"role": "assistant", "content": val[1]})
+    for user_msg, assistant_msg in history:
+        if user_msg:  # User message
+            messages.append({"role": "user", "content": user_msg})
+        if assistant_msg:  # Assistant message
+            messages.append({"role": "assistant", "content": assistant_msg})
     messages.append({"role": "user", "content": message})
     
     # Tokenize the input messages
@@ -33,22 +33,22 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
         tokenize=True,
         add_generation_prompt=True,  # Required for generation
         return_tensors="pt",
-    ).to("cuda")
+    ).to("cuda" if torch.cuda.is_available() else "cpu")
 
     # Initialize a TextStreamer for streaming output
     text_streamer = TextStreamer(tokenizer, skip_prompt=True)
 
     # Generate the model's response
     response = ""
-    for output in model.generate(
-        input_ids=inputs,
+    for token_ids in model.generate(
+        input_ids=inputs.input_ids,
         streamer=text_streamer,
         max_new_tokens=max_tokens,
         use_cache=True,
         temperature=temperature,
         top_p=top_p,
     ):
-        token = tokenizer.decode(output, skip_special_tokens=True)
+        token = tokenizer.decode(token_ids[-1:], skip_special_tokens=True)  # Decode only the last token
         response += token
         yield response