Update app.py

app.py

@@ -25,6 +25,7 @@ model = AutoModelForCausalLM.from_pretrained(model_name,
                                              quantization_config=quantization_config, 
                                              device_map="auto")
 model.eval()
+max_token_length = 4096
 
 @spaces.GPU(duration=90)
 def generate(
@@ -47,9 +48,9 @@ def generate(
     conversation.append({"role": "user", "content": message})
 
     input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
-    if input_ids.shape[1] > 4096:
-        input_ids = input_ids[:, -4096:]
-        gr.Warning(f"Trimmed input from conversation as it was longer than {4096} tokens.")
+    if input_ids.shape[1] > max_token_length:
+        input_ids = input_ids[:, -max_token_length:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {max_token_length} tokens.")
     input_ids = input_ids.to(model.device)
 
     streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)