Update app.py

app.py

@@ -24,6 +24,7 @@ base_model = AutoPeftModelForCausalLM.from_pretrained(
 )
 """
 model = AutoPeftModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+model.to("cuda")
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
 tokenizer.pad_token_id = tokenizer.eos_token_id
 
@@ -54,7 +55,8 @@ train_data = train_data["train"]
 def generate_answer(question):
     #inputs = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
     inputs = tokenizer(question, return_tensors="pt")
-    outputs = model.generate(**inputs, max_length=250, num_return_sequences=1, do_sample=True)
+    inputs.to("cuda")
+    outputs = model.generate(**inputs, max_length=2048, num_return_sequences=1, do_sample=True)
     answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
     return answer
 
@@ -63,8 +65,8 @@ iface = gr.Interface(
     inputs="text",
     outputs="text",
     title="Straight Outta Logs",
-    examples = [prepare_sample_text(test_data[0], tokenizer, start=0, end=3), prepare_sample_text(test_data[4], tokenizer, start=0, end=5), prepare_sample_text(test_data[50], tokenizer, start=0, end=1)]
-    description="Use the examples or copy own Sepsis Case example",
+    examples = [prepare_sample_text(test_data[0], tokenizer, start=0, end=3), prepare_sample_text(test_data[4], tokenizer, start=0, end=5), prepare_sample_text(test_data[50], tokenizer, start=0, end=1)],
+    description="Use the examples or copy own sepsis case example",
 )
 
 iface.launch(share=True)  # Deploy the interface