Update app.py

app.py

@@ -19,7 +19,7 @@ print(zero.device) # <-- 'cpu' 🤔
 model_id = 'FINGU-AI/Qwen-Orpo-v1'              #attn_implementation="flash_attention_2",
 model = AutoModelForCausalLM.from_pretrained(model_id,attn_implementation="sdpa",  torch_dtype= torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+# streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 model.to('cuda')
 
 # terminators = [
@@ -44,9 +44,10 @@ def inference(query):
 ]
 
     tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to("cuda")
-
-    outputs = model.generate(tokenized_chat, **generation_params, streamer=streamer)
-    return outputs
+    outputs = model.generate(tokenized_chat, **generation_params)
+    decoded_outputs = tokenizer.batch_decode(outputs, skip_specail_tokens=True)
+    assistant_response = decoded_outputs[0].split("assistant:")[-1].strip()
+    return assistant_response
 
 examples = ['How can options strategies such as straddles, strangles, and spreads be used to hedge against market volatility?',
            'How do changes in interest rates, inflation, and GDP growth impact stock and bond markets?',