Update app.py

app.py

@@ -19,7 +19,7 @@ print(zero.device) # <-- 'cpu' 🤔
 model_id = 'FINGU-AI/Qwen-Orpo-v1'              #attn_implementation="flash_attention_2",
 model = AutoModelForCausalLM.from_pretrained(model_id,attn_implementation="sdpa",  torch_dtype= torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-# streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 model.to('cuda')
 
 # terminators = [
@@ -44,10 +44,12 @@ def inference(query):
 ]
 
     tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to("cuda")
-    outputs = model.generate(tokenized_chat, **generation_params)
-    decoded_outputs = tokenizer.batch_decode(outputs, skip_specail_tokens=True)
-    assistant_response = decoded_outputs[0].split("assistant:")[-1].strip()
-    return assistant_response
+    # outputs = model.generate(tokenized_chat, **generation_params)
+    # decoded_outputs = tokenizer.batch_decode(outputs, skip_specail_tokens=True)
+    # assistant_response = decoded_outputs[0].split("assistant:")[-1].strip()
+    # return assistant_response
+    outputs = model.generate(tokenized_chat, **generation_params, streamer=streamer)
+    return outputs
 
 examples = ['How can options strategies such as straddles, strangles, and spreads be used to hedge against market volatility?',
            'How do changes in interest rates, inflation, and GDP growth impact stock and bond markets?',