Create inference.py

app/inference.py

@@ -0,0 +1,34 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
+from peft import PeftModel
+import torch
+
+ADAPTER_PATH = "adapter"
+BASE_MODEL = "Qwen/Qwen2-0.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL,
+    device_map="auto",
+    trust_remote_code=True,
+    torch_dtype=torch.float16
+)
+model = PeftModel.from_pretrained(model, ADAPTER_PATH)
+model.eval()
+
+streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+def generate_response(prompt: str) -> str:
+    formatted = f"<|im_start|>system\nYou are a helpful AI assistant.<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
+    inputs = tokenizer(formatted, return_tensors="pt").to(model.device)
+    with torch.no_grad():
+        output = model.generate(
+            **inputs,
+            max_new_tokens=512,
+            temperature=0.7,
+            top_p=0.9,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id
+        )
+    decoded = tokenizer.decode(output[0], skip_special_tokens=True)
+    answer = decoded.split("<|im_start|>assistant\n")[-1].strip()
+    return answer