Update app/main.py

app/main.py

@@ -1,31 +1,23 @@
 from fastapi import FastAPI, Request
 from pydantic import BaseModel
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM
 from peft import PeftModel
 import torch
 
 app = FastAPI()
 
 # ✅ Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
+tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2", use_auth_token=True)
 tokenizer.pad_token = tokenizer.eos_token
 
-# ✅ Setup quantization config
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.float16
-)
-
-# ✅ Load base model
+# ✅ Load base model without quantization (for CPU)
 model = AutoModelForCausalLM.from_pretrained(
     "mistralai/Mistral-7B-Instruct-v0.2",
-    device_map="auto",
-    quantization_config=bnb_config
+    torch_dtype=torch.float32,
+    use_auth_token=True
 )
 
-# ✅ Load LoRA adapter (ensure it's downloaded)
+# ✅ Load LoRA adapter
 ADAPTER_DIR = "./adapter/version 1"
 model = PeftModel.from_pretrained(model, ADAPTER_DIR)
 model.eval()
@@ -59,4 +51,4 @@ async def chat(req: ChatRequest):
     )
     response = tokenizer.decode(output[0], skip_special_tokens=True)
     reply = response.split("### Assistant:")[-1].strip()
-    return {"response": reply}
+    return {"response": reply}