Update app.py

app.py

@@ -32,10 +32,10 @@ class StopOnTokens(StoppingCriteria):
 
 def initialize_model():
     quantization_config = BitsAndBytesConfig(
-        load_in_8bit=True,
-        bnb_8bit_compute_dtype=torch.bfloat16,
-        bnb_8bit_quant_type="nf4",
-        bnb_8bit_use_double_quant=True,
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_use_double_quant=True,
     )
 
     tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
@@ -47,7 +47,7 @@ def initialize_model():
         quantization_config=quantization_config,
         torch_dtype=torch.bfloat16,
         trust_remote_code=True
-    )
+    ).to("cuda")
 
     return model, tokenizer