Update app.py

app.py

@@ -16,7 +16,14 @@ print("********************")
 print(device)
 print("********************")
 
-model = AutoModelForCausalLM.from_pretrained(model_id, config=config, torch_dtype=torch.float16)
+bnb_config = transformers.BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type='nf4',
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_compute_dtype=bfloat16
+    )
+
+model = AutoModelForCausalLM.from_pretrained(model_id, config=config, quantization_config=bnb_config)
 model = model.to(device)
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)