Update app.py

app.py

@@ -34,10 +34,10 @@ h3 {
 device = "cuda" # for GPU usage or "cpu" for CPU usage
 
 quantization_config = BitsAndBytesConfig(
-    load_in_8bit=True,
-    bnb_8bit_compute_dtype=torch.bfloat16,
-    bnb_8bit_use_double_quant=True,
-    bnb_8bit_quant_type= "nf4")
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type= "nf4")
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 model = AutoModelForCausalLM.from_pretrained(