updated app.py

app.py

@@ -20,19 +20,19 @@ books_db_client = books_db.as_retriever()
 # Initialize the model and tokenizer
 model_name = "stabilityai/stablelm-zephyr-3b"
 
-bnb_config = transformers.BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type='nf4',
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_compute_dtype=torch.bfloat16
-)
+# bnb_config = transformers.BitsAndBytesConfig(
+#     load_in_4bit=True,
+#     bnb_4bit_quant_type='nf4',
+#     bnb_4bit_use_double_quant=True,
+#     bnb_4bit_compute_dtype=torch.bfloat16
+# )
 
 model_config = transformers.AutoConfig.from_pretrained(model_name, max_new_tokens=1024)
 model = transformers.AutoModelForCausalLM.from_pretrained(
     model_name,
     trust_remote_code=True,
     config=model_config,
-    quantization_config=bnb_config,
+    # quantization_config=bnb_config,
     device_map=device,
 )