Update app.py

app.py

@@ -94,8 +94,11 @@ db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embeddings)
 
 MODEL_NAME = "tiiuae/falcon-7b-instruct"
 
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+)
 model = AutoModelForCausalLM.from_pretrained(
-    MODEL_NAME, trust_remote_code=True,load_in_8bit=True, device_map="auto",offload_folder="offload"
+    MODEL_NAME, trust_remote_code=True, device_map="auto",offload_folder="offload",quantization_config=bnb_config
 )
 model = model.eval()