Update app.py

app.py

@@ -30,20 +30,18 @@ import gradio as gr
 from PIL import Image
 import torch
 
-
-
-# # Create a configuration for quantization
-# quantization_config = BitsAndBytesConfig(
-#     load_in_4bit=True,   
-#     bnb_4bit_compute_dtype="float16",  
-#     bnb_4bit_use_double_quant=True,    
-#     bnb_4bit_quant_type="nf4",         
-# )
+# Create a configuration for quantization
+quantization_config = BitsAndBytesConfig(
+    load_in_8bit=True,   
+    bnb_8bit_compute_dtype="float16",  
+    bnb_8bit_use_double_quant=True,    
+    bnb_8bit_quant_type="nf4",         
+)
 
 # Load the model and processor
 model = Qwen2VLForConditionalGeneration.from_pretrained(
     "Qwen/QVQ-72B-Preview", device_map="auto", 
- #   quantization_config=quantization_config,
+     quantization_config=quantization_config,
      offload_folder="offload",
 )
 processor = AutoProcessor.from_pretrained("Qwen/QVQ-72B-Preview")