Update app.py

app.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 
-# 
+#
 # This space is created by SANJOG GHONGE for testing and learning purpose.
 #
 # If you want to remove this space or credits please contact me on my email id [[email protected]].
@@ -24,14 +24,25 @@
 #
 # -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, BitsAndBytesConfig
 from qwen_vl_utils import process_vision_info
 import gradio as gr
 from PIL import Image
+import torch
+print(torch.cuda.memory_summary())
+
+
+# Create a configuration for quantization
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,   # Set to True for 4-bit quantization
+    bnb_4bit_compute_dtype="float16",  # Use float16 for faster computations
+    bnb_4bit_use_double_quant=True,    # Optional: Double quantization for memory savings
+    bnb_4bit_quant_type="nf4",         # NormalFloat4 (nf4) is better for performance
+)
 
 # Load the model and processor
 model = Qwen2VLForConditionalGeneration.from_pretrained(
-    "Qwen/QVQ-72B-Preview", torch_dtype="torch.float16", device_map="auto", load_in_4bit=True,
+    "Qwen/QVQ-72B-Preview", device_map="auto", quantization_config=quantization_config,
 )
 processor = AutoProcessor.from_pretrained("Qwen/QVQ-72B-Preview")
 
@@ -83,7 +94,7 @@ def process_image_and_question(image, question):
 
 # Define the Gradio interface
 with gr.Blocks() as demo:
-    gr.Markdown("# Sanjog Image and Question Answering\nProvide an image (JPG/PNG) and a related question to get an answer.")
+    gr.Markdown("# Sanjog Test : Image and Question Answering\nProvide an image (JPG/PNG) and a related question to get an answer.")
 
     with gr.Row():
         with gr.Column():
@@ -105,51 +116,3 @@ with gr.Blocks() as demo:
 demo.launch()
 
 
-# ------------------------------------------------------------------------------------------------------------------------------------
-
-
-
-# import gradio as gr
-# from transformers import AutoProcessor, AutoModelForImageTextToText
-
-# # Load the processor and model
-# model_name = "Qwen/QVQ-72B-Preview"
-# processor = AutoProcessor.from_pretrained(model_name)
-# model = AutoModelForImageTextToText.from_pretrained(model_name)
-
-# # Define the prediction function
-# def process_image_and_question(image, question):
-#     if image is None or not question:
-#         return "Please provide both an image and a question."
-    
-#     # Process the inputs
-#     inputs = processor(images=image, text=question, return_tensors="pt")
-    
-#     # Generate the output
-#     outputs = model.generate(**inputs)
-#     answer = processor.batch_decode(outputs, skip_special_tokens=True)[0]
-    
-#     return answer
-
-# # Define the Gradio interface
-# with gr.Blocks() as demo:
-#     gr.Markdown("# Image and Question Answering\nProvide an image (JPG/PNG) and a related question to get an answer.")
-
-#     with gr.Row():
-#         with gr.Column():
-#             image_input = gr.Image(type="pil", label="Upload Image (JPG/PNG)")
-#             question_input = gr.Textbox(label="Enter your question")
-
-#         with gr.Column():
-#             output_box = gr.Textbox(label="Result", interactive=False)
-
-#     with gr.Row():
-#         clear_button = gr.Button("Clear")
-#         submit_button = gr.Button("Submit")
-
-#     # Define button functionality
-#     clear_button.click(lambda: (None, "", ""), inputs=[], outputs=[image_input, question_input, output_box])
-#     submit_button.click(process_image_and_question, inputs=[image_input, question_input], outputs=output_box)
-
-# # Launch the interface
-# demo.launch()