10b

app.py

@@ -14,10 +14,12 @@ import numpy as np
 import spaces
 
 
-adapter_id = "merve/paligemma2-3b-vqav2"
-model_id = "google/paligemma2-3b-pt-448"
+#adapter_id = "merve/paligemma2-3b-vqav2"
+adapter_id = "google/paligemma2-10b-pt-448"
+model_id = "google/paligemma2-10b-pt-448"
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-model = PaliGemmaForConditionalGeneration.from_pretrained(adapter_id).eval().to(device)
+dtype = torch.bfloat16
+model = PaliGemmaForConditionalGeneration.from_pretrained(adapter_id, device_map='cuda', torch_dtype=dtype).eval()
 processor = PaliGemmaProcessor.from_pretrained(model_id)
 
 ###### Transformers Inference
@@ -28,7 +30,7 @@ def infer(
     max_new_tokens: int
 ) -> str:
     text = "answer en " + text
-    inputs = processor(text=text, images=image, return_tensors="pt").to(device)
+    inputs = processor(text=text, images=image, return_tensors="pt").to(device=device, dtype=dtype)
     with torch.inference_mode():
       generated_ids = model.generate(
           **inputs,
@@ -71,8 +73,8 @@ with gr.Blocks(css="style.css") as demo:
             label="Max New Tokens",
             info="Set to larger for longer generation.",
             minimum=20,
-            maximum=160,
-            value=80,
+            maximum=1600,
+            value=256,
             step=10,
         )