Test deepseek

app.py

@@ -8,6 +8,11 @@ from PIL import Image, ImageDraw
 from io import BytesIO
 import re
 
+from deepseek_vl.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM
+from deepseek_vl.utils.io import load_pil_images
+from transformers import AutoModelForCausalLM
+
+
 
 models = {
     "OS-Copilot/OS-Atlas-Base-7B": Qwen2VLForConditionalGeneration.from_pretrained("OS-Copilot/OS-Atlas-Base-7B", torch_dtype="auto", device_map="auto"),
@@ -49,8 +54,78 @@ def rescale_bounding_boxes(bounding_boxes, original_width, original_height, scal
     return rescaled_boxes
 
 
+def deepseek():
+        
+    # specify the path to the model
+    model_path = "deepseek-ai/deepseek-vl2-small"
+    vl_chat_processor: DeepseekVLV2Processor = DeepseekVLV2Processor.from_pretrained(model_path)
+    tokenizer = vl_chat_processor.tokenizer
+
+    vl_gpt: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)
+    vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
+
+    ## single image conversation example
+    conversation = [
+        {
+            "role": "<|User|>",
+            "content": "<image>\n<|ref|>The giraffe at the back.<|/ref|>.",
+            "images": ["./images/visual_grounding.jpeg"],
+        },
+        {"role": "<|Assistant|>", "content": ""},
+    ]
+
+    ## multiple images (or in-context learning) conversation example
+    # conversation = [
+    #     {
+    #         "role": "User",
+    #         "content": "<image_placeholder>A dog wearing nothing in the foreground, "
+    #                    "<image_placeholder>a dog wearing a santa hat, "
+    #                    "<image_placeholder>a dog wearing a wizard outfit, and "
+    #                    "<image_placeholder>what's the dog wearing?",
+    #         "images": [
+    #             "images/dog_a.png",
+    #             "images/dog_b.png",
+    #             "images/dog_c.png",
+    #             "images/dog_d.png",
+    #         ],
+    #     },
+    #     {"role": "Assistant", "content": ""}
+    # ]
+
+    # load images and prepare for inputs
+    pil_images = load_pil_images(conversation)
+    prepare_inputs = vl_chat_processor(
+        conversations=conversation,
+        images=pil_images,
+        force_batchify=True,
+        system_prompt=""
+    ).to(vl_gpt.device)
+
+    # run image encoder to get the image embeddings
+    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
+
+    # run the model to get the response
+    outputs = vl_gpt.language_model.generate(
+        inputs_embeds=inputs_embeds,
+        attention_mask=prepare_inputs.attention_mask,
+        pad_token_id=tokenizer.eos_token_id,
+        bos_token_id=tokenizer.bos_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        max_new_tokens=512,
+        do_sample=False,
+        use_cache=True
+    )
+
+    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
+    print(f"{prepare_inputs['sft_format'][0]}", answer)
+
 @spaces.GPU
 def run_example(image, text_input, model_id="OS-Copilot/OS-Atlas-Base-7B"):
+
+    deepseek()
+    
+
+def run_example_old(image, text_input, model_id="OS-Copilot/OS-Atlas-Base-7B"):
     model = models[model_id].eval()
     processor = processors[model_id]
     prompt = f"In this UI screenshot, what is the position of the element corresponding to the command \"{text_input}\" (with bbox)?"

requirements.txt

@@ -5,4 +5,5 @@ torch
 torchvision
 transformers
 accelerate==0.30.0
-qwen-vl-utils
+qwen-vl-utils
+deepseek_vl