fix: show annotated image in web ui

Signed-off-by: ivelin <[email protected]>

app.py

@@ -95,7 +95,7 @@ def translate_point_coords_from_out_to_in(point=None, input_image_size=None, out
         f"translated point={point}, resized_image_size: {resized_width, resized_height}")
 
 
-def process_refexp(image: Image, prompt: str, model_revision: str = 'main', return_annotated_image: bool = False):
+def process_refexp(image: Image, prompt: str, model_revision: str = 'main', return_annotated_image: bool = True):
 
     print(f"(image, prompt): {image}, {prompt}")
 
@@ -205,28 +205,28 @@ def process_refexp(image: Image, prompt: str, model_revision: str = 'main', retu
 title = "Demo: GuardianUI RefExp Click"
 description = "Gradio Demo for Donut RefExp task, an instance of `VisionEncoderDecoderModel` fine-tuned on [UIBert RefExp](https://huggingface.co/datasets/ivelin/ui_refexp_saved) Dataset (UI Referring Expression). To use it, simply upload your image and type a prompt and click 'submit', or click one of the examples to load them. Optionally enter value for model git revision; latest checkpoint will be used by default."
 article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2111.15664' target='_blank'>Donut: OCR-free Document Understanding Transformer</a> | <a href='https://github.com/clovaai/donut' target='_blank'>Github Repo</a></p>"
-examples = [["example_1.jpg", "select the setting icon from top right corner", "", False],
-            ["example_1.jpg", "click on down arrow beside the entertainment", "", False],
-            ["example_1.jpg", "select the down arrow button beside lifestyle", "", False],
-            ["example_1.jpg", "click on the image beside the option traffic", "", False],
-            ["example_3.jpg", "select the third row first image", "", False],
-            ["example_3.jpg", "click the tick mark on the first image", "", False],
-            ["example_3.jpg", "select the ninth image", "", False],
-            ["example_3.jpg", "select the add icon", "", False],
-            ["example_3.jpg", "click the first image", "", False],
-            ["val-image-4.jpg", 'select 4153365454', "", False],
-            ['val-image-4.jpg', 'go to cell', "", False],
-            ['val-image-4.jpg', 'select number above cell', "", False],
-            ["val-image-1.jpg", "select calendar option", "", False],
-            ["val-image-1.jpg", "select photos&videos option", "", False],
-            ["val-image-2.jpg", "click on change store", "", False],
-            ["val-image-2.jpg", "click on shop menu at the bottom", "", False],
-            ["val-image-3.jpg", "click on image above short meow", "", False],
-            ["val-image-3.jpg", "go to cat sounds", "", False],
-            ["example_2.jpg", "click on green color button", "", False],
-            ["example_2.jpg", "click on text which is beside call now", "", False],
-            ["example_2.jpg", "click on more button", "", False],
-            ["example_2.jpg", "enter the text field next to the name", "", False],
+examples = [["example_1.jpg", "select the setting icon from top right corner", "", True],
+            ["example_1.jpg", "click on down arrow beside the entertainment", "", True],
+            ["example_1.jpg", "select the down arrow button beside lifestyle", "", True],
+            ["example_1.jpg", "click on the image beside the option traffic", "", True],
+            ["example_3.jpg", "select the third row first image", "", True],
+            ["example_3.jpg", "click the tick mark on the first image", "", True],
+            ["example_3.jpg", "select the ninth image", "", True],
+            ["example_3.jpg", "select the add icon", "", True],
+            ["example_3.jpg", "click the first image", "", True],
+            ["val-image-4.jpg", 'select 4153365454', "", True],
+            ['val-image-4.jpg', 'go to cell', "", True],
+            ['val-image-4.jpg', 'select number above cell', "", True],
+            ["val-image-1.jpg", "select calendar option", "", True],
+            ["val-image-1.jpg", "select photos&videos option", "", True],
+            ["val-image-2.jpg", "click on change store", "", True],
+            ["val-image-2.jpg", "click on shop menu at the bottom", "", True],
+            ["val-image-3.jpg", "click on image above short meow", "", True],
+            ["val-image-3.jpg", "go to cat sounds", "", True],
+            ["example_2.jpg", "click on green color button", "", True],
+            ["example_2.jpg", "click on text which is beside call now", "", True],
+            ["example_2.jpg", "click on more button", "", True],
+            ["example_2.jpg", "enter the text field next to the name", "", True],
             ]
 
 demo = gr.Interface(fn=process_refexp,

flagged/log.csv

@@ -0,0 +1,2 @@
+image,prompt,model_revision,Return Annotated Image,output 0,output 1,flag,username,timestamp
+/root/ui-refexp-click/flagged/image/tmp1ah3uhhm.png,click on browser wallet option,,,,/root/ui-refexp-click/flagged/output 1/tmpyazt_rro.json,,,2023-02-28 20:42:56.524619

flagged/output 1/tmpyazt_rro.json

@@ -0,0 +1 @@
+{"x": 0.5035532994923858, "y": 0.32, "decoder output sequence (before x,y adjustment)": "<s_prompt> click on browser wallet option</s_prompt><s_target_center><s_x> 0.31</s_x><s_y> 0.32</s_y></s_target_center>"}