fix: gradio update

Signed-off-by: ivelin <[email protected]>

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+flagged/

app.py

@@ -7,18 +7,26 @@ import html
 from transformers import DonutProcessor, VisionEncoderDecoderModel
 
 
-global model, processor, device
+global model, loaded_revision, processor, device
+model = None
+previous_revision = None
+processor = None
+device = None
+loaded_revision = None
 
 
 def load_model(pretrained_revision: str = 'main'):
-    global model, processor, device
+    global model, loaded_revision, processor, device
     pretrained_repo_name = 'ivelin/donut-refexp-click'
     # revision can be git commit hash, branch or tag
     # use 'main' for latest revision
     print(
         f"Loading model checkpoint from repo: {pretrained_repo_name}, revision: {pretrained_revision}")
+    if processor is None or loaded_revision is None or loaded_revision != pretrained_revision:
+        loaded_revision = pretrained_revision
+
     processor = DonutProcessor.from_pretrained(
-        pretrained_repo_name, revision=pretrained_revision, use_auth_token="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")
+        pretrained_repo_name, revision=pretrained_revision)  # , use_auth_token="...")
     processor.image_processor.do_align_long_axis = False
     # do not manipulate image size and position
     processor.image_processor.do_resize = False
@@ -28,8 +36,8 @@ def load_model(pretrained_revision: str = 'main'):
     processor.image_processor.do_normalize = True
     print(f'processor image size: {processor.image_processor.size}')
     model = VisionEncoderDecoderModel.from_pretrained(
-        pretrained_repo_name, use_auth_token="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", revision=pretrained_revision)
-
+        pretrained_repo_name, revision=pretrained_revision)  # use_auth_token="...",
+    print(f'model checkpoint loaded')
     device = "cuda" if torch.cuda.is_available() else "cpu"
     model.to(device)
 
@@ -90,7 +98,7 @@ def translate_point_coords_from_out_to_in(point=None, input_image_size=None, out
         f"translated point={point}, resized_image_size: {resized_width, resized_height}")
 
 
-def process_refexp(image, prompt: str, model_revision: str = 'main'):
+def process_refexp(image, prompt: str, model_revision: str = 'main', return_annotated_image: bool = True):
 
     print(f"(image, prompt): {image}, {prompt}")
 
@@ -183,13 +191,16 @@ def process_refexp(image, prompt: str, model_revision: str = 'main'):
     print(
         f"to image pixel values: x, y: {x, y}")
 
-    # draw center point circle
-    img1 = ImageDraw.Draw(image)
-
-    r = 30
-    shape = [(x-r, y-r), (x+r, y+r)]
-    img1.ellipse(shape, outline="green", width=20)
-    img1.ellipse(shape, outline="white", width=10)
+    if return_annotated_image:
+        # draw center point circle
+        img1 = ImageDraw.Draw(image)
+        r = 30
+        shape = [(x-r, y-r), (x+r, y+r)]
+        img1.ellipse(shape, outline="green", width=20)
+        img1.ellipse(shape, outline="white", width=10)
+    else:
+        # do not return image if its an API call to save bandwidth
+        image = None
 
     return image, center_point
 
@@ -197,33 +208,34 @@ def process_refexp(image, prompt: str, model_revision: str = 'main'):
 title = "Demo: GuardianUI RefExp Click"
 description = "Gradio Demo for Donut RefExp task, an instance of `VisionEncoderDecoderModel` fine-tuned on [UIBert RefExp](https://huggingface.co/datasets/ivelin/ui_refexp_saved) Dataset (UI Referring Expression). To use it, simply upload your image and type a prompt and click 'submit', or click one of the examples to load them. Optionally enter value for model git revision; latest checkpoint will be used by default."
 article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2111.15664' target='_blank'>Donut: OCR-free Document Understanding Transformer</a> | <a href='https://github.com/clovaai/donut' target='_blank'>Github Repo</a></p>"
-examples = [["example_1.jpg", "select the setting icon from top right corner", ""],
-            ["example_1.jpg", "click on down arrow beside the entertainment", ""],
-            ["example_1.jpg", "select the down arrow button beside lifestyle", ""],
-            ["example_1.jpg", "click on the image beside the option traffic", ""],
-            ["example_3.jpg", "select the third row first image", ""],
-            ["example_3.jpg", "click the tick mark on the first image", ""],
-            ["example_3.jpg", "select the ninth image", ""],
-            ["example_3.jpg", "select the add icon", ""],
-            ["example_3.jpg", "click the first image", ""],
-            ["val-image-4.jpg", 'select 4153365454', ""],
-            ['val-image-4.jpg', 'go to cell', ""],
-            ['val-image-4.jpg', 'select number above cell', ""],
-            ["val-image-1.jpg", "select calendar option", ""],
-            ["val-image-1.jpg", "select photos&videos option", ""],
-            ["val-image-2.jpg", "click on change store", ""],
-            ["val-image-2.jpg", "click on shop menu at the bottom", ""],
-            ["val-image-3.jpg", "click on image above short meow", ""],
-            ["val-image-3.jpg", "go to cat sounds", ""],
-            ["example_2.jpg", "click on green color button", ""],
-            ["example_2.jpg", "click on text which is beside call now", ""],
-            ["example_2.jpg", "click on more button", ""],
-            ["example_2.jpg", "enter the text field next to the name", ""],
+examples = [["example_1.jpg", "select the setting icon from top right corner", "", True],
+            ["example_1.jpg", "click on down arrow beside the entertainment", "", True],
+            ["example_1.jpg", "select the down arrow button beside lifestyle", "", True],
+            ["example_1.jpg", "click on the image beside the option traffic", "", True],
+            ["example_3.jpg", "select the third row first image", "", True],
+            ["example_3.jpg", "click the tick mark on the first image", "", True],
+            ["example_3.jpg", "select the ninth image", "", True],
+            ["example_3.jpg", "select the add icon", "", True],
+            ["example_3.jpg", "click the first image", "", True],
+            ["val-image-4.jpg", 'select 4153365454', "", True],
+            ['val-image-4.jpg', 'go to cell', "", True],
+            ['val-image-4.jpg', 'select number above cell', "", True],
+            ["val-image-1.jpg", "select calendar option", "", True],
+            ["val-image-1.jpg", "select photos&videos option", "", True],
+            ["val-image-2.jpg", "click on change store", "", True],
+            ["val-image-2.jpg", "click on shop menu at the bottom", "", True],
+            ["val-image-3.jpg", "click on image above short meow", "", True],
+            ["val-image-3.jpg", "go to cat sounds", "", True],
+            ["example_2.jpg", "click on green color button", "", True],
+            ["example_2.jpg", "click on text which is beside call now", "", True],
+            ["example_2.jpg", "click on more button", "", True],
+            ["example_2.jpg", "enter the text field next to the name", "", True],
             ]
 
 demo = gr.Interface(fn=process_refexp,
-                    inputs=[gr.Image(type="pil"), "text", "text"],
-                    outputs=[gr.Image(type="pil", shape=(820)), "json"],
+                    inputs=[gr.Image(type="pil"), "text", "text", gr.Checkbox(
+                        value=True, label="Return Annotated Image", visible=False)],
+                    outputs=[gr.Image(type="pil"), "json"],
                     title=title,
                     description=description,
                     article=article,
@@ -232,4 +244,5 @@ demo = gr.Interface(fn=process_refexp,
                     cache_examples=False
                     )
 
-demo.launch()  # share=True when running in a Jupyter Notebook
+# share=True when running in a Jupyter Notebook
+demo.launch(server_name="0.0.0.0")

start-ec2-service.sh

@@ -0,0 +1,10 @@
+echo "Starting a service on a aws ec2 instance"
+echo "Start this script within tmux session in order to keep it running after the EC2 console closes"
+source activate pytorch
+cd ui-refexp-click/
+pip3 install -r requirements.txt
+python3 -m app 
+echo "\n\n\n-----\n\n\n"
+echo "Press CTRL+B then D to detach from terminal without killing the app"
+echo "Then use 'tmux attach' to reconnect to the virtual terminal"
+