Remove video and add examples and handedness results

README.md

@@ -4,7 +4,7 @@ emoji: 🙌
 colorFrom: pink
 colorTo: purple
 sdk: gradio
-sdk_version: 3.0.20
+sdk_version: 3.0.26
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,12 +1,10 @@
-from os import stat
 import gradio as gr
-from matplotlib.pyplot import draw
 import mediapipe as mp
 import numpy as np
-import tempfile
-import mediapy as media
 import log_utils
 from functools import lru_cache
+import cv2
+from google.protobuf.json_format import MessageToDict
 
 logger = log_utils.get_logger()
 
@@ -34,13 +32,16 @@ def get_model(static_image_mode, max_num_hands, model_complexity, min_detection_
     min_tracking_confidence=min_tracking_conf,
   )
 
-def draw_landmarks(model, img, selected_connections, draw_background):
-  results = model.process(img)
-  output_img = img if draw_background else np.zeros_like(img)  
+def draw_landmarks(model, img, selected_connections, draw_background, flip_image):
+  img_to_process = cv2.flip(img, 1) if flip_image else img 
+  results = model.process(img_to_process)
+  output_img = img_to_process if draw_background else np.zeros_like(img_to_process)  
   if results.multi_hand_landmarks:
     for hand_landmarks in results.multi_hand_landmarks:
-      mp_draw.draw_landmarks(output_img, hand_landmarks, connections[selected_connections])    
-  return output_img
+      mp_draw.draw_landmarks(output_img, hand_landmarks, connections[selected_connections])
+  if flip_image:
+    output_img = cv2.flip(output_img, 1)        
+  return output_img, [MessageToDict(h) for _, h in enumerate(results.multi_handedness or [])]
 
 def process_image(
     img, 
@@ -51,31 +52,14 @@ def process_image(
     min_tracking_conf, 
     selected_connections, 
     draw_background,
+    flip_image,
   ):
   logger.info(f"Processing image with connections: {selected_connections}, draw background: {draw_background}")
   model = get_model(static_image_mode, max_num_hands, model_complexity, min_detection_conf, min_tracking_conf)
-  return draw_landmarks(model, img, selected_connections, draw_background)
-
-def process_video(
-    video_path,
-    static_image_mode, 
-    max_num_hands, 
-    model_complexity, 
-    min_detection_conf, 
-    min_tracking_conf, 
-    selected_connections, 
-    draw_background,
-  ):
-  logger.info(f"Processing video with connections: {selected_connections}, draw background: {draw_background}")
-  model = get_model(static_image_mode, max_num_hands, model_complexity, min_detection_conf, min_tracking_conf)
-  with tempfile.NamedTemporaryFile() as f:
-    out_path = f"{f.name}.{video_path.split('.')[-1]}"
-    with media.VideoReader(video_path) as r:
-      with media.VideoWriter(
-          out_path, shape=r.shape, fps=r.fps, bps=r.bps) as w:
-        for image in r:
-          w.add_image(draw_landmarks(model, image, selected_connections, draw_background))
-    return out_path
+  img, multi_handedness = draw_landmarks(model, img, selected_connections, draw_background, flip_image)
+  left_hand_count = len([h for h in multi_handedness if h['classification'][0]['label'] == 'Left'])
+  right_hand_count = len(multi_handedness) - left_hand_count
+  return img, multi_handedness, left_hand_count, right_hand_count
 
 
 demo = gr.Blocks()
@@ -113,6 +97,7 @@ with demo:
       ## Step 2: Set processing parameters
       """)  
     draw_background = gr.Checkbox(value=True, label="Draw background?")
+    flip_image = gr.Checkbox(value=True, label="Flip image? (Note that handedness is determined assuming the input image is mirrored, i.e., taken with a front-facing/selfie camera with images flipped horizontally. If it is not the case, please swap the handedness output in the application.)")
     connection_keys = list(connections.keys())
     selected_connections = gr.Dropdown(
       label="Select connections to draw", 
@@ -121,27 +106,27 @@ with demo:
       )
 
     gr.Markdown("""
-      ## Step 3: Select an image or video
+      ## Step 3: Select an image
       """)  
     with gr.Tabs():
       with gr.TabItem(label="Upload an image"):
         uploaded_image = gr.Image(type="numpy")
+        example_image = gr.Examples(examples=[['examples/example-01.jpg'], ['examples/example-02.jpg'], ['examples/example-03.jpg']], inputs=[uploaded_image])
         submit_uploaded_image = gr.Button(value="Process Image")
       with gr.TabItem(label="Take a picture"):
         camera_picture = gr.Image(source="webcam", type="numpy")
         submit_camera_picture = gr.Button(value="Process Image")  
-      with gr.TabItem(label="Record a video"):
-        recorded_video = gr.Video(source="webcam", format="mp4")
-        submit_recorded_video = gr.Button(value="Process Video")
-      with gr.TabItem(label="Upload a video"):
-        uploaded_video = gr.Video(format="mp4")
-        submit_uploaded_video = gr.Button(value="Process Video")  
 
   gr.Markdown(""" 
-      ## Step 4: View results
+      ## Step 4: View results 
       """)  
   with gr.Column():
-    processed_video = gr.Video()
+    with gr.Row():
+      with gr.Column():
+        left_hands = gr.Number(label="Left hands detected")
+      with gr.Column():
+        right_hands = gr.Number(label="Right hands detected")  
+    multi_handedness = gr.JSON(label="Raw results")
     processed_image = gr.Image()
 
   gr.Markdown('<img id="visitor-badge" alt="visitor badge" src="https://visitor-badge.glitch.me/badge?page_id=kristyc.mediapipe-hands" />')
@@ -153,10 +138,10 @@ with demo:
     min_tracking_conf, 
     selected_connections, 
     draw_background,
+    flip_image,
   ]
-  submit_uploaded_image.click(fn=process_image, inputs=[uploaded_image, *setting_inputs], outputs=[processed_image])
-  submit_camera_picture.click(fn=process_image, inputs=[camera_picture, *setting_inputs], outputs=[processed_image])
-  submit_recorded_video.click(fn=process_video, inputs=[recorded_video, *setting_inputs], outputs=[processed_video])
-  submit_uploaded_video.click(fn=process_video, inputs=[recorded_video, *setting_inputs], outputs=[processed_video])
+  outputs = [processed_image, multi_handedness, left_hands, right_hands]
+  submit_uploaded_image.click(fn=process_image, inputs=[uploaded_image, *setting_inputs], outputs=outputs)
+  submit_camera_picture.click(fn=process_image, inputs=[camera_picture, *setting_inputs], outputs=outputs)
 
 demo.launch()

requirements.txt

@@ -1,4 +1,4 @@
 mediapipe==0.8.10.1
-gradio==3.0.20
+gradio==3.0.26
 opencv-contrib-python==4.6.0.66
 mediapy==1.0.3