# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import glob
import pandas as pd
import numpy as np
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands


# +
# Done
def gesture_preprocess(landmark):
    """
    convert landmarks for trainable data
    66 features
    X (21): 0-20
    Y (21): 21-41
    Z (21): 42-62
    X,Y,Z range (3): 63-65
    
    params landmark: mediapipe landmark for 1 hand
    params label: str
    return: np.array (1,66)
    """ 
    lm_x = np.array([])
    lm_y = np.array([])
    lm_z = np.array([])
    for hlm in landmark.landmark:
        lm_x = np.append(lm_x, hlm.x)
        lm_y = np.append(lm_y, hlm.y)
        lm_z = np.append(lm_z, hlm.z)
    data_gest = [lm_x, lm_y, lm_z]
    x_rng, y_rng, z_rng = lm_x.max()-lm_x.min(), lm_y.max()-lm_y.min(), lm_z.max()-lm_z.min()
    data_gest = np.ravel([(k-k.min())/(k.max()-k.min()) for i, k in enumerate(data_gest)])
    data_gest = np.append(data_gest, [x_rng, y_rng, z_rng])
    return data_gest.reshape(1,-1)
    
# for lm in IMAGE_RESULTS[0].multi_hand_landmarks:
#     print(gesture_preprocess(lm))
    


# +
# Done
def data2df(data, train, label, y=None):
    '''
    add data to training df / label df
    
    param data: np.array
    param train: df
    param label: np.array
    param y: str
    return data: df, np.array
    '''
    data_gest = gesture_preprocess(data)
    train = pd.concat([train, pd.DataFrame(data_gest)])
    label = np.append(label, f'{y}')
    return train, label

# train_df = pd.DataFrame()
# label = np.array([])
# for lm in IMAGE_RESULTS[0].multi_hand_landmarks:
#     train_df, label = data2df(lm, train_df, label, 'rock')
# train_df
# label


# +
# For webcam input:
train_df = pd.DataFrame()
label = np.array([])
label_class = -1

cap = cv2.VideoCapture(0)
with mp_hands.Hands(
    max_num_hands=1,
    model_complexity=1,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5) as hands:
    while cap.isOpened():
        # hotkeys for generating dataset
        if cv2.waitKey(10) == ord('f'):
            label_class = 0  # hit
            print("recording hit")
        elif cv2.waitKey(10) == ord('g'):
            label_class = 1  # stand
            print("recording stand")
        elif cv2.waitKey(10) == ord('h'):
            label_class = 2  # split
            print("recording split")
        elif cv2.waitKey(10) == ord('j'):
            label_class = 3  # reset
            print("recording reset")
        elif cv2.waitKey(10) == ord('k'):
            label_class = -1
            print("recording stopped")
            
        success, image = cap.read()
        if not success:
            print("Ignoring empty camera frame.")
            # If loading a video, use 'break' instead of 'continue'.
            continue
        # To improve performance, optionally mark the image as not writeable to
        # pass by reference.
        image.flags.writeable = False
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        results = hands.process(image)
        
        # Draw the hand annotations on the image.
        image.flags.writeable = True
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                if label_class >= 0:
                    train_df, label = data2df(hand_landmarks, train_df, label, f'{label_class}')
                    print('training added')
                mp_drawing.draw_landmarks(
                    image,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())
#         cv2.imshow('MediaPipe Hands', cv2.flip(image, 1))
        cv2.imshow('MediaPipe Hands', image)
        if cv2.waitKey(5) & 0xFF == 27:
            break
cap.release()
cv2.destroyAllWindows()
# -

# export training data
train_df.to_csv('train_X.csv', header=False, index=False)
pd.DataFrame(label).to_csv('train_y.csv', header=False, index=False)
# label.tofile('train_y.csv', sep=',')
print(train_df.shape)
print(label.shape)

# +
# For static images:
filenames = glob.glob("test_imgs/*.jpg")
IMAGE_FILES = [cv2.imread(img) for img in filenames]
IMAGE_RESULTS = []

test_idx = 0
with mp_hands.Hands(
    static_image_mode=True,
    max_num_hands=2,
    min_detection_confidence=0.5) as hands:
    for idx, image in enumerate(IMAGE_FILES[0:test_idx+1]):
        # Read an image, flip it around y-axis for correct handedness output (see
        # above).
#         image = cv2.flip(cv2.imread(file), 1)
        # Convert the BGR image to RGB before processing.
        results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
        IMAGE_RESULTS.append(results)
        
        # Print handedness and draw hand landmarks on the image.
        print('Handedness:', results.multi_handedness)
        if not results.multi_hand_landmarks:
            continue
        image_height, image_width, _ = image.shape
        annotated_image = image.copy()
        for hand_landmarks in results.multi_hand_landmarks:
            print('hand_landmarks:', hand_landmarks)
#             train_df, label = data2df(hand_landmarks, train_df, label, f'{label}')
            f'Index finger tip coordinates: (',
            f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width}, '
            f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height})'
            )
        mp_drawing.draw_landmarks(
            annotated_image,
            hand_landmarks,
            mp_hands.HAND_CONNECTIONS,
            mp_drawing_styles.get_default_hand_landmarks_style(),
            mp_drawing_styles.get_default_hand_connections_style())
        
        clf_result = ''
        cv2.putText(image, 'Fedex', (x, y-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (36,255,12), 2)
        cv2.imshow('debug gesture', annotated_image)
        cv2.waitKey(0)
cv2.destroyAllWindows()
#         cv2.imwrite(
#         '/tmp/annotated_image' + str(idx) + '.png', annotated_image)

StarcoderdataJupyterScriptsDedupFiltered