init

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.task filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+files

app.py

@@ -0,0 +1,185 @@
+import spaces
+import gradio as gr
+
+
+'''
+
+'''
+from gradio_utils import clear_old_files,read_file
+from face_mesh_spinning import process_face_mesh_spinning
+from mp_estimate  import mean_std_label,estimate_horizontal,estimate_vertical,estimate_horizontal_points,estimate_vertical_points
+
+def process_images(image,draw_type,center_scaleup,animation_direction,
+                   z_multiply,inner_eyes,inner_mouth,
+        progress=gr.Progress(track_tqdm=True)):
+    
+    clear_old_files()
+
+    if image==None:
+        raise gr.Error("need image")
+
+    result,face_landmarker_result,rotated_points = process_face_mesh_spinning(image,draw_type,center_scaleup,animation_direction,z_multiply,inner_eyes,inner_mouth)
+    
+    return result
+
+
+css="""
+#col-left {
+    margin: 0 auto;
+    max-width: 640px;
+}
+#col-right {
+    margin: 0 auto;
+    max-width: 640px;
+}
+.grid-container {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap:10px
+}
+
+.image {
+  width: 128px; 
+  height: 128px; 
+  object-fit: cover; 
+}
+
+.text {
+  font-size: 16px;
+}
+"""
+
+from glibvision.cv2_utils  import pil_to_bgr_image,copy_image
+from mp_utils import extract_landmark,get_pixel_cordinate
+import numpy as np
+# TODO move mp_util
+def extract_landmark_double_check(numpy_image,double_check=True,center_index=4,extract_matrix=True):#4 is nose-tip
+    mp_image,face_landmarker_result = extract_landmark(numpy_image,"face_landmarker.task",0,0,extract_matrix)
+    h,w = numpy_image.shape[:2]
+    second_mp_image,first_landmarker_result = None,None
+    numpy_view = mp_image.numpy_view()
+    if double_check:
+        root_cordinate = get_pixel_cordinate(face_landmarker_result.face_landmarks,center_index,w,h)
+        diff_center_x = int(w/2 - root_cordinate[0])
+        diff_center_y = int(h/2 - root_cordinate[1])
+        base = np.zeros_like(numpy_view)
+        copy_image(base,numpy_view,diff_center_x,diff_center_y)
+        first_landmarker_result = face_landmarker_result
+        second_mp_image,face_landmarker_result = extract_landmark(base,"face_landmarker.task",0,0,extract_matrix)
+    return mp_image,face_landmarker_result,second_mp_image,first_landmarker_result
+
+#css=css,
+
+from scipy.spatial.transform import Rotation as R
+def calculate_angle(image,double_check,ignore_x,order):
+    cv2_base_image = pil_to_bgr_image(image)
+    mp_image,face_landmarker_result,_,_ = extract_landmark_double_check(cv2_base_image,double_check)
+    if len(face_landmarker_result.facial_transformation_matrixes)>0:
+        transformation_matrix=face_landmarker_result.facial_transformation_matrixes[0]
+            
+        rotation_matrix, translation_vector = transformation_matrix[:3, :3],transformation_matrix[:3, 3]
+        
+        r = R.from_matrix(rotation_matrix)
+        euler_angles = r.as_euler(order, degrees=True)
+        label = f"Mediapipe Euler yxz: {euler_angles}"
+        if ignore_x:
+            euler_angles[1]=0
+
+        result = [label,0,0,0]
+        for i,ch in enumerate(order.lower()):
+            if ch == "x":
+                result[1] = -euler_angles[i]
+            elif ch == "y":
+                result[2] = euler_angles[i]
+            elif ch == "z":
+                result[3] = euler_angles[i]
+
+        return result
+        return label,-euler_angles[1],euler_angles[0],euler_angles[2]
+    return "",0,0,0
+
+def change_animation(animation):
+     if animation:
+          return gr.Column(visible=True),gr.Column(visible=False)
+     else:
+          return gr.Column(visible=False),gr.Column(visible=True)
+with gr.Blocks(css=css, elem_id="demo-container") as demo:
+    with gr.Column():
+        gr.HTML(read_file("demo_header.html"))
+        gr.HTML(read_file("demo_tools.html"))
+    with gr.Row():
+                with gr.Column():
+                    image = gr.Image(height=800,sources=['upload','clipboard'],image_mode='RGB',elem_id="image_upload", type="pil", label="Image")
+
+                    with gr.Row(elem_id="prompt-container",  equal_height=False):
+                        with gr.Row():
+                            btn = gr.Button("Rotate Mesh", elem_id="run_button",variant="primary")
+                    
+                    
+                        
+                    with gr.Accordion(label="Advanced Settings", open=True):
+                       
+                        draw_type = gr.Radio(label="Draw type",choices=["Dot","Line","Line+Fill","Image"],value="Line",info="making image animation,take over 60 sec and limited frame only")
+                        with gr.Row( equal_height=True):
+                             inner_eyes=gr.Checkbox(label="Inner Eyes",value=True)
+                             inner_mouth=gr.Checkbox(label="Inner Mouth",value=True)
+                        with gr.Row( equal_height=True):
+                            
+                            center_scaleup = gr.Checkbox(label="ScaleUp/Fit",value=True,info="center is nose-tip,Zoomed face usually make small")
+                            z_multiply = gr.Slider(info="Nose height",
+                            label="Depth-Multiply",
+                            minimum=0.1,
+                            maximum=1.5,
+                            step=0.01,
+                            value=0.8)
+                        animation_column = gr.Column(visible=True)
+                        with animation_column:
+                            with gr.Row( equal_height=True):
+                                animation_direction = gr.Radio(label="Animation Direction",choices=["X","Y","Z"],value="Y")
+
+                        
+                                
+                              
+
+                                         
+                with gr.Column():
+                    result_image = gr.Image(height=760,label="Result", elem_id="output-animation",image_mode='RGBA')
+                   
+                    
+                    
+                    
+   
+    btn.click(fn=process_images, inputs=[image,draw_type,center_scaleup,animation_direction,
+                                         z_multiply,inner_eyes,inner_mouth,
+                                         ],outputs=[result_image,
+                                                    
+                                                    ] ,api_name='infer')
+    
+    example_images = [
+                     ["examples/02316230.jpg","examples/02316230.webp"],
+                    ["examples/00003245_00.jpg","examples/00003245_00.webp"],
+                   ["examples/00827009.jpg","examples/00827009.webp"],
+                     ["examples/00002062.jpg","examples/00002062.webp"],
+                    ["examples/00824008.jpg","examples/00824008.webp"],
+                    ["examples/00825000.jpg","examples/00825000.webp"],
+                    ["examples/00826007.jpg","examples/00826007.webp"],
+                     ["examples/00824006.jpg","examples/00824006.webp"],
+                   
+                     ["examples/00002200.jpg","examples/00002200.webp"],
+                    ["examples/00005259.jpg","examples/00005259.webp"],
+                    ["examples/00018022.jpg","examples/00018022.webp"],
+                    ["examples/img-above.jpg","examples/img-above.webp"],
+                     ["examples/00100265.jpg","examples/00100265.webp"],
+                      ["examples/00039259.jpg","examples/00039259.webp"],
+                     
+                ]
+    example1=gr.Examples(
+                examples = example_images,label="Image",
+                inputs=[image,result_image],examples_per_page=8
+    )
+  
+    gr.HTML(read_file("demo_footer.html"))
+
+    if __name__ == "__main__":
+        demo.launch()

demo_footer.html

@@ -0,0 +1,3 @@
+<div>
+    <P> Images are generated with <a href="https://huggingface.co/black-forest-labs/FLUX.1-schnell">FLUX.1-schnell</a> and licensed under <a href="http://www.apache.org/licenses/LICENSE-2.0">the Apache 2.0 License</a>
+</div>

demo_header.html

@@ -0,0 +1,15 @@
+<div style="text-align: center;">
+    <h1>
+        Mediapipe Head 2D-Spinning
+    </h1>
+    <div  class="grid-container">
+        <img src="https://akjava.github.io/AIDiagramChatWithVoice-FaceCharacter/webp/128/00544245.webp" alt="Mediapipe Face Detection" class="image">
+       
+        <p class="text">
+            This Space use <a href="http://www.apache.org/licenses/LICENSE-2.0">the Apache 2.0</a> Licensed <a href="https://ai.google.dev/edge/mediapipe/solutions/vision/face_landmarker">Mediapipe FaceLandmarker</a> <br>
+            opposite side face quality is low<br>
+            <a href ="https://huggingface.co/spaces/Akjava/mediapipe-face-mesh-3d">3D Version</a> is much faster but seems hard to mixout 2d.<br>
+        </p>
+    </div>
+    
+</div>

demo_tools.html

@@ -0,0 +1,7 @@
+<div style="text-align: center;">
+    <p>
+        This is part of <a href="https://huggingface.co/collections/Akjava/mediapipe-tools-672ffe8ee7b62763c31b70c7">Mediapipe Tools Collection</a> 
+        
+    </p>
+    <p></p> 
+</div>

face_landmarker.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff
+size 3758596

face_landmarker.task.txt

@@ -0,0 +1,8 @@
+Face landmark detection
+https://ai.google.dev/edge/mediapipe/solutions/vision/face_landmarker
+
+model card page is 
+https://storage.googleapis.com/mediapipe-assets/MediaPipe%20BlazeFace%20Model%20Card%20(Short%20Range).pdf
+
+license is Apache2.0
+https://www.apache.org/licenses/LICENSE-2.0.html

face_mesh_spinning.py

@@ -0,0 +1,393 @@
+import subprocess
+from PIL import Image,ImageOps,ImageDraw,ImageFilter
+import json
+import os
+import time
+import io
+from mp_utils import get_pixel_cordinate_list,extract_landmark,get_pixel_cordinate,get_normalized_xyz
+from glibvision.draw_utils import points_to_box,box_to_xy,plus_point,calculate_distance
+
+import numpy as np
+from glibvision.pil_utils import fill_points,create_color_image,draw_box
+
+import glibvision.pil_utils
+
+from gradio_utils import save_image,save_buffer,clear_old_files ,read_file
+
+
+import math
+import mp_triangles
+
+
+from glibvision.cv2_utils import create_color_image as cv2_create_color_image,copy_image,pil_to_bgr_image
+import cv2
+#TODO move to CV2
+
+# i'm not sure this is fast
+def apply_affine_transformation_to_triangle_add(src_tri, dst_tri, src_img, dst_img):
+    src_tri_np = np.float32(src_tri)
+    dst_tri_np = np.float32(dst_tri)
+
+    h_dst, w_dst = dst_img.shape[:2]
+
+    M = cv2.getAffineTransform(src_tri_np, dst_tri_np)
+
+    dst_mask = np.zeros((h_dst, w_dst), dtype=np.uint8)
+    cv2.fillPoly(dst_mask, [np.int32(dst_tri)], 255)
+
+    transformed = cv2.warpAffine(src_img, M, (w_dst, h_dst))
+
+    transformed = transformed * (dst_mask[:, :, np.newaxis] / 255).astype(np.uint8)  
+    dst_background = dst_img * (1 - (dst_mask[:, :, np.newaxis] / 255)).astype(np.uint8)
+    dst_img = transformed + dst_background
+
+    return dst_img
+
+def apply_affine_transformation_to_triangle_add(src_tri, dst_tri, src_img, dst_img):
+    src_tri_np = np.float32(src_tri)
+    dst_tri_np = np.float32(dst_tri)
+
+    assert src_tri_np.shape == (3, 2), f"src_tri_np の形状が不正 {src_tri_np.shape}"
+    assert dst_tri_np.shape == (3, 2), f"dst_tri_np の形状が不正 {dst_tri_np.shape}"
+
+
+    # 透視変換行列の計算
+    M = cv2.getAffineTransform(src_tri_np, dst_tri_np)
+
+    # 画像のサイズ
+    h_src, w_src = src_img.shape[:2]
+    h_dst, w_dst = dst_img.shape[:2]
+
+    # 元画像から三角形領域を切り抜くマスク生成
+    #src_mask = np.zeros((h_src, w_src), dtype=np.uint8)
+    #cv2.fillPoly(src_mask, [np.int32(src_tri)], 255)
+
+    # Not 元画像の三角形領域のみをマスクで抽出
+    src_triangle = src_img #cv2.bitwise_and(src_img, src_img, mask=src_mask)
+
+    # 変換行列を使って元画像の三角形領域を目標画像のサイズへ変換
+   
+    transformed = cv2.warpAffine(src_triangle, M, (w_dst, h_dst))
+    #print(f"dst_img={dst_img.shape}")
+    #print(f"transformed={transformed.shape}")
+    # 変換後のマスクの生成
+    dst_mask = np.zeros((h_dst, w_dst), dtype=np.uint8)
+    cv2.fillPoly(dst_mask, [np.int32(dst_tri)], 255)
+    transformed = cv2.bitwise_and(transformed, transformed, mask=dst_mask)
+
+    # 目標画像のマスク領域をクリアするためにデストのインバートマスクを作成
+    dst_mask_inv = cv2.bitwise_not(dst_mask)
+
+    # 目標画像のマスク部分をクリア
+    dst_background = cv2.bitwise_and(dst_img, dst_img, mask=dst_mask_inv)
+
+    # 変換された元画像の三角形部分と目標画像の背景部分を合成
+    dst_img = cv2.add(dst_background, transformed)
+
+    return dst_img
+
+# TODO move PIL
+def process_create_webp(images,duration=100, loop=0,quality=85):
+    frames = []
+    for image_file in images:
+        frames.append(image_file)
+    
+    output_buffer = io.BytesIO()
+    frames[0].save(output_buffer, 
+                   save_all=True, 
+                   append_images=frames[1:], 
+                   duration=duration, 
+                   loop=loop, 
+                   format='WebP',
+                   quality=quality
+                   )
+
+    return output_buffer.getvalue()
+# TODO move numpy
+def rotate_point_euler(point, angles,order="xyz"):
+  """
+  オイラー角を使って3Dポイントを回転させる関数
+
+  Args:
+    point: 回転させる3Dポイント (x, y, z)
+    angles: 各軸周りの回転角度 (rx, ry, rz) [ラジアン]
+
+  Returns:
+    回転後の3Dポイント (x', y', z')
+  """
+
+  rx, ry, rz = angles
+  point = np.array(point)
+
+  # X軸周りの回転
+  Rx = np.array([
+      [1, 0, 0],
+      [0, np.cos(rx), -np.sin(rx)],
+      [0, np.sin(rx), np.cos(rx)]
+  ])
+
+  # Y軸周りの回転
+  Ry = np.array([
+      [np.cos(ry), 0, np.sin(ry)],
+      [0, 1, 0],
+      [-np.sin(ry), 0, np.cos(ry)]
+  ])
+
+  # Z軸周りの回転
+  Rz = np.array([
+      [np.cos(rz), -np.sin(rz), 0],
+      [np.sin(rz), np.cos(rz), 0],
+      [0, 0, 1]
+  ])
+
+  # 回転行列の合成 (Z軸 -> Y軸 -> X軸 の順で回転)
+  order = order.lower()
+  if order == "xyz":
+    R = Rx @ Ry @ Rz
+  elif order == "xzy":
+    R = Rx @ Rz @ Ry
+  elif order == "yxz":
+    R = Ry @ Rx @ Rz
+  elif order == "yzx":
+    R = Ry @ Rz @ Rx
+  elif order == "zxy":
+    R = Rz @ Rx @ Ry
+  else:
+    R = Rz @ Ry @ Rx
+      
+ 
+
+  # 回転後のポイントを計算
+  rotated_point = R @ point
+
+  return rotated_point
+
+
+def process_face_mesh_spinning(image,draw_type,center_scaleup,animation_direction,z_multiply=0.8,inner_eyes=False,inner_mouth=False):
+    animation = True
+    offset_x = 0
+    offset_y = 0
+    # use when center_scaleup is True,scale is 0.45(half-size:0.5-margin/ nosetip-to-top or nosetip-to-bottom
+    scale_up = 1.0
+
+    face_landmarker_result = None
+
+
+    if image == None:#app stop support none image,if mode still image,make problem
+        #  Box for no Image Case
+        image_width = 512
+        image_height = 512
+        #image = create_color_image(image_width,image_height,(0,0,0))
+        points = [(-0.25,-0.25,0),(0.25,-0.25,0),
+            (0.25,0.25,0),(-0.25,0.25,0)
+            ]
+        normalized_center_point = [0.5,0.5]
+    else:
+        image_width = image.width
+        image_height = image.height
+        
+
+                
+
+        mp_image,face_landmarker_result = extract_landmark(image,"face_landmarker.task",0,0,True)
+
+        def rotate_image():
+            return None,face_landmarker_result,None
+                    
+        #return rotate_image()
+        # cordinate eyes
+        # cordinate all
+        landmark_points = [get_normalized_xyz(face_landmarker_result.face_landmarks,i) for i in range(0,468)]
+        # do centering
+        normalized_center_point = landmark_points[4]
+        normalized_top_point = landmark_points[10]
+        normalized_bottom_point = landmark_points[152]
+        
+
+        offset_x = normalized_center_point[0]
+        offset_y = normalized_center_point[1]
+        offset_z = normalized_center_point[2]
+
+        #need aspect?
+        points = [[point[0]-offset_x,point[1]-offset_y,point[2]*z_multiply] for point in landmark_points]
+
+
+    # split xy-cordinate and z-depth
+    def split_points_xy_z(points,width,height,center_x,center_y):
+        xys = []
+        zs = []
+        for point in points:
+            xys.append(
+                   [
+                        point[0]*width*scale_up+center_x,
+                        point[1]*height*scale_up+center_y
+                   ]
+              )
+            zs.append(point[2])
+        return xys,zs
+    
+    def draw_grid_in_center(draw,cx,cy,grid_size,grid_color,width=1,draw_horizontal=True,draw_vertical=True):
+        w = image.width
+        h = image.height
+        x_minus_divide = cx//grid_size
+        x_plus_divide = (w -cx)//grid_size
+        y_minus_divide = cy//grid_size
+        y_plus_divide = (h -cx)//grid_size
+        for i in range(-x_minus_divide,x_plus_divide+1):
+            draw.line([(cx+i*grid_size,0),(cx+i*grid_size,h)],fill=grid_color,width=width)
+        for i in range(-y_minus_divide,y_plus_divide+1):
+            draw.line([(0,cy+i*grid_size),(w,cy+i*grid_size)],fill=grid_color,width=width)
+
+    def draw_grid(image,cx=512,cy=512,first_color=(255,0,0)):
+        w = image.width
+        h = image.height
+        second_grid_size=100
+        second_color = (128,128,128)
+        draw = ImageDraw.Draw(image)
+        draw_grid_in_center(draw,cx,cy,20,(100,100,100))
+        draw_grid_in_center(draw,cx,cy,100,(192,192,192))
+        
+        
+
+        draw.line([(cx,0),(cx,image.height)],fill=first_color)
+        draw.line([(0,cy),(image.width,cy)],fill=first_color)
+
+    def create_triangle_image(points,width,height,center_x,center_y,line_color=(255,255,255),fill_color=None):
+        
+        cordinates,angled_depth = split_points_xy_z(points,width,height,center_x,center_y)
+       
+        img = create_color_image(width,height,(0,0,0))
+        draw = ImageDraw.Draw(img)
+        triangles = mp_triangles.get_triangles_copy(True,inner_eyes,inner_eyes,inner_mouth)
+
+        triangles.sort(key=lambda triangle: sum(angled_depth[index] for index in triangle) / len(triangle)
+                       ,reverse=True)
+        for triangle in triangles:
+            triangle_cordinates = [cordinates[index] for index in triangle]
+            glibvision.pil_utils.image_draw_points(draw,triangle_cordinates,line_color,fill_color)
+        
+        
+        return img
+    
+    def create_texture_image(image,origin_points,angled_points,width,height,center_x,center_y,line_color=(255,255,255),fill_color=None):
+        cv2_image = pil_to_bgr_image(image)
+        #print(f"shape={cv2_image.shape}")
+        #cv2.imwrite("tmp.jpg",cv2_image)
+        original_cordinates = []
+        cordinates,angled_depth = split_points_xy_z(angled_points,width,height,center_x,center_y)
+        # original point need offset
+        for point in origin_points:
+            original_cordinates.append(
+                   [
+                        (point[0]+offset_x)*width,
+                        (point[1]+offset_y)*height
+                   ]
+            )
+        if cv2_image.shape[2]==3:
+            cv2_bg_img = cv2_create_color_image(cv2_image,(0,0,0))
+        else:
+            cv2_bg_img = cv2_create_color_image(cv2_image,(0,0,0,0))
+        
+        triangles = mp_triangles.get_triangles_copy(True,inner_eyes,inner_eyes,inner_mouth)
+
+        triangles.sort(key=lambda triangle: sum(angled_depth[index] for index in triangle) / len(triangle)
+                       ,reverse=True)
+           
+        for triangle in triangles:
+                triangle_cordinates = [cordinates[index] for index in triangle]
+                origin_triangle_cordinates = [original_cordinates[index] for index in triangle]
+                
+                cv2_bg_img=apply_affine_transformation_to_triangle_add(origin_triangle_cordinates,triangle_cordinates,cv2_image,cv2_bg_img)
+            
+        img= Image.fromarray(cv2.cvtColor(cv2_bg_img, cv2.COLOR_RGBA2BGRA))
+        
+        return img
+    
+    def create_point_image(points,width,height,center_x,center_y):
+        cordinates,_ = split_points_xy_z(points,width,height,center_x,center_y)
+        img = create_color_image(width,height,(0,0,0))
+        glibvision.pil_utils.draw_points(img,cordinates,None,None,3,(255,0,0),3)
+       
+        return img
+    
+    def angled_points(points,angles,order="xyz"):
+        angled_cordinates = []
+        for point in points:
+            rotated_np_point = rotate_point_euler(point,angles,order)
+            angled_cordinates.append(
+                   [
+                       rotated_np_point[0],
+                       rotated_np_point[1],rotated_np_point[2]
+                   ]
+              )
+        return angled_cordinates
+    
+
+    frames = []
+
+
+    #frames.append(create_point_image(points))
+    frame_duration=100
+    start_angle=0
+    end_angle=360
+    step_angle=10
+    
+    if draw_type == "Image":
+        start_angle=-90
+        end_angle=90
+        step_angle=30
+
+    if not animation:
+        start_angle=0
+        end_angle=0
+        step_angle=360
+    if image == None:
+        draw_type="Dot"
+
+    
+    if center_scaleup and image!=None:
+        top_distance = calculate_distance(normalized_center_point,normalized_top_point)
+        bottom_distance = calculate_distance(normalized_center_point,normalized_bottom_point)
+        distance = top_distance if top_distance>bottom_distance else bottom_distance
+        #small_size = image_width if image_width<image_height else image_height
+        
+        scale_up = 0.45 / distance #half - margin
+        if image_height>image_width:
+            scale_up *= image_width/image_height
+        #print(scale_up)
+        face_center_x = int(0.5* image_width)#half
+        face_center_y = int(0.5* image_height)
+    else:
+        scale_up = 1.0
+        face_center_x = int(normalized_center_point[0]* image_width)
+        face_center_y = int(normalized_center_point[1]* image_height)
+        
+    
+
+    rotated_points = None
+    
+    if animation:
+        for i in range(start_angle,end_angle,step_angle):
+            if animation_direction == "X":
+                angles = [math.radians(i),0,0]
+            elif animation_direction == "Y":
+                angles = [0,math.radians(i),0]
+            else:
+                angles = [0,0,math.radians(i)]
+           
+            if draw_type == "Dot":
+                frames.append(create_point_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y))
+            elif draw_type == "Line":
+                frames.append(create_triangle_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y))
+            elif draw_type == "Line+Fill":
+                frames.append(create_triangle_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y,(128,128,128),(200,200,200)))
+            elif draw_type == "Image":
+                frame_duration=500
+                frames.append(create_texture_image(image,points,angled_points(points,angles),image_width,image_height,face_center_x,face_center_y))
+        webp = process_create_webp(frames,frame_duration)
+        path = save_buffer(webp)
+    
+
+
+    
+    return path,face_landmarker_result,rotated_points

glibvision/common_utils.py

@@ -0,0 +1,112 @@
+import os
+def check_exists_files(files,dirs,exit_on_error=True):
+    if files is not None:
+        if isinstance(files, str):  
+            files = [files]
+        for file in files:
+            if not os.path.isfile(file):
+                print(f"File {file} not found")
+                if exit_on_error:
+                    exit(1)
+                else:
+                    return 1
+    if dirs is not None:
+        if isinstance(dirs, str):  
+            dirs = [dirs]
+        for dir in dirs:
+            if not os.path.isdir(dir):
+                print(f"Dir {dir} not found")
+                if exit_on_error:
+                    exit(1)
+                else:
+                    return 1
+    return 0
+
+image_extensions =[".jpg"]
+
+def add_name_suffix(file_name,suffix,replace_suffix=False):
+    if not suffix.startswith("_"):#force add
+        suffix="_"+suffix
+
+    name,ext = os.path.splitext(file_name)
+    if replace_suffix:
+        index = name.rfind("_")
+        if index!=-1:
+            return f"{name[0:index]}{suffix}{ext}"
+        
+    return f"{name}{suffix}{ext}"
+
+def replace_extension(file_name,new_extension,suffix=None,replace_suffix=False):
+    if not new_extension.startswith("."):
+        new_extension="."+new_extension
+
+    name,ext = os.path.splitext(file_name)
+    new_file = f"{name}{new_extension}"
+    if suffix:
+        return add_name_suffix(name+new_extension,suffix,replace_suffix)
+    return new_file
+
+def list_digit_images(input_dir,sort=True):
+    digit_images = []
+    global image_extensions
+    files = os.listdir(input_dir)
+    for file in files:
+        if file.endswith(".jpg"):#TODO check image
+            base,ext = os.path.splitext(file)
+            if not base.isdigit():
+                continue
+            digit_images.append(file) 
+
+    if sort:
+        digit_images.sort()
+
+    return digit_images
+def list_suffix_images(input_dir,suffix,is_digit=True,sort=True):
+    digit_images = []
+    global image_extensions
+    files = os.listdir(input_dir)
+    for file in files:
+        if file.endswith(".jpg"):#TODO check image
+            base,ext = os.path.splitext(file)
+            if base.endswith(suffix):
+                if is_digit:
+                    if not base.replace(suffix,"").isdigit():
+                        continue
+                digit_images.append(file) 
+
+    if sort:
+        digit_images.sort()
+
+    return digit_images
+
+import time
+
+class ProgressTracker:
+    """
+    処理の進捗状況を追跡し、経過時間と残り時間を表示するクラス。
+    """
+
+    def __init__(self,key, total_target):
+        """
+        コンストラクタ
+
+        Args:
+            total_target (int): 処理対象の総数
+        """
+        self.key = key
+        self.total_target = total_target
+        self.complete_target = 0
+        self.start_time = time.time()
+
+    def update(self):
+        """
+        進捗を1つ進める。
+        経過時間と残り時間を表示する。
+        """
+        self.complete_target += 1
+        current_time = time.time()
+        consumed_time = current_time - self.start_time
+        remain_time = (consumed_time / self.complete_target) * (self.total_target - self.complete_target) if self.complete_target > 0 else 0 
+        print(f"stepped {self.key} {self.total_target} of {self.complete_target}, consumed {(consumed_time / 60):.1f} min, remain {(remain_time / 60):.1f} min")
+
+    

glibvision/cv2_utils.py

@@ -0,0 +1,182 @@
+import cv2
+import numpy as np
+
+
+#2024-11-30 copy paste
+def draw_bbox(image,box,color=(255,0,0),thickness=1):
+  if thickness==0:
+    return
+  
+  left = int(box[0])
+  top = int(box[1])
+  right = int(box[0]+box[2])
+  bottom = int(box[1]+box[3])
+  box_points =[(left,top),(right,top),(right,bottom),(left,bottom)]
+  
+  cv2.polylines(image, [np.array(box_points)], isClosed=True, color=color, thickness=thickness)
+
+
+def to_int_points(points):
+  int_points=[]
+  for point in points:
+    int_points.append([int(point[0]),int(point[1])])
+  return int_points
+
+def draw_text(img, text, point, font_scale=0.5, color=(200, 200, 200), thickness=1):
+  font = cv2.FONT_HERSHEY_SIMPLEX
+  cv2.putText(img, str(text), point, font, font_scale, color, thickness, cv2.LINE_AA)
+
+plot_text_color = (200, 200, 200)
+plot_text_font_scale = 0.5
+plot_index = 1
+plot_text = True
+
+def set_plot_text(is_plot,text_font_scale,text_color):
+  global plot_index,plot_text,plot_text_font_scale,plot_text_color
+  plot_text = is_plot
+  plot_index = 1
+  plot_text_font_scale = text_font_scale
+  plot_text_color = text_color
+
+def plot_points(image,points,isClosed=False,circle_size=3,circle_color=(255,0,0),line_size=1,line_color=(0,0,255)):
+    global plot_index,plot_text
+    int_points = to_int_points(points)
+    if circle_size>0:
+      for point in int_points:
+        cv2.circle(image,point,circle_size,circle_color,-1)
+        if plot_text:
+          draw_text(image,plot_index,point,plot_text_font_scale,plot_text_color)
+        plot_index+=1
+    if line_size>0:
+      cv2.polylines(image, [np.array(int_points)], isClosed=isClosed, color=line_color, thickness=line_size)
+
+def fill_points(image,points,thickness=1,line_color=(255,255,255),fill_color = (255,255,255)):
+    np_points = np.array(points,dtype=np.int32)
+    cv2.fillPoly(image, [np_points], fill_color)
+    cv2.polylines(image, [np_points], isClosed=True, color=line_color, thickness=thickness)
+
+def get_image_size(cv2_image):
+    return cv2_image.shape[:2]
+
+def get_channel(np_array):
+    return np_array.shape[2] if np_array.ndim == 3 else 1 
+
+def get_numpy_text(np_array,key=""):
+    channel = get_channel(np_array)
+    return f"{key} shape = {np_array.shape} channel = {channel} ndim = {np_array.ndim} size = {np_array.size}"
+
+
+def gray3d_to_2d(grayscale: np.ndarray) -> np.ndarray:
+    channel = get_channel(grayscale)
+    if channel!=1:
+        raise ValueError(f"color maybe rgb or rgba {get_numpy_text(grayscale)}")
+    """
+    3 次元グレースケール画像 (チャンネル数 1) を 2 次元に変換する。
+
+    Args:
+        grayscale (np.ndarray): 3 次元グレースケール画像 (チャンネル数 1)。
+
+    Returns:
+        np.ndarray: 2 次元グレースケール画像。
+    """
+
+    if grayscale.ndim == 2:
+        return grayscale
+    return np.squeeze(grayscale)
+
+def blend_rgb_images(image1: np.ndarray, image2: np.ndarray, mask: np.ndarray) -> np.ndarray:
+    """
+    2 つの RGB 画像をマスク画像を使用してブレンドする。
+
+    Args:
+        image1 (np.ndarray): 最初の画像 (RGB)。
+        image2 (np.ndarray): 2 番目の画像 (RGB)。
+        mask (np.ndarray): マスク画像 (グレースケール)。
+
+    Returns:
+        np.ndarray: ブレンドされた画像 (RGB)。
+
+    Raises:
+        ValueError: 入力画像の形状が一致しない場合。
+    """
+
+    if image1.shape != image2.shape or image1.shape[:2] != mask.shape:
+        raise ValueError("入力画像の形状が一致しません。")
+
+    # 画像を float 型に変換
+    image1 = image1.astype(float)
+    image2 = image2.astype(float)
+
+    # マスクを 3 チャンネルに変換し、0-1 の範囲にスケール
+    alpha = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR).astype(float) / 255.0
+
+    # ブレンド計算
+    blended = (1 - alpha) * image1 + alpha * image2
+
+    return blended.astype(np.uint8)
+
+def create_color_image(img,color=(255,255,255)):
+    mask = np.zeros_like(img)
+    
+    h, w = img.shape[:2]
+    cv2.rectangle(mask, (0, 0), (w, h), color, -1)
+    return mask
+#RGB Image use np.array(image, dtype=np.uint8)
+def pil_to_bgr_image(image):
+  np_image = np.array(image, dtype=np.uint8)
+  if np_image.shape[2] == 4:
+      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_RGBA2BGRA)
+  else:
+      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)
+  return bgr_img
+
+def bgr_to_rgb(np_image):
+  if np_image.shape[2] == 4:
+      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_RBGRA2RGBA)
+  else:
+      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_BGR2RGB)
+  return bgr_img
+
+def crop(image,bbox):
+   x,y,width,height = bbox
+   return image[y:y+height, x:x+width]
+#not check safe
+def paste(image,replace_image,x,y):
+   height,width = replace_image.shape[:2]
+   image[y:y+height, x:x+width] = replace_image
+
+def copy_image(img1: np.ndarray, img2: np.ndarray, x: int, y: int) -> None:
+    # チャネル数と次元数のチェック
+    if img1.ndim != 3 or img2.ndim != 3:
+        raise ValueError("Both img1 and img2 must be 3-dimensional arrays.")
+    elif img1.shape[2] != img2.shape[2]:
+        raise ValueError(f"img1 and img2 must have the same number of channels. img1 has {img1.shape[2]} channels, but img2 has {img2.shape[1]} channels.")
+
+    # Type check
+    if not isinstance(img1, np.ndarray) or not isinstance(img2, np.ndarray):
+        raise TypeError("img1 and img2 must be NumPy arrays.")
+
+    if x>=0:
+      offset_x=0
+      w = min(img1.shape[1]-x,img2.shape[1])
+    else:
+      w = min(img1.shape[1],img2.shape[1]+x)
+      offset_x=int(-x)
+      x = 0
+
+    if y>=0:
+      h = min(img1.shape[0]-y,img2.shape[0])
+      offset_y=0
+    else:
+      h = min(img1.shape[0]-y,img2.shape[0]+y)
+      offset_y=int(-y)
+      y = 0
+    x=int(x)
+    y=int(y)
+    h=int(h)
+    w=int(w)
+   
+    
+    print(f"img1 {img1.shape} img2{img2.shape} x={x} y={y} w={w} h={h}")
+    # Paste the overlapping part
+    img1[y:y+h, x:x+w] = img2[offset_y:h+offset_y, offset_x:w+offset_x]

glibvision/draw_utils.py

@@ -0,0 +1,42 @@
+# DrawUtils 
+# not PIL,CV2,Numpy drawing method
+import math
+# 2024-11-29 add calculate_distance
+def points_to_box(points):
+  x1=float('inf')
+  x2=0
+  y1=float('inf')
+  y2=0
+  for point in points:
+    if point[0]<x1:
+      x1=point[0]
+    if point[0]>x2:
+      x2=point[0]
+    if point[1]<y1:
+      y1=point[1]
+    if point[1]>y2:
+      y2=point[1]
+  return [x1,y1,x2-x1,y2-y1]
+
+def box_to_point(box):
+  return [
+    [box[0],box[1]], 
+    [box[0]+box[2],box[1]],
+    [box[0]+box[2],box[1]+box[3]],
+    [box[0],box[1]+box[3]]
+  ]
+
+def plus_point(base_pt,add_pt):
+  return [base_pt[0]+add_pt[0],base_pt[1]+add_pt[1]]
+
+def box_to_xy(box):
+  return [box[0],box[1],box[2]+box[0],box[3]+box[1]]
+
+def to_int_points(points):
+  int_points=[]
+  for point in points:
+    int_points.append([int(point[0]),int(point[1])])
+  return int_points
+
+def calculate_distance(xy, xy2):
+    return math.sqrt((xy2[0] - xy[0])**2 + (xy2[1] - xy[1])**2)

glibvision/glandmark_utils.py

@@ -0,0 +1,48 @@
+
+import os
+
+#simple single version
+def bbox_to_glandmarks(file_name,bbox,points = None):
+     base,ext = os.path.splitext(file_name)
+     glandmark = {"image":{
+            "boxes":[{
+                "left":int(bbox[0]),"top":int(bbox[1]),"width":int(bbox[2]),"height":int(bbox[3])
+            }],
+            "file":file_name,
+            "id":int(base) 
+            # width,height ignore here
+            }}
+     if points is not None:
+          parts=[
+          ]
+          for point in points:
+               parts.append({"x":int(point[0]),"y":int(point[1])})
+          glandmark["image"]["boxes"][0]["parts"] = parts
+     return glandmark
+
+#technically this is not g-landmark/dlib ,
+def convert_to_landmark_group_json(points):
+     if len(points)!=68:
+          print(f"points must be 68 but {len(points)}")
+          return None
+     new_points=list(points)
+     
+     result = [ # possible multi person ,just possible any func support multi person
+         
+          {    # index start 0 but index-number start 1
+               "chin":new_points[0:17],
+               "left_eyebrow":new_points[17:22],
+               "right_eyebrow":new_points[22:27],
+               "nose_bridge":new_points[27:31],
+               "nose_tip":new_points[31:36],
+               "left_eye":new_points[36:42],
+               "right_eye":new_points[42:48],
+               
+               # lip points customized structure  
+               # MIT licensed face_recognition
+               # https://github.com/ageitgey/face_recognition
+               "top_lip":new_points[48:55]+[new_points[64]]+[new_points[63]]+[new_points[62]]+[new_points[61]]+[new_points[60]],
+               "bottom_lip":new_points[54:60]+[new_points[48]]+[new_points[60]]+[new_points[67]]+[new_points[66]]+[new_points[65]]+[new_points[64]],
+          }
+     ]
+     return result

glibvision/mediapipe_utils.py

@@ -0,0 +1,17 @@
+import math
+from .numpy_utils import rotate_point_euler
+# mediapipe utils work with other glibvisions
+
+def rotate_points(points,angles,order="xyz",is_degree=False):
+    if is_degree:
+       angles = [math.radians(float(value)) for value in angles]
+    rotated_cordinates = []
+    for point in points:
+        rotated_np_point = rotate_point_euler(point,angles,order)
+        rotated_cordinates.append(
+                [
+                    rotated_np_point[0],
+                    rotated_np_point[1],rotated_np_point[2]
+                ]
+          )
+    return rotated_cordinates

glibvision/numpy_utils.py

@@ -0,0 +1,187 @@
+import numpy as np
+
+#2024-12-03  rotate_point_euler
+#2024-12-04 load_data
+def load_data(filepath):
+    """
+    カンマ区切りのテキストファイルからデータをNumPy配列に読み込みます。
+
+    Args:
+        filepath: データファイルのパス
+
+    Returns:
+        NumPy配列: 読み込まれたデータ。エラーが発生した場合はNone。
+    """
+    try:
+        data = np.loadtxt(filepath, delimiter=",")
+        return data
+    except (FileNotFoundError, ValueError) as e:
+        print(f"Error loading data: {e}")
+        return None
+def rotate_point_euler(point, angles,order="xyz",is_degree=False):
+  
+  """
+  オイラー角を使って3Dポイントを回転させる関数
+
+  Args:
+    point: 回転させる3Dポイント (x, y, z)
+    angles: 各軸周りの回転角度 (rx, ry, rz) [ラジアン]
+
+  Returns:
+    回転後の3Dポイント (x', y', z')
+  """
+  if is_degree:
+       angles = [np.deg2rad(value) for value in angles]
+  rx, ry, rz = angles
+  point = np.array(point)
+
+  # X軸周りの回転
+  Rx = np.array([
+      [1, 0, 0],
+      [0, np.cos(rx), -np.sin(rx)],
+      [0, np.sin(rx), np.cos(rx)]
+  ])
+
+  # Y軸周りの回転
+  Ry = np.array([
+      [np.cos(ry), 0, np.sin(ry)],
+      [0, 1, 0],
+      [-np.sin(ry), 0, np.cos(ry)]
+  ])
+
+  # Z軸周りの回転
+  Rz = np.array([
+      [np.cos(rz), -np.sin(rz), 0],
+      [np.sin(rz), np.cos(rz), 0],
+      [0, 0, 1]
+  ])
+
+  # 回転行列の合成 (Z軸 -> Y軸 -> X軸 の順で回転)
+  order = order.lower()
+  if order == "xyz":
+    R = Rx @ Ry @ Rz
+  elif order == "xzy":
+    R = Rx @ Rz @ Ry
+  elif order == "yxz":
+    R = Ry @ Rx @ Rz
+  elif order == "yzx":
+    R = Ry @ Rz @ Rx
+  elif order == "zxy":
+    R = Rz @ Rx @ Ry
+  else:#zyx
+    R = Rz @ Ry @ Rx
+      
+ 
+
+  # 回転後のポイントを計算
+  rotated_point = R @ point
+
+  return rotated_point
+
+def apply_binary_mask_to_color(base_image,color,mask):
+    """
+    二値マスクを使用して、画像の一部を別の画像にコピーする。
+
+    Args:
+        base_image (np.ndarray): コピー先の画像。
+        paste_image (np.ndarray): コピー元の画像。
+        mask (np.ndarray): 二値マスク画像。
+
+    Returns:
+        np.ndarray: マスクを適用した画像。
+
+    """
+    # TODO check all shape
+    #print_numpy(base_image)
+    #print_numpy(paste_image)
+    #print_numpy(mask)
+    if mask.ndim == 2:
+        condition = mask == 255
+    else:
+        condition = mask[:,:,0] == 255
+    
+    base_image[condition] = color
+    return base_image
+
+def apply_binary_mask_to_image(base_image,paste_image,mask):
+    """
+    二値マスクを使用して、画像の一部を別の画像にコピーする。
+
+    Args:
+        base_image (np.ndarray): コピー先の画像。
+        paste_image (np.ndarray): コピー元の画像。
+        mask (np.ndarray): 二値マスク画像。
+
+    Returns:
+        np.ndarray: マスクを適用した画像。
+
+    """
+    # TODO check all shape
+    #print_numpy(base_image)
+    #print_numpy(paste_image)
+    #print_numpy(mask)
+    if mask.ndim == 2:
+        condition = mask == 255
+    else:
+        condition = mask[:,:,0] == 255
+    
+    base_image[condition] = paste_image[condition]
+    return base_image
+
+def pil_to_numpy(image):
+    return np.array(image, dtype=np.uint8)
+
+def extruce_points(points,index,ratio=1.5):
+    """
+    indexのポイントをratio倍だけ、点群の中心から、外側に膨らます。
+    """
+    center_point = np.mean(points, axis=0)
+    if index < 0 or index > len(points):
+        raise ValueError(f"index must be range(0,{len(points)} but value = {index})")
+    point1 =points[index]
+    print(f"center = {center_point}")
+    vec_to_center = point1 - center_point
+    return vec_to_center*ratio + center_point
+
+
+def bulge_polygon(points, bulge_factor=0.1,isClosed=True):
+    """
+    ポリゴンの辺の中間に点を追加し、外側に膨らませる
+    ndarrayを返すので注意
+    """
+    # 入力 points を NumPy 配列に変換
+    points = np.array(points)
+
+    # ポリゴン全体の重心を求める
+    center_point = np.mean(points, axis=0)
+    #print(f"center = {center_point}")
+    new_points = []
+    num_points = len(points)
+    for i in range(num_points):
+        if i == num_points -1 and not isClosed:
+            break
+        p1 = points[i]
+        #print(f"p{i} = {p1}")
+        # 重心から頂点へのベクトル
+        #vec_to_center = p1 - center_point
+        
+        # 辺のベクトルを求める
+        mid_diff = points[(i + 1) % num_points] - p1
+        mid = p1+(mid_diff/2)
+
+        #print(f"mid = {mid}")
+        out_vec = mid - center_point
+
+        # 重心からのベクトルに bulge_vec を加算
+        new_point = mid + out_vec * bulge_factor
+        
+        new_points.append(p1)
+        new_points.append(new_point.astype(np.int32))
+
+    return np.array(new_points)
+
+
+# image.shape rgb are (1024,1024,3) use 1024,1024 as 2-dimensional
+def create_2d_image(shape):
+    grayscale_image = np.zeros(shape[:2], dtype=np.uint8)
+    return grayscale_image

glibvision/pil_utils.py

@@ -0,0 +1,38 @@
+from PIL import Image,ImageDraw
+from .draw_utils import box_to_xy,to_int_points,box_to_point
+#ver-2024-11-18
+def create_color_image(width, height, color=(255,255,255)):
+    if color == None:
+        color = (0,0,0)
+    
+    if len(color )== 3:
+        mode ="RGB"
+    elif len(color )== 4:
+        mode ="RGBA"
+
+    img = Image.new(mode, (width, height), color)
+    return img
+
+def fill_points(image,points,color=(255,255,255)):
+    return draw_points(image,points,fill=color)
+
+def draw_points(image,points,outline=None,fill=None,width=1,plot_color=None,plot_size=3):
+    draw = ImageDraw.Draw(image)
+    image_draw_points(draw,points,outline,fill,width,plot_color,plot_size)
+    return image
+
+def image_draw_points(draw,points,outline=None,fill=None,width=1,plot_color=None,plot_size=3):
+    int_points = [(int(x), int(y)) for x, y in points]
+    if outline is not  None or fill is not None:
+        draw.polygon(int_points, outline=outline,fill=fill,width=width)
+    if plot_color!=None:
+        print(int_points,plot_size,plot_color)
+        for point in int_points:
+            draw.circle(point,plot_size,fill=plot_color)
+
+def draw_box(image,box,outline=None,fill=None):
+    points = to_int_points(box_to_point(box))
+    return draw_points(image,points,outline,fill)
+
+def from_numpy(numpy_array):
+    return Image.fromarray(numpy_array)

gradio_utils.py

@@ -0,0 +1,68 @@
+
+
+import os
+import time
+import io
+import hashlib
+
+#2024-11-28 support bytes get_buffer_id,save_buffer
+def clear_old_files(dir="files",passed_time=60*60):
+    try:
+        files = os.listdir(dir)
+        current_time = time.time()
+        for file in files:
+            file_path = os.path.join(dir,file)
+            
+            ctime = os.stat(file_path).st_ctime
+            diff = current_time - ctime
+            #print(f"ctime={ctime},current_time={current_time},passed_time={passed_time},diff={diff}")
+            if diff > passed_time:
+                os.remove(file_path)
+    except:
+            print("maybe still gallery using error")
+
+def get_buffer_id(buffer,length=32):
+    if isinstance(buffer,bytes):
+        value = buffer
+    else:
+        value=buffer.getvalue()
+    hash_object = hashlib.sha256(value)
+    hex_dig = hash_object.hexdigest()
+    unique_id = hex_dig[:length]
+    return unique_id
+
+def get_image_id(image):
+    buffer = io.BytesIO()
+    image.save(buffer, format='PNG')
+    return get_buffer_id(buffer)
+
+def save_image(image,extension="jpg",dir_name="files"):
+    id = get_image_id(image)
+    os.makedirs(dir_name,exist_ok=True)
+    file_path = f"{dir_name}/{id}.{extension}"
+    
+    image.save(file_path)
+    return file_path
+
+def save_buffer(buffer,extension="webp",dir_name="files"):
+    id = get_buffer_id(buffer)
+    os.makedirs(dir_name,exist_ok=True)
+    file_path = f"{dir_name}/{id}.{extension}"
+    
+    with open(file_path,"wb") as f:
+        if isinstance(buffer,bytes):
+             f.write(buffer)
+        else:
+         f.write(buffer.getvalue())
+    return file_path
+
+def write_file(file_path,text):
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(text)
+
+def read_file(file_path):
+    """read the text of target file
+    """
+    with open(file_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    return content

mp_estimate.py

@@ -0,0 +1,250 @@
+#2024-12-04 add forehead_chin_points_pair,estimate_rotatios
+#formart is first,second,middle
+#2024-12-05 deg to rad
+#2024-12-06 get_feature_ratios_cordinate
+#2024-12-08 create_detail_labels
+horizontal_points_pair = [
+        [
+            "inner-eye",133,362,6
+        ],
+        [
+            "outer-eye",33,263,168
+        ],
+        [
+            "mouth",61,291,13
+        ],
+         [
+            "eyeblow",105,334,9
+        ],[
+            "nose",98,327,2
+        ],[
+            "contour",143,372,6
+        ],
+        [
+            "chin",32,262,200
+        ], [
+            "cheek",123,352,5
+        ], [
+            "cheek2",192,416,0
+        ], [
+            "nose1",129,358,1
+        ], [
+            "nose2",47,277,195
+        ], [
+            "cheek3",206,426,2
+        ], [
+            "cheek4",101,330,5
+        ], [
+            "cheek5",153,380,6
+        ]
+    ]
+def angle_between_points_and_x_axis(A, B):
+        """
+        2点A, Bを結ぶ線分とx軸の正方向との角度を計算する
+
+        Args:
+            A: A点の座標 (x, y) のタプルまたはNumPy配列
+            B: B点の座標 (x, y) のタプルまたはNumPy配列
+
+        Returns:
+            角度（ラジアン）
+        """
+        x = B[0] - A[0]
+        y = B[1] - A[1]
+        return np.arctan2(y, x)
+
+vertical_points_pair=[
+    ["forehead-chin",8,1,199]
+]
+#formart is first,second,third
+feature_ratios_indices=[
+        ["forehead",67,69,66],
+        ["forehead",10,151,9],
+        ["forehead",297,299,296],
+         #["forehead-chin",8,1,199],
+         #["middle-chin",168,199,2],
+        ["middle",168,195,2],
+        ["right",153,101,206],
+        ["right2",133,47,129],
+        ["left",380,330,426],
+        ["left2",362,277,358],
+        ["right-contour",143,123,192],
+        ["left-contour",372,352,416],
+         ["nose",4,1,2],
+    ]
+
+feature_angles_indices =[
+    ["forehead1",9,6],
+    ["forehead2",69,299],
+    ["eyes1",133,362],
+    ["eyes2",133,33],
+    ["eyes3",362,263],
+    ["nose1",6,2],
+    ["nose1",98,327],
+    ["nose1",2,1],
+    ["nose1",1,6],
+    ["lip",61,291],
+    ["lip",0,17],
+    ["jaw",152,199],
+    ["jaw",194,418],
+    ["cheek",118,214],
+    ["cheek",347,434],
+    ["contour",389,397],
+     ["contour",127,172],
+]
+def get_feature_angles_cordinate(face_landmarks,angles=feature_angles_indices):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return get_feature_angles_cordinate_points(points,angles)
+
+def get_feature_angles_cordinate_points(points,angles=feature_angles_indices):
+    cordinates=[]
+    result_angles = []
+    for indices in angles:
+        points_cordinate = get_points_by_indices(points,indices[1:])#first one is label
+        angle_rad =angle_between_points_and_x_axis(points_cordinate[0][:2],points_cordinate[1][:2])
+        result_angles.append(angle_rad)
+        cordinates.append(points_cordinate)
+    return cordinates,result_angles
+
+def get_feature_ratios_cordinate(face_landmarks,ratios=feature_ratios_indices):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return get_feature_angles_cordinate_points(points,ratios)
+
+def ratios_cordinates(cordinates):
+    
+    distance_a = calculate_distance(cordinates[0],cordinates[1])
+    distance_b = calculate_distance(cordinates[1],cordinates[2])
+    if distance_a == 0 or distance_b == 0:
+        return 0
+    else:
+        return distance_a/distance_b
+    
+def get_feature_ratios_cordinate_points(points,ratios=feature_ratios_indices):
+    cordinates=[]
+    result_ratios = []
+    for indices in ratios:
+        points_cordinate = get_points_by_indices(points,indices[1:])#first one is label
+        result_ratios.append(ratios_cordinates(points_cordinate))
+        cordinates.append(points_cordinate)
+    return cordinates,result_ratios
+
+
+#vertical-format
+forehead_chin_points_pair=[
+    [
+        "forehead-chin",8,1,199
+    ]
+]
+horizontal_contour_points_pair=[
+    [
+        "contour",143,6,372
+    ]
+]
+import math
+def calculate_distance(xy, xy2):
+    return math.sqrt((xy2[0] - xy[0])**2 + (xy2[1] - xy[1])**2)
+
+def create_detail_labels(values,radian=False,pair_data=horizontal_points_pair):
+    assert len(values) == len(pair_data)
+    lines = []
+    for i,value in enumerate(values):
+        if radian:
+            value=math.degrees(value)
+        lines.append(f"{pair_data[i][0]} = {value:.2f}")
+    return "\n".join(lines)
+
+import numpy as np
+from mp_utils import get_normalized_cordinate
+def estimate_horizontal(face_landmarks,pair_data = horizontal_points_pair):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return estimate_horizontal_points(points,pair_data)
+
+def get_points_by_indices(face_landmark_points,indices):
+    points = [face_landmark_points[index] for index in indices]
+    return points
+
+def normalized_to_pixel(cordinates,width,height):
+    pixel_point = [[pt[0]*width,pt[1]*height] for pt in cordinates]
+    return pixel_point
+
+def estimate_horizontal_points(face_landmark_points,pair_data = horizontal_points_pair):
+    z_angles=[]
+    y_ratios = []
+    cordinates = []
+    for compare_point in pair_data:
+        points_cordinate = get_points_by_indices(face_landmark_points,compare_point[1:])#first one is label
+        cordinates.append(points_cordinate)
+        angle_rad =angle_between_points_and_x_axis(points_cordinate[0][:2],points_cordinate[1][:2])
+        #angle_deg = np.degrees(angle_rad)
+        z_angles.append(angle_rad)
+        right_distance = calculate_distance(points_cordinate[0],points_cordinate[2])
+        left_distance = calculate_distance(points_cordinate[1],points_cordinate[2])
+        y_ratios.append(left_distance/(right_distance+left_distance))
+    return z_angles,y_ratios,cordinates,pair_data
+
+def estimate_vertical(face_landmarks,pair_data = vertical_points_pair):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return estimate_vertical_points(points,pair_data)
+
+
+def estimate_rotations_v2(face_landmarker_result):
+    points = get_normalized_landmarks(face_landmarker_result.face_landmarks,True)
+    values1_text=estimate_rotations_point(points)
+    result3,ratios = get_feature_ratios_cordinate_points(points)
+    key_cordinates,angles = get_feature_angles_cordinate_points(points)
+    angles_str=[str(angle) for angle in angles]
+    ratios_str=[str(ratio) for ratio in ratios]            
+    return f"{values1_text},{','.join(angles_str)},{','.join(ratios_str)}"
+    
+from mp_utils import get_normalized_landmarks 
+def estimate_rotations(face_landmarker_result):
+    points = get_normalized_landmarks(face_landmarker_result.face_landmarks,True)
+    return estimate_rotations_point(points)
+def estimate_rotations_point(points):
+    z_angles,y_ratios,h_cordinates,_ =estimate_horizontal_points(points)
+    z_angle = np.mean(z_angles)
+    y_ratio = np.mean(y_ratios)
+    _,x_ratios,h_cordinates,_ =estimate_vertical_points(points)
+    x_ratio = np.mean(x_ratios)
+
+    x_angle,_,_,_ =estimate_vertical_points(points,forehead_chin_points_pair)
+    x_angle=np.mean(x_angle)
+    
+    length_ratio = estimate_ratio(points)
+
+    result = f"{x_ratio:.6f},{y_ratio:.6f},{z_angle:.6f},{x_angle:.6f},{length_ratio:.6f}"
+    return result
+
+def estimate_ratio(face_landmark_points,a_line=forehead_chin_points_pair,b_line=horizontal_contour_points_pair):
+    points_cordinate_a = get_points_by_indices(face_landmark_points,a_line[0][1:])#for campatible
+    points_cordinate_b = get_points_by_indices(face_landmark_points,b_line[0][1:])
+    
+    distance_a = calculate_distance(points_cordinate_a[0],points_cordinate_a[2])
+    distance_b = calculate_distance(points_cordinate_b[0],points_cordinate_b[2])
+    if distance_a == 0 or distance_b == 0:
+        return 0
+    else:
+        return distance_a/distance_b
+
+def estimate_vertical_points(face_landmarks,pair_data = vertical_points_pair):
+    angles = []
+    ratios = []
+    cordinates = []
+    for compare_point in pair_data:
+        points_cordinate = get_points_by_indices(face_landmarks,compare_point[1:])#first one is label
+        cordinates.append(points_cordinate)
+        angle_rad =angle_between_points_and_x_axis(points_cordinate[0][:2],points_cordinate[2][:2])
+        #angle_deg = np.degrees(angle_rad)
+        angles.append(angle_rad)
+        up_distance = calculate_distance(points_cordinate[0],points_cordinate[1])
+        down_distance = calculate_distance(points_cordinate[1],points_cordinate[2])
+        ratios.append(down_distance/(down_distance+up_distance))
+    return angles,ratios,cordinates,pair_data
+def mean_std_label(values,radian=False):
+    mean_value = np.mean(values)
+    std_value = np.std(values)
+    if radian:
+        mean_value = math.degrees(mean_value)
+        std_value = math.degrees(std_value)
+    value_text = f"mean:{mean_value:.3f} std:{std_value:.3f}"
+    return value_text

mp_triangles.py

@@ -0,0 +1,1016 @@
+'''
+I don't know the license,I'll made script if i have spare time. 
+see https://stackoverflow.com/questions/69858216/mediapipe-facemesh-vertices-mapping
+so I wrote a simple program to generate the vertice tuples from it. The result is in the given json string. You're welcome to copy it if it helps.
+'''
+
+#2024-12-01 add hole-triangles
+#02
+
+INNER_MOUTH =[
+    [78,191,95],
+    [191,95,80],
+    [80,88,95],
+    [80,81,88],
+    [88,81,178],
+    [81,178,82],
+    [178,82,87],
+    [82,87,13],
+    [87,14,13],
+    [13,312,317],
+    [14,317,13],
+    [312,402,311],
+    [317,402,312],
+    [311,402,318],
+    [311,318,310],
+    [310,324,318],
+    [310,324,415],
+    [308,415,324]
+]
+
+INNER_LEFT_EYES=[
+    [33,246,7],
+
+    [246,7,163],
+    [246,163,161],
+    [161,163,144],
+    [161,144,160],
+    [160,144,145],
+    [160,145,159],
+    [159,145,153],
+    [159,153,158],
+    [158,153,154],
+    [158,154,157],
+    [157,154,155],
+    [157,155,173],
+
+    [173,155,133]
+]
+INNER_RIGHT_EYES=[
+    [362,398,382],
+
+    [382,398,384],
+    [382,384,381],
+    [381,384,385],
+    [381,385,380],
+    [380,385,386],
+    [380,386,374],
+    [374,386,387],
+    [374,387,373],
+    [373,387,388],
+    [373,388,390],
+    [390,388,466],
+    [390,249,466],
+
+    [466,263,249]
+]
+
+RIGHT_CONTOURS = [
+    [152, 175, 199],  # LINE_RIGHT_CONTOUR_0
+    [148, 171, 208],  # LINE_RIGHT_CONTOUR_1
+    [176, 140, 32],   # LINE_RIGHT_CONTOUR_2
+    [149, 170, 211],  # LINE_RIGHT_CONTOUR_3
+    [150, 169, 210],  # LINE_RIGHT_CONTOUR_4
+    [136, 135, 214],  # LINE_RIGHT_CONTOUR_5
+    [172, 138, 192],  # LINE_RIGHT_CONTOUR_6
+    [58, 215, 213],   # LINE_RIGHT_CONTOUR_7
+    [132, 177, 147],  # LINE_RIGHT_CONTOUR_8
+    [93, 137, 123],   # LINE_RIGHT_CONTOUR_9
+    [234, 227, 116],  # LINE_RIGHT_CONTOUR_10
+    [127, 34, 143],   # LINE_RIGHT_CONTOUR_11
+    [162, 139, 156],  # LINE_RIGHT_CONTOUR_12
+    [21, 71, 70],    # LINE_RIGHT_CONTOUR_13
+    [54, 68, 63],    # LINE_RIGHT_CONTOUR_14
+    [103, 104, 105],  # LINE_RIGHT_CONTOUR_15
+    [67, 69, 66],    # LINE_RIGHT_CONTOUR_16
+    [109, 108, 107],  # LINE_RIGHT_CONTOUR_17
+    [10, 151, 9]     # LINE_RIGHT_CONTOUR_18
+]
+
+LEFT_CONTOURS = [
+    [377, 396, 428],  # LINE_LEFT_CONTOUR_1
+    [400, 369, 262],  # LINE_LEFT_CONTOUR_2
+    [378, 395, 431],  # LINE_LEFT_CONTOUR_3
+    [379, 394, 430],  # LINE_LEFT_CONTOUR_4
+    [365, 364, 434],  # LINE_LEFT_CONTOUR_5
+    [397, 367, 416],  # LINE_LEFT_CONTOUR_6
+    [288, 435, 433],  # LINE_LEFT_CONTOUR_7
+    [361, 401, 376],  # LINE_LEFT_CONTOUR_8
+    [323, 366, 352],  # LINE_LEFT_CONTOUR_9
+    [454, 447, 345],  # LINE_LEFT_CONTOUR_10
+    [356, 264, 372],  # LINE_LEFT_CONTOUR_11
+    [389, 368, 383],  # LINE_LEFT_CONTOUR_12
+    [251, 301, 300],  # LINE_LEFT_CONTOUR_13
+    [284, 298, 293],  # LINE_LEFT_CONTOUR_14
+    [332, 333, 334],  # LINE_LEFT_CONTOUR_15
+    [297, 299, 296],  # LINE_LEFT_CONTOUR_16
+    [338, 337, 336]   # LINE_LEFT_CONTOUR_17
+]
+def get_triangles_copy(base=True,left_eye=False,right_eye=False,mouth=False):
+    triangles = []
+    if base:
+        triangles += mesh_triangle_indices
+    if left_eye:
+        triangles += INNER_LEFT_EYES
+    if right_eye:
+        triangles += INNER_RIGHT_EYES
+    if mouth:
+        triangles += INNER_MOUTH
+    return triangles
+
+
+def contour_to_triangles(is_right=True,down_up=True):
+    triangles = []
+    if is_right:
+        if down_up:
+            contours = RIGHT_CONTOURS
+        else:
+            contours = RIGHT_CONTOURS[::-1]
+    else:
+        if down_up:
+            contours = LEFT_CONTOURS
+        else:
+            contours = LEFT_CONTOURS[::-1]
+
+    sorted_mesh_triangle_indices = []
+    for triangle in mesh_triangle_indices:
+        sorted_mesh_triangle_indices.append(sorted(triangle))
+
+    # no way to know how triangle made even in future.
+    for i in range(len(contours)-1):
+        first_line = contours[i]
+        second_line = contours[i+1]
+        #outer
+        triangles.append([first_line[0],first_line[1],second_line[0]])
+        triangles.append([second_line[0],second_line[1],first_line[1]])
+        triangles.append([first_line[0],first_line[1],second_line[1]])
+        triangles.append([second_line[0],second_line[1],first_line[0]])
+
+        #inner
+        triangles.append([first_line[1],first_line[2],second_line[1]])
+        triangles.append([second_line[1],second_line[2],first_line[2]])
+        triangles.append([first_line[1],first_line[2],second_line[2]])
+        triangles.append([second_line[1],second_line[2],first_line[1]])
+
+    exist_triangles = []
+    for triangle in triangles:
+        sorted_triangle = sorted(triangle)
+        if sorted_triangle in sorted_mesh_triangle_indices:
+            exist_triangles.append(triangle)
+    return exist_triangles
+
+
+mesh_triangle_indices=[
+    [127,  34, 139],
+    [ 11,   0,  37],
+    [232, 231, 120],
+    [ 72,  37,  39],
+    [128, 121,  47],
+    [232, 121, 128],
+    [104,  69,  67],
+    [175, 171, 148],
+    [118,  50, 101],
+    [ 73,  39,  40],
+    [  9, 151, 108],
+    [ 48, 115, 131],
+    [194, 204, 211],
+    [ 74,  40, 185],
+    [ 80,  42, 183],
+    [ 40,  92, 186],
+    [230, 229, 118],
+    [202, 212, 214],
+    [ 83,  18,  17],
+    [ 76,  61, 146],
+    [160,  29,  30],
+    [ 56, 157, 173],
+    [106, 204, 194],
+    [135, 214, 192],
+    [203, 165,  98],
+    [ 21,  71,  68],
+    [ 51,  45,   4],
+    [144,  24,  23],
+    [ 77, 146,  91],
+    [205,  50, 187],
+    [201, 200,  18],
+    [ 91, 106, 182],
+    [ 90,  91, 181],
+    [ 85,  84,  17],
+    [206, 203,  36],
+    [148, 171, 140],
+    [ 92,  40,  39],
+    [193, 189, 244],
+    [159, 158,  28],
+    [247, 246, 161],
+    [236,   3, 196],
+    [ 54,  68, 104],
+    [193, 168,   8],
+    [117, 228,  31],
+    [189, 193,  55],
+    [ 98,  97,  99],
+    [126,  47, 100],
+    [166,  79, 218],
+    [155, 154,  26],
+    [209,  49, 131],
+    [135, 136, 150],
+    [ 47, 126, 217],
+    [223,  52,  53],
+    [ 45,  51, 134],
+    [211, 170, 140],
+    [ 67,  69, 108],
+    [ 43, 106,  91],
+    [230, 119, 120],
+    [226, 130, 247],
+    [ 63,  53,  52],
+    [238,  20, 242],
+    [ 46,  70, 156],
+    [ 78,  62,  96],
+    [ 46,  53,  63],
+    [143,  34, 227],
+    [123, 117, 111],
+    [ 44, 125,  19],
+    [236, 134,  51],
+    [216, 206, 205],
+    [154, 153,  22],
+    [ 39,  37, 167],
+    [200, 201, 208],
+    [ 36, 142, 100],
+    [ 57, 212, 202],
+    [ 20,  60,  99],
+    [ 28, 158, 157],
+    [ 35, 226, 113],
+    [160, 159,  27],
+    [204, 202, 210],
+    [113, 225,  46],
+    [ 43, 202, 204],
+    [ 62,  76,  77],
+    [137, 123, 116],
+    [ 41,  38,  72],
+    [203, 129, 142],
+    [ 64,  98, 240],
+    [ 49, 102,  64],
+    [ 41,  73,  74],
+    [212, 216, 207],
+    [ 42,  74, 184],
+    [169, 170, 211],
+    [170, 149, 176],
+    [105,  66,  69],
+    [122,   6, 168],
+    [123, 147, 187],
+    [ 96,  77,  90],
+    [ 65,  55, 107],
+    [ 89,  90, 180],
+    [101, 100, 120],
+    [ 63, 105, 104],
+    [ 93, 137, 227],
+    [ 15,  86,  85],
+    [129, 102,  49],
+    [ 14,  87,  86],
+    [ 55,   8,   9],
+    [100,  47, 121],
+    [145,  23,  22],
+    [ 88,  89, 179],
+    [  6, 122, 196],
+    [ 88,  95,  96],
+    [138, 172, 136],
+    [215,  58, 172],
+    [115,  48, 219],
+    [ 42,  80,  81],
+    [195,   3,  51],
+    [ 43, 146,  61],
+    [171, 175, 199],
+    [ 81,  82,  38],
+    [ 53,  46, 225],
+    [144, 163, 110],
+    [ 52,  65,  66],
+    [229, 228, 117],
+    [ 34, 127, 234],
+    [107, 108,  69],
+    [109, 108, 151],
+    [ 48,  64, 235],
+    [ 62,  78, 191],
+    [129, 209, 126],
+    [111,  35, 143],
+    [117, 123,  50],
+    [222,  65,  52],
+    [ 19, 125, 141],
+    [221,  55,  65],
+    [  3, 195, 197],
+    [ 25,   7,  33],
+    [220, 237,  44],
+    [ 70,  71, 139],
+    [122, 193, 245],
+    [247, 130,  33],
+    [ 71,  21, 162],
+    [170, 169, 150],
+    [188, 174, 196],
+    [216, 186,  92],
+    [  2,  97, 167],
+    [141, 125, 241],
+    [164, 167,  37],
+    [ 72,  38,  12],
+    [ 38,  82,  13],
+    [ 63,  68,  71],
+    [226,  35, 111],
+    [101,  50, 205],
+    [206,  92, 165],
+    [209, 198, 217],
+    [165, 167,  97],
+    [220, 115, 218],
+    [133, 112, 243],
+    [239, 238, 241],
+    [214, 135, 169],
+    [190, 173, 133],
+    [171, 208,  32],
+    [125,  44, 237],
+    [ 86,  87, 178],
+    [ 85,  86, 179],
+    [ 84,  85, 180],
+    [ 83,  84, 181],
+    [201,  83, 182],
+    [137,  93, 132],
+    [ 76,  62, 183],
+    [ 61,  76, 184],
+    [ 57,  61, 185],
+    [212,  57, 186],
+    [214, 207, 187],
+    [ 34, 143, 156],
+    [ 79, 239, 237],
+    [123, 137, 177],
+    [ 44,   1,   4],
+    [201, 194,  32],
+    [ 64, 102, 129],
+    [213, 215, 138],
+    [ 59, 166, 219],
+    [242,  99,  97],
+    [  2,  94, 141],
+    [ 75,  59, 235],
+    [ 24, 110, 228],
+    [ 25, 130, 226],
+    [ 23,  24, 229],
+    [ 22,  23, 230],
+    [ 26,  22, 231],
+    [112,  26, 232],
+    [189, 190, 243],
+    [221,  56, 190],
+    [ 28,  56, 221],
+    [ 27,  28, 222],
+    [ 29,  27, 223],
+    [ 30,  29, 224],
+    [247,  30, 225],
+    [238,  79,  20],
+    [166,  59,  75],
+    [ 60,  75, 240],
+    [147, 177, 215],
+    [ 20,  79, 166],
+    [187, 147, 213],
+    [112, 233, 244],
+    [233, 128, 245],
+    [128, 114, 188],
+    [114, 217, 174],
+    [131, 115, 220],
+    [217, 198, 236],
+    [198, 131, 134],
+    [177, 132,  58],
+    [143,  35, 124],
+    [110, 163,   7],
+    [228, 110,  25],
+    [356, 389, 368],
+    [ 11, 302, 267],
+    [452, 350, 349],
+    [302, 303, 269],
+    [357, 343, 277],
+    [452, 453, 357],
+    [333, 332, 297],
+    [175, 152, 377],
+    [347, 348, 330],
+    [303, 304, 270],
+    [  9, 336, 337],
+    [278, 279, 360],
+    [418, 262, 431],
+    [304, 408, 409],
+    [310, 415, 407],
+    [270, 409, 410],
+    [450, 348, 347],
+    [422, 430, 434],
+    [313, 314,  17],
+    [306, 307, 375],
+    [387, 388, 260],
+    [286, 414, 398],
+    [335, 406, 418],
+    [364, 367, 416],
+    [423, 358, 327],
+    [251, 284, 298],
+    [281,   5,   4],
+    [373, 374, 253],
+    [307, 320, 321],
+    [425, 427, 411],
+    [421, 313,  18],
+    [321, 405, 406],
+    [320, 404, 405],
+    [315,  16,  17],
+    [426, 425, 266],
+    [377, 400, 369],
+    [322, 391, 269],
+    [417, 465, 464],
+    [386, 257, 258],
+    [466, 260, 388],
+    [456, 399, 419],
+    [284, 332, 333],
+    [417, 285,   8],
+    [346, 340, 261],
+    [413, 441, 285],
+    [327, 460, 328],
+    [355, 371, 329],
+    [392, 439, 438],
+    [382, 341, 256],
+    [429, 420, 360],
+    [364, 394, 379],
+    [277, 343, 437],
+    [443, 444, 283],
+    [275, 440, 363],
+    [431, 262, 369],
+    [297, 338, 337],
+    [273, 375, 321],
+    [450, 451, 349],
+    [446, 342, 467],
+    [293, 334, 282],
+    [458, 461, 462],
+    [276, 353, 383],
+    [308, 324, 325],
+    [276, 300, 293],
+    [372, 345, 447],
+    [352, 345, 340],
+    [274,   1,  19],
+    [456, 248, 281],
+    [436, 427, 425],
+    [381, 256, 252],
+    [269, 391, 393],
+    [200, 199, 428],
+    [266, 330, 329],
+    [287, 273, 422],
+    [250, 462, 328],
+    [258, 286, 384],
+    [265, 353, 342],
+    [387, 259, 257],
+    [424, 431, 430],
+    [342, 353, 276],
+    [273, 335, 424],
+    [292, 325, 307],
+    [366, 447, 345],
+    [271, 303, 302],
+    [423, 266, 371],
+    [294, 455, 460],
+    [279, 278, 294],
+    [271, 272, 304],
+    [432, 434, 427],
+    [272, 407, 408],
+    [394, 430, 431],
+    [395, 369, 400],
+    [334, 333, 299],
+    [351, 417, 168],
+    [352, 280, 411],
+    [325, 319, 320],
+    [295, 296, 336],
+    [319, 403, 404],
+    [330, 348, 349],
+    [293, 298, 333],
+    [323, 454, 447],
+    [ 15,  16, 315],
+    [358, 429, 279],
+    [ 14,  15, 316],
+    [285, 336,   9],
+    [329, 349, 350],
+    [374, 380, 252],
+    [318, 402, 403],
+    [  6, 197, 419],
+    [318, 319, 325],
+    [367, 364, 365],
+    [435, 367, 397],
+    [344, 438, 439],
+    [272, 271, 311],
+    [195,   5, 281],
+    [273, 287, 291],
+    [396, 428, 199],
+    [311, 271, 268],
+    [283, 444, 445],
+    [373, 254, 339],
+    [282, 334, 296],
+    [449, 347, 346],
+    [264, 447, 454],
+    [336, 296, 299],
+    [338,  10, 151],
+    [278, 439, 455],
+    [292, 407, 415],
+    [358, 371, 355],
+    [340, 345, 372],
+    [346, 347, 280],
+    [442, 443, 282],
+    [ 19,  94, 370],
+    [441, 442, 295],
+    [248, 419, 197],
+    [263, 255, 359],
+    [440, 275, 274],
+    [300, 383, 368],
+    [351, 412, 465],
+    [263, 467, 466],
+    [301, 368, 389],
+    [395, 378, 379],
+    [412, 351, 419],
+    [436, 426, 322],
+    [  2, 164, 393],
+    [370, 462, 461],
+    [164,   0, 267],
+    [302,  11,  12],
+    [268,  12,  13],
+    [293, 300, 301],
+    [446, 261, 340],
+    [330, 266, 425],
+    [426, 423, 391],
+    [429, 355, 437],
+    [391, 327, 326],
+    [440, 457, 438],
+    [341, 382, 362],
+    [459, 457, 461],
+    [434, 430, 394],
+    [414, 463, 362],
+    [396, 369, 262],
+    [354, 461, 457],
+    [316, 403, 402],
+    [315, 404, 403],
+    [314, 405, 404],
+    [313, 406, 405],
+    [421, 418, 406],
+    [366, 401, 361],
+    [306, 408, 407],
+    [291, 409, 408],
+    [287, 410, 409],
+    [432, 436, 410],
+    [434, 416, 411],
+    [264, 368, 383],
+    [309, 438, 457],
+    [352, 376, 401],
+    [274, 275,   4],
+    [421, 428, 262],
+    [294, 327, 358],
+    [433, 416, 367],
+    [289, 455, 439],
+    [462, 370, 326],
+    [  2, 326, 370],
+    [305, 460, 455],
+    [254, 449, 448],
+    [255, 261, 446],
+    [253, 450, 449],
+    [252, 451, 450],
+    [256, 452, 451],
+    [341, 453, 452],
+    [413, 464, 463],
+    [441, 413, 414],
+    [258, 442, 441],
+    [257, 443, 442],
+    [259, 444, 443],
+    [260, 445, 444],
+    [467, 342, 445],
+    [459, 458, 250],
+    [289, 392, 290],
+    [290, 328, 460],
+    [376, 433, 435],
+    [250, 290, 392],
+    [411, 416, 433],
+    [341, 463, 464],
+    [453, 464, 465],
+    [357, 465, 412],
+    [343, 412, 399],
+    [360, 363, 440],
+    [437, 399, 456],
+    [420, 456, 363],
+    [401, 435, 288],
+    [372, 383, 353],
+    [339, 255, 249],
+    [448, 261, 255],
+    [133, 243, 190],
+    [133, 155, 112],
+    [ 33, 246, 247],
+    [ 33, 130,  25],
+    [398, 384, 286],
+    [362, 398, 414],
+    [362, 463, 341],
+    [263, 359, 467],
+    [263, 249, 255],
+    [466, 467, 260],
+    [ 75,  60, 166],
+    [238, 239,  79],
+    [162, 127, 139],
+    [ 72,  11,  37],
+    [121, 232, 120],
+    [ 73,  72,  39],
+    [114, 128,  47],
+    [233, 232, 128],
+    [103, 104,  67],
+    [152, 175, 148],
+    [119, 118, 101],
+    [ 74,  73,  40],
+    [107,   9, 108],
+    [ 49,  48, 131],
+    [ 32, 194, 211],
+    [184,  74, 185],
+    [191,  80, 183],
+    [185,  40, 186],
+    [119, 230, 118],
+    [210, 202, 214],
+    [ 84,  83,  17],
+    [ 77,  76, 146],
+    [161, 160,  30],
+    [190,  56, 173],
+    [182, 106, 194],
+    [138, 135, 192],
+    [129, 203,  98],
+    [ 54,  21,  68],
+    [  5,  51,   4],
+    [145, 144,  23],
+    [ 90,  77,  91],
+    [207, 205, 187],
+    [ 83, 201,  18],
+    [181,  91, 182],
+    [180,  90, 181],
+    [ 16,  85,  17],
+    [205, 206,  36],
+    [176, 148, 140],
+    [165,  92,  39],
+    [245, 193, 244],
+    [ 27, 159,  28],
+    [ 30, 247, 161],
+    [174, 236, 196],
+    [103,  54, 104],
+    [ 55, 193,   8],
+    [111, 117,  31],
+    [221, 189,  55],
+    [240,  98,  99],
+    [142, 126, 100],
+    [219, 166, 218],
+    [112, 155,  26],
+    [198, 209, 131],
+    [169, 135, 150],
+    [114,  47, 217],
+    [224, 223,  53],
+    [220,  45, 134],
+    [ 32, 211, 140],
+    [109,  67, 108],
+    [146,  43,  91],
+    [231, 230, 120],
+    [113, 226, 247],
+    [105,  63,  52],
+    [241, 238, 242],
+    [124,  46, 156],
+    [ 95,  78,  96],
+    [ 70,  46,  63],
+    [116, 143, 227],
+    [116, 123, 111],
+    [  1,  44,  19],
+    [  3, 236,  51],
+    [207, 216, 205],
+    [ 26, 154,  22],
+    [165,  39, 167],
+    [199, 200, 208],
+    [101,  36, 100],
+    [ 43,  57, 202],
+    [242,  20,  99],
+    [ 56,  28, 157],
+    [124,  35, 113],
+    [ 29, 160,  27],
+    [211, 204, 210],
+    [124, 113,  46],
+    [106,  43, 204],
+    [ 96,  62,  77],
+    [227, 137, 116],
+    [ 73,  41,  72],
+    [ 36, 203, 142],
+    [235,  64, 240],
+    [ 48,  49,  64],
+    [ 42,  41,  74],
+    [214, 212, 207],
+    [183,  42, 184],
+    [210, 169, 211],
+    [140, 170, 176],
+    [104, 105,  69],
+    [193, 122, 168],
+    [ 50, 123, 187],
+    [ 89,  96,  90],
+    [ 66,  65, 107],
+    [179,  89, 180],
+    [119, 101, 120],
+    [ 68,  63, 104],
+    [234,  93, 227],
+    [ 16,  15,  85],
+    [209, 129,  49],
+    [ 15,  14,  86],
+    [107,  55,   9],
+    [120, 100, 121],
+    [153, 145,  22],
+    [178,  88, 179],
+    [197,   6, 196],
+    [ 89,  88,  96],
+    [135, 138, 136],
+    [138, 215, 172],
+    [218, 115, 219],
+    [ 41,  42,  81],
+    [  5, 195,  51],
+    [ 57,  43,  61],
+    [208, 171, 199],
+    [ 41,  81,  38],
+    [224,  53, 225],
+    [ 24, 144, 110],
+    [105,  52,  66],
+    [118, 229, 117],
+    [227,  34, 234],
+    [ 66, 107,  69],
+    [ 10, 109, 151],
+    [219,  48, 235],
+    [183,  62, 191],
+    [142, 129, 126],
+    [116, 111, 143],
+    [118, 117,  50],
+    [223, 222,  52],
+    [ 94,  19, 141],
+    [222, 221,  65],
+    [196,   3, 197],
+    [ 45, 220,  44],
+    [156,  70, 139],
+    [188, 122, 245],
+    [139,  71, 162],
+    [149, 170, 150],
+    [122, 188, 196],
+    [206, 216,  92],
+    [164,   2, 167],
+    [242, 141, 241],
+    [  0, 164,  37],
+    [ 11,  72,  12],
+    [ 12,  38,  13],
+    [ 70,  63,  71],
+    [ 31, 226, 111],
+    [ 36, 101, 205],
+    [203, 206, 165],
+    [126, 209, 217],
+    [ 98, 165,  97],
+    [237, 220, 218],
+    [237, 239, 241],
+    [210, 214, 169],
+    [140, 171,  32],
+    [241, 125, 237],
+    [179,  86, 178],
+    [180,  85, 179],
+    [181,  84, 180],
+    [182,  83, 181],
+    [194, 201, 182],
+    [177, 137, 132],
+    [184,  76, 183],
+    [185,  61, 184],
+    [186,  57, 185],
+    [216, 212, 186],
+    [192, 214, 187],
+    [139,  34, 156],
+    [218,  79, 237],
+    [147, 123, 177],
+    [ 45,  44,   4],
+    [208, 201,  32],
+    [ 98,  64, 129],
+    [192, 213, 138],
+    [235,  59, 219],
+    [141, 242,  97],
+    [ 97,   2, 141],
+    [240,  75, 235],
+    [229,  24, 228],
+    [ 31,  25, 226],
+    [230,  23, 229],
+    [231,  22, 230],
+    [232,  26, 231],
+    [233, 112, 232],
+    [244, 189, 243],
+    [189, 221, 190],
+    [222,  28, 221],
+    [223,  27, 222],
+    [224,  29, 223],
+    [225,  30, 224],
+    [113, 247, 225],
+    [ 99,  60, 240],
+    [213, 147, 215],
+    [ 60,  20, 166],
+    [192, 187, 213],
+    [243, 112, 244],
+    [244, 233, 245],
+    [245, 128, 188],
+    [188, 114, 174],
+    [134, 131, 220],
+    [174, 217, 236],
+    [236, 198, 134],
+    [215, 177,  58],
+    [156, 143, 124],
+    [ 25, 110,   7],
+    [ 31, 228,  25],
+    [264, 356, 368],
+    [  0,  11, 267],
+    [451, 452, 349],
+    [267, 302, 269],
+    [350, 357, 277],
+    [350, 452, 357],
+    [299, 333, 297],
+    [396, 175, 377],
+    [280, 347, 330],
+    [269, 303, 270],
+    [151,   9, 337],
+    [344, 278, 360],
+    [424, 418, 431],
+    [270, 304, 409],
+    [272, 310, 407],
+    [322, 270, 410],
+    [449, 450, 347],
+    [432, 422, 434],
+    [ 18, 313,  17],
+    [291, 306, 375],
+    [259, 387, 260],
+    [424, 335, 418],
+    [434, 364, 416],
+    [391, 423, 327],
+    [301, 251, 298],
+    [275, 281,   4],
+    [254, 373, 253],
+    [375, 307, 321],
+    [280, 425, 411],
+    [200, 421,  18],
+    [335, 321, 406],
+    [321, 320, 405],
+    [314, 315,  17],
+    [423, 426, 266],
+    [396, 377, 369],
+    [270, 322, 269],
+    [413, 417, 464],
+    [385, 386, 258],
+    [248, 456, 419],
+    [298, 284, 333],
+    [168, 417,   8],
+    [448, 346, 261],
+    [417, 413, 285],
+    [326, 327, 328],
+    [277, 355, 329],
+    [309, 392, 438],
+    [381, 382, 256],
+    [279, 429, 360],
+    [365, 364, 379],
+    [355, 277, 437],
+    [282, 443, 283],
+    [281, 275, 363],
+    [395, 431, 369],
+    [299, 297, 337],
+    [335, 273, 321],
+    [348, 450, 349],
+    [359, 446, 467],
+    [283, 293, 282],
+    [250, 458, 462],
+    [300, 276, 383],
+    [292, 308, 325],
+    [283, 276, 293],
+    [264, 372, 447],
+    [346, 352, 340],
+    [354, 274,  19],
+    [363, 456, 281],
+    [426, 436, 425],
+    [380, 381, 252],
+    [267, 269, 393],
+    [421, 200, 428],
+    [371, 266, 329],
+    [432, 287, 422],
+    [290, 250, 328],
+    [385, 258, 384],
+    [446, 265, 342],
+    [386, 387, 257],
+    [422, 424, 430],
+    [445, 342, 276],
+    [422, 273, 424],
+    [306, 292, 307],
+    [352, 366, 345],
+    [268, 271, 302],
+    [358, 423, 371],
+    [327, 294, 460],
+    [331, 279, 294],
+    [303, 271, 304],
+    [436, 432, 427],
+    [304, 272, 408],
+    [395, 394, 431],
+    [378, 395, 400],
+    [296, 334, 299],
+    [  6, 351, 168],
+    [376, 352, 411],
+    [307, 325, 320],
+    [285, 295, 336],
+    [320, 319, 404],
+    [329, 330, 349],
+    [334, 293, 333],
+    [366, 323, 447],
+    [316,  15, 315],
+    [331, 358, 279],
+    [317,  14, 316],
+    [  8, 285,   9],
+    [277, 329, 350],
+    [253, 374, 252],
+    [319, 318, 403],
+    [351,   6, 419],
+    [324, 318, 325],
+    [397, 367, 365],
+    [288, 435, 397],
+    [278, 344, 439],
+    [310, 272, 311],
+    [248, 195, 281],
+    [375, 273, 291],
+    [175, 396, 199],
+    [312, 311, 268],
+    [276, 283, 445],
+    [390, 373, 339],
+    [295, 282, 296],
+    [448, 449, 346],
+    [356, 264, 454],
+    [337, 336, 299],
+    [337, 338, 151],
+    [294, 278, 455],
+    [308, 292, 415],
+    [429, 358, 355],
+    [265, 340, 372],
+    [352, 346, 280],
+    [295, 442, 282],
+    [354,  19, 370],
+    [285, 441, 295],
+    [195, 248, 197],
+    [457, 440, 274],
+    [301, 300, 368],
+    [417, 351, 465],
+    [251, 301, 389],
+    [394, 395, 379],
+    [399, 412, 419],
+    [410, 436, 322],
+    [326,   2, 393],
+    [354, 370, 461],
+    [393, 164, 267],
+    [268, 302,  12],
+    [312, 268,  13],
+    [298, 293, 301],
+    [265, 446, 340],
+    [280, 330, 425],
+    [322, 426, 391],
+    [420, 429, 437],
+    [393, 391, 326],
+    [344, 440, 438],
+    [458, 459, 461],
+    [364, 434, 394],
+    [428, 396, 262],
+    [274, 354, 457],
+    [317, 316, 402],
+    [316, 315, 403],
+    [315, 314, 404],
+    [314, 313, 405],
+    [313, 421, 406],
+    [323, 366, 361],
+    [292, 306, 407],
+    [306, 291, 408],
+    [291, 287, 409],
+    [287, 432, 410],
+    [427, 434, 411],
+    [372, 264, 383],
+    [459, 309, 457],
+    [366, 352, 401],
+    [  1, 274,   4],
+    [418, 421, 262],
+    [331, 294, 358],
+    [435, 433, 367],
+    [392, 289, 439],
+    [328, 462, 326],
+    [ 94,   2, 370],
+    [289, 305, 455],
+    [339, 254, 448],
+    [359, 255, 446],
+    [254, 253, 449],
+    [253, 252, 450],
+    [252, 256, 451],
+    [256, 341, 452],
+    [414, 413, 463],
+    [286, 441, 414],
+    [286, 258, 441],
+    [258, 257, 442],
+    [257, 259, 443],
+    [259, 260, 444],
+    [260, 467, 445],
+    [309, 459, 250],
+    [305, 289, 290],
+    [305, 290, 460],
+    [401, 376, 435],
+    [309, 250, 392],
+    [376, 411, 433],
+    [453, 341, 464],
+    [357, 453, 465],
+    [343, 357, 412],
+    [437, 343, 399],
+    [344, 360, 440],
+    [420, 437, 456],
+    [360, 420, 363],
+    [361, 401, 288],
+    [265, 372, 353],
+    [390, 339, 249],
+    [339, 448, 255]
+]

mp_utils.py

@@ -0,0 +1,164 @@
+import math
+
+import mediapipe as mp
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe import solutions
+import numpy as np
+
+# 2024-11-27 -extract_landmark :add args 
+# add get_pixel_xyz
+# 2024-11-28 add get_normalized_xyz
+# 2024-11-30 add get_normalized_landmarks,sort_triangles_by_depth
+# 2024-12-04 add get_normalized_landmarks args
+def calculate_distance(p1, p2):
+  """
+
+  """
+  return math.sqrt((p2[0] - p1[0])**2 + (p2[1] - p1[1])**2)
+
+
+
+def to_int_points(points):
+    ints=[]
+    for pt in points:
+        #print(pt)
+        value = [int(pt[0]),int(pt[1])]
+        #print(value)
+        ints.append(value)
+    return ints
+
+debug = False
+def divide_line_to_points(points,divided): # return divided + 1
+    total_length = 0
+    line_length_list = []
+    for i in range(len(points)-1):
+        pt_length = calculate_distance(points[i],points[i+1])
+        total_length += pt_length
+        line_length_list.append(pt_length)
+     
+    splited_length = total_length/divided
+
+    def get_new_point(index,lerp):
+        pt1 = points[index]
+        pt2 = points[index+1]
+        diff = [pt2[0] - pt1[0], pt2[1]-pt1[1]]
+        new_point = [pt1[0]+diff[0]*lerp,pt1[1]+diff[1]*lerp]
+        if debug:
+          print(f"pt1 ={pt1}  pt2 ={pt2} diff={diff} new_point={new_point}")
+
+        return new_point
+
+    if debug:
+      print(f"{total_length} splitted = {splited_length} line-length-list = {len(line_length_list)}")
+    splited_points=[points[0]]
+    for i in range(1,divided):
+        need_length = splited_length*i
+        if debug:
+          print(f"{i} need length = {need_length}")
+        current_length = 0
+        for j in range(len(line_length_list)):
+            line_length = line_length_list[j]
+            current_length+=line_length
+            if current_length>need_length:
+                if debug:
+                  print(f"over need length index = {j} current={current_length}")
+                diff = current_length - need_length
+            
+                lerp_point = 1.0 - (diff/line_length)
+                if debug:
+                  print(f"over = {diff} lerp ={lerp_point}")
+                new_point = get_new_point(j,lerp_point)
+                
+                splited_points.append(new_point)
+                break
+        
+    splited_points.append(points[-1]) # last one
+    splited_points=to_int_points(splited_points)    
+         
+    if debug:
+      print(f"sp={len(splited_points)}")
+    return splited_points
+
+
+
+def expand_bbox(bbox,left=5,top=5,right=5,bottom=5):
+   left_pixel = bbox[2]*(float(left)/100)
+   top_pixel = bbox[3]*(float(top)/100)
+   right_pixel = bbox[2]*(float(right)/100)
+   bottom_pixel = bbox[3]*(float(bottom)/100)
+   new_box = list(bbox)
+   new_box[0] -=left_pixel
+   new_box[1] -=top_pixel
+   new_box[2] +=left_pixel+right_pixel
+   new_box[3] +=top_pixel+bottom_pixel
+   return new_box
+
+#normalized value index see mp_constants
+def get_normalized_cordinate(face_landmarks_list,index):
+    x=face_landmarks_list[0][index].x
+    y=face_landmarks_list[0][index].y
+    return x,y
+
+def get_normalized_xyz(face_landmarks_list,index):
+    x=face_landmarks_list[0][index].x
+    y=face_landmarks_list[0][index].y
+    z=face_landmarks_list[0][index].z
+    return x,y,z
+
+def get_normalized_landmarks(face_landmarks_list,recentering=False,recentering_index=4,z_multiply=0.8):
+   cordinates = [get_normalized_xyz(face_landmarks_list,i) for i in range(0,468)]
+   if recentering:
+      normalized_center_point = cordinates[recentering_index]
+      offset_x = normalized_center_point[0]
+      offset_y = normalized_center_point[1]
+
+        #need aspect?
+      cordinates = [[point[0]-offset_x,point[1]-offset_y,point[2]*z_multiply] for point in cordinates]
+
+   return cordinates
+
+
+
+def sort_triangles_by_depth(landmark_points,mesh_triangle_indices):
+   assert len(landmark_points) == 468
+   mesh_triangle_indices.sort(key=lambda triangle: sum(landmark_points[index][2] for index in triangle) / len(triangle)
+                       ,reverse=True)
+# z is normalized
+def get_pixel_xyz(face_landmarks_list,landmark,width,height):
+    point = get_normalized_cordinate(face_landmarks_list,landmark)
+    z = y=face_landmarks_list[0][landmark].z
+    return int(point[0]*width),int(point[1]*height),z
+
+def get_pixel_cordinate(face_landmarks_list,landmark,width,height):
+    point = get_normalized_cordinate(face_landmarks_list,landmark)
+    return int(point[0]*width),int(point[1]*height)
+
+def get_pixel_cordinate_list(face_landmarks_list,indices,width,height):
+   cordinates = []
+   for index in indices:
+      cordinates.append(get_pixel_cordinate(face_landmarks_list,index,width,height))
+   return cordinates
+
+def extract_landmark(image_data,model_path="face_landmarker.task",min_face_detection_confidence=0, min_face_presence_confidence=0,output_facial_transformation_matrixes=False):
+  BaseOptions = mp.tasks.BaseOptions
+  FaceLandmarker = mp.tasks.vision.FaceLandmarker
+  FaceLandmarkerOptions = mp.tasks.vision.FaceLandmarkerOptions
+  VisionRunningMode = mp.tasks.vision.RunningMode
+
+  options = FaceLandmarkerOptions(
+      base_options=BaseOptions(model_asset_path=model_path),
+      running_mode=VisionRunningMode.IMAGE
+      ,min_face_detection_confidence=min_face_detection_confidence, min_face_presence_confidence=min_face_presence_confidence,
+      output_facial_transformation_matrixes=output_facial_transformation_matrixes
+      )
+  
+  with FaceLandmarker.create_from_options(options) as landmarker:
+    if isinstance(image_data,str):
+        mp_image = mp.Image.create_from_file(image_data)
+    else:
+        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np.asarray(image_data))
+    face_landmarker_result = landmarker.detect(mp_image)
+    return mp_image,face_landmarker_result
+