face_mesh_spinning.py

import subprocess
from PIL import Image,ImageOps,ImageDraw,ImageFilter
import json
import os
import time
import io
from mp_utils import get_pixel_cordinate_list,extract_landmark,get_pixel_cordinate,get_normalized_xyz
from glibvision.draw_utils import points_to_box,box_to_xy,plus_point,calculate_distance

import numpy as np
from glibvision.pil_utils import fill_points,create_color_image,draw_box

import glibvision.pil_utils

from gradio_utils import save_image,save_buffer,clear_old_files ,read_file


import math
import mp_triangles


from glibvision.cv2_utils import create_color_image as cv2_create_color_image,copy_image,pil_to_bgr_image
import cv2
#TODO move to CV2

# i'm not sure this is fast
def apply_affine_transformation_to_triangle_add(src_tri, dst_tri, src_img, dst_img):
    src_tri_np = np.float32(src_tri)
    dst_tri_np = np.float32(dst_tri)

    h_dst, w_dst = dst_img.shape[:2]

    M = cv2.getAffineTransform(src_tri_np, dst_tri_np)

    dst_mask = np.zeros((h_dst, w_dst), dtype=np.uint8)
    cv2.fillPoly(dst_mask, [np.int32(dst_tri)], 255)

    transformed = cv2.warpAffine(src_img, M, (w_dst, h_dst))

    transformed = transformed * (dst_mask[:, :, np.newaxis] / 255).astype(np.uint8)  
    dst_background = dst_img * (1 - (dst_mask[:, :, np.newaxis] / 255)).astype(np.uint8)
    dst_img = transformed + dst_background

    return dst_img

def apply_affine_transformation_to_triangle_add(src_tri, dst_tri, src_img, dst_img):
    src_tri_np = np.float32(src_tri)
    dst_tri_np = np.float32(dst_tri)

    assert src_tri_np.shape == (3, 2), f"src_tri_np の形状が不正 {src_tri_np.shape}"
    assert dst_tri_np.shape == (3, 2), f"dst_tri_np の形状が不正 {dst_tri_np.shape}"


    # 透視変換行列の計算
    M = cv2.getAffineTransform(src_tri_np, dst_tri_np)

    # 画像のサイズ
    h_src, w_src = src_img.shape[:2]
    h_dst, w_dst = dst_img.shape[:2]

    # 元画像から三角形領域を切り抜くマスク生成
    #src_mask = np.zeros((h_src, w_src), dtype=np.uint8)
    #cv2.fillPoly(src_mask, [np.int32(src_tri)], 255)

    # Not 元画像の三角形領域のみをマスクで抽出
    src_triangle = src_img #cv2.bitwise_and(src_img, src_img, mask=src_mask)

    # 変換行列を使って元画像の三角形領域を目標画像のサイズへ変換
   
    transformed = cv2.warpAffine(src_triangle, M, (w_dst, h_dst))
    #print(f"dst_img={dst_img.shape}")
    #print(f"transformed={transformed.shape}")
    # 変換後のマスクの生成
    dst_mask = np.zeros((h_dst, w_dst), dtype=np.uint8)
    cv2.fillPoly(dst_mask, [np.int32(dst_tri)], 255)
    transformed = cv2.bitwise_and(transformed, transformed, mask=dst_mask)

    # 目標画像のマスク領域をクリアするためにデストのインバートマスクを作成
    dst_mask_inv = cv2.bitwise_not(dst_mask)

    # 目標画像のマスク部分をクリア
    dst_background = cv2.bitwise_and(dst_img, dst_img, mask=dst_mask_inv)

    # 変換された元画像の三角形部分と目標画像の背景部分を合成
    dst_img = cv2.add(dst_background, transformed)

    return dst_img

# TODO move PIL
def process_create_webp(images,duration=100, loop=0,quality=85):
    frames = []
    for image_file in images:
        frames.append(image_file)
    
    output_buffer = io.BytesIO()
    frames[0].save(output_buffer, 
                   save_all=True, 
                   append_images=frames[1:], 
                   duration=duration, 
                   loop=loop, 
                   format='WebP',
                   quality=quality
                   )

    return output_buffer.getvalue()
# TODO move numpy
def rotate_point_euler(point, angles,order="xyz"):
  """
  オイラー角を使って3Dポイントを回転させる関数

  Args:
    point: 回転させる3Dポイント (x, y, z)
    angles: 各軸周りの回転角度 (rx, ry, rz) [ラジアン]

  Returns:
    回転後の3Dポイント (x', y', z')
  """

  rx, ry, rz = angles
  point = np.array(point)

  # X軸周りの回転
  Rx = np.array([
      [1, 0, 0],
      [0, np.cos(rx), -np.sin(rx)],
      [0, np.sin(rx), np.cos(rx)]
  ])

  # Y軸周りの回転
  Ry = np.array([
      [np.cos(ry), 0, np.sin(ry)],
      [0, 1, 0],
      [-np.sin(ry), 0, np.cos(ry)]
  ])

  # Z軸周りの回転
  Rz = np.array([
      [np.cos(rz), -np.sin(rz), 0],
      [np.sin(rz), np.cos(rz), 0],
      [0, 0, 1]
  ])

  # 回転行列の合成 (Z軸 -> Y軸 -> X軸 の順で回転)
  order = order.lower()
  if order == "xyz":
    R = Rx @ Ry @ Rz
  elif order == "xzy":
    R = Rx @ Rz @ Ry
  elif order == "yxz":
    R = Ry @ Rx @ Rz
  elif order == "yzx":
    R = Ry @ Rz @ Rx
  elif order == "zxy":
    R = Rz @ Rx @ Ry
  else:
    R = Rz @ Ry @ Rx
      
 

  # 回転後のポイントを計算
  rotated_point = R @ point

  return rotated_point


def process_face_mesh_spinning(image,draw_type,center_scaleup,animation_direction,z_multiply=0.8,inner_eyes=False,inner_mouth=False):
    animation = True
    offset_x = 0
    offset_y = 0
    # use when center_scaleup is True,scale is 0.45(half-size:0.5-margin/ nosetip-to-top or nosetip-to-bottom
    scale_up = 1.0

    face_landmarker_result = None


    if image == None:#app stop support none image,if mode still image,make problem
        #  Box for no Image Case
        image_width = 512
        image_height = 512
        #image = create_color_image(image_width,image_height,(0,0,0))
        points = [(-0.25,-0.25,0),(0.25,-0.25,0),
            (0.25,0.25,0),(-0.25,0.25,0)
            ]
        normalized_center_point = [0.5,0.5]
    else:
        image_width = image.width
        image_height = image.height
        

                

        mp_image,face_landmarker_result = extract_landmark(image,"face_landmarker.task",0,0,True)

        def rotate_image():
            return None,face_landmarker_result,None
                    
        #return rotate_image()
        # cordinate eyes
        # cordinate all
        landmark_points = [get_normalized_xyz(face_landmarker_result.face_landmarks,i) for i in range(0,468)]
        # do centering
        normalized_center_point = landmark_points[4]
        normalized_top_point = landmark_points[10]
        normalized_bottom_point = landmark_points[152]
        

        offset_x = normalized_center_point[0]
        offset_y = normalized_center_point[1]
        offset_z = normalized_center_point[2]

        #need aspect?
        points = [[point[0]-offset_x,point[1]-offset_y,point[2]*z_multiply] for point in landmark_points]


    # split xy-cordinate and z-depth
    def split_points_xy_z(points,width,height,center_x,center_y):
        xys = []
        zs = []
        for point in points:
            xys.append(
                   [
                        point[0]*width*scale_up+center_x,
                        point[1]*height*scale_up+center_y
                   ]
              )
            zs.append(point[2])
        return xys,zs
    
    def draw_grid_in_center(draw,cx,cy,grid_size,grid_color,width=1,draw_horizontal=True,draw_vertical=True):
        w = image.width
        h = image.height
        x_minus_divide = cx//grid_size
        x_plus_divide = (w -cx)//grid_size
        y_minus_divide = cy//grid_size
        y_plus_divide = (h -cx)//grid_size
        for i in range(-x_minus_divide,x_plus_divide+1):
            draw.line([(cx+i*grid_size,0),(cx+i*grid_size,h)],fill=grid_color,width=width)
        for i in range(-y_minus_divide,y_plus_divide+1):
            draw.line([(0,cy+i*grid_size),(w,cy+i*grid_size)],fill=grid_color,width=width)

    def draw_grid(image,cx=512,cy=512,first_color=(255,0,0)):
        w = image.width
        h = image.height
        second_grid_size=100
        second_color = (128,128,128)
        draw = ImageDraw.Draw(image)
        draw_grid_in_center(draw,cx,cy,20,(100,100,100))
        draw_grid_in_center(draw,cx,cy,100,(192,192,192))
        
        

        draw.line([(cx,0),(cx,image.height)],fill=first_color)
        draw.line([(0,cy),(image.width,cy)],fill=first_color)

    def create_triangle_image(points,width,height,center_x,center_y,line_color=(255,255,255),fill_color=None):
        
        cordinates,angled_depth = split_points_xy_z(points,width,height,center_x,center_y)
       
        img = create_color_image(width,height,(0,0,0))
        draw = ImageDraw.Draw(img)
        triangles = mp_triangles.get_triangles_copy(True,inner_eyes,inner_eyes,inner_mouth)

        triangles.sort(key=lambda triangle: sum(angled_depth[index] for index in triangle) / len(triangle)
                       ,reverse=True)
        for triangle in triangles:
            triangle_cordinates = [cordinates[index] for index in triangle]
            glibvision.pil_utils.image_draw_points(draw,triangle_cordinates,line_color,fill_color)
        
        
        return img
    
    def create_texture_image(image,origin_points,angled_points,width,height,center_x,center_y,line_color=(255,255,255),fill_color=None):
        cv2_image = pil_to_bgr_image(image)
        #print(f"shape={cv2_image.shape}")
        #cv2.imwrite("tmp.jpg",cv2_image)
        original_cordinates = []
        cordinates,angled_depth = split_points_xy_z(angled_points,width,height,center_x,center_y)
        # original point need offset
        for point in origin_points:
            original_cordinates.append(
                   [
                        (point[0]+offset_x)*width,
                        (point[1]+offset_y)*height
                   ]
            )
        if cv2_image.shape[2]==3:
            cv2_bg_img = cv2_create_color_image(cv2_image,(0,0,0))
        else:
            cv2_bg_img = cv2_create_color_image(cv2_image,(0,0,0,0))
        
        triangles = mp_triangles.get_triangles_copy(True,inner_eyes,inner_eyes,inner_mouth)

        triangles.sort(key=lambda triangle: sum(angled_depth[index] for index in triangle) / len(triangle)
                       ,reverse=True)
           
        for triangle in triangles:
                triangle_cordinates = [cordinates[index] for index in triangle]
                origin_triangle_cordinates = [original_cordinates[index] for index in triangle]
                
                cv2_bg_img=apply_affine_transformation_to_triangle_add(origin_triangle_cordinates,triangle_cordinates,cv2_image,cv2_bg_img)
            
        img= Image.fromarray(cv2.cvtColor(cv2_bg_img, cv2.COLOR_RGBA2BGRA))
        
        return img
    
    def create_point_image(points,width,height,center_x,center_y):
        cordinates,_ = split_points_xy_z(points,width,height,center_x,center_y)
        img = create_color_image(width,height,(0,0,0))
        glibvision.pil_utils.draw_points(img,cordinates,None,None,3,(255,0,0),3)
       
        return img
    
    def angled_points(points,angles,order="xyz"):
        angled_cordinates = []
        for point in points:
            rotated_np_point = rotate_point_euler(point,angles,order)
            angled_cordinates.append(
                   [
                       rotated_np_point[0],
                       rotated_np_point[1],rotated_np_point[2]
                   ]
              )
        return angled_cordinates
    

    frames = []


    #frames.append(create_point_image(points))
    frame_duration=100
    start_angle=0
    end_angle=360
    step_angle=10
    
    if draw_type == "Image":
        start_angle=-90
        end_angle=90
        step_angle=30

    if not animation:
        start_angle=0
        end_angle=0
        step_angle=360
    if image == None:
        draw_type="Dot"

    
    if center_scaleup and image!=None:
        top_distance = calculate_distance(normalized_center_point,normalized_top_point)
        bottom_distance = calculate_distance(normalized_center_point,normalized_bottom_point)
        distance = top_distance if top_distance>bottom_distance else bottom_distance
        #small_size = image_width if image_width<image_height else image_height
        
        scale_up = 0.45 / distance #half - margin
        if image_height>image_width:
            scale_up *= image_width/image_height
        #print(scale_up)
        face_center_x = int(0.5* image_width)#half
        face_center_y = int(0.5* image_height)
    else:
        scale_up = 1.0
        face_center_x = int(normalized_center_point[0]* image_width)
        face_center_y = int(normalized_center_point[1]* image_height)
        
    

    rotated_points = None
    
    if animation:
        for i in range(start_angle,end_angle,step_angle):
            if animation_direction == "X":
                angles = [math.radians(i),0,0]
            elif animation_direction == "Y":
                angles = [0,math.radians(i),0]
            else:
                angles = [0,0,math.radians(i)]
           
            if draw_type == "Dot":
                frames.append(create_point_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y))
            elif draw_type == "Line":
                frames.append(create_triangle_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y))
            elif draw_type == "Line+Fill":
                frames.append(create_triangle_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y,(128,128,128),(200,200,200)))
            elif draw_type == "Image":
                frame_duration=500
                frames.append(create_texture_image(image,points,angled_points(points,angles),image_width,image_height,face_center_x,face_center_y))
        webp = process_create_webp(frames,frame_duration)
        path = save_buffer(webp)
    


    
    return path,face_landmarker_result,rotated_points