data_process/lib/preprocess.py

"""This script contains the image preprocessing code for Deep3DFaceRecon_pytorch."""

import numpy as np
from scipy.io import loadmat
from PIL import Image
import cv2
import os
from skimage import transform as trans
import torch
import warnings

warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)
warnings.filterwarnings("ignore", category=FutureWarning)


def POS(xp, x):
    """
    Calculate translation and scale using least squares for image alignment.

    Args:
        xp (np.ndarray): Target points, shape (2, N).
        x (np.ndarray): Source points, shape (2, N).

    Returns:
        tuple: Translation vector (t) and scale factor (s).
    """
    npts = xp.shape[1]
    A = np.zeros([2 * npts, 8])

    A[0:2 * npts - 1:2, 0:3] = x.T
    A[0:2 * npts - 1:2, 3] = 1
    A[1:2 * npts:2, 4:7] = x.T
    A[1:2 * npts:2, 7] = 1

    b = xp.T.reshape([2 * npts, 1])
    k, _, _, _ = np.linalg.lstsq(A, b, rcond=None)

    R1, R2 = k[:3], k[4:7]
    sTx, sTy = k[3], k[7]
    s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2
    t = np.array([sTx, sTy])

    return t, s


def BBRegression(points, params):
    """
    Perform bounding box regression for 68 landmark detection.

    Args:
        points (np.ndarray): Facial landmarks, shape (5, 2).
        params (dict): Regression parameters.

    Returns:
        np.ndarray: Bounding box [x, y, w, h].
    """
    w1, b1, w2, b2 = params['W1'], params['B1'], params['W2'], params['B2']
    data = points.reshape([5, 2])
    data_mean = np.mean(data, axis=0)

    data -= data_mean
    rms = np.sqrt(np.sum(data ** 2) / 5)
    data /= rms
    data = data.reshape([1, 10]).T

    inputs = np.matmul(w1, data) + b1
    inputs = 2 / (1 + np.exp(-2 * inputs)) - 1
    inputs = np.matmul(w2, inputs) + b2
    inputs = inputs.T

    x, y = inputs[:, 0] * rms + data_mean[0], inputs[:, 1] * rms + data_mean[1]
    w = (224 / inputs[:, 2]) * rms

    return np.array([x, y, w, w]).reshape([4])


def img_padding(img, box):
    """
    Pad image to avoid cropping issues.

    Args:
        img (np.ndarray): Input image.
        box (np.ndarray): Bounding box [x, y, w, h].

    Returns:
        tuple: Padded image, updated bounding box, success flag.
    """
    success = True
    bbox = box.copy()
    h, w = img.shape[:2]
    padded_img = np.zeros([2 * h, 2 * w, 3])

    padded_img[h // 2: h + h // 2, w // 2: w + w // 2] = img
    bbox[:2] += [w // 2, h // 2]

    if bbox[0] < 0 or bbox[1] < 0:
        success = False

    return padded_img, bbox, success


def crop(img, bbox):
    """
    Crop image based on bounding box.

    Args:
        img (np.ndarray): Input image.
        bbox (np.ndarray): Bounding box [x, y, w, h].

    Returns:
        tuple: Cropped image, scale factor.
    """
    padded_img, padded_bbox, flag = img_padding(img, bbox)
    if not flag:
        return padded_img, 0

    x, y, w, h = padded_bbox
    cropped_img = padded_img[y:y + h, x:x + w]
    cropped_img = cv2.resize(cropped_img.astype(np.uint8), (224, 224), interpolation=cv2.INTER_CUBIC)

    return cropped_img, 224 / w


def scale_trans(img, lm, t, s):
    """
    Apply scaling and translation to the image and landmarks.

    Args:
        img (np.ndarray): Input image.
        lm (np.ndarray): Landmarks.
        t (np.ndarray): Translation vector.
        s (float): Scale factor.

    Returns:
        tuple: Transformed image, inverse scale parameters.
    """
    img_h, img_w = img.shape[:2]
    M_s = np.array([[1, 0, -t[0] + img_w // 2 + 0.5], [0, 1, -img_h // 2 + t[1]]], dtype=np.float32)
    img = cv2.warpAffine(img, M_s, (img_w, img_h))

    w, h = int(img_w / s * 100), int(img_h / s * 100)
    img = cv2.resize(img, (w, h))

    lm = np.stack([lm[:, 0] - t[0] + img_w // 2, lm[:, 1] - t[1] + img_h // 2], axis=1) / s * 100
    bbox = [w // 2 - 112, h // 2 - 112, 224, 224]

    cropped_img, scale2 = crop(img, bbox)
    assert scale2 != 0

    t1 = np.array([bbox[0], bbox[1]])
    scale = s / 100
    t2 = np.array([t[0] - img_w / 2, t[1] - img_h / 2])

    return cropped_img, (scale / scale2, scale * t1 + t2)


def align_for_lm(img, five_points):
    """
    Align facial image using facial landmarks for landmark detection refinement.

    Args:
        img: Input facial image (numpy array)
        five_points: Facial landmark coordinates (5 points, 10 values)

    Returns:
        crop_img: Cropped and aligned facial image
        scale: Scaling factor applied during cropping
        bbox: Bounding box coordinates [x, y, width, height]

    Process:
        1. Predict optimal face bounding box using landmark regression
        2. Crop and align image based on predicted bounding box
    """
    # Reshape landmarks to 1x10 array (5 points x 2 coordinates)
    five_points = np.array(five_points).reshape([1, 10])

    # Load bounding box regressor parameters (MATLAB format)
    params = loadmat('util/BBRegressorParam_r.mat')  # Contains regression weights

    # Predict optimal face bounding box using regression model
    bbox = BBRegression(five_points, params)  # Returns [x, y, width, height]

    # Verify valid bounding box prediction
    assert bbox[2] != 0, "Invalid bounding box width (zero detected)"

    # Convert to integer coordinates for cropping
    bbox = np.round(bbox).astype(np.int32)

    # Crop image and get scaling factor
    crop_img, scale = crop(img, bbox)  # crop() should handle boundary checks

    return crop_img, scale, bbox


def resize_n_crop_img(img, lm, ldmk_3d, t, s, s_3d, target_size=224., mask=None):
    """
    Resize and center-crop image with corresponding landmark transformation

    Args:
        img: PIL.Image - Input image
        lm: np.array - Facial landmarks in original image coordinates [N, 2]
        t: tuple - (tx, ty) translation parameters
        s: float - Scaling factor
        target_size: float - Output image dimensions (square)
        mask: PIL.Image - Optional mask image

    Returns:
        img: PIL.Image - Processed image
        lm: np.array - Transformed landmarks [N, 2]
        mask: PIL.Image - Processed mask (or None)
        left: int - Left crop coordinate
        up: int - Top crop coordinate
    """
    # Original image dimensions
    w0, h0 = img.size

    # Calculate scaled dimensions
    w = (w0 * s).astype(np.int32)
    h = (h0 * s).astype(np.int32)

    w_3d = (w0 * s_3d).astype(np.int32)
    h_3d = (h0 * s_3d).astype(np.int32)

    # Calculate crop coordinates after scaling and translation
    # Horizontal crop window
    left = (w / 2 - target_size / 2 + (t[0] - w0 / 2) * s).astype(np.int32)
    right = left + target_size

    # Vertical crop window (note inverted Y-axis in images)
    up = (h / 2 - target_size / 2 + (h0 / 2 - t[1]) * s).astype(np.int32)
    below = up + target_size
    left = int(left)
    up = int(up)
    right = int(right)
    below = int(below)
    # Resize and crop main image
    img = img.resize((w, h), resample=Image.BICUBIC)
    img = img.crop((left, up, right, below))

    # Process mask if provided
    if mask is not None:
        mask = mask.resize((w, h), resample=Image.BICUBIC)
        mask = mask.crop((left, up, right, below))

    # Transform landmarks to cropped coordinates
    # 1. Adjust for translation and original image center
    # 2. Apply scaling
    # 3. Adjust for final crop offset
    lm = np.stack([lm[:, 0] - t[0] + w0 / 2,
                   lm[:, 1] - t[1] + h0 / 2], axis=1) * s
    crop_offset = np.array([(w / 2 - target_size / 2),
                            (h / 2 - target_size / 2)])
    lm = lm - crop_offset.reshape(1, 2)

    ldmk_3d = np.stack([ldmk_3d[:, 0] - t[0] + w0 / 2, ldmk_3d[:, 1] -
                        t[1] + h0 / 2], axis=1) * s_3d
    ldmk_3d = ldmk_3d - np.reshape(
        np.array([(w_3d / 2 - 512 / 2), (h_3d / 2 - 512 / 2)]), [1, 2])

    return img, lm, mask, left, up, ldmk_3d


def extract_5p(lm):
    """
    Extract 5-point facial landmarks from 68 landmarks.

    Args:
        lm (np.ndarray): 68 facial landmarks.

    Returns:
        np.ndarray: 5-point landmarks.
    """
    lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1
    lm5p = np.stack([
        lm[lm_idx[0], :],
        np.mean(lm[lm_idx[[1, 2]], :], axis=0),
        np.mean(lm[lm_idx[[3, 4]], :], axis=0),
        lm[lm_idx[5], :],
        lm[lm_idx[6], :]
    ], axis=0)

    return lm5p[[1, 2, 0, 3, 4], :]


def align_img(img, lm, lm3D, ldmk_3d, mask=None, target_size=224., rescale_factor=102., rescale_factor_3D=218.):
    """
    Align facial image using 2D-3D landmark correspondence

    Args:
        img: PIL.Image - Input facial image (H, W, 3)
        lm: np.array - Facial landmarks (68, 2) in image coordinates (y-axis inverted)
        lm3D: np.array - 3D reference landmarks (5, 3) for pose estimation
        mask: PIL.Image - Optional facial mask (H, W, 3)
        target_size: float - Output image dimensions (square)
        rescale_factor: float - Normalization factor for face scale

    Returns:
        trans_params: np.array - [raw_W, raw_H, scale, tx, ty] transformation parameters
        img_new: PIL.Image - Aligned image (target_size, target_size, 3)
        lm_new: np.array - Transformed landmarks (68, 2)
        mask_new: PIL.Image - Aligned mask (target_size, target_size)
        crop_left: int - Left crop coordinate
        crop_up: int - Top crop coordinate
        s: float - Final scaling factor

    Process:
        1. Extract 5-point landmarks if needed
        2. Estimate face scale and translation using POS algorithm
        3. Resize and crop image with landmark adjustment
    """
    # Original image dimensions
    w0, h0 = img.size

    # Extract 5 facial landmarks if not provided
    if lm.shape[0] != 5:
        lm5p = extract_5p(lm)  # Convert 68-point to 5-point landmarks
    else:
        lm5p = lm

    # Calculate scale and translation using PnP algorithm
    # POS (Perspective-n-Point algorithm) implementation
    t, s = POS(lm5p.T, lm3D.T)  # Returns translation vector and scale factor
    s_3d = rescale_factor_3D / s
    s = rescale_factor / s  # Normalize scale using reference face size
    # Apply geometric transformation
    img_new, lm_new, mask_new, crop_left, crop_up, ldmk_3d_align = resize_n_crop_img(
        img,
        lm,
        ldmk_3d,
        t,
        s,
        s_3d=s_3d,
        target_size=target_size,
        mask=mask
    )

    # Package transformation parameters [original_w, original_h, scale, tx, ty]
    trans_params = np.array([w0, h0, s, t[0][0], t[1][0]])

    return trans_params, img_new, lm_new, mask_new, crop_left, crop_up, s, ldmk_3d_align


def estimate_norm(lm_68p, H):
    """
    Estimate similarity transformation matrix for face alignment.

    Args:
        lm_68p (np.ndarray): 68 facial landmarks.
        H (int): Image height.

    Returns:
        np.ndarray: Transformation matrix (2, 3).
    """
    lm = extract_5p(lm_68p)
    lm[:, -1] = H - 1 - lm[:, -1]

    tform = trans.SimilarityTransform()
    src = np.array([
        [38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366],
        [41.5493, 92.3655], [70.7299, 92.2041]
    ], dtype=np.float32)

    tform.estimate(lm, src)
    M = tform.params

    return M[0:2, :] if np.linalg.det(M) != 0 else np.eye(2, 3)


def estimate_norm_torch(lm_68p, H):
    """
    Estimate similarity transformation matrix for face alignment using PyTorch.

    Args:
        lm_68p (torch.Tensor): 68 facial landmarks.
        H (int): Image height.

    Returns:
        torch.Tensor: Transformation matrices.
    """
    lm_68p_ = lm_68p.detach().cpu().numpy()
    M = [estimate_norm(lm, H) for lm in lm_68p_]

    return torch.tensor(np.array(M), dtype=torch.float32, device=lm_68p.device)