core/data/camera_pose_utils.py

import copy
import numpy as np
import torch
from scipy.spatial.transform import Rotation as R


def get_opencv_from_blender(matrix_world, fov, image_size):
    # convert matrix_world to opencv format extrinsics
    opencv_world_to_cam = matrix_world.inverse()
    opencv_world_to_cam[1, :] *= -1
    opencv_world_to_cam[2, :] *= -1
    R, T = opencv_world_to_cam[:3, :3], opencv_world_to_cam[:3, 3]
    R, T = R.unsqueeze(0), T.unsqueeze(0)

    # convert fov to opencv format intrinsics
    focal = 1 / np.tan(fov / 2)
    intrinsics = np.diag(np.array([focal, focal, 1])).astype(np.float32)
    opencv_cam_matrix = torch.from_numpy(intrinsics).unsqueeze(0).float()
    opencv_cam_matrix[:, :2, -1] += torch.tensor([image_size / 2, image_size / 2])
    opencv_cam_matrix[:, [0, 1], [0, 1]] *= image_size / 2

    return R, T, opencv_cam_matrix


def cartesian_to_spherical(xyz):
    xy = xyz[:, 0] ** 2 + xyz[:, 1] ** 2
    z = np.sqrt(xy + xyz[:, 2] ** 2)
    # for elevation angle defined from z-axis down
    theta = np.arctan2(np.sqrt(xy), xyz[:, 2])
    azimuth = np.arctan2(xyz[:, 1], xyz[:, 0])
    return np.stack([theta, azimuth, z], axis=-1)


def spherical_to_cartesian(spherical_coords):
    # convert from spherical to cartesian coordinates
    theta, azimuth, radius = spherical_coords.T
    x = radius * np.sin(theta) * np.cos(azimuth)
    y = radius * np.sin(theta) * np.sin(azimuth)
    z = radius * np.cos(theta)
    return np.stack([x, y, z], axis=-1)


def look_at(eye, center, up):
    # Create a normalized direction vector from eye to center
    f = np.array(center) - np.array(eye)
    f /= np.linalg.norm(f)

    # Create a normalized right vector
    up_norm = np.array(up) / np.linalg.norm(up)
    s = np.cross(f, up_norm)
    s /= np.linalg.norm(s)

    # Recompute the up vector
    u = np.cross(s, f)

    # Create rotation matrix R
    R = np.array([[s[0], s[1], s[2]], [u[0], u[1], u[2]], [-f[0], -f[1], -f[2]]])

    # Create translation vector T
    T = -np.dot(R, np.array(eye))

    return R, T


def get_blender_from_spherical(elevation, azimuth):
    """Generates blender camera from spherical coordinates."""

    cartesian_coords = spherical_to_cartesian(np.array([[elevation, azimuth, 3.5]]))

    # get camera rotation
    center = np.array([0, 0, 0])
    eye = cartesian_coords[0]
    up = np.array([0, 0, 1])

    R, T = look_at(eye, center, up)
    R = R.T
    T = -np.dot(R, T)
    RT = np.concatenate([R, T.reshape(3, 1)], axis=-1)

    blender_cam = torch.from_numpy(RT).float()
    blender_cam = torch.cat([blender_cam, torch.tensor([[0, 0, 0, 1]])], dim=0)
    print(blender_cam)
    return blender_cam


def invert_pose(r, t):
    r_inv = r.T
    t_inv = -np.dot(r_inv, t)
    return r_inv, t_inv


def transform_pose_sequence_to_relative(poses, as_z_up=False):
    """
    poses: a sequence of 3*4 C2W camera pose matrices
    as_z_up: output in z-up format. If False, the output is in y-up format
    """
    r0, t0 = poses[0][:3, :3], poses[0][:3, 3]
    # r0_inv, t0_inv = invert_pose(r0, t0)
    r0_inv = r0.T
    new_rt0 = np.hstack([np.eye(3, 3), np.zeros((3, 1))])
    if as_z_up:
        new_rt0 = c2w_y_up_to_z_up(new_rt0)
    transformed_poses = [new_rt0]
    for pose in poses[1:]:
        r, t = pose[:3, :3], pose[:3, 3]
        new_r = np.dot(r0_inv, r)
        new_t = np.dot(r0_inv, t - t0)
        new_rt = np.hstack([new_r, new_t[:, None]])
        if as_z_up:
            new_rt = c2w_y_up_to_z_up(new_rt)
        transformed_poses.append(new_rt)
    return transformed_poses


def c2w_y_up_to_z_up(c2w_3x4):
    R_y_up_to_z_up = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]])

    R = c2w_3x4[:, :3]
    t = c2w_3x4[:, 3]

    R_z_up = R_y_up_to_z_up @ R
    t_z_up = R_y_up_to_z_up @ t

    T_z_up = np.hstack((R_z_up, t_z_up.reshape(3, 1)))

    return T_z_up


def transform_pose_sequence_to_relative_w2c(poses):
    new_rt_list = []
    first_frame_rt = copy.deepcopy(poses[0])
    first_frame_r_inv = first_frame_rt[:, :3].T
    first_frame_t = first_frame_rt[:, -1]
    for rt in poses:
        rt[:, :3] = np.matmul(rt[:, :3], first_frame_r_inv)
        rt[:, -1] = rt[:, -1] - np.matmul(rt[:, :3], first_frame_t)
        new_rt_list.append(copy.deepcopy(rt))
    return new_rt_list


def transform_pose_sequence_to_relative_c2w(poses):
    first_frame_rt = poses[0]
    first_frame_r_inv = first_frame_rt[:, :3].T
    first_frame_t = first_frame_rt[:, -1]
    rotations = poses[:, :, :3]
    translations = poses[:, :, 3]

    # Compute new rotations and translations in batch
    new_rotations = torch.matmul(first_frame_r_inv, rotations)
    new_translations = torch.matmul(
        first_frame_r_inv, (translations - first_frame_t.unsqueeze(0)).unsqueeze(-1)
    )
    # Concatenate new rotations and translations
    new_rt = torch.cat([new_rotations, new_translations], dim=-1)

    return new_rt


def convert_w2c_between_c2w(poses):
    rotations = poses[:, :, :3]
    translations = poses[:, :, 3]
    new_rotations = rotations.transpose(-1, -2)
    new_translations = torch.matmul(-new_rotations, translations.unsqueeze(-1))
    new_rt = torch.cat([new_rotations, new_translations], dim=-1)
    return new_rt


def slerp(q1, q2, t):
    """
    Performs spherical linear interpolation (SLERP) between two quaternions.

    Args:
        q1 (torch.Tensor): Start quaternion (4,).
        q2 (torch.Tensor): End quaternion (4,).
        t (float or torch.Tensor): Interpolation parameter in [0, 1].

    Returns:
        torch.Tensor: Interpolated quaternion (4,).
    """
    q1 = q1 / torch.linalg.norm(q1)  # Normalize q1
    q2 = q2 / torch.linalg.norm(q2)  # Normalize q2

    dot = torch.dot(q1, q2)

    # Ensure shortest path (flip q2 if needed)
    if dot < 0.0:
        q2 = -q2
        dot = -dot

    # Avoid numerical precision issues
    dot = torch.clamp(dot, -1.0, 1.0)

    theta = torch.acos(dot)  # Angle between q1 and q2

    if theta < 1e-6:  # If very close, use linear interpolation
        return (1 - t) * q1 + t * q2

    sin_theta = torch.sin(theta)

    return (torch.sin((1 - t) * theta) / sin_theta) * q1 + (
        torch.sin(t * theta) / sin_theta
    ) * q2


def interpolate_camera_poses(c2w: torch.Tensor, factor: int) -> torch.Tensor:
    """
    Interpolates a sequence of camera c2w poses to N times the length of the original sequence.

    Args:
        c2w (torch.Tensor): Input camera poses of shape (N, 3, 4).
        factor (int): The upsampling factor (e.g., 2 for doubling the length).

    Returns:
        torch.Tensor: Interpolated camera poses of shape (N * factor, 3, 4).
    """
    assert c2w.ndim == 3 and c2w.shape[1:] == (
        3,
        4,
    ), "Input tensor must have shape (N, 3, 4)."
    assert factor > 1, "Upsampling factor must be greater than 1."

    N = c2w.shape[0]
    new_length = N * factor

    # Extract rotations (R) and translations (T)
    rotations = c2w[:, :3, :3]  # Shape (N, 3, 3)
    translations = c2w[:, :3, 3]  # Shape (N, 3)

    # Convert rotations to quaternions for interpolation
    quaternions = torch.tensor(
        R.from_matrix(rotations.numpy()).as_quat()
    )  # Shape (N, 4)

    # Initialize interpolated quaternions and translations
    interpolated_quats = []
    interpolated_translations = []

    # Perform interpolation
    for i in range(N - 1):
        # Start and end quaternions and translations for this segment
        q1, q2 = quaternions[i], quaternions[i + 1]
        t1, t2 = translations[i], translations[i + 1]

        # Time steps for interpolation within this segment
        t_values = torch.linspace(0, 1, factor, dtype=torch.float32)

        # Interpolate quaternions using SLERP
        for t in t_values:
            interpolated_quats.append(slerp(q1, q2, t))

        # Interpolate translations linearly
        interp_t = t1 * (1 - t_values[:, None]) + t2 * t_values[:, None]
        interpolated_translations.append(interp_t)

    interpolated_quats.append(quaternions[0])
    interpolated_translations.append(translations[0].unsqueeze(0))
    # Add the last pose (end of sequence)
    interpolated_quats.append(quaternions[-1])
    interpolated_translations.append(translations[-1].unsqueeze(0))  # Add as 2D tensor

    # Combine interpolated results
    interpolated_quats = torch.stack(interpolated_quats, dim=0)  # Shape (new_length, 4)
    interpolated_translations = torch.cat(
        interpolated_translations, dim=0
    )  # Shape (new_length, 3)

    # Convert quaternions back to rotation matrices
    interpolated_rotations = torch.tensor(
        R.from_quat(interpolated_quats.numpy()).as_matrix()
    )  # Shape (new_length, 3, 3)

    # Form final c2w matrix
    interpolated_c2w = torch.zeros((new_length, 3, 4), dtype=torch.float32)
    interpolated_c2w[:, :3, :3] = interpolated_rotations
    interpolated_c2w[:, :3, 3] = interpolated_translations

    return interpolated_c2w