core/data/combined_multi_view_dataset.py

import PIL
import numpy as np
import torch
from PIL import Image

from .camera_pose_utils import (
    convert_w2c_between_c2w,
    transform_pose_sequence_to_relative_c2w,
)


def get_ray_embeddings(
    poses, size_h=256, size_w=256, fov_xy_list=None, focal_xy_list=None
):
    """
    poses: sequence of cameras poses (y-up format)
    """
    use_focal = False
    if fov_xy_list is None or fov_xy_list[0] is None or fov_xy_list[0][0] is None:
        assert focal_xy_list is not None
        use_focal = True

    rays_embeddings = []
    for i in range(poses.shape[0]):
        cur_pose = poses[i]
        if use_focal:
            rays_o, rays_d = get_rays(
                # [h, w, 3]
                cur_pose,
                size_h,
                size_w,
                focal_xy=focal_xy_list[i],
            )
        else:
            rays_o, rays_d = get_rays(
                cur_pose, size_h, size_w, fov_xy=fov_xy_list[i]
            )  # [h, w, 3]

        rays_plucker = torch.cat(
            [torch.cross(rays_o, rays_d, dim=-1), rays_d], dim=-1
        )  # [h, w, 6]
        rays_embeddings.append(rays_plucker)

    rays_embeddings = (
        torch.stack(rays_embeddings, dim=0).permute(0, 3, 1, 2).contiguous()
    )  # [V, 6, h, w]
    return rays_embeddings


def get_rays(pose, h, w, fov_xy=None, focal_xy=None, opengl=True):
    x, y = torch.meshgrid(
        torch.arange(w, device=pose.device),
        torch.arange(h, device=pose.device),
        indexing="xy",
    )
    x = x.flatten()
    y = y.flatten()

    cx = w * 0.5
    cy = h * 0.5

    # print("fov_xy=", fov_xy)
    # print("focal_xy=", focal_xy)

    if focal_xy is None:
        assert fov_xy is not None, "fov_x/y and focal_x/y cannot both be None."
        focal_x = w * 0.5 / np.tan(0.5 * np.deg2rad(fov_xy[0]))
        focal_y = h * 0.5 / np.tan(0.5 * np.deg2rad(fov_xy[1]))
    else:
        assert (
            len(focal_xy) == 2
        ), "focal_xy should be a list-like object containing only two elements (focal length in x and y direction)."
        focal_x = w * focal_xy[0]
        focal_y = h * focal_xy[1]

    camera_dirs = torch.nn.functional.pad(
        torch.stack(
            [
                (x - cx + 0.5) / focal_x,
                (y - cy + 0.5) / focal_y * (-1.0 if opengl else 1.0),
            ],
            dim=-1,
        ),
        (0, 1),
        value=(-1.0 if opengl else 1.0),
    )  # [hw, 3]

    rays_d = camera_dirs @ pose[:3, :3].transpose(0, 1)  # [hw, 3]
    rays_o = pose[:3, 3].unsqueeze(0).expand_as(rays_d)  # [hw, 3]

    rays_o = rays_o.view(h, w, 3)
    rays_d = safe_normalize(rays_d).view(h, w, 3)

    return rays_o, rays_d


def safe_normalize(x, eps=1e-20):
    return x / length(x, eps)


def length(x, eps=1e-20):
    if isinstance(x, np.ndarray):
        return np.sqrt(np.maximum(np.sum(x * x, axis=-1, keepdims=True), eps))
    else:
        return torch.sqrt(torch.clamp(dot(x, x), min=eps))


def dot(x, y):
    if isinstance(x, np.ndarray):
        return np.sum(x * y, -1, keepdims=True)
    else:
        return torch.sum(x * y, -1, keepdim=True)


def extend_list_by_repeating(original_list, target_length, repeat_idx, at_front):
    if not original_list:
        raise ValueError("The original list cannot be empty.")

    extended_list = []
    original_length = len(original_list)
    for i in range(target_length - original_length):
        extended_list.append(original_list[repeat_idx])

    if at_front:
        extended_list.extend(original_list)
        return extended_list
    else:
        original_list.extend(extended_list)
        return original_list


def select_evenly_spaced_elements(arr, x):
    if x <= 0 or len(arr) == 0:
        return []

    # Calculate step size as the ratio of length of the list and x
    step = len(arr) / x

    # Pick elements at indices that are multiples of step (round them to nearest integer)
    selected_elements = [arr[round(i * step)] for i in range(x)]

    return selected_elements


def convert_co3d_annotation_to_opengl_pose_and_intrinsics(frame_annotation):
    p = frame_annotation.viewpoint.principal_point
    f = frame_annotation.viewpoint.focal_length
    h, w = frame_annotation.image.size
    K = np.eye(3)
    s = (min(h, w) - 1) / 2
    if frame_annotation.viewpoint.intrinsics_format == "ndc_norm_image_bounds":
        K[0, 0] = f[0] * (w - 1) / 2
        K[1, 1] = f[1] * (h - 1) / 2
    elif frame_annotation.viewpoint.intrinsics_format == "ndc_isotropic":
        K[0, 0] = f[0] * s / 2
        K[1, 1] = f[1] * s / 2
    else:
        assert (
            False
        ), f"Invalid intrinsics_format: {frame_annotation.viewpoint.intrinsics_format}"
    K[0, 2] = -p[0] * s + (w - 1) / 2
    K[1, 2] = -p[1] * s + (h - 1) / 2

    R = np.array(frame_annotation.viewpoint.R).T  # note the transpose here
    T = np.array(frame_annotation.viewpoint.T)
    pose = np.concatenate([R, T[:, None]], 1)
    # Need to be converted into OpenGL format. Flip the direction of x, z axis
    pose = np.diag([-1, 1, -1]).astype(np.float32) @ pose
    return pose, K


def normalize_w2c_camera_pose_sequence(
    target_camera_poses,
    condition_camera_poses=None,
    output_c2w=False,
    translation_norm_mode="div_by_max",
):
    """
    Normalize camera pose sequence so that the first frame is identity rotation and zero translation,
    and the translation scale is normalized by the farest point from the first frame (to one).
    :param target_camera_poses: W2C poses tensor in [N, 3, 4]
    :param condition_camera_poses: W2C poses tensor in [N, 3, 4]
    :return: Tuple(Tensor, Tensor), the normalized `target_camera_poses` and `condition_camera_poses`
    """
    # Normalize at w2c, all poses should be in w2c in UnifiedFrame
    num_target_views = target_camera_poses.size(0)
    if condition_camera_poses is not None:
        all_poses = torch.concat([target_camera_poses, condition_camera_poses], dim=0)
    else:
        all_poses = target_camera_poses
    # Convert W2C to C2W
    normalized_poses = transform_pose_sequence_to_relative_c2w(
        convert_w2c_between_c2w(all_poses)
    )
    # Here normalized_poses is C2W
    if not output_c2w:
        # Convert from C2W back to W2C if output_c2w is False.
        normalized_poses = convert_w2c_between_c2w(normalized_poses)

    t_norms = torch.linalg.norm(normalized_poses[:, :, 3], ord=2, dim=-1)
    # print("t_norms=", t_norms)
    largest_t_norm = torch.max(t_norms)

    # print("largest_t_norm=", largest_t_norm)
    # normalized_poses[:, :, 3] -= first_t.unsqueeze(0).repeat(normalized_poses.size(0), 1)
    if translation_norm_mode == "div_by_max_plus_one":
        # Always add a constant component to the translation norm
        largest_t_norm = largest_t_norm + 1.0
    elif translation_norm_mode == "div_by_max":
        largest_t_norm = largest_t_norm
        if largest_t_norm <= 0.05:
            largest_t_norm = 0.05
    elif translation_norm_mode == "disabled":
        largest_t_norm = 1.0
    else:
        assert False, f"Invalid translation_norm_mode: {translation_norm_mode}."
    normalized_poses[:, :, 3] /= largest_t_norm

    target_camera_poses = normalized_poses[:num_target_views]
    if condition_camera_poses is not None:
        condition_camera_poses = normalized_poses[num_target_views:]
    else:
        condition_camera_poses = None
    # print("After First condition:", condition_camera_poses[0])
    # print("After First target:", target_camera_poses[0])
    return target_camera_poses, condition_camera_poses


def central_crop_pil_image(_image, crop_size, use_central_padding=False):
    if use_central_padding:
        # Determine the new size
        _w, _h = _image.size
        new_size = max(_w, _h)
        # Create a new image with white background
        new_image = Image.new("RGB", (new_size, new_size), (255, 255, 255))
        # Calculate the position to paste the original image
        paste_position = ((new_size - _w) // 2, (new_size - _h) // 2)
        # Paste the original image onto the new image
        new_image.paste(_image, paste_position)
        _image = new_image
    # get the new size again if padded
    _w, _h = _image.size
    scale = crop_size / min(_h, _w)
    # resize shortest side to crop_size
    _w_out, _h_out = int(scale * _w), int(scale * _h)
    _image = _image.resize(
        (_w_out, _h_out),
        resample=(
            PIL.Image.Resampling.LANCZOS if scale < 1 else PIL.Image.Resampling.BICUBIC
        ),
    )
    # center crop
    margin_w = (_image.size[0] - crop_size) // 2
    margin_h = (_image.size[1] - crop_size) // 2
    _image = _image.crop(
        (margin_w, margin_h, margin_w + crop_size, margin_h + crop_size)
    )
    return _image


def crop_and_resize(
    image: Image.Image, target_width: int, target_height: int
) -> Image.Image:
    """
    Crops and resizes an image while preserving the aspect ratio.

    Args:
        image (Image.Image): Input PIL image to be cropped and resized.
        target_width (int): Target width of the output image.
        target_height (int): Target height of the output image.

    Returns:
        Image.Image: Cropped and resized image.
    """
    # Original dimensions
    original_width, original_height = image.size
    original_aspect = original_width / original_height
    target_aspect = target_width / target_height

    # Calculate crop box to maintain aspect ratio
    if original_aspect > target_aspect:
        # Crop horizontally
        new_width = int(original_height * target_aspect)
        new_height = original_height
        left = (original_width - new_width) / 2
        top = 0
        right = left + new_width
        bottom = original_height
    else:
        # Crop vertically
        new_width = original_width
        new_height = int(original_width / target_aspect)
        left = 0
        top = (original_height - new_height) / 2
        right = original_width
        bottom = top + new_height

    # Crop and resize
    cropped_image = image.crop((left, top, right, bottom))
    resized_image = cropped_image.resize((target_width, target_height), Image.LANCZOS)

    return resized_image


def calculate_fov_after_resize(
    fov_x: float,
    fov_y: float,
    original_width: int,
    original_height: int,
    target_width: int,
    target_height: int,
) -> (float, float):
    """
    Calculates the new field of view after cropping and resizing an image.

    Args:
        fov_x (float): Original field of view in the x-direction (horizontal).
        fov_y (float): Original field of view in the y-direction (vertical).
        original_width (int): Original width of the image.
        original_height (int): Original height of the image.
        target_width (int): Target width of the output image.
        target_height (int): Target height of the output image.

    Returns:
        (float, float): New field of view (fov_x, fov_y) after cropping and resizing.
    """
    original_aspect = original_width / original_height
    target_aspect = target_width / target_height

    if original_aspect > target_aspect:
        # Crop horizontally
        new_width = int(original_height * target_aspect)
        new_fov_x = fov_x * (new_width / original_width)
        new_fov_y = fov_y
    else:
        # Crop vertically
        new_height = int(original_width / target_aspect)
        new_fov_y = fov_y * (new_height / original_height)
        new_fov_x = fov_x

    return new_fov_x, new_fov_y