inference/depth.py

# # Example usage
# import torch
# import numpy as np
# from PIL import Image
# from torchvision import transforms
# from config import LABELS_TO_IDS
# from utils.vis_utils import visualize_mask_with_overlay

# import torch
# import torch.nn.functional as F
# import numpy as np
# import cv2

# TASK = 'depth'
# VERSION = 'sapiens_0.3b'

# model_path = get_model_path(TASK, VERSION)
# print(model_path)

# model = torch.jit.load(model_path)
# model.eval()
# model.to("cuda")


# def get_depth(image, depth_model, input_shape=(3, 1024, 768), device="cuda"):
#     # Preprocess the image
#     img = preprocess_image(image, input_shape)
    
#     # Run the model
#     with torch.no_grad():
#         result = depth_model(img.to(device))
    
#     # Post-process the output
#     depth_map = post_process_depth(result, (image.shape[0], image.shape[1]))
    
#     # Visualize the depth map
#     depth_image = visualize_depth(depth_map)
    
#     return depth_image, depth_map

# def preprocess_image(image, input_shape):
#     img = cv2.resize(image, (input_shape[2], input_shape[1]), interpolation=cv2.INTER_LINEAR).transpose(2, 0, 1)
#     img = torch.from_numpy(img)
#     img = img[[2, 1, 0], ...].float()
#     mean = torch.tensor([123.5, 116.5, 103.5]).view(-1, 1, 1)
#     std = torch.tensor([58.5, 57.0, 57.5]).view(-1, 1, 1)
#     img = (img - mean) / std
#     return img.unsqueeze(0)

# def post_process_depth(result, original_shape):
#     # Check the dimensionality of the result
#     if result.dim() == 3:
#         result = result.unsqueeze(0)
#     elif result.dim() == 4:
#         pass
#     else:
#         raise ValueError(f"Unexpected result dimension: {result.dim()}")
    
#     # Ensure we're interpolating to the correct dimensions
#     seg_logits = F.interpolate(result, size=original_shape, mode="bilinear", align_corners=False).squeeze(0)
#     depth_map = seg_logits.data.float().cpu().numpy()
    
#     # If depth_map has an extra dimension, squeeze it
#     if depth_map.ndim == 3 and depth_map.shape[0] == 1:
#         depth_map = depth_map.squeeze(0)
    
#     return depth_map

# def visualize_depth(depth_map):
#     # Normalize the depth map
#     min_val, max_val = np.nanmin(depth_map), np.nanmax(depth_map)
#     depth_normalized = 1 - ((depth_map - min_val) / (max_val - min_val))
    
#     # Convert to uint8
#     depth_normalized = (depth_normalized * 255).astype(np.uint8)
    
#     # Apply colormap
#     depth_colored = cv2.applyColorMap(depth_normalized, cv2.COLORMAP_INFERNO)
    
#     return depth_colored

# # You can add the surface normal calculation if needed
# def calculate_surface_normal(depth_map):
#     kernel_size = 7
#     grad_x = cv2.Sobel(depth_map.astype(np.float32), cv2.CV_32F, 1, 0, ksize=kernel_size)
#     grad_y = cv2.Sobel(depth_map.astype(np.float32), cv2.CV_32F, 0, 1, ksize=kernel_size)
#     z = np.full(grad_x.shape, -1)
#     normals = np.dstack((-grad_x, -grad_y, z))

#     normals_mag = np.linalg.norm(normals, axis=2, keepdims=True)
#     with np.errstate(divide="ignore", invalid="ignore"):
#         normals_normalized = normals / (normals_mag + 1e-5)

#     normals_normalized = np.nan_to_num(normals_normalized, nan=-1, posinf=-1, neginf=-1)
#     normal_from_depth = ((normals_normalized + 1) / 2 * 255).astype(np.uint8)
#     normal_from_depth = normal_from_depth[:, :, ::-1]  # RGB to BGR for cv2

#     return normal_from_depth

# from utils.vis_utils import resize_image

# pil_image = Image.open('/home/user/app/assets/image.webp')

# # Load and process an image
# image = cv2.imread('/home/user/app/assets/frame.png')
# depth_image, depth_map = get_depth(image, model)

# surface_normal = calculate_surface_normal(depth_map)
# cv2.imwrite("output_surface_normal.jpg", surface_normal)
# # Save the results
# output_im = cv2.imwrite("output_depth_image2.jpg", depth_image)

import torch
import torch.nn.functional as F
import numpy as np
import cv2
from PIL import Image
from config import SAPIENS_LITE_MODELS_PATH

def load_model(task, version):
    try:
        model_path = SAPIENS_LITE_MODELS_PATH[task][version]
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        model = torch.jit.load(model_path)
        model.eval()
        model.to(device)
        return model, device
    except KeyError as e:
        print(f"Error: Tarea o versión inválida. {e}")
        return None, None

def preprocess_image(image, input_shape):
    img = cv2.resize(image, (input_shape[2], input_shape[1]), interpolation=cv2.INTER_LINEAR).transpose(2, 0, 1)
    img = torch.from_numpy(img)
    img = img[[2, 1, 0], ...].float()
    mean = torch.tensor([123.5, 116.5, 103.5]).view(-1, 1, 1)
    std = torch.tensor([58.5, 57.0, 57.5]).view(-1, 1, 1)
    img = (img - mean) / std
    return img.unsqueeze(0)

def post_process_depth(result, original_shape):
    if result.dim() == 3:
        result = result.unsqueeze(0)
    elif result.dim() == 4:
        pass
    else:
        raise ValueError(f"Unexpected result dimension: {result.dim()}")
    
    seg_logits = F.interpolate(result, size=original_shape, mode="bilinear", align_corners=False).squeeze(0)
    depth_map = seg_logits.data.float().cpu().numpy()
    
    if depth_map.ndim == 3 and depth_map.shape[0] == 1:
        depth_map = depth_map.squeeze(0)
    
    return depth_map

def visualize_depth(depth_map):
    min_val, max_val = np.nanmin(depth_map), np.nanmax(depth_map)
    depth_normalized = 1 - ((depth_map - min_val) / (max_val - min_val))
    depth_normalized = (depth_normalized * 255).astype(np.uint8)
    depth_colored = cv2.applyColorMap(depth_normalized, cv2.COLORMAP_INFERNO)
    return depth_colored

def calculate_surface_normal(depth_map):
    kernel_size = 7
    grad_x = cv2.Sobel(depth_map.astype(np.float32), cv2.CV_32F, 1, 0, ksize=kernel_size)
    grad_y = cv2.Sobel(depth_map.astype(np.float32), cv2.CV_32F, 0, 1, ksize=kernel_size)
    z = np.full(grad_x.shape, -1)
    normals = np.dstack((-grad_x, -grad_y, z))

    normals_mag = np.linalg.norm(normals, axis=2, keepdims=True)
    with np.errstate(divide="ignore", invalid="ignore"):
        normals_normalized = normals / (normals_mag + 1e-5)

    normals_normalized = np.nan_to_num(normals_normalized, nan=-1, posinf=-1, neginf=-1)
    normal_from_depth = ((normals_normalized + 1) / 2 * 255).astype(np.uint8)
    normal_from_depth = normal_from_depth[:, :, ::-1]  # RGB to BGR for cv2

    return normal_from_depth

def process_image_or_video(input_data, task='depth', version='sapiens_0.3b'):
    model, device = load_model(task, version)
    if model is None or device is None:
        return None

    input_shape = (3, 1024, 768)

    def process_frame(frame):
        if isinstance(frame, Image.Image):
            frame = np.array(frame)
        
        if frame.shape[2] == 4:  # RGBA
            frame = cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB)
        
        img = preprocess_image(frame, input_shape)
        
        with torch.no_grad():
            result = model(img.to(device))
        
        depth_map = post_process_depth(result, (frame.shape[0], frame.shape[1]))
        depth_image = visualize_depth(depth_map)
        
        return Image.fromarray(cv2.cvtColor(depth_image, cv2.COLOR_BGR2RGB))

    if isinstance(input_data, np.ndarray):  # Video frame
        return process_frame(input_data)
    elif isinstance(input_data, Image.Image):  # Imagen
        return process_frame(input_data)
    else:
        print("Tipo de entrada no soportado. Por favor, proporcione una imagen PIL o un frame de video numpy.")
        return None