import os from pathlib import Path import gradio as gr import numpy as np import open3d as o3d import torch from PIL import Image from transformers import DPTForDepthEstimation, DPTImageProcessor # Initialize the image processor and depth estimation model image_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large") depth_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large", ignore_mismatched_sizes=True) def process_image(image_path, resized_width=800, z_scale=208): """ Processes the input image to generate a depth map and a 3D mesh reconstruction. Args: image_path (str): The file path to the input image. Returns: list: A list containing the depth image, 3D mesh reconstruction, and GLTF file path. """ image_path = Path(image_path) if not image_path.exists(): raise ValueError("Image file not found") # Load and resize the image image_raw = Image.open(image_path).convert("RGB") print(f"Original size: {image_raw.size}") resized_height = int(resized_width * image_raw.size[1] / image_raw.size[0]) image = image_raw.resize((resized_width, resized_height), Image.Resampling.LANCZOS) print(f"Resized size: {image.size}") # Prepare image for the model encoding = image_processor(image, return_tensors="pt") # Perform depth estimation with torch.no_grad(): outputs = depth_model(**encoding) predicted_depth = outputs.predicted_depth # Interpolate depth to match the image size prediction = torch.nn.functional.interpolate( predicted_depth.unsqueeze(1), size=(image.height, image.width), mode="bicubic", align_corners=False, ).squeeze() # Normalize the depth image to 8-bit if torch.cuda.is_available(): prediction = prediction.numpy() else: prediction = prediction.cpu().numpy() depth_min, depth_max = prediction.min(), prediction.max() depth_image = ((prediction - depth_min) / (depth_max - depth_min) * 255).astype("uint8") try: gltf_path = create_3d_obj(np.array(image), prediction, image_path, depth=10, z_scale=z_scale) except Exception: gltf_path = create_3d_obj(np.array(image), prediction, image_path, depth=8, z_scale=z_scale) img = Image.fromarray(depth_image) if torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.ipc_collect() return [img, gltf_path, gltf_path] def create_3d_obj(rgb_image, raw_depth, image_path, depth=10, z_scale=200): """ Creates a 3D object from RGB and depth images. Args: rgb_image (np.ndarray): The RGB image as a NumPy array. raw_depth (np.ndarray): The raw depth data. image_path (Path): The path to the original image. depth (int, optional): Depth parameter for Poisson reconstruction. Defaults to 10. z_scale (float, optional): Scaling factor for the Z-axis. Defaults to 200. Returns: str: The file path to the saved GLTF model. """ # Normalize the depth image depth_image = ((raw_depth - raw_depth.min()) / (raw_depth.max() - raw_depth.min()) * 255).astype("uint8") depth_o3d = o3d.geometry.Image(depth_image) image_o3d = o3d.geometry.Image(rgb_image) # Create RGBD image rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( image_o3d, depth_o3d, convert_rgb_to_intensity=False ) height, width = depth_image.shape # Define camera intrinsics camera_intrinsic = o3d.camera.PinholeCameraIntrinsic( width, height, fx=z_scale, fy=z_scale, cx=width / 2.0, cy=height / 2.0, ) # Generate point cloud from RGBD image pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic) # Scale the Z dimension points = np.asarray(pcd.points) depth_scaled = ((raw_depth - raw_depth.min()) / (raw_depth.max() - raw_depth.min())) * (z_scale*100) z_values = depth_scaled.flatten()[:len(points)] points[:, 2] *= z_values pcd.points = o3d.utility.Vector3dVector(points) # Estimate and orient normals pcd.estimate_normals( search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=60) ) pcd.orient_normals_towards_camera_location(camera_location=np.array([0.0, 0.0, 1.5 ])) # Apply transformations pcd.transform([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]) pcd.transform([[-1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) # Perform Poisson surface reconstruction print(f"Running Poisson surface reconstruction with depth {depth}") mesh_raw, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson( pcd, depth=depth, width=0, scale=1.1, linear_fit=True ) print(f"Raw mesh vertices: {len(mesh_raw.vertices)}, triangles: {len(mesh_raw.triangles)}") # Simplify the mesh using vertex clustering voxel_size = max(mesh_raw.get_max_bound() - mesh_raw.get_min_bound()) / (max(width, height) * 0.8) mesh = mesh_raw.simplify_vertex_clustering( voxel_size=voxel_size, contraction=o3d.geometry.SimplificationContraction.Average, ) print(f"Simplified mesh vertices: {len(mesh.vertices)}, triangles: {len(mesh.triangles)}") # Crop the mesh to the bounding box of the point cloud bbox = pcd.get_axis_aligned_bounding_box() mesh_crop = mesh.crop(bbox) # Save the mesh as a GLTF file temp_dir = Path.cwd() / "models" temp_dir.mkdir(exist_ok=True) gltf_path = str(temp_dir / f"{image_path.stem}.gltf") o3d.io.write_triangle_mesh(gltf_path, mesh_crop, write_triangle_uvs=True) return gltf_path # Define Gradio interface components title = "Zero-Shot Depth Estimation with DPT + 3D Point Cloud" description = ( "This demo by Charles Fettinger is an update to the original " "DPT Demo. " "It uses the DPT model to predict the depth of an image and then uses 3D Point Cloud to create a 3D object." ) # Create Gradio sliders for resized_width and z_scale resized_width_slider = gr.Slider( minimum=256, maximum=1760, step=16, value=800, label="Resized Width", info="Resize the image based upon width, preserving the aspect ratio" ) z_scale_slider = gr.Slider( minimum=0.2, maximum=3.0, step=0.01, value=0.5, label="Z-Scale", info="Scale the amount of 3D model depth, short or tall (can distort)." ) examples = [["examples/" + img] for img in os.listdir("examples/")] process_image.zerogpu = True #gr.set_static_paths(paths=["models/","examples/"]) iface = gr.Interface( fn=process_image, inputs=[ gr.Image(type="filepath", label="Input Image"), resized_width_slider, z_scale_slider ], outputs=[ gr.Image(label="Predicted Depth", type="pil"), gr.Model3D(label="3D Mesh Reconstruction", clear_color=[1.0, 1.0, 1.0, 1.0]), gr.File(label="3D GLTF"), ], title=title, description=description, examples=examples, examples_per_page=15, flagging_mode=None, allow_flagging="never", cache_examples=False, delete_cache=(86400,86400), theme="Surn/Beeuty", show_progress = 'full' ) if __name__ == "__main__": iface.launch(debug=True, show_api=False, favicon_path="./favicon.ico", allowed_paths=["models/","examples/"])