point-cloud

app.py

@@ -2,14 +2,14 @@ import gradio as gr
 from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
 import supervision as sv
 from inference import DepthPredictor, SegmentPredictor
-from utils import create_3d_obj
+from utils import create_3d_obj, create_3d_pc
 import numpy as np
 
 
 def snap(image, video):
     depth_predictor = DepthPredictor()
     depth_result = depth_predictor.predict(image)
-    gltf_path = create_3d_obj(np.array(image), depth_result)
+    gltf_path = create_3d_pc(np.array(image), depth_result)
     #segment_predictor = SegmentPredictor()
     #sam_result = segment_predictor.predict(image)
     return [image, gltf_path, gltf_path]#[depth_result, gltf_path, gltf_path]

utils.py

@@ -50,4 +50,38 @@ def create_3d_obj(rgb_image, depth_image, depth=10):
     gltf_path = './image.gltf'
     o3d.io.write_triangle_mesh(
         gltf_path, mesh_crop, write_triangle_uvs=True)
-    return gltf_path
+    return gltf_path
+
+
+def create_3d_pc(rgb_image, depth_image, image_path, depth=10):
+    depth_image = depth_image.astype(np.float32)  # Convert depth map to float32
+    depth_o3d = o3d.geometry.Image(depth_image)
+    image_o3d = o3d.geometry.Image(rgb_image)
+    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
+        image_o3d, depth_o3d, convert_rgb_to_intensity=False)
+
+    w = int(depth_image.shape[1])
+    h = int(depth_image.shape[0])
+
+    # Specify camera intrinsic parameters (modify based on actual camera)
+    fx = 500
+    fy = 500
+    cx = w / 2
+    cy = h / 2
+
+    camera_intrinsic = o3d.camera.PinholeCameraIntrinsic(w, h, fx, fy, cx, cy)
+
+    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
+        rgbd_image, camera_intrinsic)
+
+    print('Estimating normals...')
+    pcd.estimate_normals(
+        search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30))
+    pcd.orient_normals_towards_camera_location(
+        camera_location=np.array([0., 0., 1000.]))
+
+    # Save the point cloud as a PLY file
+    ply_path = image_path.with_suffix('.ply')
+    o3d.io.write_point_cloud(str(ply_path), pcd)
+
+    return str(ply_path)  # Return the file path where the PLY file is saved