Spaces:
Runtime error
Runtime error
File size: 6,042 Bytes
7598e8a a979122 1689431 7598e8a c1883e2 7598e8a c1883e2 7598e8a 9917d3b b7a1e25 9917d3b d140d89 9917d3b d140d89 a979122 4b43677 a979122 8f90f14 1689431 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 |
import numpy as np
import open3d as o3d
import open3d as o3d
import plotly.express as px
import numpy as np
import pandas as pd
from inference import DepthPredictor
def create_3d_obj(rgb_image, depth_image, depth=10, path='./image.gltf'):
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False)
w = int(depth_image.shape[1])
h = int(depth_image.shape[0])
camera_intrinsic = o3d.camera.PinholeCameraIntrinsic()
camera_intrinsic.set_intrinsics(w, h, 500, 500, w/2, h/2)
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
rgbd_image, camera_intrinsic)
print('normals')
pcd.normals = o3d.utility.Vector3dVector(
np.zeros((1, 3))) # invalidate existing normals
pcd.estimate_normals(
search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30))
pcd.orient_normals_towards_camera_location(
camera_location=np.array([0., 0., 1000.]))
pcd.transform([[1, 0, 0, 0],
[0, -1, 0, 0],
[0, 0, -1, 0],
[0, 0, 0, 1]])
pcd.transform([[-1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1]])
print('run Poisson surface reconstruction')
with o3d.utility.VerbosityContextManager(o3d.utility.VerbosityLevel.Debug) as cm:
mesh_raw, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
pcd, depth=depth, width=0, scale=1.1, linear_fit=True)
voxel_size = max(mesh_raw.get_max_bound() - mesh_raw.get_min_bound()) / 256
print(f'voxel_size = {voxel_size:e}')
mesh = mesh_raw.simplify_vertex_clustering(
voxel_size=voxel_size,
contraction=o3d.geometry.SimplificationContraction.Average)
# vertices_to_remove = densities < np.quantile(densities, 0.001)
# mesh.remove_vertices_by_mask(vertices_to_remove)
bbox = pcd.get_axis_aligned_bounding_box()
mesh_crop = mesh.crop(bbox)
gltf_path = path
o3d.io.write_triangle_mesh(
gltf_path, mesh_crop, write_triangle_uvs=True)
return gltf_path
def create_3d_pc(rgb_image, depth_image, depth=10):
depth_image = depth_image.astype(np.float32) # Convert depth map to float32
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False)
w = int(depth_image.shape[1])
h = int(depth_image.shape[0])
# Specify camera intrinsic parameters (modify based on actual camera)
fx = 500
fy = 500
cx = w / 2
cy = h / 2
camera_intrinsic = o3d.camera.PinholeCameraIntrinsic(w, h, fx, fy, cx, cy)
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
rgbd_image, camera_intrinsic)
print('Estimating normals...')
pcd.estimate_normals(
search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30))
pcd.orient_normals_towards_camera_location(
camera_location=np.array([0., 0., 1000.]))
# Save the point cloud as a PLY file
filename = "pc.pcd"
o3d.io.write_point_cloud(filename, pcd)
return filename # Return the file path where the PLY file is saved
def point_cloud(rgb_image, depth_image):
# Step 2: Create an RGBD image from the RGB and depth image
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(image_o3d, depth_o3d, convert_rgb_to_intensity=False)
# Step 3: Create a PointCloud from the RGBD image
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, o3d.camera.PinholeCameraIntrinsic(o3d.camera.PinholeCameraIntrinsicParameters.PrimeSenseDefault))
# Step 4: Convert PointCloud data to a NumPy array
points = np.asarray(pcd.points)
colors = np.asarray(pcd.colors)
# Step 5: Create a DataFrame from the NumPy arrays
data = {'x': points[:, 0], 'y': points[:, 1], 'z': points[:, 2],
'red': colors[:, 0], 'green': colors[:, 1], 'blue': colors[:, 2]}
df = pd.DataFrame(data)
# Step 6: Create a 3D scatter plot using Plotly Express
fig = px.scatter_3d(df, x='x', y='y', z='z', color='red', size_max=0.1)
return fig
def array_PCL(rgb_image, depth_image):
FX_RGB = 5.1885790117450188e+02
FY_RGB = 5.1946961112127485e+02
CX_RGB = 3.2558244941119034e+0
CY_RGB = 2.5373616633400465e+02
FX_DEPTH = FX_RGB
FY_DEPTH = FY_RGB
CX_DEPTH = CX_RGB
CY_DEPTH = CY_RGB
height = depth_image.shape[0]
width = depth_image.shape[1]
# compute indices:
jj = np.tile(range(width), height)
ii = np.repeat(range(height), width)
# Compute constants:
xx = (jj - CX_DEPTH) / FX_DEPTH
yy = (ii - CY_DEPTH) / FY_DEPTH
# transform depth image to vector of z:
length = height * width
z = depth_image.reshape(length)
# compute point cloud
pcd = np.dstack((xx * z, yy * z, z)).reshape((length, 3))
#cam_RGB = np.apply_along_axis(np.linalg.inv(R).dot, 1, pcd) - np.linalg.inv(R).dot(T)
xx_rgb = ((rgb_image[:, 0] * FX_RGB) / rgb_image[:, 2] + CX_RGB + width / 2).astype(int).clip(0, width - 1)
yy_rgb = ((rgb_image[:, 1] * FY_RGB) / rgb_image[:, 2] + CY_RGB).astype(int).clip(0, height - 1)
colors = rgb_image[yy_rgb, xx_rgb]/255
return pcd, colors
def generate_PCL(image):
depth_predictor = DepthPredictor()
depth_result = depth_predictor.predict(image)
pcd, colors = array_PCL(image, depth_result)
fig = px.scatter_3d(x=pcd[:, 0], y=pcd[:, 1], z=pcd[:, 2], color=colors, size_max=0.1)
return fig
def plot_PCL(rgb_image, depth_image):
pcd, colors = array_PCL(rgb_image, depth_image)
fig = px.scatter_3d(x=pcd[:, 0], y=pcd[:, 1], z=pcd[:, 2], color=colors, size_max=0.1)
return fig |