Spaces:
Sleeping
Sleeping
File size: 7,910 Bytes
b3da277 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 |
import numpy as np
import open3d as o3d
import open3d as o3d
import plotly.express as px
import numpy as np
import pandas as pd
from inference import DepthPredictor
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
def create_3d_obj(rgb_image, depth_image, depth=10, path="./image.gltf"):
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False
)
w = int(depth_image.shape[1])
h = int(depth_image.shape[0])
camera_intrinsic = o3d.camera.PinholeCameraIntrinsic()
camera_intrinsic.set_intrinsics(w, h, 500, 500, w / 2, h / 2)
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic)
print("normals")
pcd.normals = o3d.utility.Vector3dVector(
np.zeros((1, 3))
) # invalidate existing normals
pcd.estimate_normals(
search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30)
)
pcd.orient_normals_towards_camera_location(
camera_location=np.array([0.0, 0.0, 1000.0])
)
pcd.transform([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])
pcd.transform([[-1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
print("run Poisson surface reconstruction")
with o3d.utility.VerbosityContextManager(o3d.utility.VerbosityLevel.Debug) as cm:
mesh_raw, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
pcd, depth=depth, width=0, scale=1.1, linear_fit=True
)
voxel_size = max(mesh_raw.get_max_bound() - mesh_raw.get_min_bound()) / 256
print(f"voxel_size = {voxel_size:e}")
mesh = mesh_raw.simplify_vertex_clustering(
voxel_size=voxel_size,
contraction=o3d.geometry.SimplificationContraction.Average,
)
# vertices_to_remove = densities < np.quantile(densities, 0.001)
# mesh.remove_vertices_by_mask(vertices_to_remove)
bbox = pcd.get_axis_aligned_bounding_box()
mesh_crop = mesh.crop(bbox)
gltf_path = path
o3d.io.write_triangle_mesh(gltf_path, mesh_crop, write_triangle_uvs=True)
return gltf_path
def create_3d_pc(rgb_image, depth_image, depth=10):
depth_image = depth_image.astype(np.float32) # Convert depth map to float32
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False
)
w = int(depth_image.shape[1])
h = int(depth_image.shape[0])
# Specify camera intrinsic parameters (modify based on actual camera)
fx = 500
fy = 500
cx = w / 2
cy = h / 2
camera_intrinsic = o3d.camera.PinholeCameraIntrinsic(w, h, fx, fy, cx, cy)
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic)
print("Estimating normals...")
pcd.estimate_normals(
search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30)
)
pcd.orient_normals_towards_camera_location(
camera_location=np.array([0.0, 0.0, 1000.0])
)
# Save the point cloud as a PLY file
filename = "pc.pcd"
o3d.io.write_point_cloud(filename, pcd)
return filename # Return the file path where the PLY file is saved
def point_cloud(rgb_image):
depth_predictor = DepthPredictor()
depth_result = depth_predictor.predict(rgb_image)
# Step 2: Create an RGBD image from the RGB and depth image
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False
)
# Step 3: Create a PointCloud from the RGBD image
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
rgbd_image,
o3d.camera.PinholeCameraIntrinsic(
o3d.camera.PinholeCameraIntrinsicParameters.PrimeSenseDefault
),
)
# Step 4: Convert PointCloud data to a NumPy array
points = np.asarray(pcd.points)
colors = np.asarray(pcd.colors)
# Step 5: Create a DataFrame from the NumPy arrays
data = {
"x": points[:, 0],
"y": points[:, 1],
"z": points[:, 2],
"red": colors[:, 0],
"green": colors[:, 1],
"blue": colors[:, 2],
}
df = pd.DataFrame(data)
size = np.zeros(len(df))
size[:] = 0.01
# Step 6: Create a 3D scatter plot using Plotly Express
fig = px.scatter_3d(df, x="x", y="y", z="z", color="red", size=size)
return fig
def array_PCL(rgb_image, depth_image):
FX_RGB = 5.1885790117450188e02
FY_RGB = 5.1946961112127485e02
CX_RGB = 3.2558244941119034e0
CY_RGB = 2.5373616633400465e02
FX_DEPTH = FX_RGB
FY_DEPTH = FY_RGB
CX_DEPTH = CX_RGB
CY_DEPTH = CY_RGB
height = depth_image.shape[0]
width = depth_image.shape[1]
# compute indices:
jj = np.tile(range(width), height)
ii = np.repeat(range(height), width)
# Compute constants:
xx = (jj - CX_DEPTH) / FX_DEPTH
yy = (ii - CY_DEPTH) / FY_DEPTH
# transform depth image to vector of z:
length = height * width
z = depth_image.reshape(length)
# compute point cloud
pcd = np.dstack((xx * z, yy * z, z)).reshape((length, 3))
# cam_RGB = np.apply_along_axis(np.linalg.inv(R).dot, 1, pcd) - np.linalg.inv(R).dot(T)
xx_rgb = (
((rgb_image[:, 0] * FX_RGB) / rgb_image[:, 2] + CX_RGB + width / 2)
.astype(int)
.clip(0, width - 1)
)
yy_rgb = (
((rgb_image[:, 1] * FY_RGB) / rgb_image[:, 2] + CY_RGB)
.astype(int)
.clip(0, height - 1)
)
# colors = rgb_image[yy_rgb, xx_rgb]/255
return pcd # , colors
def generate_PCL(image):
depth_predictor = DepthPredictor()
depth_result = depth_predictor.predict(image)
image = np.array(image)
pcd = array_PCL(image, depth_result)
fig = px.scatter_3d(x=pcd[:, 0], y=pcd[:, 1], z=pcd[:, 2], size_max=0.01)
return fig
def plot_PCL(rgb_image, depth_image):
pcd, colors = array_PCL(rgb_image, depth_image)
fig = px.scatter_3d(
x=pcd[:, 0], y=pcd[:, 1], z=pcd[:, 2], color=colors, size_max=0.1
)
return fig
def PCL3(image):
depth_predictor = DepthPredictor()
depth_result = depth_predictor.predict(image)
image = np.array(image)
# Step 2: Create an RGBD image from the RGB and depth image
depth_o3d = o3d.geometry.Image(depth_result)
image_o3d = o3d.geometry.Image(image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False
)
# Step 3: Create a PointCloud from the RGBD image
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
rgbd_image,
o3d.camera.PinholeCameraIntrinsic(
o3d.camera.PinholeCameraIntrinsicParameters.PrimeSenseDefault
),
)
# Step 4: Convert PointCloud data to a NumPy array
vis = o3d.visualization.Visualizer()
vis.add_geometry(pcd)
# Step 4: Convert PointCloud data to a NumPy array
points = np.asarray(pcd.points)
colors = np.asarray(pcd.colors)
sizes = np.zeros(colors.shape)
sizes[:] = 0.01
colors = [tuple(c) for c in colors]
fig = plt.figure()
# ax = fig.add_subplot(111, projection='3d')
ax = Axes3D(fig)
print("plotting...")
ax.scatter(points[:, 0], points[:, 1], points[:, 2], c=colors, s=0.01)
print("Plot Succesful")
# data = {'x': points[:, 0], 'y': points[:, 1], 'z': points[:, 2], 'sizes': sizes[:, 0]}
# df = pd.DataFrame(data)
# Step 6: Create a 3D scatter plot using Plotly Express
# fig = px.scatter_3d(df, x='x', y='y', z='z', color=colors, size="sizes")
return fig
import numpy as np
|