initial 3d-re files

app.py

@@ -0,0 +1,14 @@
+import gradio as gr
+from depth_detection import depth_detection
+from mesh_network import mesh_network
+from point_cloud import point_cloud
+
+def reconstruction3d(input_image):
+
+    new_image, output = depth_detection(input_image,pad=16)
+    pcd_img = point_cloud(new_image, output)
+    output_mesh = mesh_network(pcd_img)
+    return output_mesh
+
+iface = gr.Interface(fn=reconstruction3d, inputs="image", outputs="image")
+iface.launch()

depth_detection.py

@@ -0,0 +1,18 @@
+import torch
+from transformers import GLPNImageProcessor, GLPNForDepthEstimation
+from image_resize import resize_img
+
+def depth_detection(image, pad=16):
+
+    feature_extractor = GLPNImageProcessor.from_pretrained("vinvino02/glpn-nyu")
+    model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu")
+    new_img = resize_img(image)
+    inputs = feature_extractor(images=new_img, return_tensors="pt")
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        predicted_depth = outputs.predicted_depth
+
+    output = predicted_depth.squeeze().cpu().numpy() * 1000.0
+    output = output[pad:-pad, pad:-pad]
+    return new_img,output

image_resize.py

@@ -0,0 +1,10 @@
+def resize_img(image):
+    new_height = 480 if image.height > 480 else image.height
+    new_height -= (new_height % 32)
+    new_width = int(new_height * image.width / image.height)
+    diff = new_width % 32
+    new_width = new_width - diff if diff < 16 else new_width + 32 - diff
+    new_size = (new_width, new_height)
+    image = image.resize(new_size)
+
+    return image

mesh_network.py

@@ -0,0 +1,18 @@
+import numpy as np
+import open3d as o3d
+
+def mesh_network(point_cloud):
+
+    pcd = point_cloud
+    cl, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=20.0)
+    pcd = pcd.select_by_index(ind)
+
+    pcd.estimate_normals()
+    pcd.orient_normals_to_align_with_direction()
+
+    mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=12, n_threads=1)[0]
+    rotation = mesh.get_rotation_matrix_from_xyz((np.pi, 0, 0))
+    mesh.rotate(rotation, center=(0, 0, 0))
+    o3d.io.write_triangle_mesh(f'./mesh_depth12.obj', mesh)
+
+    return mesh

point_cloud.py

@@ -0,0 +1,20 @@
+import numpy as np
+import open3d as o3d
+
+def point_cloud(image, dp_image):
+
+    width, height = image.size
+
+    depth_image = (dp_image * 255 / np.max(dp_image)).astype('uint8')
+    image = np.array(image)
+
+    depth_o3d = o3d.geometry.Image(depth_image)
+    image_o3d = o3d.geometry.Image(image)
+    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(image_o3d, depth_o3d, convert_rgb_to_intensity=False)
+
+    camera_intrinsic = o3d.camera.PinholeCameraIntrinsic()
+    camera_intrinsic.set_intrinsics(width, height, 500, 500, width/2, height/2)
+
+    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic)
+    o3d.io.write_point_cloud("output_donut.ply", pcd)
+    return pcd