Spaces:
Runtime error
Runtime error
File size: 5,556 Bytes
30ded18 2479715 30ded18 2479715 30ded18 2479715 ee8887e 2479715 f055421 0bed186 2479715 0461f1c ee8887e 0461f1c 2479715 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
import gradio as gr
from transformers import DPTFeatureExtractor, DPTForDepthEstimation
import torch
import numpy as np
from PIL import Image
import open3d as o3d
from pathlib import Path
import os
feature_extractor = DPTFeatureExtractor.from_pretrained("Intel/dpt-large")
model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large")
def process_image(image_path, experiment_id):
image_path = Path(image_path)
image_raw = Image.open(image_path)
image = image_raw.resize(
(800, int(800 * image_raw.size[1] / image_raw.size[0])),
Image.Resampling.LANCZOS)
# prepare image for the model
encoding = feature_extractor(image, return_tensors="pt")
# forward pass
with torch.no_grad():
outputs = model(**encoding)
predicted_depth = outputs.predicted_depth
# interpolate to original size
prediction = torch.nn.functional.interpolate(
predicted_depth.unsqueeze(1),
size=image.size[::-1],
mode="bicubic",
align_corners=False,
).squeeze()
output = prediction.cpu().numpy()
depth_image = (output * 255 / np.max(output)).astype('uint8')
try:
gltf_path = create_3d_obj(np.array(image), depth_image, image_path)
img = Image.fromarray(depth_image)
return [img, gltf_path, gltf_path]
except Exception as e:
gltf_path = create_3d_obj(
np.array(image), depth_image, image_path, depth=8)
img = Image.fromarray(depth_image)
return [img, gltf_path, gltf_path]
except:
print("Error reconstructing 3D model")
raise Exception("Error reconstructing 3D model")
def create_3d_obj(rgb_image, depth_image, image_path, depth=10):
depth_o3d = o3d.geometry.Image(depth_image)
image_o3d = o3d.geometry.Image(rgb_image)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
image_o3d, depth_o3d, convert_rgb_to_intensity=False)
w = int(depth_image.shape[1])
h = int(depth_image.shape[0])
camera_intrinsic = o3d.camera.PinholeCameraIntrinsic()
camera_intrinsic.set_intrinsics(w, h, 500, 500, w/2, h/2)
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
rgbd_image, camera_intrinsic)
print('normals')
pcd.normals = o3d.utility.Vector3dVector(
np.zeros((1, 3))) # invalidate existing normals
pcd.estimate_normals(
search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30))
pcd.orient_normals_towards_camera_location(
camera_location=np.array([0., 0., 1000.]))
pcd.transform([[1, 0, 0, 0],
[0, -1, 0, 0],
[0, 0, -1, 0],
[0, 0, 0, 1]])
pcd.transform([[-1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1]])
print('run Poisson surface reconstruction')
with o3d.utility.VerbosityContextManager(o3d.utility.VerbosityLevel.Debug) as cm:
mesh_raw, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
pcd, depth=depth, width=0, scale=1.1, linear_fit=True)
voxel_size = max(mesh_raw.get_max_bound() - mesh_raw.get_min_bound()) / 256
print(f'voxel_size = {voxel_size:e}')
mesh = mesh_raw.simplify_vertex_clustering(
voxel_size=voxel_size,
contraction=o3d.geometry.SimplificationContraction.Average)
# vertices_to_remove = densities < np.quantile(densities, 0.001)
# mesh.remove_vertices_by_mask(vertices_to_remove)
bbox = pcd.get_axis_aligned_bounding_box()
mesh_crop = mesh.crop(bbox)
gltf_path = f'./{image_path.stem}.gltf'
o3d.io.write_triangle_mesh(
gltf_path, mesh_crop, write_triangle_uvs=True)
return gltf_path
title = "LIVE: Towards Layer-wise Image Vectorization (CVPR 2022 Oral)"
description = "This demo shows the effectiveness of LIVE <a href='' target='_blank'>Paper</a>. Given the input image, LIVE is able to progressively build the SVG output with a layer-wise representation." \
"<br>NOTE: for efficiency, we resize input images to 240x240 for Huggingface Space. "
examples = [["examples/" + img] for img in os.listdir("examples/")]
iface = gr.Interface(fn=process_image,
inputs=[gr.inputs.Image(type="filepath", label="Input Image"),
gr.inputs.Radio(
[
'add 5 paths one by one',
'add 8 paths one by one',
'add [1,3,5,7] paths',
'add [1,2,4,8,16,32,32,32...] total 128 paths',
'add [1,2,4,8,16,32,32,32...] total 256 paths',
],
type="value",
default='add 5 paths one by one',
label='experiment id'
)
],
outputs=[gr.outputs.Image(label="predicted depth", type="pil"),
gr.outputs.Image3D(label="3d mesh reconstruction", clear_color=[
1.0, 1.0, 1.0, 1.0]),
gr.outputs.File(label="3d gLTF")],
title=title,
description=description,
examples=examples,
allow_flagging="never")
iface.launch(debug=True, enable_queue=False, cache_examples=True)
|