import os import base64 import tempfile from typing import Any import torch import numpy as np from PIL import Image import gradio as gr import trimesh from transparent_background import Remover # Import and setup SPAR3D os.system("USE_CUDA=1 pip install -vv --no-build-isolation ./texture_baker ./uv_unwrapper") import spar3d.utils as spar3d_utils from spar3d.system import SPAR3D # Constants COND_WIDTH = 512 COND_HEIGHT = 512 COND_DISTANCE = 2.2 COND_FOVY = 0.591627 BACKGROUND_COLOR = [0.5, 0.5, 0.5] # Initialize models device = spar3d_utils.get_device() bg_remover = Remover() model = SPAR3D.from_pretrained( "stabilityai/stable-point-aware-3d", config_name="config.yaml", weight_name="model.safetensors" ).eval().to(device) # Initialize camera parameters c2w_cond = spar3d_utils.default_cond_c2w(COND_DISTANCE) intrinsic, intrinsic_normed_cond = spar3d_utils.create_intrinsic_from_fov_rad( COND_FOVY, COND_HEIGHT, COND_WIDTH ) def create_batch(input_image: Image) -> dict[str, Any]: """Prepare image batch for model input.""" img_cond = ( torch.from_numpy( np.asarray(input_image.resize((COND_WIDTH, COND_HEIGHT))).astype(np.float32) / 255.0 ) .float() .clip(0, 1) ) mask_cond = img_cond[:, :, -1:] rgb_cond = torch.lerp( torch.tensor(BACKGROUND_COLOR)[None, None, :], img_cond[:, :, :3], mask_cond ) batch = { "rgb_cond": rgb_cond.unsqueeze(0), "mask_cond": mask_cond.unsqueeze(0), "c2w_cond": c2w_cond.unsqueeze(0), "intrinsic_cond": intrinsic.unsqueeze(0), "intrinsic_normed_cond": intrinsic_normed_cond.unsqueeze(0), } return batch def process_image(image_base64: str) -> str: """Process image and return GLB as base64.""" try: # Decode base64 image image_data = base64.b64decode(image_base64) input_image = Image.open(tempfile.SpooledTemporaryFile(suffix='.png')) input_image.frombytes(image_data) # Remove background if needed if input_image.mode != 'RGBA': input_image = bg_remover.process(input_image.convert("RGB")) # Auto crop input_image = spar3d_utils.foreground_crop( input_image, crop_ratio=1.3, # Default padding ratio newsize=(COND_WIDTH, COND_HEIGHT), no_crop=False ) # Prepare batch batch = create_batch(input_image) batch = {k: v.to(device) for k, v in batch.items()} # Generate mesh with torch.no_grad(): with torch.autocast(device_type=device, dtype=torch.bfloat16) if "cuda" in device else nullcontext(): trimesh_mesh, _ = model.generate_mesh( batch, texture_resolution=1024, remesh="none", vertex_count=-1, estimate_illumination=False ) trimesh_mesh = trimesh_mesh[0] # Export to GLB temp_file = tempfile.NamedTemporaryFile(suffix='.glb', delete=False) trimesh_mesh.export(temp_file.name, file_type="glb", include_normals=True) # Convert to base64 with open(temp_file.name, 'rb') as f: glb_base64 = base64.b64encode(f.read()).decode('utf-8') # Cleanup os.unlink(temp_file.name) return glb_base64 except Exception as e: return str(e) # Create Gradio interface demo = gr.Interface( fn=process_image, inputs=gr.Text(label="Base64 Image"), outputs=gr.Text(label="Base64 GLB"), title="SPAR3D Image to GLB Converter", description="Upload a base64-encoded image and get back a base64-encoded GLB file" ) if __name__ == "__main__": demo.launch(share=False)