diff --git a/LHM/__pycache__/__init__.cpython-310.pyc b/LHM/__pycache__/__init__.cpython-310.pyc
index 0c803f3a225c2fce2481220c11b979b25e9f99e6..8cc17c1c32a0a2309ece27dad6948c9dabff4527 100644
Binary files a/LHM/__pycache__/__init__.cpython-310.pyc and b/LHM/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/datasets/__pycache__/__init__.cpython-310.pyc b/LHM/datasets/__pycache__/__init__.cpython-310.pyc
index 3daadbd8d38a642cbc6b9b1b0922c9cabd12654c..88a182922209087636cf6d4fbc18eaae77970bdb 100644
Binary files a/LHM/datasets/__pycache__/__init__.cpython-310.pyc and b/LHM/datasets/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc b/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc
index c544b69ddfd4d20701c43b0ff80c1574dcb358cf..051133a02d98a9e573c6e62e0f4c568fa2c74a70 100644
Binary files a/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc and b/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc differ
diff --git a/LHM/datasets/__pycache__/mixer.cpython-310.pyc b/LHM/datasets/__pycache__/mixer.cpython-310.pyc
index ce221761110fad472b0ff6a2edfd97a7c34d41c4..084687a91fba917611e9ec1c75a3491b76851343 100644
Binary files a/LHM/datasets/__pycache__/mixer.cpython-310.pyc and b/LHM/datasets/__pycache__/mixer.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc b/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc
index b1d21eff8d1c75aacb6ac1647038022327f89d04..c9664ecb175cee3ce70f145063437157b6e1b70f 100644
Binary files a/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc and b/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/__init__.cpython-310.pyc b/LHM/models/__pycache__/__init__.cpython-310.pyc
index ee964eb382708d44566c1a1e3dcf4f8d811bfa6e..71cb36df64a56747064973d3f4911811fe0b0ecb 100644
Binary files a/LHM/models/__pycache__/__init__.cpython-310.pyc and b/LHM/models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/arcface_utils.cpython-310.pyc b/LHM/models/__pycache__/arcface_utils.cpython-310.pyc
index 918310444421a8d3a5fa48610c43aca7b3b4c399..45deda4f1ee7ab373095ce6216caee2985697789 100644
Binary files a/LHM/models/__pycache__/arcface_utils.cpython-310.pyc and b/LHM/models/__pycache__/arcface_utils.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/embedder.cpython-310.pyc b/LHM/models/__pycache__/embedder.cpython-310.pyc
index b8286e8990552f59c7b2febb6784c9ffd1a5c7ac..515eeeb446ba0b187a900d74774155ff70535ecf 100644
Binary files a/LHM/models/__pycache__/embedder.cpython-310.pyc and b/LHM/models/__pycache__/embedder.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc b/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc
index 87bfffd653c15f4c65968b511449b2f098905399..0ce3402cd4d1cdbd67811ad713038d4745f544c3 100644
Binary files a/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc and b/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/transformer.cpython-310.pyc b/LHM/models/__pycache__/transformer.cpython-310.pyc
index 6fa52d1a979dc61aac4efcf10a4ffb1fffdc70c5..cdc53869967c9103bf68d68a045ee3cf86f42383 100644
Binary files a/LHM/models/__pycache__/transformer.cpython-310.pyc and b/LHM/models/__pycache__/transformer.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/transformer_dit.cpython-310.pyc b/LHM/models/__pycache__/transformer_dit.cpython-310.pyc
index 53fb80d90c57e8d51fb0d987e1028cb470f0efca..44ca673da5f40a985e17eff02ee14bc92e61ab88 100644
Binary files a/LHM/models/__pycache__/transformer_dit.cpython-310.pyc and b/LHM/models/__pycache__/transformer_dit.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/utils.cpython-310.pyc b/LHM/models/__pycache__/utils.cpython-310.pyc
index 5da13881b53ed53c479210d3da3d149cdfdb28ac..5d7c1fcab28b461b2ceac30eb8a43b7424f4e4b8 100644
Binary files a/LHM/models/__pycache__/utils.cpython-310.pyc and b/LHM/models/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc
index 2a3fabeeecd4c5c46b57795df41927a404745ffd..8cb55e47ffa256976807b5e48f297cdba48adfcd 100644
Binary files a/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc b/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc
index 1dd76881592f8c3fcc42c73c4507ba5f49f27e47..792907277f9df41bf0d73673c89423a0636a1f7a 100644
Binary files a/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc and b/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc differ
diff --git a/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc b/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc
index 8f1e68efc91c5590eb9f93060f55b96665f58fa0..bd0433b1241136e7f36b72628e13076242a038ad 100644
Binary files a/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc and b/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc
index eda3789eecdbca8b2e66fc0b6c54de2880dbfea2..6e92d765f701924db46c0b1f8677cc96e99420ee 100644
Binary files a/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc
index 39a1a5cdb91b89ac3dfe2c4d0cf20e93be475062..aef519a379beb67ba5a6c86897ed37eeacfb659f 100644
Binary files a/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc b/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc
index d4f8d50bbd81bce4a739bc542fc4146689e2d1ab..feacc1792ffbe4b0fe6f0378d1667172d30c7e0f 100644
Binary files a/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc and b/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc b/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc
index 3def0e0eeeaddff9a927965b76ef14b87947ee9e..ab5ecb1b51d618fd3c35a1934ed320bf5e222c43 100644
Binary files a/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc and b/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc
index 0ef26a81331d5ba4ffac8cfc264f14dab43bbdcc..871fa70f7edd236f252ec20a9ca65d946573f3fe 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc
index 9e6b22f8082c3771b88b434020062dee6d821cee..b125bc0a3a5550724ca9deb0357deb0f88ae91d7 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc
index c4be456f6ae50e1d45c568cff5010cd2acd8458e..c1649b79909a1ca08d1b165974942f40d3ebb697 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc
index c212f98eff800a8905d213176bd5bcd5db89d495..56a8cb7bbb3ff6ec1f08348761e727fc7f3f6891 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc
index f2112186e397f8e40039e0bf690b481fc20cc0a8..be75f08dbd36f623603b24dfbd59fe021fd5d196 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc
index ee8289876a5a4f4db74f1c30ccbb366f95cb897c..e9c9b84c963a4f75938e501b4001d73042718bcc 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc
index b545e879ef03bce565ad9f6c5be889b06dba6ac7..c7d94312d2420deebe383779dc890c3ba360646e 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc
index abfc52222787e11a589e0d90bc04f3977e9c9412..fda5ac4b5513879a284874e11809717ae1565e56 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc
index 338912b657173d5d5fc70148b10433ddd4ca773c..90d3e3f9580cf9187810fac783c439ff2829cc68 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc
index ab29cf0203c66b883d323e69835c1b41e0438c68..0759c3676f32ae59ba02b93b5198e84fab80851b 100644
Binary files a/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc b/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc
index 7cca874973e78a5e725dc9e1b72c08afad6863d2..6f53f7abca6a50156d17ade3fe6fd22e8a2f8c08 100644
Binary files a/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc and b/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc b/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc
index 93b9174a17f9e95ccc7cd5818be44e96a7e49eb9..ed57afaabfc2003e7ffd2eff7d946edc69ba0bdb 100644
Binary files a/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc and b/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc b/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc
index 56b68659e92637c14d6dd86a9efc0a9865942760..18929d48120239f84d57a04fd06ef273b29849f5 100644
Binary files a/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc and b/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc b/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc
index b38b0c1a775f659eb24f21503e7b8c4c6bba8914..d95f53467b0faddcad40004608c3c9c1bb3f7f5a 100644
Binary files a/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc and b/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc b/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc
index 6a711eca87c4123c0721b35aa66684ef236b1654..f5c98c01875a644d7d72fc136f67f39065c79bbd 100644
Binary files a/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc and b/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc b/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc
index adc5205064b76c441c5acd7cb4b8162c49293f89..d2ea2f50aa1348e2a8a7f97e402e78395b206aa4 100644
Binary files a/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc and b/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc b/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc
index 2b73e6d3101093cf1e03e8811e162273048c1a39..c9b7d1fa611f39e2dfae5dbbea770235c634253e 100644
Binary files a/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc and b/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc b/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc
index 513c995ff7f7fda0b7ce5512aa7d6e75cab81daf..6702bdfc0b5cb3a252c9990014d8af5d697955a4 100644
Binary files a/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc and b/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/gs_renderer.py b/LHM/models/rendering/gs_renderer.py
index 962a6b35bc877744106d5be7f95d7f7790d09aa8..debf9f600d6ec47fc4c89d8dfebb994cc3aad64f 100644
--- a/LHM/models/rendering/gs_renderer.py
+++ b/LHM/models/rendering/gs_renderer.py
@@ -829,14 +829,15 @@ class GS3DRenderer(nn.Module):
# Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means
screenspace_points = (
torch.zeros_like(
- gs.xyz, dtype=gs.xyz.dtype, requires_grad=False, device=self.device
+ gs.xyz, dtype=gs.xyz.dtype, requires_grad=True, device=self.device
)
+ 0
)
- # try:
- # screenspace_points.retain_grad()
- # except:
- # pass
+
+ try:
+ screenspace_points.retain_grad()
+ except:
+ pass
bg_color = background_color
# Set up rasterization configuration
@@ -855,7 +856,7 @@ class GS3DRenderer(nn.Module):
sh_degree=self.sh_degree,
campos=viewpoint_camera.camera_center,
prefiltered=False,
- debug=False,
+ debug=True,
)
rasterizer = GaussianRasterizer(raster_settings=raster_settings)
@@ -883,11 +884,13 @@ class GS3DRenderer(nn.Module):
colors_precomp = None
shs = gs.shs
+ # print(shs, colors_precomp)
+ # print(means3D.device, means2D.device, opacity.device, rotations.device, self.device)
+ # print(means3D.dtype, means2D.dtype, rotations.dtype, opacity.dtype)
+ # print(means3D.shape, means2D.shape, rotations.shape, opacity.shape)
# Rasterize visible Gaussians to image, obtain their radii (on screen).
# NOTE that dadong tries to regress rgb not shs
# with torch.autocast(device_type=self.device.type, dtype=torch.float32):
- print(means3D.device, means2D.device, colors_precomp.device, opacity.device, rotations.device, self.device)
- print(means3D.dtype, means2D.dtype, colors_precomp.dtype)
rendered_image, radii, rendered_depth, rendered_alpha = rasterizer(
means3D=means3D,
means2D=means2D,
@@ -937,7 +940,7 @@ class GS3DRenderer(nn.Module):
# ret["comp_mask"] = rendered_mask.permute(1, 2, 0)
return ret
- @torch.no_grad()
+
def animate_gs_model(
self, gs_attr: GaussianAppOutput, query_points, smplx_data, debug=False
):
@@ -1066,7 +1069,7 @@ class GS3DRenderer(nn.Module):
gs_list.append(gs_copy)
return gs_list, cano_gs_list
- @torch.no_grad()
+
def forward_gs_attr(self, x, query_points, smplx_data, debug=False, x_fine=None):
"""
x: [N, C] Float[Tensor, "Np Cp"],
@@ -1083,7 +1086,7 @@ class GS3DRenderer(nn.Module):
gs_attr: GaussianAppOutput = self.gs_net(x, query_points, x_fine)
return gs_attr
- @torch.no_grad()
+
def get_query_points(self, smplx_data, device):
with torch.no_grad():
with torch.autocast(device_type=device.type, dtype=torch.float32):
@@ -1115,7 +1118,7 @@ class GS3DRenderer(nn.Module):
pcl_embed.to(dtype=latent_feat.dtype), latent_feat, extra_info
)
return gs_feats
- @torch.no_grad()
+
def query_latent_feat(
self,
positions: Float[Tensor, "*B N1 3"],
@@ -1140,7 +1143,7 @@ class GS3DRenderer(nn.Module):
)
return gs_feats, positions, smplx_data
- @torch.no_grad()
+
def forward_single_batch(
self,
gs_list: list[GaussianModel],
@@ -1183,7 +1186,7 @@ class GS3DRenderer(nn.Module):
return out
- @torch.no_grad()
+
def forward_cano_batch(
self,
gs_list: list[GaussianModel],
@@ -1289,7 +1292,7 @@ class GS3DRenderer(nn.Module):
:, vidx : vidx + 1
] # e.g. body_pose: [1, N_v, 21, 3] -> [1, 1, 21, 3]
return smpl_data_single_view
- @torch.no_grad()
+
def forward_gs(
self,
gs_hidden_features: Float[Tensor, "B Np Cp"],
@@ -1397,7 +1400,7 @@ class GS3DRenderer(nn.Module):
) # [B, NV, H, W, 3] -> [B, NV, 1, H, W]
return out
- @torch.no_grad()
+
def forward(
self,
gs_hidden_features: Float[Tensor, "B Np Cp"],
diff --git a/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc
index bd272672f473dec0c9e00a7d74f041ebb5463606..e069f5599d5181e82b2d9597a58e9fb7dbcf46a8 100644
Binary files a/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc
index 925c7329ea83a5f48d19734ccc9d0477de43eab4..4f4e5128f5a04d00e5a40d82638db0f8bb71f084 100644
Binary files a/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc
index 405eaa92bf6c22723a4a3a9d78724f55567417de..956fb914c7e64c25be4d2569c6f0c0bc6bb0621b 100644
Binary files a/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc
index 702bdcb0f7196759f00d71a5116f1fc786c24f9e..954195223a88280232b6bb27a23dcf8af9050e59 100644
Binary files a/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc
index 846e7f156719e7678ad636eda399d502f0ce64de..d28996f6bce1d0a4f057f42d12bf0a083a1e9adf 100644
Binary files a/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc
index 8ca713402f98961f63de80772159f3c75a257ad0..d9a9b7e6fd964e1066359a5f47cbdb2abaa3d513 100644
Binary files a/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc
index 41302159e6b94e811ec1b9f8c497ca90d3bc52af..809b9da80ee932436203cc88b6fda1793c1dcc75 100644
Binary files a/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc
index 9dabbe1090995c2712e344c353946c23bc404a56..d0c3f9eba6356901b06b4c5aa4afe450f537f36e 100644
Binary files a/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/outputs/__pycache__/base.cpython-310.pyc b/LHM/outputs/__pycache__/base.cpython-310.pyc
index 659870af7755ad5eaf3ea72e71293a98d89059d3..fcb721ce5675c33ee1bcbabb2951c13c90e14009 100644
Binary files a/LHM/outputs/__pycache__/base.cpython-310.pyc and b/LHM/outputs/__pycache__/base.cpython-310.pyc differ
diff --git a/LHM/outputs/__pycache__/output.cpython-310.pyc b/LHM/outputs/__pycache__/output.cpython-310.pyc
index 7c190d2d1756bbf9f48cc588849e7e1c75a9168a..70498c7b9ce18437d1de8a6723ef800fc14c4887 100644
Binary files a/LHM/outputs/__pycache__/output.cpython-310.pyc and b/LHM/outputs/__pycache__/output.cpython-310.pyc differ
diff --git a/LHM/runners/__pycache__/__init__.cpython-310.pyc b/LHM/runners/__pycache__/__init__.cpython-310.pyc
index b060ba3788d899bc1d23c743230fc55467335d84..61774f01ef4d3b2383b675be33e4fe0209aa82e0 100644
Binary files a/LHM/runners/__pycache__/__init__.cpython-310.pyc and b/LHM/runners/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/runners/__pycache__/abstract.cpython-310.pyc b/LHM/runners/__pycache__/abstract.cpython-310.pyc
index 670a8bc5af4f6cdd97f1a9b78f8e6305cff71000..5167786d224a9562c221edd69970afc5861ba0c9 100644
Binary files a/LHM/runners/__pycache__/abstract.cpython-310.pyc and b/LHM/runners/__pycache__/abstract.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc b/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc
index 950db5b426678ba25ef22d0a9ec7ca391ef269d7..ab728ac5350ff43a73e2d54e31d566ce680e713b 100644
Binary files a/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc and b/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc b/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc
index cd448a4a63a54616d0e2e0a34f107f0e2b8eb884..e364332b0a28e0a4d025a9401a44c83ae29dec17 100644
Binary files a/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc and b/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc b/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc
index 3ef631994eadbae02765b1ba43611bfec4bd52af..0f516da3931786f74bc27af31e6258e3c6a45164 100644
Binary files a/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc and b/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/utils.cpython-310.pyc b/LHM/runners/infer/__pycache__/utils.cpython-310.pyc
index 37bfc8efe26cd74101a96b818e9555f4359c5ca2..b5dbb1bca7fffa6ca35bb693bfabee77f8e6ae33 100644
Binary files a/LHM/runners/infer/__pycache__/utils.cpython-310.pyc and b/LHM/runners/infer/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/__init__.cpython-310.pyc b/LHM/utils/__pycache__/__init__.cpython-310.pyc
index 177b34f106b777c9fe551e6203f073243a8a6227..027f4cc03eba7247b5eab202af47fb3ed9a25080 100644
Binary files a/LHM/utils/__pycache__/__init__.cpython-310.pyc and b/LHM/utils/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/face_detector.cpython-310.pyc b/LHM/utils/__pycache__/face_detector.cpython-310.pyc
index 210280b3797f674896f0aa81723f276b993eae57..110ad03a54ea6e6bdd1ea45bbcaa82f6a6a86a3f 100644
Binary files a/LHM/utils/__pycache__/face_detector.cpython-310.pyc and b/LHM/utils/__pycache__/face_detector.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc b/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc
index 72db6324131c5f6e854f5c0f0f64587998c447b1..999087705d342e19acc83506c306c7370330e687 100644
Binary files a/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc and b/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/hf_hub.cpython-310.pyc b/LHM/utils/__pycache__/hf_hub.cpython-310.pyc
index 65951d3b611b5ef20bc38540e8dddb31edb79c35..cbdbb9d2e32cb38f3919320eabe574002a47c7a1 100644
Binary files a/LHM/utils/__pycache__/hf_hub.cpython-310.pyc and b/LHM/utils/__pycache__/hf_hub.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/logging.cpython-310.pyc b/LHM/utils/__pycache__/logging.cpython-310.pyc
index b060540c1a6f5493a4de11f634d4141c4624d4e0..7b6c6068ae074b74d26f1bb72d3c2b71cdba9318 100644
Binary files a/LHM/utils/__pycache__/logging.cpython-310.pyc and b/LHM/utils/__pycache__/logging.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/registry.cpython-310.pyc b/LHM/utils/__pycache__/registry.cpython-310.pyc
index 5468c597f6cc4e6d0b28b6fca89f626bd6c26ff9..1364412db53c44d6314fe32cda7b69f1e80b55e9 100644
Binary files a/LHM/utils/__pycache__/registry.cpython-310.pyc and b/LHM/utils/__pycache__/registry.cpython-310.pyc differ
diff --git a/app.py b/app.py
index c48d898a023df04b3c90d0c38291eab86c12c119..a6b526a241e84c029a50546e53de57fdf08384a6 100644
--- a/app.py
+++ b/app.py
@@ -13,772 +13,777 @@
# limitations under the License.
-# import os
-# os.system("rm -rf /data-nvme/zerogpu-offload/")
-# import cv2
-# import time
-# from PIL import Image
-# import numpy as np
-# import gradio as gr
-# import base64
-# import spaces
-# import torch
-# torch._dynamo.config.disable = True
-# import subprocess
-# import os
-# import argparse
-# from omegaconf import OmegaConf
-# from rembg import remove
-# from engine.pose_estimation.pose_estimator import PoseEstimator
-# from LHM.utils.face_detector import VGGHeadDetector
-# from LHM.utils.hf_hub import wrap_model_hub
-# from LHM.runners.infer.utils import (
-# calc_new_tgt_size_by_aspect,
-# center_crop_according_to_mask,
-# prepare_motion_seqs,
-# resize_image_keepaspect_np,
-# )
-# from engine.SegmentAPI.base import Bbox
-
-# def get_bbox(mask):
-# height, width = mask.shape
-# pha = mask / 255.0
-# pha[pha < 0.5] = 0.0
-# pha[pha >= 0.5] = 1.0
-
-# # obtain bbox
-# _h, _w = np.where(pha == 1)
-
-# whwh = [
-# _w.min().item(),
-# _h.min().item(),
-# _w.max().item(),
-# _h.max().item(),
-# ]
-
-# box = Bbox(whwh)
-
-# # scale box to 1.05
-# scale_box = box.scale(1.1, width=width, height=height)
-# return scale_box
-
-# def infer_preprocess_image(
-# rgb_path,
-# mask,
-# intr,
-# pad_ratio,
-# bg_color,
-# max_tgt_size,
-# aspect_standard,
-# enlarge_ratio,
-# render_tgt_size,
-# multiply,
-# need_mask=True,
-# ):
-# """inferece
-# image, _, _ = preprocess_image(image_path, mask_path=None, intr=None, pad_ratio=0, bg_color=1.0,
-# max_tgt_size=896, aspect_standard=aspect_standard, enlarge_ratio=[1.0, 1.0],
-# render_tgt_size=source_size, multiply=14, need_mask=True)
-
-# """
-
-# rgb = np.array(Image.open(rgb_path))
-# rgb_raw = rgb.copy()
-
-# bbox = get_bbox(mask)
-# bbox_list = bbox.get_box()
-
-# rgb = rgb[bbox_list[1] : bbox_list[3], bbox_list[0] : bbox_list[2]]
-# mask = mask[bbox_list[1] : bbox_list[3], bbox_list[0] : bbox_list[2]]
-
-# h, w, _ = rgb.shape
-# assert w < h
-# cur_ratio = h / w
-# scale_ratio = cur_ratio / aspect_standard
-
-# target_w = int(min(w * scale_ratio, h))
-# offset_w = (target_w - w) // 2
-# # resize to target ratio.
-# if offset_w > 0:
-# rgb = np.pad(
-# rgb,
-# ((0, 0), (offset_w, offset_w), (0, 0)),
-# mode="constant",
-# constant_values=255,
-# )
-# mask = np.pad(
-# mask,
-# ((0, 0), (offset_w, offset_w)),
-# mode="constant",
-# constant_values=0,
-# )
-# else:
-# offset_w = -offset_w
-# rgb = rgb[:,offset_w:-offset_w,:]
-# mask = mask[:,offset_w:-offset_w]
-
-# # resize to target ratio.
-
-# rgb = np.pad(
-# rgb,
-# ((0, 0), (offset_w, offset_w), (0, 0)),
-# mode="constant",
-# constant_values=255,
-# )
-
-# mask = np.pad(
-# mask,
-# ((0, 0), (offset_w, offset_w)),
-# mode="constant",
-# constant_values=0,
-# )
-
-# rgb = rgb / 255.0 # normalize to [0, 1]
-# mask = mask / 255.0
-
-# mask = (mask > 0.5).astype(np.float32)
-# rgb = rgb[:, :, :3] * mask[:, :, None] + bg_color * (1 - mask[:, :, None])
-
-# # resize to specific size require by preprocessor of smplx-estimator.
-# rgb = resize_image_keepaspect_np(rgb, max_tgt_size)
-# mask = resize_image_keepaspect_np(mask, max_tgt_size)
-
-# # crop image to enlarge human area.
-# rgb, mask, offset_x, offset_y = center_crop_according_to_mask(
-# rgb, mask, aspect_standard, enlarge_ratio
-# )
-# if intr is not None:
-# intr[0, 2] -= offset_x
-# intr[1, 2] -= offset_y
-
-# # resize to render_tgt_size for training
-
-# tgt_hw_size, ratio_y, ratio_x = calc_new_tgt_size_by_aspect(
-# cur_hw=rgb.shape[:2],
-# aspect_standard=aspect_standard,
-# tgt_size=render_tgt_size,
-# multiply=multiply,
-# )
-
-# rgb = cv2.resize(
-# rgb, dsize=(tgt_hw_size[1], tgt_hw_size[0]), interpolation=cv2.INTER_AREA
-# )
-# mask = cv2.resize(
-# mask, dsize=(tgt_hw_size[1], tgt_hw_size[0]), interpolation=cv2.INTER_AREA
-# )
-
-# if intr is not None:
-
-# # ******************** Merge *********************** #
-# intr = scale_intrs(intr, ratio_x=ratio_x, ratio_y=ratio_y)
-# assert (
-# abs(intr[0, 2] * 2 - rgb.shape[1]) < 2.5
-# ), f"{intr[0, 2] * 2}, {rgb.shape[1]}"
-# assert (
-# abs(intr[1, 2] * 2 - rgb.shape[0]) < 2.5
-# ), f"{intr[1, 2] * 2}, {rgb.shape[0]}"
-
-# # ******************** Merge *********************** #
-# intr[0, 2] = rgb.shape[1] // 2
-# intr[1, 2] = rgb.shape[0] // 2
-
-# rgb = torch.from_numpy(rgb).float().permute(2, 0, 1).unsqueeze(0) # [1, 3, H, W]
-# mask = (
-# torch.from_numpy(mask[:, :, None]).float().permute(2, 0, 1).unsqueeze(0)
-# ) # [1, 1, H, W]
-# return rgb, mask, intr
-
-# def parse_configs():
-
-# parser = argparse.ArgumentParser()
-# parser.add_argument("--config", type=str)
-# parser.add_argument("--infer", type=str)
-# args, unknown = parser.parse_known_args()
-
-# cfg = OmegaConf.create()
-# cli_cfg = OmegaConf.from_cli(unknown)
-
-# # parse from ENV
-# if os.environ.get("APP_INFER") is not None:
-# args.infer = os.environ.get("APP_INFER")
-# if os.environ.get("APP_MODEL_NAME") is not None:
-# cli_cfg.model_name = os.environ.get("APP_MODEL_NAME")
-
-# args.config = args.infer if args.config is None else args.config
-
-# if args.config is not None:
-# cfg_train = OmegaConf.load(args.config)
-# cfg.source_size = cfg_train.dataset.source_image_res
-# try:
-# cfg.src_head_size = cfg_train.dataset.src_head_size
-# except:
-# cfg.src_head_size = 112
-# cfg.render_size = cfg_train.dataset.render_image.high
-# _relative_path = os.path.join(
-# cfg_train.experiment.parent,
-# cfg_train.experiment.child,
-# os.path.basename(cli_cfg.model_name).split("_")[-1],
-# )
-
-# cfg.save_tmp_dump = os.path.join("exps", "save_tmp", _relative_path)
-# cfg.image_dump = os.path.join("exps", "images", _relative_path)
-# cfg.video_dump = os.path.join("exps", "videos", _relative_path) # output path
-
-# if args.infer is not None:
-# cfg_infer = OmegaConf.load(args.infer)
-# cfg.merge_with(cfg_infer)
-# cfg.setdefault(
-# "save_tmp_dump", os.path.join("exps", cli_cfg.model_name, "save_tmp")
-# )
-# cfg.setdefault("image_dump", os.path.join("exps", cli_cfg.model_name, "images"))
-# cfg.setdefault(
-# "video_dump", os.path.join("dumps", cli_cfg.model_name, "videos")
-# )
-# cfg.setdefault("mesh_dump", os.path.join("dumps", cli_cfg.model_name, "meshes"))
-
-# cfg.motion_video_read_fps = 6
-# cfg.merge_with(cli_cfg)
-
-# cfg.setdefault("logger", "INFO")
-
-# assert cfg.model_name is not None, "model_name is required"
-
-# return cfg, cfg_train
-
-# def _build_model(cfg):
-# from LHM.models import model_dict
-
-# hf_model_cls = wrap_model_hub(model_dict["human_lrm_sapdino_bh_sd3_5"])
-# model = hf_model_cls.from_pretrained(cfg.model_name)
-
-# return model
-
-# def launch_pretrained():
-# from huggingface_hub import snapshot_download, hf_hub_download
-# hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='assets.tar', local_dir="./")
-# os.system("tar -xvf assets.tar && rm assets.tar")
-# hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='LHM-0.5B.tar', local_dir="./")
-# os.system("tar -xvf LHM-0.5B.tar && rm LHM-0.5B.tar")
-# hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='LHM_prior_model.tar', local_dir="./")
-# os.system("tar -xvf LHM_prior_model.tar && rm LHM_prior_model.tar")
-
-# def launch_env_not_compile_with_cuda():
-# os.system("pip install chumpy")
-# os.system("pip uninstall -y basicsr")
-# os.system("pip install git+https://github.com/hitsz-zuoqi/BasicSR/")
-# # os.system("pip install -e ./third_party/sam2")
-# os.system("pip install numpy==1.23.0")
-# # os.system("pip install git+https://github.com/hitsz-zuoqi/sam2/")
-# # os.system("pip install git+https://github.com/ashawkey/diff-gaussian-rasterization/")
-# # os.system("pip install git+https://github.com/camenduru/simple-knn/")
-# os.system("pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py310_cu121_pyt251/download.html")
-
-
-# def animation_infer(renderer, gs_model_list, query_points, smplx_params, render_c2ws, render_intrs, render_bg_colors):
-# '''Inference code avoid repeat forward.
-# '''
-# render_h, render_w = int(render_intrs[0, 0, 1, 2] * 2), int(
-# render_intrs[0, 0, 0, 2] * 2
-# )
-# # render target views
-# render_res_list = []
-# num_views = render_c2ws.shape[1]
-# start_time = time.time()
-
-# # render target views
-# render_res_list = []
-
-# for view_idx in range(num_views):
-# render_res = renderer.forward_animate_gs(
-# gs_model_list,
-# query_points,
-# renderer.get_single_view_smpl_data(smplx_params, view_idx),
-# render_c2ws[:, view_idx : view_idx + 1],
-# render_intrs[:, view_idx : view_idx + 1],
-# render_h,
-# render_w,
-# render_bg_colors[:, view_idx : view_idx + 1],
-# )
-# render_res_list.append(render_res)
-# print(
-# f"time elpased(animate gs model per frame):{(time.time() - start_time)/num_views}"
-# )
-
-# out = defaultdict(list)
-# for res in render_res_list:
-# for k, v in res.items():
-# if isinstance(v[0], torch.Tensor):
-# out[k].append(v.detach().cpu())
-# else:
-# out[k].append(v)
-# for k, v in out.items():
-# # print(f"out key:{k}")
-# if isinstance(v[0], torch.Tensor):
-# out[k] = torch.concat(v, dim=1)
-# if k in ["comp_rgb", "comp_mask", "comp_depth"]:
-# out[k] = out[k][0].permute(
-# 0, 2, 3, 1
-# ) # [1, Nv, 3, H, W] -> [Nv, 3, H, W] - > [Nv, H, W, 3]
-# else:
-# out[k] = v
-# return out
-
-# def assert_input_image(input_image):
-# if input_image is None:
-# raise gr.Error("No image selected or uploaded!")
-
-# def prepare_working_dir():
-# import tempfile
-# working_dir = tempfile.TemporaryDirectory()
-# return working_dir
-
-# def init_preprocessor():
-# from LHM.utils.preprocess import Preprocessor
-# global preprocessor
-# preprocessor = Preprocessor()
-
-# def preprocess_fn(image_in: np.ndarray, remove_bg: bool, recenter: bool, working_dir):
-# image_raw = os.path.join(working_dir.name, "raw.png")
-# with Image.fromarray(image_in) as img:
-# img.save(image_raw)
-# image_out = os.path.join(working_dir.name, "rembg.png")
-# success = preprocessor.preprocess(image_path=image_raw, save_path=image_out, rmbg=remove_bg, recenter=recenter)
-# assert success, f"Failed under preprocess_fn!"
-# return image_out
-
-# def get_image_base64(path):
-# with open(path, "rb") as image_file:
-# encoded_string = base64.b64encode(image_file.read()).decode()
-# return f"data:image/png;base64,{encoded_string}"
-
-
-# def demo_lhm(pose_estimator, face_detector, lhm, cfg):
-
-# @spaces.GPU
-# def core_fn(image: str, video_params, working_dir):
-# image_raw = os.path.join(working_dir.name, "raw.png")
-# with Image.fromarray(image) as img:
-# img.save(image_raw)
+import os
+os.system("rm -rf /data-nvme/zerogpu-offload/")
+os.system("pip install chumpy")
+os.system("pip uninstall -y basicsr")
+os.system("pip install git+https://github.com/hitsz-zuoqi/BasicSR/")
+os.system("pip install numpy==1.23.0")
+os.system("pip install ./wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl")
+os.system("pip install ./wheels/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl")
+
+import cv2
+import time
+from PIL import Image
+import numpy as np
+import gradio as gr
+import base64
+import spaces
+import torch
+torch._dynamo.config.disable = True
+import subprocess
+import os
+import argparse
+from omegaconf import OmegaConf
+from rembg import remove
+from engine.pose_estimation.pose_estimator import PoseEstimator
+from LHM.utils.face_detector import VGGHeadDetector
+from LHM.utils.hf_hub import wrap_model_hub
+from LHM.runners.infer.utils import (
+ calc_new_tgt_size_by_aspect,
+ center_crop_according_to_mask,
+ prepare_motion_seqs,
+ resize_image_keepaspect_np,
+)
+from LHM.utils.ffmpeg_utils import images_to_video
+from engine.SegmentAPI.base import Bbox
+
+def get_bbox(mask):
+ height, width = mask.shape
+ pha = mask / 255.0
+ pha[pha < 0.5] = 0.0
+ pha[pha >= 0.5] = 1.0
+
+ # obtain bbox
+ _h, _w = np.where(pha == 1)
+
+ whwh = [
+ _w.min().item(),
+ _h.min().item(),
+ _w.max().item(),
+ _h.max().item(),
+ ]
+
+ box = Bbox(whwh)
+
+ # scale box to 1.05
+ scale_box = box.scale(1.1, width=width, height=height)
+ return scale_box
+
+def infer_preprocess_image(
+ rgb_path,
+ mask,
+ intr,
+ pad_ratio,
+ bg_color,
+ max_tgt_size,
+ aspect_standard,
+ enlarge_ratio,
+ render_tgt_size,
+ multiply,
+ need_mask=True,
+):
+ """inferece
+ image, _, _ = preprocess_image(image_path, mask_path=None, intr=None, pad_ratio=0, bg_color=1.0,
+ max_tgt_size=896, aspect_standard=aspect_standard, enlarge_ratio=[1.0, 1.0],
+ render_tgt_size=source_size, multiply=14, need_mask=True)
+
+ """
+
+ rgb = np.array(Image.open(rgb_path))
+ rgb_raw = rgb.copy()
+
+ bbox = get_bbox(mask)
+ bbox_list = bbox.get_box()
+
+ rgb = rgb[bbox_list[1] : bbox_list[3], bbox_list[0] : bbox_list[2]]
+ mask = mask[bbox_list[1] : bbox_list[3], bbox_list[0] : bbox_list[2]]
+
+ h, w, _ = rgb.shape
+ assert w < h
+ cur_ratio = h / w
+ scale_ratio = cur_ratio / aspect_standard
+
+ target_w = int(min(w * scale_ratio, h))
+ offset_w = (target_w - w) // 2
+ # resize to target ratio.
+ if offset_w > 0:
+ rgb = np.pad(
+ rgb,
+ ((0, 0), (offset_w, offset_w), (0, 0)),
+ mode="constant",
+ constant_values=255,
+ )
+ mask = np.pad(
+ mask,
+ ((0, 0), (offset_w, offset_w)),
+ mode="constant",
+ constant_values=0,
+ )
+ else:
+ offset_w = -offset_w
+ rgb = rgb[:,offset_w:-offset_w,:]
+ mask = mask[:,offset_w:-offset_w]
+
+ # resize to target ratio.
+
+ rgb = np.pad(
+ rgb,
+ ((0, 0), (offset_w, offset_w), (0, 0)),
+ mode="constant",
+ constant_values=255,
+ )
+
+ mask = np.pad(
+ mask,
+ ((0, 0), (offset_w, offset_w)),
+ mode="constant",
+ constant_values=0,
+ )
+
+ rgb = rgb / 255.0 # normalize to [0, 1]
+ mask = mask / 255.0
+
+ mask = (mask > 0.5).astype(np.float32)
+ rgb = rgb[:, :, :3] * mask[:, :, None] + bg_color * (1 - mask[:, :, None])
+
+ # resize to specific size require by preprocessor of smplx-estimator.
+ rgb = resize_image_keepaspect_np(rgb, max_tgt_size)
+ mask = resize_image_keepaspect_np(mask, max_tgt_size)
+
+ # crop image to enlarge human area.
+ rgb, mask, offset_x, offset_y = center_crop_according_to_mask(
+ rgb, mask, aspect_standard, enlarge_ratio
+ )
+ if intr is not None:
+ intr[0, 2] -= offset_x
+ intr[1, 2] -= offset_y
+
+ # resize to render_tgt_size for training
+
+ tgt_hw_size, ratio_y, ratio_x = calc_new_tgt_size_by_aspect(
+ cur_hw=rgb.shape[:2],
+ aspect_standard=aspect_standard,
+ tgt_size=render_tgt_size,
+ multiply=multiply,
+ )
+
+ rgb = cv2.resize(
+ rgb, dsize=(tgt_hw_size[1], tgt_hw_size[0]), interpolation=cv2.INTER_AREA
+ )
+ mask = cv2.resize(
+ mask, dsize=(tgt_hw_size[1], tgt_hw_size[0]), interpolation=cv2.INTER_AREA
+ )
+
+ if intr is not None:
+
+ # ******************** Merge *********************** #
+ intr = scale_intrs(intr, ratio_x=ratio_x, ratio_y=ratio_y)
+ assert (
+ abs(intr[0, 2] * 2 - rgb.shape[1]) < 2.5
+ ), f"{intr[0, 2] * 2}, {rgb.shape[1]}"
+ assert (
+ abs(intr[1, 2] * 2 - rgb.shape[0]) < 2.5
+ ), f"{intr[1, 2] * 2}, {rgb.shape[0]}"
+
+ # ******************** Merge *********************** #
+ intr[0, 2] = rgb.shape[1] // 2
+ intr[1, 2] = rgb.shape[0] // 2
+
+ rgb = torch.from_numpy(rgb).float().permute(2, 0, 1).unsqueeze(0) # [1, 3, H, W]
+ mask = (
+ torch.from_numpy(mask[:, :, None]).float().permute(2, 0, 1).unsqueeze(0)
+ ) # [1, 1, H, W]
+ return rgb, mask, intr
+
+def parse_configs():
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--config", type=str)
+ parser.add_argument("--infer", type=str)
+ args, unknown = parser.parse_known_args()
+
+ cfg = OmegaConf.create()
+ cli_cfg = OmegaConf.from_cli(unknown)
+
+ # parse from ENV
+ if os.environ.get("APP_INFER") is not None:
+ args.infer = os.environ.get("APP_INFER")
+ if os.environ.get("APP_MODEL_NAME") is not None:
+ cli_cfg.model_name = os.environ.get("APP_MODEL_NAME")
+
+ args.config = args.infer if args.config is None else args.config
+
+ if args.config is not None:
+ cfg_train = OmegaConf.load(args.config)
+ cfg.source_size = cfg_train.dataset.source_image_res
+ try:
+ cfg.src_head_size = cfg_train.dataset.src_head_size
+ except:
+ cfg.src_head_size = 112
+ cfg.render_size = cfg_train.dataset.render_image.high
+ _relative_path = os.path.join(
+ cfg_train.experiment.parent,
+ cfg_train.experiment.child,
+ os.path.basename(cli_cfg.model_name).split("_")[-1],
+ )
+
+ cfg.save_tmp_dump = os.path.join("exps", "save_tmp", _relative_path)
+ cfg.image_dump = os.path.join("exps", "images", _relative_path)
+ cfg.video_dump = os.path.join("exps", "videos", _relative_path) # output path
+
+ if args.infer is not None:
+ cfg_infer = OmegaConf.load(args.infer)
+ cfg.merge_with(cfg_infer)
+ cfg.setdefault(
+ "save_tmp_dump", os.path.join("exps", cli_cfg.model_name, "save_tmp")
+ )
+ cfg.setdefault("image_dump", os.path.join("exps", cli_cfg.model_name, "images"))
+ cfg.setdefault(
+ "video_dump", os.path.join("dumps", cli_cfg.model_name, "videos")
+ )
+ cfg.setdefault("mesh_dump", os.path.join("dumps", cli_cfg.model_name, "meshes"))
+
+ cfg.motion_video_read_fps = 6
+ cfg.merge_with(cli_cfg)
+
+ cfg.setdefault("logger", "INFO")
+
+ assert cfg.model_name is not None, "model_name is required"
+
+ return cfg, cfg_train
+
+def _build_model(cfg):
+ from LHM.models import model_dict
+
+ hf_model_cls = wrap_model_hub(model_dict["human_lrm_sapdino_bh_sd3_5"])
+ model = hf_model_cls.from_pretrained(cfg.model_name)
+
+ return model
+
+def launch_pretrained():
+ from huggingface_hub import snapshot_download, hf_hub_download
+ hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='assets.tar', local_dir="./")
+ os.system("tar -xvf assets.tar && rm assets.tar")
+ hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='LHM-0.5B.tar', local_dir="./")
+ os.system("tar -xvf LHM-0.5B.tar && rm LHM-0.5B.tar")
+ hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='LHM_prior_model.tar', local_dir="./")
+ os.system("tar -xvf LHM_prior_model.tar && rm LHM_prior_model.tar")
+
+def launch_env_not_compile_with_cuda():
+ os.system("pip install chumpy")
+ os.system("pip uninstall -y basicsr")
+ os.system("pip install git+https://github.com/hitsz-zuoqi/BasicSR/")
+ os.system("pip install numpy==1.23.0")
+ # os.system("pip install git+https://github.com/hitsz-zuoqi/sam2/")
+ # os.system("pip install git+https://github.com/ashawkey/diff-gaussian-rasterization/")
+ # os.system("pip install git+https://github.com/camenduru/simple-knn/")
+ # os.system("pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py310_cu121_pyt240/download.html")
+
+
+def animation_infer(renderer, gs_model_list, query_points, smplx_params, render_c2ws, render_intrs, render_bg_colors):
+ '''Inference code avoid repeat forward.
+ '''
+ render_h, render_w = int(render_intrs[0, 0, 1, 2] * 2), int(
+ render_intrs[0, 0, 0, 2] * 2
+ )
+ # render target views
+ render_res_list = []
+ num_views = render_c2ws.shape[1]
+ start_time = time.time()
+
+ # render target views
+ render_res_list = []
+
+ for view_idx in range(num_views):
+ render_res = renderer.forward_animate_gs(
+ gs_model_list,
+ query_points,
+ renderer.get_single_view_smpl_data(smplx_params, view_idx),
+ render_c2ws[:, view_idx : view_idx + 1],
+ render_intrs[:, view_idx : view_idx + 1],
+ render_h,
+ render_w,
+ render_bg_colors[:, view_idx : view_idx + 1],
+ )
+ render_res_list.append(render_res)
+ print(
+ f"time elpased(animate gs model per frame):{(time.time() - start_time)/num_views}"
+ )
+
+ out = defaultdict(list)
+ for res in render_res_list:
+ for k, v in res.items():
+ if isinstance(v[0], torch.Tensor):
+ out[k].append(v.detach().cpu())
+ else:
+ out[k].append(v)
+ for k, v in out.items():
+ # print(f"out key:{k}")
+ if isinstance(v[0], torch.Tensor):
+ out[k] = torch.concat(v, dim=1)
+ if k in ["comp_rgb", "comp_mask", "comp_depth"]:
+ out[k] = out[k][0].permute(
+ 0, 2, 3, 1
+ ) # [1, Nv, 3, H, W] -> [Nv, 3, H, W] - > [Nv, H, W, 3]
+ else:
+ out[k] = v
+ return out
+
+def assert_input_image(input_image):
+ if input_image is None:
+ raise gr.Error("No image selected or uploaded!")
+
+def prepare_working_dir():
+ import tempfile
+ working_dir = tempfile.TemporaryDirectory()
+ return working_dir
+
+def init_preprocessor():
+ from LHM.utils.preprocess import Preprocessor
+ global preprocessor
+ preprocessor = Preprocessor()
+
+def preprocess_fn(image_in: np.ndarray, remove_bg: bool, recenter: bool, working_dir):
+ image_raw = os.path.join(working_dir.name, "raw.png")
+ with Image.fromarray(image_in) as img:
+ img.save(image_raw)
+ image_out = os.path.join(working_dir.name, "rembg.png")
+ success = preprocessor.preprocess(image_path=image_raw, save_path=image_out, rmbg=remove_bg, recenter=recenter)
+ assert success, f"Failed under preprocess_fn!"
+ return image_out
+
+def get_image_base64(path):
+ with open(path, "rb") as image_file:
+ encoded_string = base64.b64encode(image_file.read()).decode()
+ return f"data:image/png;base64,{encoded_string}"
+
+
+def demo_lhm(pose_estimator, face_detector, lhm, cfg):
+
+ @spaces.GPU
+ def core_fn(image: str, video_params, working_dir):
+ image_raw = os.path.join(working_dir.name, "raw.png")
+ with Image.fromarray(image) as img:
+ img.save(image_raw)
-# base_vid = os.path.basename(video_params).split("_")[0]
-# smplx_params_dir = os.path.join("./assets/sample_motion", base_vid, "smplx_params")
+ base_vid = os.path.basename(video_params).split("_")[0]
+ smplx_params_dir = os.path.join("./assets/sample_motion", base_vid, "smplx_params")
-# dump_video_path = os.path.join(working_dir.name, "output.mp4")
-# dump_image_path = os.path.join(working_dir.name, "output.png")
+ dump_video_path = os.path.join(working_dir.name, "output.mp4")
+ dump_image_path = os.path.join(working_dir.name, "output.png")
-# # prepare dump paths
-# omit_prefix = os.path.dirname(image_raw)
-# image_name = os.path.basename(image_raw)
-# uid = image_name.split(".")[0]
-# subdir_path = os.path.dirname(image_raw).replace(omit_prefix, "")
-# subdir_path = (
-# subdir_path[1:] if subdir_path.startswith("/") else subdir_path
-# )
-# print("subdir_path and uid:", subdir_path, uid)
+ # prepare dump paths
+ omit_prefix = os.path.dirname(image_raw)
+ image_name = os.path.basename(image_raw)
+ uid = image_name.split(".")[0]
+ subdir_path = os.path.dirname(image_raw).replace(omit_prefix, "")
+ subdir_path = (
+ subdir_path[1:] if subdir_path.startswith("/") else subdir_path
+ )
+ print("subdir_path and uid:", subdir_path, uid)
-# motion_seqs_dir = smplx_params_dir
+ motion_seqs_dir = smplx_params_dir
-# motion_name = os.path.dirname(
-# motion_seqs_dir[:-1] if motion_seqs_dir[-1] == "/" else motion_seqs_dir
-# )
-
-# motion_name = os.path.basename(motion_name)
-
-# dump_image_dir = os.path.dirname(dump_image_path)
-# os.makedirs(dump_image_dir, exist_ok=True)
-
-# print(image_raw, motion_seqs_dir, dump_image_dir, dump_video_path)
-
-# dump_tmp_dir = dump_image_dir
-
-# shape_pose = pose_estimator(image_raw)
-# assert shape_pose.is_full_body, f"The input image is illegal, {shape_pose.msg}"
-
-# if os.path.exists(dump_video_path):
-# return dump_image_path, dump_video_path
-# source_size = cfg.source_size
-# render_size = cfg.render_size
-# render_fps = 30
-
-# aspect_standard = 5.0 / 3
-# motion_img_need_mask = cfg.get("motion_img_need_mask", False) # False
-# vis_motion = cfg.get("vis_motion", False) # False
-
-
-# input_np = cv2.imread(image_raw)
-# output_np = remove(input_np)
-# # cv2.imwrite("./vis.png", output_np)
-# parsing_mask = output_np[:,:,3]
-
-# # prepare reference image
-# image, _, _ = infer_preprocess_image(
-# image_raw,
-# mask=parsing_mask,
-# intr=None,
-# pad_ratio=0,
-# bg_color=1.0,
-# max_tgt_size=896,
-# aspect_standard=aspect_standard,
-# enlarge_ratio=[1.0, 1.0],
-# render_tgt_size=source_size,
-# multiply=14,
-# need_mask=True,
-# )
-
-# try:
-# rgb = np.array(Image.open(image_path))
-# rgb = torch.from_numpy(rgb).permute(2, 0, 1)
-# bbox = face_detector.detect_face(rgb)
-# head_rgb = rgb[:, int(bbox[1]) : int(bbox[3]), int(bbox[0]) : int(bbox[2])]
-# head_rgb = head_rgb.permute(1, 2, 0)
-# src_head_rgb = head_rgb.cpu().numpy()
-# except:
-# print("w/o head input!")
-# src_head_rgb = np.zeros((112, 112, 3), dtype=np.uint8)
-
-# # resize to dino size
-# try:
-# src_head_rgb = cv2.resize(
-# src_head_rgb,
-# dsize=(cfg.src_head_size, cfg.src_head_size),
-# interpolation=cv2.INTER_AREA,
-# ) # resize to dino size
-# except:
-# src_head_rgb = np.zeros(
-# (cfg.src_head_size, cfg.src_head_size, 3), dtype=np.uint8
-# )
-
-# src_head_rgb = (
-# torch.from_numpy(src_head_rgb / 255.0).float().permute(2, 0, 1).unsqueeze(0)
-# ) # [1, 3, H, W]
-
-# save_ref_img_path = os.path.join(
-# dump_tmp_dir, "output.png"
-# )
-# vis_ref_img = (image[0].permute(1, 2, 0).cpu().detach().numpy() * 255).astype(
-# np.uint8
-# )
-# Image.fromarray(vis_ref_img).save(save_ref_img_path)
-
-# # read motion seq
-# motion_name = os.path.dirname(
-# motion_seqs_dir[:-1] if motion_seqs_dir[-1] == "/" else motion_seqs_dir
-# )
-# motion_name = os.path.basename(motion_name)
-
-# motion_seq = prepare_motion_seqs(
-# motion_seqs_dir,
-# None,
-# save_root=dump_tmp_dir,
-# fps=30,
-# bg_color=1.0,
-# aspect_standard=aspect_standard,
-# enlarge_ratio=[1.0, 1, 0],
-# render_image_res=render_size,
-# multiply=16,
-# need_mask=motion_img_need_mask,
-# vis_motion=vis_motion,
-# )
-
-# camera_size = len(motion_seq["motion_seqs"])
-# shape_param = shape_pose.beta
-
-# device = "cuda"
-# dtype = torch.float32
-# shape_param = torch.tensor(shape_param, dtype=dtype).unsqueeze(0)
-
-# lhm.to(dtype)
-
-# smplx_params = motion_seq['smplx_params']
-# smplx_params['betas'] = shape_param.to(device)
-
-# gs_model_list, query_points, transform_mat_neutral_pose = lhm.infer_single_view(
-# image.unsqueeze(0).to(device, dtype),
-# src_head_rgb.unsqueeze(0).to(device, dtype),
-# None,
-# None,
-# render_c2ws=motion_seq["render_c2ws"].to(device),
-# render_intrs=motion_seq["render_intrs"].to(device),
-# render_bg_colors=motion_seq["render_bg_colors"].to(device),
-# smplx_params={
-# k: v.to(device) for k, v in smplx_params.items()
-# },
-# )
-
-
-# # rendering !!!!
-
-# start_time = time.time()
-# batch_dict = dict()
-# batch_size = 40 # avoid memeory out!
-
-# for batch_i in range(0, camera_size, batch_size):
-# with torch.no_grad():
-# # TODO check device and dtype
-# # dict_keys(['comp_rgb', 'comp_rgb_bg', 'comp_mask', 'comp_depth', '3dgs'])
-# keys = [
-# "root_pose",
-# "body_pose",
-# "jaw_pose",
-# "leye_pose",
-# "reye_pose",
-# "lhand_pose",
-# "rhand_pose",
-# "trans",
-# "focal",
-# "princpt",
-# "img_size_wh",
-# "expr",
-# ]
-# batch_smplx_params = dict()
-# batch_smplx_params["betas"] = shape_param.to(device)
-# batch_smplx_params['transform_mat_neutral_pose'] = transform_mat_neutral_pose
-# for key in keys:
-# batch_smplx_params[key] = motion_seq["smplx_params"][key][
-# :, batch_i : batch_i + batch_size
-# ].to(device)
-
-# res = lhm.animation_infer(gs_model_list, query_points, batch_smplx_params,
-# render_c2ws=motion_seq["render_c2ws"][
-# :, batch_i : batch_i + batch_size
-# ].to(device),
-# render_intrs=motion_seq["render_intrs"][
-# :, batch_i : batch_i + batch_size
-# ].to(device),
-# render_bg_colors=motion_seq["render_bg_colors"][
-# :, batch_i : batch_i + batch_size
-# ].to(device),
-# )
-
-# for accumulate_key in ["comp_rgb", "comp_mask"]:
-# if accumulate_key not in batch_dict:
-# batch_dict[accumulate_key] = []
-# batch_dict[accumulate_key].append(res[accumulate_key].detach().cpu())
-# del res
-# torch.cuda.empty_cache()
-
-# for accumulate_key in ["comp_rgb", "comp_mask"]:
-# batch_dict[accumulate_key] = torch.cat(batch_dict[accumulate_key], dim=0)
-
-# print(f"time elapsed: {time.time() - start_time}")
-# rgb = batch_dict["comp_rgb"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
-# mask = batch_dict["comp_mask"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
-# mask[mask < 0.5] = 0.0
-
-# rgb = rgb * mask + (1 - mask) * 1
-# rgb = np.clip(rgb * 255, 0, 255).astype(np.uint8)
-
-# if vis_motion:
-# # print(rgb.shape, motion_seq["vis_motion_render"].shape)
-
-# vis_ref_img = np.tile(
-# cv2.resize(vis_ref_img, (rgb[0].shape[1], rgb[0].shape[0]))[
-# None, :, :, :
-# ],
-# (rgb.shape[0], 1, 1, 1),
-# )
-# rgb = np.concatenate(
-# [rgb, motion_seq["vis_motion_render"], vis_ref_img], axis=2
-# )
-
-# os.makedirs(os.path.dirname(dump_video_path), exist_ok=True)
-
-# images_to_video(
-# rgb,
-# output_path=dump_video_path,
-# fps=render_fps,
-# gradio_codec=False,
-# verbose=True,
-# )
-
-# # self.infer_single(
-# # image_path,
-# # motion_seqs_dir=motion_seqs_dir,
-# # motion_img_dir=None,
-# # motion_video_read_fps=30,
-# # export_video=False,
-# # export_mesh=False,
-# # dump_tmp_dir=dump_image_dir,
-# # dump_image_dir=dump_image_dir,
-# # dump_video_path=dump_video_path,
-# # shape_param=shape_pose.beta,
-# # )
-
-# # status = spaces.GPU(infer_impl(
-# # gradio_demo_image=image_raw,
-# # gradio_motion_file=smplx_params_dir,
-# # gradio_masked_image=dump_image_path,
-# # gradio_video_save_path=dump_video_path
-# # ))
-
-# return dump_image_path, dump_video_path
-# # if status:
-# # return dump_image_path, dump_video_path
-# # else:
-# # return None, None
-
-# _TITLE = '''LHM: Large Animatable Human Model'''
-
-# _DESCRIPTION = '''
-# Reconstruct a human avatar in 0.2 seconds with A100!
-# '''
-
-# with gr.Blocks(analytics_enabled=False) as demo:
-
-# #
-# logo_url = "./assets/rgba_logo_new.png"
-# logo_base64 = get_image_base64(logo_url)
-# gr.HTML(
-# f"""
-#
-#
-#
Large Animatable Human Model
-#
-#
Notes: Please input full-body image in case of detection errors.
"""
-# )
-
-# # DISPLAY
-# with gr.Row():
-
-# with gr.Column(variant='panel', scale=1):
-# with gr.Tabs(elem_id="openlrm_input_image"):
-# with gr.TabItem('Input Image'):
-# with gr.Row():
-# input_image = gr.Image(label="Input Image", image_mode="RGBA", height=480, width=270, sources="upload", type="numpy", elem_id="content_image")
-# # EXAMPLES
-# with gr.Row():
-# examples = [
-# ['assets/sample_input/joker.jpg'],
-# ['assets/sample_input/anime.png'],
-# ['assets/sample_input/basket.png'],
-# ['assets/sample_input/ai_woman1.JPG'],
-# ['assets/sample_input/anime2.JPG'],
-# ['assets/sample_input/anime3.JPG'],
-# ['assets/sample_input/boy1.png'],
-# ['assets/sample_input/choplin.jpg'],
-# ['assets/sample_input/eins.JPG'],
-# ['assets/sample_input/girl1.png'],
-# ['assets/sample_input/girl2.png'],
-# ['assets/sample_input/robot.jpg'],
-# ]
-# gr.Examples(
-# examples=examples,
-# inputs=[input_image],
-# examples_per_page=20,
-# )
-
-# with gr.Column():
-# with gr.Tabs(elem_id="openlrm_input_video"):
-# with gr.TabItem('Input Video'):
-# with gr.Row():
-# video_input = gr.Video(label="Input Video",height=480, width=270, interactive=False)
-
-# examples = [
-# # './assets/sample_motion/danaotiangong/danaotiangong_origin.mp4',
-# './assets/sample_motion/ex5/ex5_origin.mp4',
-# './assets/sample_motion/girl2/girl2_origin.mp4',
-# './assets/sample_motion/jntm/jntm_origin.mp4',
-# './assets/sample_motion/mimo1/mimo1_origin.mp4',
-# './assets/sample_motion/mimo2/mimo2_origin.mp4',
-# './assets/sample_motion/mimo4/mimo4_origin.mp4',
-# './assets/sample_motion/mimo5/mimo5_origin.mp4',
-# './assets/sample_motion/mimo6/mimo6_origin.mp4',
-# './assets/sample_motion/nezha/nezha_origin.mp4',
-# './assets/sample_motion/taiji/taiji_origin.mp4'
-# ]
-
-# gr.Examples(
-# examples=examples,
-# inputs=[video_input],
-# examples_per_page=20,
-# )
-# with gr.Column(variant='panel', scale=1):
-# with gr.Tabs(elem_id="openlrm_processed_image"):
-# with gr.TabItem('Processed Image'):
-# with gr.Row():
-# processed_image = gr.Image(label="Processed Image", image_mode="RGBA", type="filepath", elem_id="processed_image", height=480, width=270, interactive=False)
-
-# with gr.Column(variant='panel', scale=1):
-# with gr.Tabs(elem_id="openlrm_render_video"):
-# with gr.TabItem('Rendered Video'):
-# with gr.Row():
-# output_video = gr.Video(label="Rendered Video", format="mp4", height=480, width=270, autoplay=True)
-
-# # SETTING
-# with gr.Row():
-# with gr.Column(variant='panel', scale=1):
-# submit = gr.Button('Generate', elem_id="openlrm_generate", variant='primary')
-
-
-# working_dir = gr.State()
-# submit.click(
-# fn=assert_input_image,
-# inputs=[input_image],
-# queue=False,
-# ).success(
-# fn=prepare_working_dir,
-# outputs=[working_dir],
-# queue=False,
-# ).success(
-# fn=core_fn,
-# inputs=[input_image, video_input, working_dir], # video_params refer to smpl dir
-# outputs=[processed_image, output_video],
-# )
-
-# demo.queue()
-# demo.launch()
-
-
-# def launch_gradio_app():
-
-# os.environ.update({
-# "APP_ENABLED": "1",
-# "APP_MODEL_NAME": "./exps/releases/video_human_benchmark/human-lrm-500M/step_060000/",
-# "APP_INFER": "./configs/inference/human-lrm-500M.yaml",
-# "APP_TYPE": "infer.human_lrm",
-# "NUMBA_THREADING_LAYER": 'omp',
-# })
-
-# # from LHM.runners import REGISTRY_RUNNERS
-# # RunnerClass = REGISTRY_RUNNERS[os.getenv("APP_TYPE")]
-# # with RunnerClass() as runner:
-# # runner.to('cuda')
-# # demo_lhm(infer_impl=runner.infer)
-
-# facedetector = VGGHeadDetector(
-# "./pretrained_models/gagatracker/vgghead/vgg_heads_l.trcd",
-# device='cpu',
-# )
-# facedetector.to('cuda')
-
-# pose_estimator = PoseEstimator(
-# "./pretrained_models/human_model_files/", device='cpu'
-# )
-# pose_estimator.to('cuda')
-# pose_estimator.device = 'cuda'
-
-# cfg, cfg_train = parse_configs()
-# lhm = _build_model(cfg)
-# lhm.to('cuda')
-
-# demo_lhm(pose_estimator, facedetector, lhm, cfg)
-
-
-
-# if __name__ == '__main__':
-# # launch_pretrained()
-# # launch_env_not_compile_with_cuda()
-# launch_gradio_app()
+ motion_name = os.path.dirname(
+ motion_seqs_dir[:-1] if motion_seqs_dir[-1] == "/" else motion_seqs_dir
+ )
+
+ motion_name = os.path.basename(motion_name)
+
+ dump_image_dir = os.path.dirname(dump_image_path)
+ os.makedirs(dump_image_dir, exist_ok=True)
+
+ print(image_raw, motion_seqs_dir, dump_image_dir, dump_video_path)
+
+ dump_tmp_dir = dump_image_dir
+
+ shape_pose = pose_estimator(image_raw)
+ assert shape_pose.is_full_body, f"The input image is illegal, {shape_pose.msg}"
+
+ if os.path.exists(dump_video_path):
+ return dump_image_path, dump_video_path
+ source_size = cfg.source_size
+ render_size = cfg.render_size
+ render_fps = 30
+
+ aspect_standard = 5.0 / 3
+ motion_img_need_mask = cfg.get("motion_img_need_mask", False) # False
+ vis_motion = cfg.get("vis_motion", False) # False
+
+
+ input_np = cv2.imread(image_raw)
+ output_np = remove(input_np)
+ # cv2.imwrite("./vis.png", output_np)
+ parsing_mask = output_np[:,:,3]
+
+ # prepare reference image
+ image, _, _ = infer_preprocess_image(
+ image_raw,
+ mask=parsing_mask,
+ intr=None,
+ pad_ratio=0,
+ bg_color=1.0,
+ max_tgt_size=896,
+ aspect_standard=aspect_standard,
+ enlarge_ratio=[1.0, 1.0],
+ render_tgt_size=source_size,
+ multiply=14,
+ need_mask=True,
+ )
+
+ try:
+ rgb = np.array(Image.open(image_path))
+ rgb = torch.from_numpy(rgb).permute(2, 0, 1)
+ bbox = face_detector.detect_face(rgb)
+ head_rgb = rgb[:, int(bbox[1]) : int(bbox[3]), int(bbox[0]) : int(bbox[2])]
+ head_rgb = head_rgb.permute(1, 2, 0)
+ src_head_rgb = head_rgb.cpu().numpy()
+ except:
+ print("w/o head input!")
+ src_head_rgb = np.zeros((112, 112, 3), dtype=np.uint8)
+
+ # resize to dino size
+ try:
+ src_head_rgb = cv2.resize(
+ src_head_rgb,
+ dsize=(cfg.src_head_size, cfg.src_head_size),
+ interpolation=cv2.INTER_AREA,
+ ) # resize to dino size
+ except:
+ src_head_rgb = np.zeros(
+ (cfg.src_head_size, cfg.src_head_size, 3), dtype=np.uint8
+ )
+
+ src_head_rgb = (
+ torch.from_numpy(src_head_rgb / 255.0).float().permute(2, 0, 1).unsqueeze(0)
+ ) # [1, 3, H, W]
+
+ save_ref_img_path = os.path.join(
+ dump_tmp_dir, "output.png"
+ )
+ vis_ref_img = (image[0].permute(1, 2, 0).cpu().detach().numpy() * 255).astype(
+ np.uint8
+ )
+ Image.fromarray(vis_ref_img).save(save_ref_img_path)
+
+ # read motion seq
+ motion_name = os.path.dirname(
+ motion_seqs_dir[:-1] if motion_seqs_dir[-1] == "/" else motion_seqs_dir
+ )
+ motion_name = os.path.basename(motion_name)
+
+ motion_seq = prepare_motion_seqs(
+ motion_seqs_dir,
+ None,
+ save_root=dump_tmp_dir,
+ fps=30,
+ bg_color=1.0,
+ aspect_standard=aspect_standard,
+ enlarge_ratio=[1.0, 1, 0],
+ render_image_res=render_size,
+ multiply=16,
+ need_mask=motion_img_need_mask,
+ vis_motion=vis_motion,
+ )
+
+ camera_size = len(motion_seq["motion_seqs"])
+ shape_param = shape_pose.beta
+
+ device = "cuda"
+ dtype = torch.float32
+ shape_param = torch.tensor(shape_param, dtype=dtype).unsqueeze(0)
+
+ lhm.to(dtype)
+
+ smplx_params = motion_seq['smplx_params']
+ smplx_params['betas'] = shape_param.to(device)
+
+ gs_model_list, query_points, transform_mat_neutral_pose = lhm.infer_single_view(
+ image.unsqueeze(0).to(device, dtype),
+ src_head_rgb.unsqueeze(0).to(device, dtype),
+ None,
+ None,
+ render_c2ws=motion_seq["render_c2ws"].to(device),
+ render_intrs=motion_seq["render_intrs"].to(device),
+ render_bg_colors=motion_seq["render_bg_colors"].to(device),
+ smplx_params={
+ k: v.to(device) for k, v in smplx_params.items()
+ },
+ )
+
+ # rendering !!!!
+ start_time = time.time()
+ batch_dict = dict()
+ batch_size = 80 # avoid memeory out!
+
+ for batch_i in range(0, camera_size, batch_size):
+ with torch.no_grad():
+ # TODO check device and dtype
+ # dict_keys(['comp_rgb', 'comp_rgb_bg', 'comp_mask', 'comp_depth', '3dgs'])
+ keys = [
+ "root_pose",
+ "body_pose",
+ "jaw_pose",
+ "leye_pose",
+ "reye_pose",
+ "lhand_pose",
+ "rhand_pose",
+ "trans",
+ "focal",
+ "princpt",
+ "img_size_wh",
+ "expr",
+ ]
+ batch_smplx_params = dict()
+ batch_smplx_params["betas"] = shape_param.to(device)
+ batch_smplx_params['transform_mat_neutral_pose'] = transform_mat_neutral_pose
+ for key in keys:
+ batch_smplx_params[key] = motion_seq["smplx_params"][key][
+ :, batch_i : batch_i + batch_size
+ ].to(device)
+
+ res = lhm.animation_infer(gs_model_list, query_points, batch_smplx_params,
+ render_c2ws=motion_seq["render_c2ws"][
+ :, batch_i : batch_i + batch_size
+ ].to(device),
+ render_intrs=motion_seq["render_intrs"][
+ :, batch_i : batch_i + batch_size
+ ].to(device),
+ render_bg_colors=motion_seq["render_bg_colors"][
+ :, batch_i : batch_i + batch_size
+ ].to(device),
+ )
+
+ for accumulate_key in ["comp_rgb", "comp_mask"]:
+ if accumulate_key not in batch_dict:
+ batch_dict[accumulate_key] = []
+ batch_dict[accumulate_key].append(res[accumulate_key].detach().cpu())
+ del res
+ torch.cuda.empty_cache()
+
+ for accumulate_key in ["comp_rgb", "comp_mask"]:
+ batch_dict[accumulate_key] = torch.cat(batch_dict[accumulate_key], dim=0)
+
+ print(f"time elapsed: {time.time() - start_time}")
+ rgb = batch_dict["comp_rgb"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
+ mask = batch_dict["comp_mask"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
+ mask[mask < 0.5] = 0.0
+
+ rgb = rgb * mask + (1 - mask) * 1
+ rgb = np.clip(rgb * 255, 0, 255).astype(np.uint8)
+
+ if vis_motion:
+ # print(rgb.shape, motion_seq["vis_motion_render"].shape)
+
+ vis_ref_img = np.tile(
+ cv2.resize(vis_ref_img, (rgb[0].shape[1], rgb[0].shape[0]))[
+ None, :, :, :
+ ],
+ (rgb.shape[0], 1, 1, 1),
+ )
+ rgb = np.concatenate(
+ [rgb, motion_seq["vis_motion_render"], vis_ref_img], axis=2
+ )
+
+ os.makedirs(os.path.dirname(dump_video_path), exist_ok=True)
+
+ images_to_video(
+ rgb,
+ output_path=dump_video_path,
+ fps=render_fps,
+ gradio_codec=False,
+ verbose=True,
+ )
+
+ # self.infer_single(
+ # image_path,
+ # motion_seqs_dir=motion_seqs_dir,
+ # motion_img_dir=None,
+ # motion_video_read_fps=30,
+ # export_video=False,
+ # export_mesh=False,
+ # dump_tmp_dir=dump_image_dir,
+ # dump_image_dir=dump_image_dir,
+ # dump_video_path=dump_video_path,
+ # shape_param=shape_pose.beta,
+ # )
+
+ # status = spaces.GPU(infer_impl(
+ # gradio_demo_image=image_raw,
+ # gradio_motion_file=smplx_params_dir,
+ # gradio_masked_image=dump_image_path,
+ # gradio_video_save_path=dump_video_path
+ # ))
+
+ return dump_image_path, dump_video_path
+ # if status:
+ # return dump_image_path, dump_video_path
+ # else:
+ # return None, None
+
+ _TITLE = '''LHM: Large Animatable Human Model'''
+
+ _DESCRIPTION = '''
+ Reconstruct a human avatar in 0.2 seconds with A100!
+ '''
+
+ with gr.Blocks(analytics_enabled=False) as demo:
+
+ #
+ logo_url = "./assets/rgba_logo_new.png"
+ logo_base64 = get_image_base64(logo_url)
+ gr.HTML(
+ f"""
+
+
+
Large Animatable Human Model
+
+
Notes: Please input full-body image in case of detection errors.
"""
+ )
+
+ # DISPLAY
+ with gr.Row():
+
+ with gr.Column(variant='panel', scale=1):
+ with gr.Tabs(elem_id="openlrm_input_image"):
+ with gr.TabItem('Input Image'):
+ with gr.Row():
+ input_image = gr.Image(label="Input Image", image_mode="RGBA", height=480, width=270, sources="upload", type="numpy", elem_id="content_image")
+ # EXAMPLES
+ with gr.Row():
+ examples = [
+ ['assets/sample_input/joker.jpg'],
+ ['assets/sample_input/anime.png'],
+ ['assets/sample_input/basket.png'],
+ ['assets/sample_input/ai_woman1.JPG'],
+ ['assets/sample_input/anime2.JPG'],
+ ['assets/sample_input/anime3.JPG'],
+ ['assets/sample_input/boy1.png'],
+ ['assets/sample_input/choplin.jpg'],
+ ['assets/sample_input/eins.JPG'],
+ ['assets/sample_input/girl1.png'],
+ ['assets/sample_input/girl2.png'],
+ ['assets/sample_input/robot.jpg'],
+ ]
+ gr.Examples(
+ examples=examples,
+ inputs=[input_image],
+ examples_per_page=20,
+ )
+
+ with gr.Column():
+ with gr.Tabs(elem_id="openlrm_input_video"):
+ with gr.TabItem('Input Video'):
+ with gr.Row():
+ video_input = gr.Video(label="Input Video",height=480, width=270, interactive=False)
+
+ examples = [
+ # './assets/sample_motion/danaotiangong/danaotiangong_origin.mp4',
+ './assets/sample_motion/ex5/ex5_origin.mp4',
+ './assets/sample_motion/girl2/girl2_origin.mp4',
+ './assets/sample_motion/jntm/jntm_origin.mp4',
+ './assets/sample_motion/mimo1/mimo1_origin.mp4',
+ './assets/sample_motion/mimo2/mimo2_origin.mp4',
+ './assets/sample_motion/mimo4/mimo4_origin.mp4',
+ './assets/sample_motion/mimo5/mimo5_origin.mp4',
+ './assets/sample_motion/mimo6/mimo6_origin.mp4',
+ './assets/sample_motion/nezha/nezha_origin.mp4',
+ './assets/sample_motion/taiji/taiji_origin.mp4'
+ ]
+
+ gr.Examples(
+ examples=examples,
+ inputs=[video_input],
+ examples_per_page=20,
+ )
+ with gr.Column(variant='panel', scale=1):
+ with gr.Tabs(elem_id="openlrm_processed_image"):
+ with gr.TabItem('Processed Image'):
+ with gr.Row():
+ processed_image = gr.Image(label="Processed Image", image_mode="RGBA", type="filepath", elem_id="processed_image", height=480, width=270, interactive=False)
+
+ with gr.Column(variant='panel', scale=1):
+ with gr.Tabs(elem_id="openlrm_render_video"):
+ with gr.TabItem('Rendered Video'):
+ with gr.Row():
+ output_video = gr.Video(label="Rendered Video", format="mp4", height=480, width=270, autoplay=True)
+
+ # SETTING
+ with gr.Row():
+ with gr.Column(variant='panel', scale=1):
+ submit = gr.Button('Generate', elem_id="openlrm_generate", variant='primary')
+
+
+ working_dir = gr.State()
+ submit.click(
+ fn=assert_input_image,
+ inputs=[input_image],
+ queue=False,
+ ).success(
+ fn=prepare_working_dir,
+ outputs=[working_dir],
+ queue=False,
+ ).success(
+ fn=core_fn,
+ inputs=[input_image, video_input, working_dir], # video_params refer to smpl dir
+ outputs=[processed_image, output_video],
+ )
+
+ demo.queue()
+ demo.launch()
+
+
+def launch_gradio_app():
+
+ os.environ.update({
+ "APP_ENABLED": "1",
+ "APP_MODEL_NAME": "./exps/releases/video_human_benchmark/human-lrm-500M/step_060000/",
+ "APP_INFER": "./configs/inference/human-lrm-500M.yaml",
+ "APP_TYPE": "infer.human_lrm",
+ "NUMBA_THREADING_LAYER": 'omp',
+ })
+
+ # from LHM.runners import REGISTRY_RUNNERS
+ # RunnerClass = REGISTRY_RUNNERS[os.getenv("APP_TYPE")]
+ # with RunnerClass() as runner:
+ # runner.to('cuda')
+ # demo_lhm(infer_impl=runner.infer)
+
+ facedetector = VGGHeadDetector(
+ "./pretrained_models/gagatracker/vgghead/vgg_heads_l.trcd",
+ device='cpu',
+ )
+ facedetector.to('cuda')
+
+ pose_estimator = PoseEstimator(
+ "./pretrained_models/human_model_files/", device='cpu'
+ )
+ pose_estimator.to('cuda')
+ pose_estimator.device = 'cuda'
+
+ cfg, cfg_train = parse_configs()
+ lhm = _build_model(cfg)
+ lhm.to('cuda')
+
+ demo_lhm(pose_estimator, facedetector, lhm, cfg)
+
+
+
+if __name__ == '__main__':
+ launch_pretrained()
+ # launch_env_not_compile_with_cuda()
+ launch_gradio_app()
-import gradio as gr
+# import gradio as gr
-def greet(name):
- return "Hello " + name + "!!"
+# def greet(name):
+# return "Hello " + name + "!!"
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
\ No newline at end of file
+# demo = gr.Interface(fn=greet, inputs="text", outputs="text")
+# demo.launch()
\ No newline at end of file
diff --git a/engine/SegmentAPI/__pycache__/base.cpython-310.pyc b/engine/SegmentAPI/__pycache__/base.cpython-310.pyc
index e52da8fc0c129d8e68b229ec9ad308e849a3eba5..6e65eff38c522b44064e0434ca24019b60d7d106 100644
Binary files a/engine/SegmentAPI/__pycache__/base.cpython-310.pyc and b/engine/SegmentAPI/__pycache__/base.cpython-310.pyc differ
diff --git a/engine/__pycache__/__init__.cpython-310.pyc b/engine/__pycache__/__init__.cpython-310.pyc
index f2d431c8c00d8fee1725d3926733d2fafdf7afbd..b0853ff2b4e846fcadc51cd0e567f73da0caa709 100644
Binary files a/engine/__pycache__/__init__.cpython-310.pyc and b/engine/__pycache__/__init__.cpython-310.pyc differ
diff --git a/engine/__pycache__/ouputs.cpython-310.pyc b/engine/__pycache__/ouputs.cpython-310.pyc
index e271f93f21c75c869cbd849d18271bbd75870284..fe01d3fcbc9e2a17b9ea9d3a0546758769e29096 100644
Binary files a/engine/__pycache__/ouputs.cpython-310.pyc and b/engine/__pycache__/ouputs.cpython-310.pyc differ
diff --git a/requirements.txt b/requirements.txt
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f1c901ed13007ba4e946d8409bccae606c01bc5b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -0,0 +1,55 @@
+einops
+roma
+accelerate
+smplx
+iopath
+# gradio
+wheel
+# chumpy==0.66
+decord==0.6.0
+diffusers
+dna==0.0.1
+gfpgan==1.3.8
+gsplat==1.4.0
+huggingface_hub==0.23.2
+imageio==2.19.3
+jaxtyping==0.2.38
+kiui==0.2.14
+kornia==0.7.2
+loguru==0.7.3
+lpips==0.1.4
+matplotlib==3.5.3
+megfile==4.1.0.post2
+numpy==1.23.0
+omegaconf==2.3.0
+open3d==0.19.0
+opencv_python
+opencv_python_headless
+Pillow==11.1.0
+plyfile
+pygltflib==1.16.2
+pyrender==0.1.45
+PyYAML==6.0.1
+rembg==2.0.63
+Requests==2.32.3
+scipy
+setuptools==74.0.0
+taming_transformers_rom1504==0.0.6
+timm==1.0.15
+
+# https://download.pytorch.org/whl/cu121/torch-2.5.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=92af92c569de5da937dd1afb45ecfdd598ec1254cf2e49e3d698cb24d71aae14
+# https://download.pytorch.org/whl/cu121/torchvision-0.20.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=304937b82c933d5155bd04d771f4b187273f67a76050bb4276b521f7e9b4c4e7
+# https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=e213ff8123e20602bd486739ffee4013338b02f9d2e0e4635a2912750854fdbe
+
+https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=28bfba084dca52a06c465d7ad0f3cc372c35fc503f3eab881cc17a5fd82914e7
+https://download.pytorch.org/whl/cu121/torchvision-0.19.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=5ee103c7eb47f8b08837e0e48b178f7ecc91d769d2b61240b90cb5aa2d06ce77
+
+--no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py310_cu121_pyt240/download.html
+
+tqdm==4.66.4
+transformers==4.41.2
+trimesh==4.4.9
+typeguard==2.13.3
+xatlas==0.0.9
+imageio-ffmpeg
+rembg[cpu]
\ No newline at end of file
diff --git a/requirements_lhm.txt b/requirements_lhm.txt
index 3de99af6b8ec7760ce1e3556f924c6c4ed3028ed..207884130435e82ff63dea63653ef0418ab15f9b 100644
--- a/requirements_lhm.txt
+++ b/requirements_lhm.txt
@@ -41,6 +41,11 @@ timm==1.0.15
# https://download.pytorch.org/whl/cu121/torchvision-0.20.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=304937b82c933d5155bd04d771f4b187273f67a76050bb4276b521f7e9b4c4e7
# https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=e213ff8123e20602bd486739ffee4013338b02f9d2e0e4635a2912750854fdbe
+https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=28bfba084dca52a06c465d7ad0f3cc372c35fc503f3eab881cc17a5fd82914e7
+https://download.pytorch.org/whl/cu121/torchvision-0.19.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=5ee103c7eb47f8b08837e0e48b178f7ecc91d769d2b61240b90cb5aa2d06ce77
+
+--no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py310_cu121_pyt240/download.html
+
tqdm==4.66.4
transformers==4.41.2
trimesh==4.4.9
diff --git a/wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl b/wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl
index da53c5019a5ede138d382278bd71a482df6b0599..db1ee9ca92a4ee01732d9e9696dfe436dedcbe96 100644
--- a/wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl
+++ b/wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:e0f3184936fcc68139947905916039ddf5973c5e3c0bd2d4680565bf89934e22
-size 3408819
+oid sha256:62216150573e09f7716dfa6e72c4f6fa6065cb60bf6a2eabaf8e5ccf8b36c03d
+size 3347247