Hugging Face's logo

Spaces:
kaifz
/
pgnd
Running on Zero

App Files Community
kaifz commited on
Commit
8b3d0f7
·
1 Parent(s): 66eb1ff

test-final

Browse files
Files changed (18) hide show
  1. app.py +38 -31
  2. src/experiments/log/gs/temp/form_video.mp4 +0 -0
  3. src/experiments/log/gs/temp/gs_pred.splat +1 -1
  4. src/experiments/log/temp/0000.png +2 -2
  5. src/experiments/log/temp/0001.png +2 -2
  6. src/experiments/log/temp/0002.png +2 -2
  7. src/experiments/log/temp/0003.png +2 -2
  8. src/experiments/log/temp/0004.png +2 -2
  9. src/experiments/log/temp/0005.png +2 -2
  10. src/experiments/log/temp/0006.png +2 -2
  11. src/experiments/log/temp/0007.png +2 -2
  12. src/experiments/log/temp/0008.png +2 -2
  13. src/experiments/log/temp/0009.png +2 -2
  14. src/experiments/log/temp/0010.png +2 -2
  15. src/experiments/log/temp/0011.png +2 -2
  16. src/experiments/log/temp/0012.png +2 -2
  17. src/experiments/log/temp/0013.png +2 -2
  18. src/experiments/log/temp/0014.png +2 -2
app.py CHANGED
@@ -948,7 +948,7 @@ class DynamicsVisualizer:
948
  self.preprocess_bg_gs()
949
  rendervar = self.reset_state(params, visualize_image=False, init=True)
950
  rendervar, rendervar_full = self.update_rendervar(rendervar)
951
- self.rendervar = rendervar
952
 
953
  im, depth = self.render(rendervar_full, 0, bg=[0.0, 0.0, 0.0])
954
  im_show = (im.permute(1, 2, 0) * 255.0).cpu().numpy().astype(np.uint8).copy()
@@ -983,6 +983,8 @@ class DynamicsVisualizer:
983
  self.gripper_params = tuple(
984
  v.detach().cpu() if isinstance(v, torch.Tensor) else v for v in self.gripper_params
985
  )
 
 
986
 
987
  form_video = gr.Video(
988
  label='Predicted video',
@@ -1000,9 +1002,9 @@ class DynamicsVisualizer:
1000
 
1001
  return form_video, form_3dgs_pred, \
1002
  self.preprocess_metadata, self.state, self.params, \
1003
- self.table_params, self.gripper_params
1004
 
1005
- def run_command(self, unit_command, preprocess_metadata, state, params, table_params, gripper_params):
1006
  import warp as wp
1007
  wp.init()
1008
  gpus = [int(gpu) for gpu in self.cfg.gpus]
@@ -1035,6 +1037,8 @@ class DynamicsVisualizer:
1035
  self.gripper_params = tuple(
1036
  v.to(self.torch_device) if isinstance(v, torch.Tensor) else v for v in self.gripper_params
1037
  )
 
 
1038
 
1039
  num_steps = 15
1040
  batch_size = 1
@@ -1078,8 +1082,8 @@ class DynamicsVisualizer:
1078
  # print(eef_xyz.cpu().numpy().tolist(), end=' ')
1079
 
1080
  self.step()
1081
- rendervar, rendervar_full = self.update_rendervar(self.rendervar)
1082
- self.rendervar = rendervar
1083
  im, depth = self.render(rendervar_full, 0, bg=[0.0, 0.0, 0.0])
1084
  im_show = (im.permute(1, 2, 0) * 255.0).cpu().numpy().astype(np.uint8).copy()
1085
 
@@ -1109,6 +1113,8 @@ class DynamicsVisualizer:
1109
  self.gripper_params = tuple(
1110
  v.detach().cpu() if isinstance(v, torch.Tensor) else v for v in self.gripper_params
1111
  )
 
 
1112
 
1113
  make_video(root / 'log/temp', root / f'log/gs/temp/form_video.mp4', '%04d.png', 5)
1114
 
@@ -1140,31 +1146,31 @@ class DynamicsVisualizer:
1140
  )
1141
  return form_video, form_3dgs_pred, \
1142
  self.preprocess_metadata, self.state, self.params, \
1143
- self.table_params, self.gripper_params
1144
 
1145
  @spaces.GPU
1146
- def on_click_run_xplus(self, preprocess_metadata, state, params, table_params, gripper_params):
1147
- return self.run_command([5.0, 0, 0], preprocess_metadata, state, params, table_params, gripper_params)
1148
 
1149
  @spaces.GPU
1150
- def on_click_run_xminus(self, preprocess_metadata, state, params, table_params, gripper_params):
1151
- return self.run_command([-5.0, 0, 0], preprocess_metadata, state, params, table_params, gripper_params)
1152
 
1153
  @spaces.GPU
1154
- def on_click_run_yplus(self, preprocess_metadata, state, params, table_params, gripper_params):
1155
- return self.run_command([0, 5.0, 0], preprocess_metadata, state, params, table_params, gripper_params)
1156
 
1157
  @spaces.GPU
1158
- def on_click_run_yminus(self, preprocess_metadata, state, params, table_params, gripper_params):
1159
- return self.run_command([0, -5.0, 0], preprocess_metadata, state, params, table_params, gripper_params)
1160
 
1161
  @spaces.GPU
1162
- def on_click_run_zplus(self, preprocess_metadata, state, params, table_params, gripper_params):
1163
- return self.run_command([0, 0, 5.0], preprocess_metadata, state, params, table_params, gripper_params)
1164
 
1165
  @spaces.GPU
1166
- def on_click_run_zminus(self, preprocess_metadata, state, params, table_params, gripper_params):
1167
- return self.run_command([0, 0, -5.0], preprocess_metadata, state, params, table_params, gripper_params)
1168
 
1169
  def launch(self, share=False):
1170
 
@@ -1174,6 +1180,7 @@ class DynamicsVisualizer:
1174
  params = gr.State(self.params)
1175
  table_params = gr.State(self.table_params)
1176
  gripper_params = gr.State(self.gripper_params)
 
1177
 
1178
  with gr.Row():
1179
  gr.Markdown("# Particle-Grid Neural Dynamics for Learning Deformable Object Models from RGB-D Videos")
@@ -1290,49 +1297,49 @@ class DynamicsVisualizer:
1290
  inputs=[],
1291
  outputs=[form_video, form_3dgs_pred,
1292
  preprocess_metadata, state, params,
1293
- table_params, gripper_params])
1294
 
1295
  run_xplus.click(self.on_click_run_xplus,
1296
  inputs=[preprocess_metadata, state, params,
1297
- table_params, gripper_params],
1298
  outputs=[form_video, form_3dgs_pred,
1299
  preprocess_metadata, state, params,
1300
- table_params, gripper_params])
1301
 
1302
  run_xminus.click(self.on_click_run_xminus,
1303
  inputs=[preprocess_metadata, state, params,
1304
- table_params, gripper_params],
1305
  outputs=[form_video, form_3dgs_pred,
1306
  preprocess_metadata, state, params,
1307
- table_params, gripper_params])
1308
 
1309
  run_yplus.click(self.on_click_run_yplus,
1310
  inputs=[preprocess_metadata, state, params,
1311
- table_params, gripper_params],
1312
  outputs=[form_video, form_3dgs_pred,
1313
  preprocess_metadata, state, params,
1314
- table_params, gripper_params])
1315
 
1316
  run_yminus.click(self.on_click_run_yminus,
1317
  inputs=[preprocess_metadata, state, params,
1318
- table_params, gripper_params],
1319
  outputs=[form_video, form_3dgs_pred,
1320
  preprocess_metadata, state, params,
1321
- table_params, gripper_params])
1322
 
1323
  run_zplus.click(self.on_click_run_zplus,
1324
  inputs=[preprocess_metadata, state, params,
1325
- table_params, gripper_params],
1326
  outputs=[form_video, form_3dgs_pred,
1327
  preprocess_metadata, state, params,
1328
- table_params, gripper_params])
1329
 
1330
  run_zminus.click(self.on_click_run_zminus,
1331
  inputs=[preprocess_metadata, state, params,
1332
- table_params, gripper_params],
1333
  outputs=[form_video, form_3dgs_pred,
1334
  preprocess_metadata, state, params,
1335
- table_params, gripper_params])
1336
 
1337
  # Set up callbacks
1338
  # run_reset_2.click(self.reset_2,
 
948
  self.preprocess_bg_gs()
949
  rendervar = self.reset_state(params, visualize_image=False, init=True)
950
  rendervar, rendervar_full = self.update_rendervar(rendervar)
951
+ # self.rendervar = rendervar
952
 
953
  im, depth = self.render(rendervar_full, 0, bg=[0.0, 0.0, 0.0])
954
  im_show = (im.permute(1, 2, 0) * 255.0).cpu().numpy().astype(np.uint8).copy()
 
983
  self.gripper_params = tuple(
984
  v.detach().cpu() if isinstance(v, torch.Tensor) else v for v in self.gripper_params
985
  )
986
+ for k, v in rendervar.items():
987
+ rendervar[k] = v.detach().cpu() if isinstance(v, torch.Tensor) else v
988
 
989
  form_video = gr.Video(
990
  label='Predicted video',
 
1002
 
1003
  return form_video, form_3dgs_pred, \
1004
  self.preprocess_metadata, self.state, self.params, \
1005
+ self.table_params, self.gripper_params, rendervar
1006
 
1007
+ def run_command(self, unit_command, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1008
  import warp as wp
1009
  wp.init()
1010
  gpus = [int(gpu) for gpu in self.cfg.gpus]
 
1037
  self.gripper_params = tuple(
1038
  v.to(self.torch_device) if isinstance(v, torch.Tensor) else v for v in self.gripper_params
1039
  )
1040
+ for k, v in rendervar.items():
1041
+ rendervar[k] = v.to(self.torch_device) if isinstance(v, torch.Tensor) else v
1042
 
1043
  num_steps = 15
1044
  batch_size = 1
 
1082
  # print(eef_xyz.cpu().numpy().tolist(), end=' ')
1083
 
1084
  self.step()
1085
+ rendervar, rendervar_full = self.update_rendervar(rendervar)
1086
+ # self.rendervar = rendervar
1087
  im, depth = self.render(rendervar_full, 0, bg=[0.0, 0.0, 0.0])
1088
  im_show = (im.permute(1, 2, 0) * 255.0).cpu().numpy().astype(np.uint8).copy()
1089
 
 
1113
  self.gripper_params = tuple(
1114
  v.detach().cpu() if isinstance(v, torch.Tensor) else v for v in self.gripper_params
1115
  )
1116
+ for k, v in rendervar.items():
1117
+ rendervar[k] = v.detach().cpu() if isinstance(v, torch.Tensor) else v
1118
 
1119
  make_video(root / 'log/temp', root / f'log/gs/temp/form_video.mp4', '%04d.png', 5)
1120
 
 
1146
  )
1147
  return form_video, form_3dgs_pred, \
1148
  self.preprocess_metadata, self.state, self.params, \
1149
+ self.table_params, self.gripper_params, rendervar
1150
 
1151
  @spaces.GPU
1152
+ def on_click_run_xplus(self, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1153
+ return self.run_command([5.0, 0, 0], preprocess_metadata, state, params, table_params, gripper_params, rendervar)
1154
 
1155
  @spaces.GPU
1156
+ def on_click_run_xminus(self, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1157
+ return self.run_command([-5.0, 0, 0], preprocess_metadata, state, params, table_params, gripper_params, rendervar)
1158
 
1159
  @spaces.GPU
1160
+ def on_click_run_yplus(self, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1161
+ return self.run_command([0, 5.0, 0], preprocess_metadata, state, params, table_params, gripper_params, rendervar)
1162
 
1163
  @spaces.GPU
1164
+ def on_click_run_yminus(self, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1165
+ return self.run_command([0, -5.0, 0], preprocess_metadata, state, params, table_params, gripper_params, rendervar)
1166
 
1167
  @spaces.GPU
1168
+ def on_click_run_zplus(self, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1169
+ return self.run_command([0, 0, 5.0], preprocess_metadata, state, params, table_params, gripper_params, rendervar)
1170
 
1171
  @spaces.GPU
1172
+ def on_click_run_zminus(self, preprocess_metadata, state, params, table_params, gripper_params, rendervar):
1173
+ return self.run_command([0, 0, -5.0], preprocess_metadata, state, params, table_params, gripper_params, rendervar)
1174
 
1175
  def launch(self, share=False):
1176
 
 
1180
  params = gr.State(self.params)
1181
  table_params = gr.State(self.table_params)
1182
  gripper_params = gr.State(self.gripper_params)
1183
+ rendervar = gr.State(None)
1184
 
1185
  with gr.Row():
1186
  gr.Markdown("# Particle-Grid Neural Dynamics for Learning Deformable Object Models from RGB-D Videos")
 
1297
  inputs=[],
1298
  outputs=[form_video, form_3dgs_pred,
1299
  preprocess_metadata, state, params,
1300
+ table_params, gripper_params, rendervar])
1301
 
1302
  run_xplus.click(self.on_click_run_xplus,
1303
  inputs=[preprocess_metadata, state, params,
1304
+ table_params, gripper_params, rendervar],
1305
  outputs=[form_video, form_3dgs_pred,
1306
  preprocess_metadata, state, params,
1307
+ table_params, gripper_params, rendervar])
1308
 
1309
  run_xminus.click(self.on_click_run_xminus,
1310
  inputs=[preprocess_metadata, state, params,
1311
+ table_params, gripper_params, rendervar],
1312
  outputs=[form_video, form_3dgs_pred,
1313
  preprocess_metadata, state, params,
1314
+ table_params, gripper_params, rendervar])
1315
 
1316
  run_yplus.click(self.on_click_run_yplus,
1317
  inputs=[preprocess_metadata, state, params,
1318
+ table_params, gripper_params, rendervar],
1319
  outputs=[form_video, form_3dgs_pred,
1320
  preprocess_metadata, state, params,
1321
+ table_params, gripper_params, rendervar])
1322
 
1323
  run_yminus.click(self.on_click_run_yminus,
1324
  inputs=[preprocess_metadata, state, params,
1325
+ table_params, gripper_params, rendervar],
1326
  outputs=[form_video, form_3dgs_pred,
1327
  preprocess_metadata, state, params,
1328
+ table_params, gripper_params, rendervar])
1329
 
1330
  run_zplus.click(self.on_click_run_zplus,
1331
  inputs=[preprocess_metadata, state, params,
1332
+ table_params, gripper_params, rendervar],
1333
  outputs=[form_video, form_3dgs_pred,
1334
  preprocess_metadata, state, params,
1335
+ table_params, gripper_params, rendervar])
1336
 
1337
  run_zminus.click(self.on_click_run_zminus,
1338
  inputs=[preprocess_metadata, state, params,
1339
+ table_params, gripper_params, rendervar],
1340
  outputs=[form_video, form_3dgs_pred,
1341
  preprocess_metadata, state, params,
1342
+ table_params, gripper_params, rendervar])
1343
 
1344
  # Set up callbacks
1345
  # run_reset_2.click(self.reset_2,
src/experiments/log/gs/temp/form_video.mp4 CHANGED
Binary files a/src/experiments/log/gs/temp/form_video.mp4 and b/src/experiments/log/gs/temp/form_video.mp4 differ
 
src/experiments/log/gs/temp/gs_pred.splat CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:111f09eae16a77a8e63af83f4adcb78c21ac301113a18fae10bcaa89432aff5f
3
  size 7675392
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da576876524c55864094c59bbe99313a167797d312af80eb3666bac7b0a77b34
3
  size 7675392
src/experiments/log/temp/0000.png CHANGED

Git LFS Details

  • SHA256: 77b3cfbc740682e220917dbc11f617077f5f1b7f3897565aa233ae20961b4e63
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB

Git LFS Details

  • SHA256: 11456b9ce9029fa0ed7527ad41efba11bb26552bec2470e7ea76dde89e0c828e
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB
src/experiments/log/temp/0001.png CHANGED

Git LFS Details

  • SHA256: cd667b7ce224a604ce20103753e79faa3d94deb1ac01d3ac2530c78df454430f
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB

Git LFS Details

  • SHA256: 1004a96994e767b8bc4cf65402ff339500e4f8fb147863ab65a39d4d828202c4
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB
src/experiments/log/temp/0002.png CHANGED

Git LFS Details

  • SHA256: 7d97b92f82aa2a4c2ab2ecdbafb27265f2b8246c41ae48906a2694f1baa234b5
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB

Git LFS Details

  • SHA256: 87a9f831a0ddf866e50446be340e01009256dfe64618f0c9a1d0256b71601daf
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB
src/experiments/log/temp/0003.png CHANGED

Git LFS Details

  • SHA256: a53ec2eb51510a425e94d6c77cfffe523b3d7b58d3f55f9c9971bb0860df14ba
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB

Git LFS Details

  • SHA256: b7d2e986c5bfe920d2e3d0fdcf81d7d3d0968f4de65bbd98b79931a852eba665
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB
src/experiments/log/temp/0004.png CHANGED

Git LFS Details

  • SHA256: 0cc56f75c675a58ec61e482d5e9cf532e7880b33f9ed43248c75e62134aa897d
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB

Git LFS Details

  • SHA256: 9806bc97e1eaf12e7f8fb6bf8b2218c9ddd59446e1ec19db1cbe9df2e3e7e2b6
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB
src/experiments/log/temp/0005.png CHANGED

Git LFS Details

  • SHA256: 25eb614538d6665089a9abf3acd61117645ecf8087e96aaf89cd9a0236f222bc
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 39da1a7f34fe222ea5e2d2c1ca5ae2a0b57b3b7b3f20d05814fd0467f774e50a
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0006.png CHANGED

Git LFS Details

  • SHA256: 4f138c4c08e933e41e4545fd75d3b400fbe8c5f51cab29b6113c09d43fdc6a8f
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 94aec7a81308466c70d03fed54eafe6a61dcaa586da999d309a9d964677c1929
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0007.png CHANGED

Git LFS Details

  • SHA256: 39e47213f688d7e8322c52cdcb90515ee2f3b948f4bcaa310a010892aba12bd8
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 83499cda333830834b88059283aad389d52d2d4e7b171518bb498d5caa847bb7
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0008.png CHANGED

Git LFS Details

  • SHA256: 8574795142c6738e04e34467f5fe6f42fc6132b681483608bebc8ed814eebba0
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: d6a408586985bbac39ba40acb2d4f4ec859157b429ecccf794844cf0a9f5efcf
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0009.png CHANGED

Git LFS Details

  • SHA256: e3dbe50211cfa3b861b3ae4d1695cfda3a886f4f710870e6134d7ff7d0beb990
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: c0346bb096cbf89297b20fc93ca9d2c4ace328231eab1ed225175ceba06f18c0
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0010.png CHANGED

Git LFS Details

  • SHA256: 37dbc2cb4ebdb788971f1676d3f86c0d51ea8b058fa709bbec16347a20e593f5
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 43ea224b2bf8dd24e4b0bd5a9e7f5f5db041f89cdfb9281d5ed9b3f5cd98d0cf
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0011.png CHANGED

Git LFS Details

  • SHA256: 111c751c5c68bc9f5343c3ccbd87ac0549fcedec5246dc47a985ed1f777afb99
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 28abd277a392e3388dfc52ef568756761c7c11f5c5ae53e8fa78ff721067ce96
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0012.png CHANGED

Git LFS Details

  • SHA256: 0f412dfad710635fd79c080705b470155a738eb7b143fe739bc6303be1ab57db
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 727674ea075279239340b6965acfb83842a43fdd4eb6c0a51eca570060b8e612
  • Pointer size: 131 Bytes
  • Size of remote file: 290 kB
src/experiments/log/temp/0013.png CHANGED

Git LFS Details

  • SHA256: 7b5ee00ff589b68b25ed014038a08a28d5214ca328ea53705b63c7a5d74093dd
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: 7616e6af30d23cdb22762d4abcea747f6d764ffc758fc791ebe4f3080ce7da45
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB
src/experiments/log/temp/0014.png CHANGED

Git LFS Details

  • SHA256: f40c996ebbd4ef62355d3e6bfe4c97ba07d1184d06b108444658fb367e55cdff
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB

Git LFS Details

  • SHA256: da6e4014b7cb38c77c140d48188c1d769d3adb5876e4bed7a3d973084696e008
  • Pointer size: 131 Bytes
  • Size of remote file: 289 kB