Upload folder using huggingface_hub

.hydra/config.yaml

@@ -0,0 +1,160 @@
+resume: true
+device: cuda
+use_amp: false
+seed: 100000
+dataset_repo_id: HumanoidTeam/aloha_real_test
+video_backend: pyav
+training:
+  offline_steps: 5010
+  num_workers: 8
+  batch_size: 64
+  eval_freq: 25000
+  log_freq: 200
+  save_checkpoint: true
+  save_freq: 25000
+  online_steps: 0
+  online_rollout_n_episodes: 1
+  online_rollout_batch_size: 1
+  online_steps_between_rollouts: 1
+  online_sampling_ratio: 0.5
+  online_env_seed: null
+  online_buffer_capacity: null
+  online_buffer_seed_size: 0
+  do_online_rollout_async: false
+  image_transforms:
+    enable: false
+    max_num_transforms: 3
+    random_order: false
+    brightness:
+      weight: 1
+      min_max:
+      - 0.8
+      - 1.2
+    contrast:
+      weight: 1
+      min_max:
+      - 0.8
+      - 1.2
+    saturation:
+      weight: 1
+      min_max:
+      - 0.5
+      - 1.5
+    hue:
+      weight: 1
+      min_max:
+      - -0.05
+      - 0.05
+    sharpness:
+      weight: 1
+      min_max:
+      - 0.8
+      - 1.2
+  grad_clip_norm: 10
+  lr: 0.0001
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  adam_betas:
+  - 0.95
+  - 0.999
+  adam_eps: 1.0e-08
+  adam_weight_decay: 1.0e-06
+  delta_timestamps:
+    observation.image: '[i / ${fps} for i in range(1 - ${policy.n_obs_steps}, 1)]'
+    observation.state: '[i / ${fps} for i in range(1 - ${policy.n_obs_steps}, 1)]'
+    action: '[i / ${fps} for i in range(1 - ${policy.n_obs_steps}, 1 - ${policy.n_obs_steps}
+      + ${policy.horizon})]'
+  drop_n_last_frames: 7
+eval:
+  n_episodes: 50
+  batch_size: 50
+  use_async_envs: false
+wandb:
+  enable: true
+  disable_artifact: false
+  project: lerobot_real_aloha
+  notes: ws-lerobot-hackathon
+fps: 10
+env:
+  name: pusht
+  task: PushT-v0
+  image_size: 96
+  state_dim: 2
+  action_dim: 2
+  fps: ${fps}
+  episode_length: 300
+  gym:
+    obs_type: pixels_agent_pos
+    render_mode: rgb_array
+    visualization_width: 384
+    visualization_height: 384
+override_dataset_stats:
+  observation.image:
+    mean:
+    - - - 0.5
+    - - - 0.5
+    - - - 0.5
+    std:
+    - - - 0.5
+    - - - 0.5
+    - - - 0.5
+  observation.state:
+    min:
+    - 13.456424
+    - 32.938293
+    max:
+    - 496.14618
+    - 510.9579
+  action:
+    min:
+    - 12.0
+    - 25.0
+    max:
+    - 511.0
+    - 511.0
+policy:
+  name: diffusion
+  n_obs_steps: 2
+  horizon: 16
+  n_action_steps: 8
+  input_shapes:
+    observation.image:
+    - 3
+    - 96
+    - 96
+    observation.state:
+    - ${env.state_dim}
+  output_shapes:
+    action:
+    - ${env.action_dim}
+  input_normalization_modes:
+    observation.image: mean_std
+    observation.state: min_max
+  output_normalization_modes:
+    action: min_max
+  vision_backbone: resnet18
+  crop_shape:
+  - 84
+  - 84
+  crop_is_random: true
+  pretrained_backbone_weights: null
+  use_group_norm: true
+  spatial_softmax_num_keypoints: 32
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  diffusion_step_embed_dim: 128
+  use_film_scale_modulation: true
+  noise_scheduler_type: DDPM
+  num_train_timesteps: 100
+  beta_schedule: squaredcos_cap_v2
+  beta_start: 0.0001
+  beta_end: 0.02
+  prediction_type: epsilon
+  clip_sample: true
+  clip_sample_range: 1.0
+  num_inference_steps: null
+  do_mask_loss_for_padding: false

.hydra/hydra.yaml

@@ -0,0 +1,160 @@
+hydra:
+  run:
+    dir: outputs/train/2024-12-06/22-28-29_real_world_act_default
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=outputs/train/2024-12-06/22-28-29_real_world_act_default
+    - hydra.mode=RUN
+    task:
+    - resume=true
+    - dataset_repo_id=HumanoidTeam/aloha_real_test
+    - training.offline_steps=5010
+  job:
+    name: default
+    chdir: null
+    override_dirname: dataset_repo_id=HumanoidTeam/aloha_real_test,resume=true,training.offline_steps=5010
+    id: ???
+    num: ???
+    config_name: default
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.2'
+    cwd: /home/ubuntu/lerobot-hackathon
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ubuntu/lerobot-hackathon/lerobot/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ubuntu/lerobot-hackathon/outputs/train/2024-12-06/22-28-29_real_world_act_default
+    choices:
+      policy: diffusion
+      env: pusht
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- resume=true
+- dataset_repo_id=HumanoidTeam/aloha_real_test
+- training.offline_steps=5010

checkpoints/005000/pretrained_model/README.md

@@ -0,0 +1,12 @@
+---
+library_name: lerobot
+tags:
+- act
+- model_hub_mixin
+- pytorch_model_hub_mixin
+- robotics
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Library: https://github.com/huggingface/lerobot
+- Docs: [More Information Needed]

checkpoints/005000/pretrained_model/config.json

@@ -0,0 +1,55 @@
+{
+  "chunk_size": 100,
+  "dim_feedforward": 3200,
+  "dim_model": 512,
+  "dropout": 0.1,
+  "feedforward_activation": "relu",
+  "input_normalization_modes": {
+    "observation.images.cam_high": "mean_std",
+    "observation.images.cam_left_wrist": "mean_std",
+    "observation.images.cam_right_wrist": "mean_std",
+    "observation.state": "mean_std"
+  },
+  "input_shapes": {
+    "observation.images.cam_high": [
+      3,
+      480,
+      640
+    ],
+    "observation.images.cam_left_wrist": [
+      3,
+      480,
+      640
+    ],
+    "observation.images.cam_right_wrist": [
+      3,
+      480,
+      640
+    ],
+    "observation.state": [
+      14
+    ]
+  },
+  "kl_weight": 10.0,
+  "latent_dim": 32,
+  "n_action_steps": 100,
+  "n_decoder_layers": 1,
+  "n_encoder_layers": 4,
+  "n_heads": 8,
+  "n_obs_steps": 1,
+  "n_vae_encoder_layers": 4,
+  "output_normalization_modes": {
+    "action": "mean_std"
+  },
+  "output_shapes": {
+    "action": [
+      14
+    ]
+  },
+  "pre_norm": false,
+  "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
+  "replace_final_stride_with_dilation": false,
+  "temporal_ensemble_coeff": null,
+  "use_vae": true,
+  "vision_backbone": "resnet18"
+}

checkpoints/005000/pretrained_model/config.yaml

@@ -0,0 +1,248 @@
+resume: false
+device: cuda
+use_amp: false
+seed: 1000
+dataset_repo_id: ccop/aloha_stationary_replay_test_v2
+video_backend: pyav
+training:
+  offline_steps: 5000
+  num_workers: 8
+  batch_size: 8
+  eval_freq: -1
+  log_freq: 200
+  save_checkpoint: true
+  save_freq: 10000
+  online_steps: 0
+  online_rollout_n_episodes: 1
+  online_rollout_batch_size: 1
+  online_steps_between_rollouts: 1
+  online_sampling_ratio: 0.5
+  online_env_seed: null
+  online_buffer_capacity: null
+  online_buffer_seed_size: 0
+  do_online_rollout_async: false
+  image_transforms:
+    enable: false
+    max_num_transforms: 3
+    random_order: false
+    brightness:
+      weight: 1
+      min_max:
+      - 0.8
+      - 1.2
+    contrast:
+      weight: 1
+      min_max:
+      - 0.8
+      - 1.2
+    saturation:
+      weight: 1
+      min_max:
+      - 0.5
+      - 1.5
+    hue:
+      weight: 1
+      min_max:
+      - -0.05
+      - 0.05
+    sharpness:
+      weight: 1
+      min_max:
+      - 0.8
+      - 1.2
+  lr: 1.0e-05
+  lr_backbone: 1.0e-05
+  weight_decay: 0.0001
+  grad_clip_norm: 10
+  delta_timestamps:
+    action:
+    - 0.0
+    - 0.03333333333333333
+    - 0.06666666666666667
+    - 0.1
+    - 0.13333333333333333
+    - 0.16666666666666666
+    - 0.2
+    - 0.23333333333333334
+    - 0.26666666666666666
+    - 0.3
+    - 0.3333333333333333
+    - 0.36666666666666664
+    - 0.4
+    - 0.43333333333333335
+    - 0.4666666666666667
+    - 0.5
+    - 0.5333333333333333
+    - 0.5666666666666667
+    - 0.6
+    - 0.6333333333333333
+    - 0.6666666666666666
+    - 0.7
+    - 0.7333333333333333
+    - 0.7666666666666667
+    - 0.8
+    - 0.8333333333333334
+    - 0.8666666666666667
+    - 0.9
+    - 0.9333333333333333
+    - 0.9666666666666667
+    - 1.0
+    - 1.0333333333333334
+    - 1.0666666666666667
+    - 1.1
+    - 1.1333333333333333
+    - 1.1666666666666667
+    - 1.2
+    - 1.2333333333333334
+    - 1.2666666666666666
+    - 1.3
+    - 1.3333333333333333
+    - 1.3666666666666667
+    - 1.4
+    - 1.4333333333333333
+    - 1.4666666666666666
+    - 1.5
+    - 1.5333333333333334
+    - 1.5666666666666667
+    - 1.6
+    - 1.6333333333333333
+    - 1.6666666666666667
+    - 1.7
+    - 1.7333333333333334
+    - 1.7666666666666666
+    - 1.8
+    - 1.8333333333333333
+    - 1.8666666666666667
+    - 1.9
+    - 1.9333333333333333
+    - 1.9666666666666666
+    - 2.0
+    - 2.033333333333333
+    - 2.066666666666667
+    - 2.1
+    - 2.1333333333333333
+    - 2.1666666666666665
+    - 2.2
+    - 2.2333333333333334
+    - 2.2666666666666666
+    - 2.3
+    - 2.3333333333333335
+    - 2.3666666666666667
+    - 2.4
+    - 2.433333333333333
+    - 2.466666666666667
+    - 2.5
+    - 2.533333333333333
+    - 2.566666666666667
+    - 2.6
+    - 2.6333333333333333
+    - 2.6666666666666665
+    - 2.7
+    - 2.7333333333333334
+    - 2.7666666666666666
+    - 2.8
+    - 2.8333333333333335
+    - 2.8666666666666667
+    - 2.9
+    - 2.933333333333333
+    - 2.966666666666667
+    - 3.0
+    - 3.033333333333333
+    - 3.066666666666667
+    - 3.1
+    - 3.1333333333333333
+    - 3.1666666666666665
+    - 3.2
+    - 3.2333333333333334
+    - 3.2666666666666666
+    - 3.3
+eval:
+  n_episodes: 50
+  batch_size: 50
+  use_async_envs: false
+wandb:
+  enable: true
+  disable_artifact: false
+  project: lerobot_real_aloha
+  notes: ws-lerobot-hackathon
+fps: 30
+env:
+  name: real_world
+  task: null
+  state_dim: 14
+  action_dim: 14
+  fps: ${fps}
+override_dataset_stats:
+  observation.images.cam_right_wrist:
+    mean:
+    - - - 0.485
+    - - - 0.456
+    - - - 0.406
+    std:
+    - - - 0.229
+    - - - 0.224
+    - - - 0.225
+  observation.images.cam_left_wrist:
+    mean:
+    - - - 0.485
+    - - - 0.456
+    - - - 0.406
+    std:
+    - - - 0.229
+    - - - 0.224
+    - - - 0.225
+  observation.images.cam_high:
+    mean:
+    - - - 0.485
+    - - - 0.456
+    - - - 0.406
+    std:
+    - - - 0.229
+    - - - 0.224
+    - - - 0.225
+policy:
+  name: act
+  n_obs_steps: 1
+  chunk_size: 100
+  n_action_steps: 100
+  input_shapes:
+    observation.images.cam_right_wrist:
+    - 3
+    - 480
+    - 640
+    observation.images.cam_left_wrist:
+    - 3
+    - 480
+    - 640
+    observation.images.cam_high:
+    - 3
+    - 480
+    - 640
+    observation.state:
+    - ${env.state_dim}
+  output_shapes:
+    action:
+    - ${env.action_dim}
+  input_normalization_modes:
+    observation.images.cam_right_wrist: mean_std
+    observation.images.cam_left_wrist: mean_std
+    observation.images.cam_high: mean_std
+    observation.state: mean_std
+  output_normalization_modes:
+    action: mean_std
+  vision_backbone: resnet18
+  pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1
+  replace_final_stride_with_dilation: false
+  pre_norm: false
+  dim_model: 512
+  n_heads: 8
+  dim_feedforward: 3200
+  feedforward_activation: relu
+  n_encoder_layers: 4
+  n_decoder_layers: 1
+  use_vae: true
+  latent_dim: 32
+  n_vae_encoder_layers: 4
+  temporal_ensemble_momentum: null
+  dropout: 0.1
+  kl_weight: 10.0

checkpoints/005000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eba28285320bc2ddcda5e0ce0be693634a20ec638d10cf364694649bfc184964
+size 206767160

checkpoints/005000/training_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af943082a3d9b919922d7de964e9f2bdc429b3d800234ea46f97ccedce9b309c
+size 413053071

wandb/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-12-07T01:07:41.167467035Z","level":"INFO","msg":"using version","core version":"0.19.0"}
+{"time":"2024-12-07T01:07:41.167480199Z","level":"INFO","msg":"created symlink","path":"outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010741-bwwpk9vl/logs/debug-core.log"}
+{"time":"2024-12-07T01:07:41.270189094Z","level":"INFO","msg":"created new stream","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270216139Z","level":"INFO","msg":"stream: started","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270250461Z","level":"INFO","msg":"writer: Do: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270250342Z","level":"INFO","msg":"sender: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270317361Z","level":"INFO","msg":"handler: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.601065068Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-12-07T01:07:46.320070438Z","level":"INFO","msg":"stream: closing","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:46.320109578Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-07T01:07:46.320581424Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-07T01:07:46.890807441Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-07T01:07:47.027194656Z","level":"INFO","msg":"handler: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:47.027227034Z","level":"INFO","msg":"sender: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:47.027228216Z","level":"INFO","msg":"writer: Close: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:47.027338658Z","level":"INFO","msg":"stream: closed","id":"bwwpk9vl"}

wandb/debug.log

@@ -0,0 +1,25 @@
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Current SDK version is 0.19.0
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Configure stats pid to 3507456
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/lerobot-hackathon/wandb/settings
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] wandb: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:_log_setup():528] Logging user logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010741-bwwpk9vl/logs/debug.log
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:_log_setup():529] Logging internal logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010741-bwwpk9vl/logs/debug-internal.log
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():639] calling init triggers
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():645] wandb.init called with sweep_config: {}
+config: {'resume': True, 'device': 'cuda', 'use_amp': False, 'seed': 1000, 'dataset_repo_id': 'ccop/aloha_stationary_replay_test_v2', 'video_backend': 'pyav', 'training': {'offline_steps': 5000, 'num_workers': 8, 'batch_size': 8, 'eval_freq': -1, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 10000, 'online_steps': 0, 'online_rollout_n_episodes': 1, 'online_rollout_batch_size': 1, 'online_steps_between_rollouts': 1, 'online_sampling_ratio': 0.5, 'online_env_seed': None, 'online_buffer_capacity': None, 'online_buffer_seed_size': 0, 'do_online_rollout_async': False, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'brightness': {'weight': 1, 'min_max': [0.8, 1.2]}, 'contrast': {'weight': 1, 'min_max': [0.8, 1.2]}, 'saturation': {'weight': 1, 'min_max': [0.5, 1.5]}, 'hue': {'weight': 1, 'min_max': [-0.05, 0.05]}, 'sharpness': {'weight': 1, 'min_max': [0.8, 1.2]}}, 'lr': 1e-05, 'lr_backbone': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10, 'delta_timestamps': {'action': [0.0, 0.03333333333333333, 0.06666666666666667, 0.1, 0.13333333333333333, 0.16666666666666666, 0.2, 0.23333333333333334, 0.26666666666666666, 0.3, 0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6, 0.6333333333333333, 0.6666666666666666, 0.7, 0.7333333333333333, 0.7666666666666667, 0.8, 0.8333333333333334, 0.8666666666666667, 0.9, 0.9333333333333333, 0.9666666666666667, 1.0, 1.0333333333333334, 1.0666666666666667, 1.1, 1.1333333333333333, 1.1666666666666667, 1.2, 1.2333333333333334, 1.2666666666666666, 1.3, 1.3333333333333333, 1.3666666666666667, 1.4, 1.4333333333333333, 1.4666666666666666, 1.5, 1.5333333333333334, 1.5666666666666667, 1.6, 1.6333333333333333, 1.6666666666666667, 1.7, 1.7333333333333334, 1.7666666666666666, 1.8, 1.8333333333333333, 1.8666666666666667, 1.9, 1.9333333333333333, 1.9666666666666666, 2.0, 2.033333333333333, 2.066666666666667, 2.1, 2.1333333333333333, 2.1666666666666665, 2.2, 2.2333333333333334, 2.2666666666666666, 2.3, 2.3333333333333335, 2.3666666666666667, 2.4, 2.433333333333333, 2.466666666666667, 2.5, 2.533333333333333, 2.566666666666667, 2.6, 2.6333333333333333, 2.6666666666666665, 2.7, 2.7333333333333334, 2.7666666666666666, 2.8, 2.8333333333333335, 2.8666666666666667, 2.9, 2.933333333333333, 2.966666666666667, 3.0, 3.033333333333333, 3.066666666666667, 3.1, 3.1333333333333333, 3.1666666666666665, 3.2, 3.2333333333333334, 3.2666666666666666, 3.3]}}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot_real_aloha', 'notes': 'ws-lerobot-hackathon'}, 'fps': 30, 'env': {'name': 'real_world', 'task': None, 'state_dim': 14, 'action_dim': 14, 'fps': 30}, 'override_dataset_stats': {'observation.images.cam_right_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_left_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_high': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}}, 'policy': {'name': 'act', 'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'input_shapes': {'observation.images.cam_right_wrist': [3, 480, 640], 'observation.images.cam_left_wrist': [3, 480, 640], 'observation.images.cam_high': [3, 480, 640], 'observation.state': [14]}, 'output_shapes': {'action': [14]}, 'input_normalization_modes': {'observation.images.cam_right_wrist': 'mean_std', 'observation.images.cam_left_wrist': 'mean_std', 'observation.images.cam_high': 'mean_std', 'observation.state': 'mean_std'}, 'output_normalization_modes': {'action': 'mean_std'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_momentum': None, 'dropout': 0.1, 'kl_weight': 10.0}}
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():688] starting backend
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():692] sending inform_init request
+2024-12-07 01:07:41,165 INFO    MainThread:3507456 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-07 01:07:41,166 INFO    MainThread:3507456 [wandb_init.py:init():705] backend started and connected
+2024-12-07 01:07:41,168 INFO    MainThread:3507456 [wandb_init.py:init():798] updated telemetry
+2024-12-07 01:07:41,174 INFO    MainThread:3507456 [wandb_init.py:init():830] communicating run to backend with 90.0 second timeout
+2024-12-07 01:07:41,592 INFO    MainThread:3507456 [wandb_init.py:init():875] run resumed
+2024-12-07 01:07:41,597 INFO    MainThread:3507456 [wandb_init.py:init():882] starting run threads in backend
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_console_start():2443] atexit reg
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_redirect():2293] redirect: wrap_raw
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_redirect():2358] Wrapping output streams.
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_redirect():2383] Redirects installed.
+2024-12-07 01:07:41,713 INFO    MainThread:3507456 [wandb_init.py:init():925] run started, returning control to user process
+2024-12-07 01:07:46,320 WARNING MsgRouterThr:3507456 [router.py:message_loop():75] message_loop has been closed

wandb/run-20241206_222830-bwwpk9vl/files/config.yaml

@@ -0,0 +1,194 @@
+_wandb:
+    value:
+        cli_version: 0.19.0
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 51
+                - 55
+            "2":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 51
+                - 55
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.10.12
+            "5": 0.19.0
+            "8":
+                - 5
+            "12": 0.19.0
+            "13": linux-x86_64
+dataset_repo_id:
+    value: ccop/aloha_stationary_replay_test_v2
+device:
+    value: cuda
+env:
+    value:
+        action_dim: 14
+        fps: 30
+        name: real_world
+        state_dim: 14
+        task: null
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        use_async_envs: false
+fps:
+    value: 30
+override_dataset_stats:
+    value:
+        observation.images.cam_high:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+        observation.images.cam_left_wrist:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+        observation.images.cam_right_wrist:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+policy:
+    value:
+        chunk_size: 100
+        dim_feedforward: 3200
+        dim_model: 512
+        dropout: 0.1
+        feedforward_activation: relu
+        input_normalization_modes:
+            observation.images.cam_high: mean_std
+            observation.images.cam_left_wrist: mean_std
+            observation.images.cam_right_wrist: mean_std
+            observation.state: mean_std
+        input_shapes:
+            observation.images.cam_high:
+                - 3
+                - 480
+                - 640
+            observation.images.cam_left_wrist:
+                - 3
+                - 480
+                - 640
+            observation.images.cam_right_wrist:
+                - 3
+                - 480
+                - 640
+            observation.state:
+                - 14
+        kl_weight: 10
+        latent_dim: 32
+        n_action_steps: 100
+        n_decoder_layers: 1
+        n_encoder_layers: 4
+        n_heads: 8
+        n_obs_steps: 1
+        n_vae_encoder_layers: 4
+        name: act
+        output_normalization_modes:
+            action: mean_std
+        output_shapes:
+            action:
+                - 14
+        pre_norm: false
+        pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1
+        replace_final_stride_with_dilation: false
+        temporal_ensemble_momentum: null
+        use_vae: true
+        vision_backbone: resnet18
+resume:
+    value: false
+seed:
+    value: 1000
+training:
+    value:
+        batch_size: 8
+        delta_timestamps:
+            action: '[i / 30 for i in range(100)]'
+        do_online_rollout_async: false
+        eval_freq: -1
+        grad_clip_norm: 10
+        image_transforms:
+            brightness:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+            contrast:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+            enable: false
+            hue:
+                min_max:
+                    - -0.05
+                    - 0.05
+                weight: 1
+            max_num_transforms: 3
+            random_order: false
+            saturation:
+                min_max:
+                    - 0.5
+                    - 1.5
+                weight: 1
+            sharpness:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+        log_freq: 200
+        lr: 1e-05
+        lr_backbone: 1e-05
+        num_workers: 8
+        offline_steps: 5000
+        online_buffer_capacity: null
+        online_buffer_seed_size: 0
+        online_env_seed: null
+        online_rollout_batch_size: 1
+        online_rollout_n_episodes: 1
+        online_sampling_ratio: 0.5
+        online_steps: 0
+        online_steps_between_rollouts: 1
+        save_checkpoint: true
+        save_freq: 10000
+        weight_decay: 0.0001
+use_amp:
+    value: false
+video_backend:
+    value: pyav
+wandb:
+    value:
+        disable_artifact: false
+        enable: true
+        notes: ws-lerobot-hackathon
+        project: lerobot_real_aloha

wandb/run-20241206_222830-bwwpk9vl/files/output.log

@@ -0,0 +1,46 @@
+Logs will be synced with wandb.
+INFO 2024-12-06 22:28:30 n/logger.py:132 Track this run --> https://wandb.ai/zaringleb/lerobot_real_aloha/runs/bwwpk9vl
+INFO 2024-12-06 22:28:30 ts/train.py:312 make_dataset
+WARNING 2024-12-06 22:28:30 s/factory.py:68 There might be a mismatch between your training dataset (dataset_repo_id='ccop/aloha_stationary_replay_test_v2') and your environment (cfg.env.name='real_world').
+Fetching 9 files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 39945.75it/s]
+INFO 2024-12-06 22:28:32 ts/train.py:328 make_policy
+WARNING 2024-12-06 22:28:32 s/factory.py:28 Hydra config is missing arguments: {'temporal_ensemble_coeff'}
+/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py:338: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  grad_scaler = GradScaler(enabled=cfg.use_amp)
+INFO 2024-12-06 22:28:33 on/logger.py:39 Output dir: outputs/train/2024-12-06/22-28-29_real_world_act_default
+INFO 2024-12-06 22:28:33 ts/train.py:349 cfg.env.task=None
+INFO 2024-12-06 22:28:33 ts/train.py:350 cfg.training.offline_steps=5000 (5K)
+INFO 2024-12-06 22:28:33 ts/train.py:351 cfg.training.online_steps=0
+INFO 2024-12-06 22:28:33 ts/train.py:352 offline_dataset.num_samples=800 (800)
+INFO 2024-12-06 22:28:33 ts/train.py:353 offline_dataset.num_episodes=1
+INFO 2024-12-06 22:28:33 ts/train.py:354 num_learnable_params=51613582 (52M)
+INFO 2024-12-06 22:28:33 ts/train.py:355 num_total_params=51613684 (52M)
+INFO 2024-12-06 22:28:33 ts/train.py:421 Start offline training on a fixed dataset
+INFO 2024-12-06 22:28:44 ts/train.py:192 step:0 smpl:8 ep:0 epch:0.01 loss:85.424 grdn:1036.794 lr:1.0e-05 updt_s:7.126 data_s:3.108
+INFO 2024-12-06 22:32:21 ts/train.py:192 step:200 smpl:2K ep:2 epch:2.01 loss:3.562 grdn:97.104 lr:1.0e-05 updt_s:1.076 data_s:2.453
+INFO 2024-12-06 22:36:00 ts/train.py:192 step:400 smpl:3K ep:4 epch:4.01 loss:2.455 grdn:86.458 lr:1.0e-05 updt_s:1.094 data_s:2.019
+INFO 2024-12-06 22:39:40 ts/train.py:192 step:600 smpl:5K ep:6 epch:6.01 loss:2.300 grdn:76.224 lr:1.0e-05 updt_s:1.088 data_s:2.352
+INFO 2024-12-06 22:43:19 ts/train.py:192 step:800 smpl:6K ep:8 epch:8.01 loss:1.897 grdn:66.798 lr:1.0e-05 updt_s:1.108 data_s:2.149
+INFO 2024-12-06 22:46:58 ts/train.py:192 step:1K smpl:8K ep:10 epch:10.01 loss:1.727 grdn:65.505 lr:1.0e-05 updt_s:1.100 data_s:1.999
+INFO 2024-12-06 22:50:37 ts/train.py:192 step:1K smpl:10K ep:12 epch:12.01 loss:1.474 grdn:56.429 lr:1.0e-05 updt_s:1.091 data_s:2.366
+INFO 2024-12-06 22:54:16 ts/train.py:192 step:1K smpl:11K ep:14 epch:14.01 loss:1.324 grdn:52.798 lr:1.0e-05 updt_s:1.104 data_s:1.794
+INFO 2024-12-06 22:57:55 ts/train.py:192 step:2K smpl:13K ep:16 epch:16.01 loss:1.275 grdn:48.997 lr:1.0e-05 updt_s:1.099 data_s:2.227
+INFO 2024-12-06 23:01:35 ts/train.py:192 step:2K smpl:14K ep:18 epch:18.01 loss:1.195 grdn:50.355 lr:1.0e-05 updt_s:1.411 data_s:2.189
+INFO 2024-12-06 23:05:13 ts/train.py:192 step:2K smpl:16K ep:20 epch:20.01 loss:1.077 grdn:40.395 lr:1.0e-05 updt_s:1.090 data_s:1.955
+INFO 2024-12-06 23:08:53 ts/train.py:192 step:2K smpl:18K ep:22 epch:22.01 loss:0.905 grdn:47.089 lr:1.0e-05 updt_s:1.090 data_s:2.396
+INFO 2024-12-06 23:12:32 ts/train.py:192 step:2K smpl:19K ep:24 epch:24.01 loss:0.878 grdn:44.676 lr:1.0e-05 updt_s:1.090 data_s:2.298
+INFO 2024-12-06 23:16:11 ts/train.py:192 step:3K smpl:21K ep:26 epch:26.01 loss:0.736 grdn:41.427 lr:1.0e-05 updt_s:1.106 data_s:2.089
+INFO 2024-12-06 23:19:50 ts/train.py:192 step:3K smpl:22K ep:28 epch:28.01 loss:0.633 grdn:33.782 lr:1.0e-05 updt_s:1.090 data_s:2.224
+INFO 2024-12-06 23:23:29 ts/train.py:192 step:3K smpl:24K ep:30 epch:30.01 loss:0.550 grdn:34.230 lr:1.0e-05 updt_s:1.103 data_s:2.172
+INFO 2024-12-06 23:27:08 ts/train.py:192 step:3K smpl:26K ep:32 epch:32.01 loss:0.572 grdn:37.149 lr:1.0e-05 updt_s:1.086 data_s:2.248
+INFO 2024-12-06 23:30:47 ts/train.py:192 step:3K smpl:27K ep:34 epch:34.01 loss:0.490 grdn:34.704 lr:1.0e-05 updt_s:1.093 data_s:1.711
+INFO 2024-12-06 23:34:25 ts/train.py:192 step:4K smpl:29K ep:36 epch:36.01 loss:0.459 grdn:28.810 lr:1.0e-05 updt_s:1.105 data_s:1.716
+INFO 2024-12-06 23:38:04 ts/train.py:192 step:4K smpl:30K ep:38 epch:38.01 loss:0.450 grdn:30.977 lr:1.0e-05 updt_s:1.095 data_s:2.136
+INFO 2024-12-06 23:41:44 ts/train.py:192 step:4K smpl:32K ep:40 epch:40.01 loss:0.376 grdn:34.368 lr:1.0e-05 updt_s:1.092 data_s:1.970
+INFO 2024-12-06 23:45:23 ts/train.py:192 step:4K smpl:34K ep:42 epch:42.01 loss:0.309 grdn:26.141 lr:1.0e-05 updt_s:1.090 data_s:2.131
+INFO 2024-12-06 23:49:02 ts/train.py:192 step:4K smpl:35K ep:44 epch:44.01 loss:0.282 grdn:21.178 lr:1.0e-05 updt_s:1.095 data_s:2.192
+INFO 2024-12-06 23:52:41 ts/train.py:192 step:5K smpl:37K ep:46 epch:46.01 loss:0.255 grdn:25.957 lr:1.0e-05 updt_s:1.078 data_s:2.502
+INFO 2024-12-06 23:56:20 ts/train.py:192 step:5K smpl:38K ep:48 epch:48.01 loss:0.250 grdn:20.830 lr:1.0e-05 updt_s:1.088 data_s:2.212
+INFO 2024-12-06 23:59:56 ts/train.py:383 Checkpoint policy after step 5000
+INFO 2024-12-06 23:59:58 ts/train.py:393 Resume training
+INFO 2024-12-06 23:59:58 ts/train.py:455 End of training

wandb/run-20241206_222830-bwwpk9vl/files/requirements.txt

@@ -0,0 +1,129 @@
+pymunk==6.9.0
+charset-normalizer==3.4.0
+pip==22.0.2
+multidict==6.1.0
+pytz==2024.2
+pandas==2.2.3
+fasteners==0.19
+einops==0.8.0
+setproctitle==1.3.4
+python-dateutil==2.9.0.post0
+pydantic==2.10.3
+dm-control==1.0.14
+MarkupSafe==3.0.2
+safetensors==0.4.5
+cloudpickle==3.1.0
+imageio-ffmpeg==0.5.1
+smmap==5.0.1
+huggingface-hub==0.26.5
+torch==2.6.0.dev20241112+cu121
+nvidia-cusolver-cu12==11.4.5.107
+inquirerpy==0.3.4
+soupsieve==2.6
+deepdiff==8.0.1
+nvidia-nvjitlink-cu12==12.4.127
+aiosignal==1.3.1
+dm-tree==0.1.8
+aiohappyeyeballs==2.4.4
+propcache==0.2.1
+gym-aloha==0.1.1
+labmaze==1.0.6
+pyav==14.0.1
+nvidia-cuda-cupti-cu12==12.1.105
+cffi==1.17.1
+importlib_metadata==8.5.0
+nvidia-cuda-runtime-cu12==12.1.105
+pycparser==2.22
+h5py==3.12.1
+nvidia-nccl-cu12==2.21.5
+orderly-set==5.2.2
+pyarrow==18.1.0
+Flask==3.1.0
+multiprocess==0.70.16
+blinker==1.9.0
+regex==2024.11.6
+Jinja2==3.1.4
+tqdm==4.67.1
+certifi==2024.8.30
+frozenlist==1.5.0
+attrs==24.2.0
+pytorch-triton==3.1.0+cf34004b8a
+omegaconf==2.3.0
+itsdangerous==2.2.0
+packaging==24.2
+pfzy==0.3.4
+pydantic_core==2.27.1
+antlr4-python3-runtime==4.9.3
+cmake==3.31.1
+imageio==2.36.1
+setuptools==59.6.0
+nvidia-cusparselt-cu12==0.6.2
+zipp==3.21.0
+torchvision==0.20.0.dev20241112+cu121
+nvidia-curand-cu12==10.3.2.106
+PyOpenGL-accelerate==3.1.7
+numpy==1.26.4
+dm-env==1.6
+nvidia-cusparse-cu12==12.1.0.106
+networkx==3.4.2
+triton==3.1.0
+termcolor==2.5.0
+gitdb==4.0.11
+torchaudio==2.5.0.dev20241112+cu121
+glfw==2.8.0
+tzdata==2024.2
+pillow==11.0.0
+idna==3.10
+pyparsing==3.2.0
+typing_extensions==4.12.2
+wcwidth==0.2.13
+sympy==1.13.1
+nvidia-cublas-cu12==12.1.3.1
+zarr==2.18.3
+protobuf==5.29.1
+annotated-types==0.7.0
+xxhash==3.5.0
+aiohttp==3.11.10
+Werkzeug==3.1.3
+mujoco==2.3.7
+gymnasium==0.29.1
+rerun-sdk==0.20.3
+nvidia-cufft-cu12==11.0.2.54
+dill==0.3.8
+scipy==1.14.1
+hydra-core==1.3.2
+fsspec==2024.9.0
+platformdirs==4.3.6
+PySocks==1.7.1
+PyOpenGL==3.1.7
+requests==2.32.3
+lxml==5.3.0
+Farama-Notifications==0.0.4
+nvidia-cuda-nvrtc-cu12==12.1.105
+numba==0.60.0
+docker-pycreds==0.4.0
+six==1.17.0
+lerobot==0.1.0
+nvidia-cudnn-cu12==9.1.0.70
+sentry-sdk==2.19.2
+absl-py==2.1.0
+wandb==0.19.0
+mpmath==1.3.0
+llvmlite==0.43.0
+numcodecs==0.13.1
+GitPython==3.1.43
+nvidia-nvtx-cu12==12.1.105
+prompt_toolkit==3.0.48
+datasets==3.1.0
+opencv-python==4.10.0.84
+filelock==3.16.1
+async-timeout==5.0.1
+diffusers==0.31.0
+yarl==1.18.3
+click==8.1.7
+asciitree==0.3.3
+psutil==6.1.0
+gdown==5.2.0
+PyYAML==6.0.2
+urllib3==2.2.3
+beautifulsoup4==4.12.3

wandb/run-20241206_222830-bwwpk9vl/files/wandb-metadata.json

@@ -0,0 +1,48 @@
+{
+  "os": "Linux-6.8.0-1015-aws-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2024-12-06T22:28:30.311945Z",
+  "args": [
+    "policy=act_aloha_real",
+    "env=aloha_real",
+    "dataset_repo_id=ccop/aloha_stationary_replay_test_v2"
+  ],
+  "program": "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py",
+  "codePath": "lerobot/scripts/train.py",
+  "git": {
+    "remote": "[email protected]:HumanoidTeam/lerobot-hackathon.git",
+    "commit": "ab8aa3f8e7b6e5486d7af26ccd319d358ffad579"
+  },
+  "email": "[email protected]",
+  "root": "outputs/train/2024-12-06/22-28-29_real_world_act_default",
+  "host": "ip-172-31-20-130",
+  "username": "humanoidteam",
+  "executable": "/home/ubuntu/lerobot-hackathon/lerobot/bin/python",
+  "codePathLocal": "lerobot/scripts/train.py",
+  "cpu_count": 4,
+  "cpu_count_logical": 8,
+  "gpu": "Tesla T4",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "311993479168",
+      "used": "216540794880"
+    }
+  },
+  "memory": {
+    "total": "33263837184"
+  },
+  "cpu": {
+    "count": 4,
+    "countLogical": 8
+  },
+  "gpu_nvidia": [
+    {
+      "name": "Tesla T4",
+      "memoryTotal": "16106127360",
+      "cudaCores": 2560,
+      "architecture": "Turing"
+    }
+  ],
+  "cudaVersion": "12.4"
+}

wandb/run-20241206_222830-bwwpk9vl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/lr":1e-05,"train/loss":0.2502947151660919,"train/is_online":false,"train/num_samples":38408,"train/num_episodes":48.01,"train/kld_loss":0.01526648923754692,"train/grad_norm":20.83009910583496,"train/dataloading_s":2.211516670999117,"_timestamp":1.7335293805980325e+09,"_step":4800,"_runtime":5488.627260612,"train/step":4800,"train/l1_loss":0.09762981534004211,"train/num_epochs":48.01,"_wandb":{"runtime":5488},"train/update_s":1.0881254139821976}

wandb/run-20241206_222830-bwwpk9vl/logs/debug-core.log

@@ -0,0 +1,15 @@
+{"time":"2024-12-06T22:28:29.883207186Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpf2z320nz/port-2539524.txt","pid":2539524,"debug":false,"disable-analytics":false}
+{"time":"2024-12-06T22:28:29.883234859Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-12-06T22:28:29.886661968Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2539524}
+{"time":"2024-12-06T22:28:29.886681547Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43495,"Zone":""}}
+{"time":"2024-12-06T22:28:30.072566654Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:55002"}
+{"time":"2024-12-06T22:28:30.312961224Z","level":"INFO","msg":"handleInformInit: received","streamId":"bwwpk9vl","id":"127.0.0.1:55002"}
+{"time":"2024-12-06T22:28:30.416433116Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bwwpk9vl","id":"127.0.0.1:55002"}
+{"time":"2024-12-06T23:59:58.939118572Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:55002"}
+{"time":"2024-12-06T23:59:58.939192863Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-12-06T23:59:58.939181535Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:55002"}
+{"time":"2024-12-06T23:59:58.939312003Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:55002"}
+{"time":"2024-12-07T00:00:06.315410708Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:43495->127.0.0.1:55002: use of closed network connection","id":"127.0.0.1:55002"}
+{"time":"2024-12-07T00:00:06.536210754Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:55002"}
+{"time":"2024-12-07T00:00:06.536232465Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:55002"}
+{"time":"2024-12-07T00:00:06.536247048Z","level":"INFO","msg":"server is closed"}

wandb/run-20241206_222830-bwwpk9vl/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-12-06T22:28:30.313104758Z","level":"INFO","msg":"using version","core version":"0.19.0"}
+{"time":"2024-12-06T22:28:30.313117284Z","level":"INFO","msg":"created symlink","path":"outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241206_222830-bwwpk9vl/logs/debug-core.log"}
+{"time":"2024-12-06T22:28:30.416399043Z","level":"INFO","msg":"created new stream","id":"bwwpk9vl"}
+{"time":"2024-12-06T22:28:30.416428271Z","level":"INFO","msg":"stream: started","id":"bwwpk9vl"}
+{"time":"2024-12-06T22:28:30.416469205Z","level":"INFO","msg":"handler: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-06T22:28:30.416473548Z","level":"INFO","msg":"sender: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-06T22:28:30.416459377Z","level":"INFO","msg":"writer: Do: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-06T22:28:30.658850895Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-12-06T23:59:58.939177282Z","level":"INFO","msg":"stream: closing","id":"bwwpk9vl"}
+{"time":"2024-12-06T23:59:58.939228097Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-06T23:59:58.939881357Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-07T00:00:06.36632207Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-07T00:00:06.536040063Z","level":"INFO","msg":"handler: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T00:00:06.536088365Z","level":"INFO","msg":"writer: Close: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T00:00:06.536107691Z","level":"INFO","msg":"sender: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T00:00:06.536149416Z","level":"INFO","msg":"stream: closed","id":"bwwpk9vl"}

wandb/run-20241206_222830-bwwpk9vl/logs/debug.log

@@ -0,0 +1,24 @@
+2024-12-06 22:28:30,306 INFO    MainThread:2539524 [wandb_setup.py:_flush():68] Current SDK version is 0.19.0
+2024-12-06 22:28:30,306 INFO    MainThread:2539524 [wandb_setup.py:_flush():68] Configure stats pid to 2539524
+2024-12-06 22:28:30,306 INFO    MainThread:2539524 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-12-06 22:28:30,306 INFO    MainThread:2539524 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/lerobot-hackathon/wandb/settings
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_setup.py:_flush():68] wandb: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_init.py:_log_setup():528] Logging user logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241206_222830-bwwpk9vl/logs/debug.log
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_init.py:_log_setup():529] Logging internal logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241206_222830-bwwpk9vl/logs/debug-internal.log
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_init.py:init():639] calling init triggers
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_init.py:init():645] wandb.init called with sweep_config: {}
+config: {'resume': False, 'device': 'cuda', 'use_amp': False, 'seed': 1000, 'dataset_repo_id': 'ccop/aloha_stationary_replay_test_v2', 'video_backend': 'pyav', 'training': {'offline_steps': 5000, 'num_workers': 8, 'batch_size': 8, 'eval_freq': -1, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 10000, 'online_steps': 0, 'online_rollout_n_episodes': 1, 'online_rollout_batch_size': 1, 'online_steps_between_rollouts': 1, 'online_sampling_ratio': 0.5, 'online_env_seed': None, 'online_buffer_capacity': None, 'online_buffer_seed_size': 0, 'do_online_rollout_async': False, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'brightness': {'weight': 1, 'min_max': [0.8, 1.2]}, 'contrast': {'weight': 1, 'min_max': [0.8, 1.2]}, 'saturation': {'weight': 1, 'min_max': [0.5, 1.5]}, 'hue': {'weight': 1, 'min_max': [-0.05, 0.05]}, 'sharpness': {'weight': 1, 'min_max': [0.8, 1.2]}}, 'lr': 1e-05, 'lr_backbone': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10, 'delta_timestamps': {'action': '[i / 30 for i in range(100)]'}}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot_real_aloha', 'notes': 'ws-lerobot-hackathon'}, 'fps': 30, 'env': {'name': 'real_world', 'task': None, 'state_dim': 14, 'action_dim': 14, 'fps': 30}, 'override_dataset_stats': {'observation.images.cam_right_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_left_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_high': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}}, 'policy': {'name': 'act', 'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'input_shapes': {'observation.images.cam_right_wrist': [3, 480, 640], 'observation.images.cam_left_wrist': [3, 480, 640], 'observation.images.cam_high': [3, 480, 640], 'observation.state': [14]}, 'output_shapes': {'action': [14]}, 'input_normalization_modes': {'observation.images.cam_right_wrist': 'mean_std', 'observation.images.cam_left_wrist': 'mean_std', 'observation.images.cam_high': 'mean_std', 'observation.state': 'mean_std'}, 'output_normalization_modes': {'action': 'mean_std'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_momentum': None, 'dropout': 0.1, 'kl_weight': 10.0}}
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_init.py:init():688] starting backend
+2024-12-06 22:28:30,307 INFO    MainThread:2539524 [wandb_init.py:init():692] sending inform_init request
+2024-12-06 22:28:30,311 INFO    MainThread:2539524 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-06 22:28:30,311 INFO    MainThread:2539524 [wandb_init.py:init():705] backend started and connected
+2024-12-06 22:28:30,313 INFO    MainThread:2539524 [wandb_init.py:init():798] updated telemetry
+2024-12-06 22:28:30,319 INFO    MainThread:2539524 [wandb_init.py:init():830] communicating run to backend with 90.0 second timeout
+2024-12-06 22:28:30,655 INFO    MainThread:2539524 [wandb_init.py:init():882] starting run threads in backend
+2024-12-06 22:28:30,767 INFO    MainThread:2539524 [wandb_run.py:_console_start():2443] atexit reg
+2024-12-06 22:28:30,767 INFO    MainThread:2539524 [wandb_run.py:_redirect():2293] redirect: wrap_raw
+2024-12-06 22:28:30,767 INFO    MainThread:2539524 [wandb_run.py:_redirect():2358] Wrapping output streams.
+2024-12-06 22:28:30,767 INFO    MainThread:2539524 [wandb_run.py:_redirect():2383] Redirects installed.
+2024-12-06 22:28:30,769 INFO    MainThread:2539524 [wandb_init.py:init():925] run started, returning control to user process
+2024-12-06 23:59:58,939 WARNING MsgRouterThr:2539524 [router.py:message_loop():75] message_loop has been closed

wandb/run-20241207_010636-bwwpk9vl/files/config.yaml

@@ -0,0 +1,296 @@
+_wandb:
+    value:
+        cli_version: 0.19.0
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 51
+                - 55
+            "2":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 51
+                - 55
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 23
+                - 55
+                - 62
+            "4": 3.10.12
+            "5": 0.19.0
+            "8":
+                - 5
+            "12": 0.19.0
+            "13": linux-x86_64
+dataset_repo_id:
+    value: ccop/aloha_stationary_replay_test_v2
+device:
+    value: cuda
+env:
+    value:
+        action_dim: 14
+        fps: 30
+        name: real_world
+        state_dim: 14
+        task: null
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        use_async_envs: false
+fps:
+    value: 30
+override_dataset_stats:
+    value:
+        observation.images.cam_high:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+        observation.images.cam_left_wrist:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+        observation.images.cam_right_wrist:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+policy:
+    value:
+        chunk_size: 100
+        dim_feedforward: 3200
+        dim_model: 512
+        dropout: 0.1
+        feedforward_activation: relu
+        input_normalization_modes:
+            observation.images.cam_high: mean_std
+            observation.images.cam_left_wrist: mean_std
+            observation.images.cam_right_wrist: mean_std
+            observation.state: mean_std
+        input_shapes:
+            observation.images.cam_high:
+                - 3
+                - 480
+                - 640
+            observation.images.cam_left_wrist:
+                - 3
+                - 480
+                - 640
+            observation.images.cam_right_wrist:
+                - 3
+                - 480
+                - 640
+            observation.state:
+                - 14
+        kl_weight: 10
+        latent_dim: 32
+        n_action_steps: 100
+        n_decoder_layers: 1
+        n_encoder_layers: 4
+        n_heads: 8
+        n_obs_steps: 1
+        n_vae_encoder_layers: 4
+        name: act
+        output_normalization_modes:
+            action: mean_std
+        output_shapes:
+            action:
+                - 14
+        pre_norm: false
+        pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1
+        replace_final_stride_with_dilation: false
+        temporal_ensemble_momentum: null
+        use_vae: true
+        vision_backbone: resnet18
+resume:
+    value: true
+seed:
+    value: 1000
+training:
+    value:
+        batch_size: 8
+        delta_timestamps:
+            action:
+                - 0
+                - 0.03333333333333333
+                - 0.06666666666666667
+                - 0.1
+                - 0.13333333333333333
+                - 0.16666666666666666
+                - 0.2
+                - 0.23333333333333334
+                - 0.26666666666666666
+                - 0.3
+                - 0.3333333333333333
+                - 0.36666666666666664
+                - 0.4
+                - 0.43333333333333335
+                - 0.4666666666666667
+                - 0.5
+                - 0.5333333333333333
+                - 0.5666666666666667
+                - 0.6
+                - 0.6333333333333333
+                - 0.6666666666666666
+                - 0.7
+                - 0.7333333333333333
+                - 0.7666666666666667
+                - 0.8
+                - 0.8333333333333334
+                - 0.8666666666666667
+                - 0.9
+                - 0.9333333333333333
+                - 0.9666666666666667
+                - 1
+                - 1.0333333333333334
+                - 1.0666666666666667
+                - 1.1
+                - 1.1333333333333333
+                - 1.1666666666666667
+                - 1.2
+                - 1.2333333333333334
+                - 1.2666666666666666
+                - 1.3
+                - 1.3333333333333333
+                - 1.3666666666666667
+                - 1.4
+                - 1.4333333333333333
+                - 1.4666666666666666
+                - 1.5
+                - 1.5333333333333334
+                - 1.5666666666666667
+                - 1.6
+                - 1.6333333333333333
+                - 1.6666666666666667
+                - 1.7
+                - 1.7333333333333334
+                - 1.7666666666666666
+                - 1.8
+                - 1.8333333333333333
+                - 1.8666666666666667
+                - 1.9
+                - 1.9333333333333333
+                - 1.9666666666666666
+                - 2
+                - 2.033333333333333
+                - 2.066666666666667
+                - 2.1
+                - 2.1333333333333333
+                - 2.1666666666666665
+                - 2.2
+                - 2.2333333333333334
+                - 2.2666666666666666
+                - 2.3
+                - 2.3333333333333335
+                - 2.3666666666666667
+                - 2.4
+                - 2.433333333333333
+                - 2.466666666666667
+                - 2.5
+                - 2.533333333333333
+                - 2.566666666666667
+                - 2.6
+                - 2.6333333333333333
+                - 2.6666666666666665
+                - 2.7
+                - 2.7333333333333334
+                - 2.7666666666666666
+                - 2.8
+                - 2.8333333333333335
+                - 2.8666666666666667
+                - 2.9
+                - 2.933333333333333
+                - 2.966666666666667
+                - 3
+                - 3.033333333333333
+                - 3.066666666666667
+                - 3.1
+                - 3.1333333333333333
+                - 3.1666666666666665
+                - 3.2
+                - 3.2333333333333334
+                - 3.2666666666666666
+                - 3.3
+        do_online_rollout_async: false
+        eval_freq: -1
+        grad_clip_norm: 10
+        image_transforms:
+            brightness:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+            contrast:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+            enable: false
+            hue:
+                min_max:
+                    - -0.05
+                    - 0.05
+                weight: 1
+            max_num_transforms: 3
+            random_order: false
+            saturation:
+                min_max:
+                    - 0.5
+                    - 1.5
+                weight: 1
+            sharpness:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+        log_freq: 200
+        lr: 1e-05
+        lr_backbone: 1e-05
+        num_workers: 8
+        offline_steps: 5000
+        online_buffer_capacity: null
+        online_buffer_seed_size: 0
+        online_env_seed: null
+        online_rollout_batch_size: 1
+        online_rollout_n_episodes: 1
+        online_sampling_ratio: 0.5
+        online_steps: 0
+        online_steps_between_rollouts: 1
+        save_checkpoint: true
+        save_freq: 10000
+        weight_decay: 0.0001
+use_amp:
+    value: false
+video_backend:
+    value: pyav
+wandb:
+    value:
+        disable_artifact: false
+        enable: true
+        notes: ws-lerobot-hackathon
+        project: lerobot_real_aloha

wandb/run-20241207_010636-bwwpk9vl/files/output.log

@@ -0,0 +1,28 @@
+Logs will be synced with wandb.
+INFO 2024-12-07 01:06:36 n/logger.py:132 Track this run --> https://wandb.ai/zaringleb/lerobot_real_aloha/runs/bwwpk9vl
+INFO 2024-12-07 01:06:36 ts/train.py:312 make_dataset
+WARNING 2024-12-07 01:06:36 s/factory.py:68 There might be a mismatch between your training dataset (dataset_repo_id='ccop/aloha_stationary_replay_test_v2') and your environment (cfg.env.name='real_world').
+Fetching 9 files: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 19774.09it/s]
+INFO 2024-12-07 01:06:39 ts/train.py:328 make_policy
+WARNING 2024-12-07 01:06:39 s/factory.py:28 Hydra config is missing arguments: {'temporal_ensemble_coeff'}
+Loading weights from local directory
+/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py:338: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  grad_scaler = GradScaler(enabled=cfg.use_amp)
+Error executing job with overrides: ['policy=act_aloha_real', 'env=aloha_real', 'dataset_repo_id=ccop/aloha_stationary_replay_test_v2', 'resume=true', 'dataset_repo_id=HumanoidTeam/aloha_real_test', 'training.offline_steps=5010']
+Traceback (most recent call last):
+  File "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py", line 652, in train_cli
+    train(
+  File "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py", line 343, in train
+    step = logger.load_last_training_state(optimizer, lr_scheduler)
+  File "/home/ubuntu/lerobot-hackathon/lerobot/common/logger.py", line 218, in load_last_training_state
+    training_state = torch.load(self.last_checkpoint_dir / self.training_state_file_name)
+  File "/home/ubuntu/lerobot-hackathon/lerobot/lib/python3.10/site-packages/torch/serialization.py", line 1444, in load
+    raise pickle.UnpicklingError(_get_wo_message(str(e))) from None
+_pickle.UnpicklingError: Weights only load failed. This file can still be loaded, to do so you have two options, do those steps only if you trust the source of the checkpoint.
+	(1) Re-running `torch.load` with `weights_only` set to `False` will likely succeed, but it can result in arbitrary code execution. Do it only if you got the file from a trusted source.
+	(2) Alternatively, to load with `weights_only=True` please check the recommended steps in the following error message.
+	WeightsUnpickler error: Unsupported global: GLOBAL numpy.core.multiarray._reconstruct was not an allowed global by default. Please use `torch.serialization.add_safe_globals([_reconstruct])` or the `torch.serialization.safe_globals([_reconstruct])` context manager to allowlist this global if you trust this class/function.
+
+Check the documentation of torch.load to learn more about types accepted by default with weights_only https://pytorch.org/docs/stable/generated/torch.load.html.
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

wandb/run-20241207_010636-bwwpk9vl/files/requirements.txt

@@ -0,0 +1,129 @@
+pymunk==6.9.0
+charset-normalizer==3.4.0
+pip==22.0.2
+multidict==6.1.0
+pytz==2024.2
+pandas==2.2.3
+fasteners==0.19
+einops==0.8.0
+setproctitle==1.3.4
+python-dateutil==2.9.0.post0
+pydantic==2.10.3
+dm-control==1.0.14
+MarkupSafe==3.0.2
+safetensors==0.4.5
+cloudpickle==3.1.0
+imageio-ffmpeg==0.5.1
+smmap==5.0.1
+huggingface-hub==0.26.5
+torch==2.6.0.dev20241112+cu121
+nvidia-cusolver-cu12==11.4.5.107
+inquirerpy==0.3.4
+soupsieve==2.6
+deepdiff==8.0.1
+nvidia-nvjitlink-cu12==12.4.127
+aiosignal==1.3.1
+dm-tree==0.1.8
+aiohappyeyeballs==2.4.4
+propcache==0.2.1
+gym-aloha==0.1.1
+labmaze==1.0.6
+pyav==14.0.1
+nvidia-cuda-cupti-cu12==12.1.105
+cffi==1.17.1
+importlib_metadata==8.5.0
+nvidia-cuda-runtime-cu12==12.1.105
+pycparser==2.22
+h5py==3.12.1
+nvidia-nccl-cu12==2.21.5
+orderly-set==5.2.2
+pyarrow==18.1.0
+Flask==3.1.0
+multiprocess==0.70.16
+blinker==1.9.0
+regex==2024.11.6
+Jinja2==3.1.4
+tqdm==4.67.1
+certifi==2024.8.30
+frozenlist==1.5.0
+attrs==24.2.0
+pytorch-triton==3.1.0+cf34004b8a
+omegaconf==2.3.0
+itsdangerous==2.2.0
+packaging==24.2
+pfzy==0.3.4
+pydantic_core==2.27.1
+antlr4-python3-runtime==4.9.3
+cmake==3.31.1
+imageio==2.36.1
+setuptools==59.6.0
+nvidia-cusparselt-cu12==0.6.2
+zipp==3.21.0
+torchvision==0.20.0.dev20241112+cu121
+nvidia-curand-cu12==10.3.2.106
+PyOpenGL-accelerate==3.1.7
+numpy==1.26.4
+dm-env==1.6
+nvidia-cusparse-cu12==12.1.0.106
+networkx==3.4.2
+triton==3.1.0
+termcolor==2.5.0
+gitdb==4.0.11
+torchaudio==2.5.0.dev20241112+cu121
+glfw==2.8.0
+tzdata==2024.2
+pillow==11.0.0
+idna==3.10
+pyparsing==3.2.0
+typing_extensions==4.12.2
+wcwidth==0.2.13
+sympy==1.13.1
+nvidia-cublas-cu12==12.1.3.1
+zarr==2.18.3
+protobuf==5.29.1
+annotated-types==0.7.0
+xxhash==3.5.0
+aiohttp==3.11.10
+Werkzeug==3.1.3
+mujoco==2.3.7
+gymnasium==0.29.1
+rerun-sdk==0.20.3
+nvidia-cufft-cu12==11.0.2.54
+dill==0.3.8
+scipy==1.14.1
+hydra-core==1.3.2
+fsspec==2024.9.0
+platformdirs==4.3.6
+PySocks==1.7.1
+PyOpenGL==3.1.7
+requests==2.32.3
+lxml==5.3.0
+Farama-Notifications==0.0.4
+nvidia-cuda-nvrtc-cu12==12.1.105
+numba==0.60.0
+docker-pycreds==0.4.0
+six==1.17.0
+lerobot==0.1.0
+nvidia-cudnn-cu12==9.1.0.70
+sentry-sdk==2.19.2
+absl-py==2.1.0
+wandb==0.19.0
+mpmath==1.3.0
+llvmlite==0.43.0
+numcodecs==0.13.1
+GitPython==3.1.43
+nvidia-nvtx-cu12==12.1.105
+prompt_toolkit==3.0.48
+datasets==3.1.0
+opencv-python==4.10.0.84
+filelock==3.16.1
+async-timeout==5.0.1
+diffusers==0.31.0
+yarl==1.18.3
+click==8.1.7
+asciitree==0.3.3
+psutil==6.1.0
+gdown==5.2.0
+PyYAML==6.0.2
+urllib3==2.2.3
+beautifulsoup4==4.12.3

wandb/run-20241207_010636-bwwpk9vl/files/wandb-metadata.json

@@ -0,0 +1,52 @@
+{
+  "os": "Linux-6.8.0-1015-aws-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2024-12-07T01:06:36.020217Z",
+  "args": [
+    "policy=act_aloha_real",
+    "env=aloha_real",
+    "dataset_repo_id=ccop/aloha_stationary_replay_test_v2",
+    "hydra.run.dir=outputs/train/2024-12-06/22-28-29_real_world_act_default",
+    "resume=true",
+    "dataset_repo_id=HumanoidTeam/aloha_real_test",
+    "training.offline_steps=5010"
+  ],
+  "program": "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py",
+  "codePath": "lerobot/scripts/train.py",
+  "git": {
+    "remote": "[email protected]:HumanoidTeam/lerobot-hackathon.git",
+    "commit": "ab8aa3f8e7b6e5486d7af26ccd319d358ffad579"
+  },
+  "email": "[email protected]",
+  "root": "outputs/train/2024-12-06/22-28-29_real_world_act_default",
+  "host": "ip-172-31-20-130",
+  "username": "humanoidteam",
+  "executable": "/home/ubuntu/lerobot-hackathon/lerobot/bin/python",
+  "codePathLocal": "lerobot/scripts/train.py",
+  "cpu_count": 4,
+  "cpu_count_logical": 8,
+  "gpu": "Tesla T4",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "311993479168",
+      "used": "217724030976"
+    }
+  },
+  "memory": {
+    "total": "33263837184"
+  },
+  "cpu": {
+    "count": 4,
+    "countLogical": 8
+  },
+  "gpu_nvidia": [
+    {
+      "name": "Tesla T4",
+      "memoryTotal": "16106127360",
+      "cudaCores": 2560,
+      "architecture": "Turing"
+    }
+  ],
+  "cudaVersion": "12.4"
+}

wandb/run-20241207_010636-bwwpk9vl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/num_episodes":48.01,"train/is_online":false,"train/num_epochs":48.01,"_runtime":5488.627260612,"train/dataloading_s":2.211516670999117,"train/loss":0.2502947151660919,"_wandb":{"runtime":5493},"train/l1_loss":0.09762981534004211,"train/lr":1e-05,"train/step":4800,"train/grad_norm":20.83009910583496,"_step":4800,"train/num_samples":38408,"_timestamp":1.7335293805980325e+09,"train/update_s":1.0881254139821976,"train/kld_loss":0.01526648923754692}

wandb/run-20241207_010636-bwwpk9vl/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-12-07T01:06:35.555739791Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpw4k2sq2e/port-3507304.txt","pid":3507304,"debug":false,"disable-analytics":false}
+{"time":"2024-12-07T01:06:35.555772863Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-12-07T01:06:35.559538633Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3507304}
+{"time":"2024-12-07T01:06:35.559558787Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41501,"Zone":""}}
+{"time":"2024-12-07T01:06:35.744706996Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:36.021210331Z","level":"INFO","msg":"handleInformInit: received","streamId":"bwwpk9vl","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:36.124059246Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bwwpk9vl","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:41.270283035Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:41.270348753Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-12-07T01:06:41.270339778Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:41.270441093Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:42.336511877Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:42.336534508Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:49972"}
+{"time":"2024-12-07T01:06:42.336548673Z","level":"INFO","msg":"server is closed"}

wandb/run-20241207_010636-bwwpk9vl/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-12-07T01:06:36.021365256Z","level":"INFO","msg":"using version","core version":"0.19.0"}
+{"time":"2024-12-07T01:06:36.021379424Z","level":"INFO","msg":"created symlink","path":"outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010636-bwwpk9vl/logs/debug-core.log"}
+{"time":"2024-12-07T01:06:36.124016018Z","level":"INFO","msg":"created new stream","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:36.124052183Z","level":"INFO","msg":"stream: started","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:36.124086504Z","level":"INFO","msg":"writer: Do: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:36.124086178Z","level":"INFO","msg":"sender: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:36.124165775Z","level":"INFO","msg":"handler: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:36.467542952Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-12-07T01:06:41.27033621Z","level":"INFO","msg":"stream: closing","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:41.270379684Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-07T01:06:41.270898953Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-07T01:06:42.191128546Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-07T01:06:42.336356796Z","level":"INFO","msg":"handler: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:42.336396896Z","level":"INFO","msg":"writer: Close: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:42.336425596Z","level":"INFO","msg":"sender: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:06:42.336454571Z","level":"INFO","msg":"stream: closed","id":"bwwpk9vl"}

wandb/run-20241207_010636-bwwpk9vl/logs/debug.log

@@ -0,0 +1,25 @@
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_setup.py:_flush():68] Current SDK version is 0.19.0
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_setup.py:_flush():68] Configure stats pid to 3507304
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/lerobot-hackathon/wandb/settings
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_setup.py:_flush():68] wandb: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_init.py:_log_setup():528] Logging user logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010636-bwwpk9vl/logs/debug.log
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_init.py:_log_setup():529] Logging internal logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010636-bwwpk9vl/logs/debug-internal.log
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_init.py:init():639] calling init triggers
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_init.py:init():645] wandb.init called with sweep_config: {}
+config: {'resume': True, 'device': 'cuda', 'use_amp': False, 'seed': 1000, 'dataset_repo_id': 'ccop/aloha_stationary_replay_test_v2', 'video_backend': 'pyav', 'training': {'offline_steps': 5000, 'num_workers': 8, 'batch_size': 8, 'eval_freq': -1, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 10000, 'online_steps': 0, 'online_rollout_n_episodes': 1, 'online_rollout_batch_size': 1, 'online_steps_between_rollouts': 1, 'online_sampling_ratio': 0.5, 'online_env_seed': None, 'online_buffer_capacity': None, 'online_buffer_seed_size': 0, 'do_online_rollout_async': False, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'brightness': {'weight': 1, 'min_max': [0.8, 1.2]}, 'contrast': {'weight': 1, 'min_max': [0.8, 1.2]}, 'saturation': {'weight': 1, 'min_max': [0.5, 1.5]}, 'hue': {'weight': 1, 'min_max': [-0.05, 0.05]}, 'sharpness': {'weight': 1, 'min_max': [0.8, 1.2]}}, 'lr': 1e-05, 'lr_backbone': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10, 'delta_timestamps': {'action': [0.0, 0.03333333333333333, 0.06666666666666667, 0.1, 0.13333333333333333, 0.16666666666666666, 0.2, 0.23333333333333334, 0.26666666666666666, 0.3, 0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6, 0.6333333333333333, 0.6666666666666666, 0.7, 0.7333333333333333, 0.7666666666666667, 0.8, 0.8333333333333334, 0.8666666666666667, 0.9, 0.9333333333333333, 0.9666666666666667, 1.0, 1.0333333333333334, 1.0666666666666667, 1.1, 1.1333333333333333, 1.1666666666666667, 1.2, 1.2333333333333334, 1.2666666666666666, 1.3, 1.3333333333333333, 1.3666666666666667, 1.4, 1.4333333333333333, 1.4666666666666666, 1.5, 1.5333333333333334, 1.5666666666666667, 1.6, 1.6333333333333333, 1.6666666666666667, 1.7, 1.7333333333333334, 1.7666666666666666, 1.8, 1.8333333333333333, 1.8666666666666667, 1.9, 1.9333333333333333, 1.9666666666666666, 2.0, 2.033333333333333, 2.066666666666667, 2.1, 2.1333333333333333, 2.1666666666666665, 2.2, 2.2333333333333334, 2.2666666666666666, 2.3, 2.3333333333333335, 2.3666666666666667, 2.4, 2.433333333333333, 2.466666666666667, 2.5, 2.533333333333333, 2.566666666666667, 2.6, 2.6333333333333333, 2.6666666666666665, 2.7, 2.7333333333333334, 2.7666666666666666, 2.8, 2.8333333333333335, 2.8666666666666667, 2.9, 2.933333333333333, 2.966666666666667, 3.0, 3.033333333333333, 3.066666666666667, 3.1, 3.1333333333333333, 3.1666666666666665, 3.2, 3.2333333333333334, 3.2666666666666666, 3.3]}}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot_real_aloha', 'notes': 'ws-lerobot-hackathon'}, 'fps': 30, 'env': {'name': 'real_world', 'task': None, 'state_dim': 14, 'action_dim': 14, 'fps': 30}, 'override_dataset_stats': {'observation.images.cam_right_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_left_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_high': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}}, 'policy': {'name': 'act', 'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'input_shapes': {'observation.images.cam_right_wrist': [3, 480, 640], 'observation.images.cam_left_wrist': [3, 480, 640], 'observation.images.cam_high': [3, 480, 640], 'observation.state': [14]}, 'output_shapes': {'action': [14]}, 'input_normalization_modes': {'observation.images.cam_right_wrist': 'mean_std', 'observation.images.cam_left_wrist': 'mean_std', 'observation.images.cam_high': 'mean_std', 'observation.state': 'mean_std'}, 'output_normalization_modes': {'action': 'mean_std'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_momentum': None, 'dropout': 0.1, 'kl_weight': 10.0}}
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_init.py:init():688] starting backend
+2024-12-07 01:06:36,015 INFO    MainThread:3507304 [wandb_init.py:init():692] sending inform_init request
+2024-12-07 01:06:36,019 INFO    MainThread:3507304 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-07 01:06:36,020 INFO    MainThread:3507304 [wandb_init.py:init():705] backend started and connected
+2024-12-07 01:06:36,022 INFO    MainThread:3507304 [wandb_init.py:init():798] updated telemetry
+2024-12-07 01:06:36,027 INFO    MainThread:3507304 [wandb_init.py:init():830] communicating run to backend with 90.0 second timeout
+2024-12-07 01:06:36,459 INFO    MainThread:3507304 [wandb_init.py:init():875] run resumed
+2024-12-07 01:06:36,463 INFO    MainThread:3507304 [wandb_init.py:init():882] starting run threads in backend
+2024-12-07 01:06:36,579 INFO    MainThread:3507304 [wandb_run.py:_console_start():2443] atexit reg
+2024-12-07 01:06:36,579 INFO    MainThread:3507304 [wandb_run.py:_redirect():2293] redirect: wrap_raw
+2024-12-07 01:06:36,580 INFO    MainThread:3507304 [wandb_run.py:_redirect():2358] Wrapping output streams.
+2024-12-07 01:06:36,580 INFO    MainThread:3507304 [wandb_run.py:_redirect():2383] Redirects installed.
+2024-12-07 01:06:36,581 INFO    MainThread:3507304 [wandb_init.py:init():925] run started, returning control to user process
+2024-12-07 01:06:41,270 WARNING MsgRouterThr:3507304 [router.py:message_loop():75] message_loop has been closed

wandb/run-20241207_010741-bwwpk9vl/files/config.yaml

@@ -0,0 +1,296 @@
+_wandb:
+    value:
+        cli_version: 0.19.0
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 51
+                - 55
+            "2":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 51
+                - 55
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 23
+                - 55
+                - 62
+            "4": 3.10.12
+            "5": 0.19.0
+            "8":
+                - 5
+            "12": 0.19.0
+            "13": linux-x86_64
+dataset_repo_id:
+    value: ccop/aloha_stationary_replay_test_v2
+device:
+    value: cuda
+env:
+    value:
+        action_dim: 14
+        fps: 30
+        name: real_world
+        state_dim: 14
+        task: null
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        use_async_envs: false
+fps:
+    value: 30
+override_dataset_stats:
+    value:
+        observation.images.cam_high:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+        observation.images.cam_left_wrist:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+        observation.images.cam_right_wrist:
+            mean:
+                - - - 0.485
+                - - - 0.456
+                - - - 0.406
+            std:
+                - - - 0.229
+                - - - 0.224
+                - - - 0.225
+policy:
+    value:
+        chunk_size: 100
+        dim_feedforward: 3200
+        dim_model: 512
+        dropout: 0.1
+        feedforward_activation: relu
+        input_normalization_modes:
+            observation.images.cam_high: mean_std
+            observation.images.cam_left_wrist: mean_std
+            observation.images.cam_right_wrist: mean_std
+            observation.state: mean_std
+        input_shapes:
+            observation.images.cam_high:
+                - 3
+                - 480
+                - 640
+            observation.images.cam_left_wrist:
+                - 3
+                - 480
+                - 640
+            observation.images.cam_right_wrist:
+                - 3
+                - 480
+                - 640
+            observation.state:
+                - 14
+        kl_weight: 10
+        latent_dim: 32
+        n_action_steps: 100
+        n_decoder_layers: 1
+        n_encoder_layers: 4
+        n_heads: 8
+        n_obs_steps: 1
+        n_vae_encoder_layers: 4
+        name: act
+        output_normalization_modes:
+            action: mean_std
+        output_shapes:
+            action:
+                - 14
+        pre_norm: false
+        pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1
+        replace_final_stride_with_dilation: false
+        temporal_ensemble_momentum: null
+        use_vae: true
+        vision_backbone: resnet18
+resume:
+    value: true
+seed:
+    value: 1000
+training:
+    value:
+        batch_size: 8
+        delta_timestamps:
+            action:
+                - 0
+                - 0.03333333333333333
+                - 0.06666666666666667
+                - 0.1
+                - 0.13333333333333333
+                - 0.16666666666666666
+                - 0.2
+                - 0.23333333333333334
+                - 0.26666666666666666
+                - 0.3
+                - 0.3333333333333333
+                - 0.36666666666666664
+                - 0.4
+                - 0.43333333333333335
+                - 0.4666666666666667
+                - 0.5
+                - 0.5333333333333333
+                - 0.5666666666666667
+                - 0.6
+                - 0.6333333333333333
+                - 0.6666666666666666
+                - 0.7
+                - 0.7333333333333333
+                - 0.7666666666666667
+                - 0.8
+                - 0.8333333333333334
+                - 0.8666666666666667
+                - 0.9
+                - 0.9333333333333333
+                - 0.9666666666666667
+                - 1
+                - 1.0333333333333334
+                - 1.0666666666666667
+                - 1.1
+                - 1.1333333333333333
+                - 1.1666666666666667
+                - 1.2
+                - 1.2333333333333334
+                - 1.2666666666666666
+                - 1.3
+                - 1.3333333333333333
+                - 1.3666666666666667
+                - 1.4
+                - 1.4333333333333333
+                - 1.4666666666666666
+                - 1.5
+                - 1.5333333333333334
+                - 1.5666666666666667
+                - 1.6
+                - 1.6333333333333333
+                - 1.6666666666666667
+                - 1.7
+                - 1.7333333333333334
+                - 1.7666666666666666
+                - 1.8
+                - 1.8333333333333333
+                - 1.8666666666666667
+                - 1.9
+                - 1.9333333333333333
+                - 1.9666666666666666
+                - 2
+                - 2.033333333333333
+                - 2.066666666666667
+                - 2.1
+                - 2.1333333333333333
+                - 2.1666666666666665
+                - 2.2
+                - 2.2333333333333334
+                - 2.2666666666666666
+                - 2.3
+                - 2.3333333333333335
+                - 2.3666666666666667
+                - 2.4
+                - 2.433333333333333
+                - 2.466666666666667
+                - 2.5
+                - 2.533333333333333
+                - 2.566666666666667
+                - 2.6
+                - 2.6333333333333333
+                - 2.6666666666666665
+                - 2.7
+                - 2.7333333333333334
+                - 2.7666666666666666
+                - 2.8
+                - 2.8333333333333335
+                - 2.8666666666666667
+                - 2.9
+                - 2.933333333333333
+                - 2.966666666666667
+                - 3
+                - 3.033333333333333
+                - 3.066666666666667
+                - 3.1
+                - 3.1333333333333333
+                - 3.1666666666666665
+                - 3.2
+                - 3.2333333333333334
+                - 3.2666666666666666
+                - 3.3
+        do_online_rollout_async: false
+        eval_freq: -1
+        grad_clip_norm: 10
+        image_transforms:
+            brightness:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+            contrast:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+            enable: false
+            hue:
+                min_max:
+                    - -0.05
+                    - 0.05
+                weight: 1
+            max_num_transforms: 3
+            random_order: false
+            saturation:
+                min_max:
+                    - 0.5
+                    - 1.5
+                weight: 1
+            sharpness:
+                min_max:
+                    - 0.8
+                    - 1.2
+                weight: 1
+        log_freq: 200
+        lr: 1e-05
+        lr_backbone: 1e-05
+        num_workers: 8
+        offline_steps: 5000
+        online_buffer_capacity: null
+        online_buffer_seed_size: 0
+        online_env_seed: null
+        online_rollout_batch_size: 1
+        online_rollout_n_episodes: 1
+        online_sampling_ratio: 0.5
+        online_steps: 0
+        online_steps_between_rollouts: 1
+        save_checkpoint: true
+        save_freq: 10000
+        weight_decay: 0.0001
+use_amp:
+    value: false
+video_backend:
+    value: pyav
+wandb:
+    value:
+        disable_artifact: false
+        enable: true
+        notes: ws-lerobot-hackathon
+        project: lerobot_real_aloha

wandb/run-20241207_010741-bwwpk9vl/files/output.log

@@ -0,0 +1,28 @@
+Logs will be synced with wandb.
+INFO 2024-12-07 01:07:41 n/logger.py:132 Track this run --> https://wandb.ai/zaringleb/lerobot_real_aloha/runs/bwwpk9vl
+INFO 2024-12-07 01:07:41 ts/train.py:312 make_dataset
+WARNING 2024-12-07 01:07:41 s/factory.py:68 There might be a mismatch between your training dataset (dataset_repo_id='ccop/aloha_stationary_replay_test_v2') and your environment (cfg.env.name='real_world').
+Fetching 9 files: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 70034.76it/s]
+INFO 2024-12-07 01:07:43 ts/train.py:328 make_policy
+WARNING 2024-12-07 01:07:44 s/factory.py:28 Hydra config is missing arguments: {'temporal_ensemble_coeff'}
+Loading weights from local directory
+/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py:338: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  grad_scaler = GradScaler(enabled=cfg.use_amp)
+Error executing job with overrides: ['resume=true', 'dataset_repo_id=HumanoidTeam/aloha_real_test', 'training.offline_steps=5010']
+Traceback (most recent call last):
+  File "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py", line 652, in train_cli
+    train(
+  File "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py", line 343, in train
+    step = logger.load_last_training_state(optimizer, lr_scheduler)
+  File "/home/ubuntu/lerobot-hackathon/lerobot/common/logger.py", line 218, in load_last_training_state
+    training_state = torch.load(self.last_checkpoint_dir / self.training_state_file_name)
+  File "/home/ubuntu/lerobot-hackathon/lerobot/lib/python3.10/site-packages/torch/serialization.py", line 1444, in load
+    raise pickle.UnpicklingError(_get_wo_message(str(e))) from None
+_pickle.UnpicklingError: Weights only load failed. This file can still be loaded, to do so you have two options, do those steps only if you trust the source of the checkpoint.
+	(1) Re-running `torch.load` with `weights_only` set to `False` will likely succeed, but it can result in arbitrary code execution. Do it only if you got the file from a trusted source.
+	(2) Alternatively, to load with `weights_only=True` please check the recommended steps in the following error message.
+	WeightsUnpickler error: Unsupported global: GLOBAL numpy.core.multiarray._reconstruct was not an allowed global by default. Please use `torch.serialization.add_safe_globals([_reconstruct])` or the `torch.serialization.safe_globals([_reconstruct])` context manager to allowlist this global if you trust this class/function.
+
+Check the documentation of torch.load to learn more about types accepted by default with weights_only https://pytorch.org/docs/stable/generated/torch.load.html.
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

wandb/run-20241207_010741-bwwpk9vl/files/requirements.txt

@@ -0,0 +1,129 @@
+pymunk==6.9.0
+charset-normalizer==3.4.0
+pip==22.0.2
+multidict==6.1.0
+pytz==2024.2
+pandas==2.2.3
+fasteners==0.19
+einops==0.8.0
+setproctitle==1.3.4
+python-dateutil==2.9.0.post0
+pydantic==2.10.3
+dm-control==1.0.14
+MarkupSafe==3.0.2
+safetensors==0.4.5
+cloudpickle==3.1.0
+imageio-ffmpeg==0.5.1
+smmap==5.0.1
+huggingface-hub==0.26.5
+torch==2.6.0.dev20241112+cu121
+nvidia-cusolver-cu12==11.4.5.107
+inquirerpy==0.3.4
+soupsieve==2.6
+deepdiff==8.0.1
+nvidia-nvjitlink-cu12==12.4.127
+aiosignal==1.3.1
+dm-tree==0.1.8
+aiohappyeyeballs==2.4.4
+propcache==0.2.1
+gym-aloha==0.1.1
+labmaze==1.0.6
+pyav==14.0.1
+nvidia-cuda-cupti-cu12==12.1.105
+cffi==1.17.1
+importlib_metadata==8.5.0
+nvidia-cuda-runtime-cu12==12.1.105
+pycparser==2.22
+h5py==3.12.1
+nvidia-nccl-cu12==2.21.5
+orderly-set==5.2.2
+pyarrow==18.1.0
+Flask==3.1.0
+multiprocess==0.70.16
+blinker==1.9.0
+regex==2024.11.6
+Jinja2==3.1.4
+tqdm==4.67.1
+certifi==2024.8.30
+frozenlist==1.5.0
+attrs==24.2.0
+pytorch-triton==3.1.0+cf34004b8a
+omegaconf==2.3.0
+itsdangerous==2.2.0
+packaging==24.2
+pfzy==0.3.4
+pydantic_core==2.27.1
+antlr4-python3-runtime==4.9.3
+cmake==3.31.1
+imageio==2.36.1
+setuptools==59.6.0
+nvidia-cusparselt-cu12==0.6.2
+zipp==3.21.0
+torchvision==0.20.0.dev20241112+cu121
+nvidia-curand-cu12==10.3.2.106
+PyOpenGL-accelerate==3.1.7
+numpy==1.26.4
+dm-env==1.6
+nvidia-cusparse-cu12==12.1.0.106
+networkx==3.4.2
+triton==3.1.0
+termcolor==2.5.0
+gitdb==4.0.11
+torchaudio==2.5.0.dev20241112+cu121
+glfw==2.8.0
+tzdata==2024.2
+pillow==11.0.0
+idna==3.10
+pyparsing==3.2.0
+typing_extensions==4.12.2
+wcwidth==0.2.13
+sympy==1.13.1
+nvidia-cublas-cu12==12.1.3.1
+zarr==2.18.3
+protobuf==5.29.1
+annotated-types==0.7.0
+xxhash==3.5.0
+aiohttp==3.11.10
+Werkzeug==3.1.3
+mujoco==2.3.7
+gymnasium==0.29.1
+rerun-sdk==0.20.3
+nvidia-cufft-cu12==11.0.2.54
+dill==0.3.8
+scipy==1.14.1
+hydra-core==1.3.2
+fsspec==2024.9.0
+platformdirs==4.3.6
+PySocks==1.7.1
+PyOpenGL==3.1.7
+requests==2.32.3
+lxml==5.3.0
+Farama-Notifications==0.0.4
+nvidia-cuda-nvrtc-cu12==12.1.105
+numba==0.60.0
+docker-pycreds==0.4.0
+six==1.17.0
+lerobot==0.1.0
+nvidia-cudnn-cu12==9.1.0.70
+sentry-sdk==2.19.2
+absl-py==2.1.0
+wandb==0.19.0
+mpmath==1.3.0
+llvmlite==0.43.0
+numcodecs==0.13.1
+GitPython==3.1.43
+nvidia-nvtx-cu12==12.1.105
+prompt_toolkit==3.0.48
+datasets==3.1.0
+opencv-python==4.10.0.84
+filelock==3.16.1
+async-timeout==5.0.1
+diffusers==0.31.0
+yarl==1.18.3
+click==8.1.7
+asciitree==0.3.3
+psutil==6.1.0
+gdown==5.2.0
+PyYAML==6.0.2
+urllib3==2.2.3
+beautifulsoup4==4.12.3

wandb/run-20241207_010741-bwwpk9vl/files/wandb-metadata.json

@@ -0,0 +1,49 @@
+{
+  "os": "Linux-6.8.0-1015-aws-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2024-12-07T01:07:41.166322Z",
+  "args": [
+    "hydra.run.dir=outputs/train/2024-12-06/22-28-29_real_world_act_default",
+    "resume=true",
+    "dataset_repo_id=HumanoidTeam/aloha_real_test",
+    "training.offline_steps=5010"
+  ],
+  "program": "/home/ubuntu/lerobot-hackathon/lerobot/scripts/train.py",
+  "codePath": "lerobot/scripts/train.py",
+  "git": {
+    "remote": "[email protected]:HumanoidTeam/lerobot-hackathon.git",
+    "commit": "ab8aa3f8e7b6e5486d7af26ccd319d358ffad579"
+  },
+  "email": "[email protected]",
+  "root": "outputs/train/2024-12-06/22-28-29_real_world_act_default",
+  "host": "ip-172-31-20-130",
+  "username": "humanoidteam",
+  "executable": "/home/ubuntu/lerobot-hackathon/lerobot/bin/python",
+  "codePathLocal": "lerobot/scripts/train.py",
+  "cpu_count": 4,
+  "cpu_count_logical": 8,
+  "gpu": "Tesla T4",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "311993479168",
+      "used": "217724108800"
+    }
+  },
+  "memory": {
+    "total": "33263837184"
+  },
+  "cpu": {
+    "count": 4,
+    "countLogical": 8
+  },
+  "gpu_nvidia": [
+    {
+      "name": "Tesla T4",
+      "memoryTotal": "16106127360",
+      "cudaCores": 2560,
+      "architecture": "Turing"
+    }
+  ],
+  "cudaVersion": "12.4"
+}

wandb/run-20241207_010741-bwwpk9vl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/update_s":1.0881254139821976,"train/step":4800,"train/num_episodes":48.01,"train/dataloading_s":2.211516670999117,"train/lr":1e-05,"_step":4800,"train/grad_norm":20.83009910583496,"train/num_epochs":48.01,"_runtime":5488.627260612,"_wandb":{"runtime":5498},"train/is_online":false,"train/kld_loss":0.01526648923754692,"train/l1_loss":0.09762981534004211,"_timestamp":1.7335293805980325e+09,"train/num_samples":38408,"train/loss":0.2502947151660919}

wandb/run-20241207_010741-bwwpk9vl/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-12-07T01:07:40.734055218Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpnmhdaztf/port-3507456.txt","pid":3507456,"debug":false,"disable-analytics":false}
+{"time":"2024-12-07T01:07:40.734088106Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-12-07T01:07:40.737816002Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3507456}
+{"time":"2024-12-07T01:07:40.737842301Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36119,"Zone":""}}
+{"time":"2024-12-07T01:07:40.923329848Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:41.167320115Z","level":"INFO","msg":"handleInformInit: received","streamId":"bwwpk9vl","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:41.270221077Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bwwpk9vl","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:46.320004505Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:46.320088721Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-12-07T01:07:46.320081333Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:46.320187736Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:47.027392524Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:47.027414185Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:47356"}
+{"time":"2024-12-07T01:07:47.027425094Z","level":"INFO","msg":"server is closed"}

wandb/run-20241207_010741-bwwpk9vl/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-12-07T01:07:41.167467035Z","level":"INFO","msg":"using version","core version":"0.19.0"}
+{"time":"2024-12-07T01:07:41.167480199Z","level":"INFO","msg":"created symlink","path":"outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010741-bwwpk9vl/logs/debug-core.log"}
+{"time":"2024-12-07T01:07:41.270189094Z","level":"INFO","msg":"created new stream","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270216139Z","level":"INFO","msg":"stream: started","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270250461Z","level":"INFO","msg":"writer: Do: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270250342Z","level":"INFO","msg":"sender: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.270317361Z","level":"INFO","msg":"handler: started","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:41.601065068Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-12-07T01:07:46.320070438Z","level":"INFO","msg":"stream: closing","id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:46.320109578Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-07T01:07:46.320581424Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-07T01:07:46.890807441Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-07T01:07:47.027194656Z","level":"INFO","msg":"handler: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:47.027227034Z","level":"INFO","msg":"sender: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:47.027228216Z","level":"INFO","msg":"writer: Close: closed","stream_id":"bwwpk9vl"}
+{"time":"2024-12-07T01:07:47.027338658Z","level":"INFO","msg":"stream: closed","id":"bwwpk9vl"}

wandb/run-20241207_010741-bwwpk9vl/logs/debug.log

@@ -0,0 +1,25 @@
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Current SDK version is 0.19.0
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Configure stats pid to 3507456
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Loading settings from /home/ubuntu/lerobot-hackathon/wandb/settings
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_setup.py:_flush():68] wandb: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:_log_setup():528] Logging user logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010741-bwwpk9vl/logs/debug.log
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:_log_setup():529] Logging internal logs to outputs/train/2024-12-06/22-28-29_real_world_act_default/wandb/run-20241207_010741-bwwpk9vl/logs/debug-internal.log
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():639] calling init triggers
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():645] wandb.init called with sweep_config: {}
+config: {'resume': True, 'device': 'cuda', 'use_amp': False, 'seed': 1000, 'dataset_repo_id': 'ccop/aloha_stationary_replay_test_v2', 'video_backend': 'pyav', 'training': {'offline_steps': 5000, 'num_workers': 8, 'batch_size': 8, 'eval_freq': -1, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 10000, 'online_steps': 0, 'online_rollout_n_episodes': 1, 'online_rollout_batch_size': 1, 'online_steps_between_rollouts': 1, 'online_sampling_ratio': 0.5, 'online_env_seed': None, 'online_buffer_capacity': None, 'online_buffer_seed_size': 0, 'do_online_rollout_async': False, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'brightness': {'weight': 1, 'min_max': [0.8, 1.2]}, 'contrast': {'weight': 1, 'min_max': [0.8, 1.2]}, 'saturation': {'weight': 1, 'min_max': [0.5, 1.5]}, 'hue': {'weight': 1, 'min_max': [-0.05, 0.05]}, 'sharpness': {'weight': 1, 'min_max': [0.8, 1.2]}}, 'lr': 1e-05, 'lr_backbone': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10, 'delta_timestamps': {'action': [0.0, 0.03333333333333333, 0.06666666666666667, 0.1, 0.13333333333333333, 0.16666666666666666, 0.2, 0.23333333333333334, 0.26666666666666666, 0.3, 0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6, 0.6333333333333333, 0.6666666666666666, 0.7, 0.7333333333333333, 0.7666666666666667, 0.8, 0.8333333333333334, 0.8666666666666667, 0.9, 0.9333333333333333, 0.9666666666666667, 1.0, 1.0333333333333334, 1.0666666666666667, 1.1, 1.1333333333333333, 1.1666666666666667, 1.2, 1.2333333333333334, 1.2666666666666666, 1.3, 1.3333333333333333, 1.3666666666666667, 1.4, 1.4333333333333333, 1.4666666666666666, 1.5, 1.5333333333333334, 1.5666666666666667, 1.6, 1.6333333333333333, 1.6666666666666667, 1.7, 1.7333333333333334, 1.7666666666666666, 1.8, 1.8333333333333333, 1.8666666666666667, 1.9, 1.9333333333333333, 1.9666666666666666, 2.0, 2.033333333333333, 2.066666666666667, 2.1, 2.1333333333333333, 2.1666666666666665, 2.2, 2.2333333333333334, 2.2666666666666666, 2.3, 2.3333333333333335, 2.3666666666666667, 2.4, 2.433333333333333, 2.466666666666667, 2.5, 2.533333333333333, 2.566666666666667, 2.6, 2.6333333333333333, 2.6666666666666665, 2.7, 2.7333333333333334, 2.7666666666666666, 2.8, 2.8333333333333335, 2.8666666666666667, 2.9, 2.933333333333333, 2.966666666666667, 3.0, 3.033333333333333, 3.066666666666667, 3.1, 3.1333333333333333, 3.1666666666666665, 3.2, 3.2333333333333334, 3.2666666666666666, 3.3]}}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot_real_aloha', 'notes': 'ws-lerobot-hackathon'}, 'fps': 30, 'env': {'name': 'real_world', 'task': None, 'state_dim': 14, 'action_dim': 14, 'fps': 30}, 'override_dataset_stats': {'observation.images.cam_right_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_left_wrist': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}, 'observation.images.cam_high': {'mean': [[[0.485]], [[0.456]], [[0.406]]], 'std': [[[0.229]], [[0.224]], [[0.225]]]}}, 'policy': {'name': 'act', 'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'input_shapes': {'observation.images.cam_right_wrist': [3, 480, 640], 'observation.images.cam_left_wrist': [3, 480, 640], 'observation.images.cam_high': [3, 480, 640], 'observation.state': [14]}, 'output_shapes': {'action': [14]}, 'input_normalization_modes': {'observation.images.cam_right_wrist': 'mean_std', 'observation.images.cam_left_wrist': 'mean_std', 'observation.images.cam_high': 'mean_std', 'observation.state': 'mean_std'}, 'output_normalization_modes': {'action': 'mean_std'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_momentum': None, 'dropout': 0.1, 'kl_weight': 10.0}}
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():688] starting backend
+2024-12-07 01:07:41,161 INFO    MainThread:3507456 [wandb_init.py:init():692] sending inform_init request
+2024-12-07 01:07:41,165 INFO    MainThread:3507456 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-07 01:07:41,166 INFO    MainThread:3507456 [wandb_init.py:init():705] backend started and connected
+2024-12-07 01:07:41,168 INFO    MainThread:3507456 [wandb_init.py:init():798] updated telemetry
+2024-12-07 01:07:41,174 INFO    MainThread:3507456 [wandb_init.py:init():830] communicating run to backend with 90.0 second timeout
+2024-12-07 01:07:41,592 INFO    MainThread:3507456 [wandb_init.py:init():875] run resumed
+2024-12-07 01:07:41,597 INFO    MainThread:3507456 [wandb_init.py:init():882] starting run threads in backend
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_console_start():2443] atexit reg
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_redirect():2293] redirect: wrap_raw
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_redirect():2358] Wrapping output streams.
+2024-12-07 01:07:41,711 INFO    MainThread:3507456 [wandb_run.py:_redirect():2383] Redirects installed.
+2024-12-07 01:07:41,713 INFO    MainThread:3507456 [wandb_init.py:init():925] run started, returning control to user process
+2024-12-07 01:07:46,320 WARNING MsgRouterThr:3507456 [router.py:message_loop():75] message_loop has been closed