Hugging Face's logo

Trelis
/
avatar-3500-b2

Diffusers
TensorBoard
Safetensors
Model card Files Metrics Community
rs545837 commited on
Commit
bb2ffb3
·
verified ·
1 Parent(s): 1a1ad5d

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .ipynb_checkpoints/config-checkpoint.yaml +56 -0
  2. checkpoint-1500/audio_proj/config.json +5 -0
  3. checkpoint-1500/audio_proj/diffusion_pytorch_model.safetensors +3 -0
  4. checkpoint-1500/diffusion_net/config.json +88 -0
  5. checkpoint-3500/audio_proj/config.json +5 -0
  6. checkpoint-3500/audio_proj/diffusion_pytorch_model.safetensors +3 -0
  7. checkpoint-3500/diffusion_net/config.json +88 -0
  8. checkpoint-3500/diffusion_net/diffusion_pytorch_model.safetensors +3 -0
  9. checkpoint-3500/image_proj/config.json +5 -0
  10. checkpoint-3500/image_proj/diffusion_pytorch_model.safetensors +3 -0
  11. checkpoint-3500/optimizer.bin +3 -0
  12. checkpoint-3500/random_states_0.pkl +3 -0
  13. checkpoint-3500/reference_net/config.json +66 -0
  14. checkpoint-3500/reference_net/diffusion_pytorch_model.safetensors +3 -0
  15. checkpoint-3500/scheduler.bin +3 -0
  16. checkpoint-5000/audio_proj/config.json +5 -0
  17. checkpoint-5000/audio_proj/diffusion_pytorch_model.safetensors +3 -0
  18. checkpoint-5000/diffusion_net/config.json +88 -0
  19. checkpoint-5000/diffusion_net/diffusion_pytorch_model.safetensors +3 -0
  20. checkpoint-5000/image_proj/config.json +5 -0
  21. checkpoint-5000/image_proj/diffusion_pytorch_model.safetensors +3 -0
  22. checkpoint-5000/optimizer.bin +3 -0
  23. checkpoint-5000/random_states_0.pkl +3 -0
  24. checkpoint-5000/reference_net/config.json +66 -0
  25. checkpoint-5000/reference_net/diffusion_pytorch_model.safetensors +3 -0
  26. checkpoint-5000/scheduler.bin +3 -0
  27. checkpoint-7000/audio_proj/config.json +5 -0
  28. checkpoint-7000/audio_proj/diffusion_pytorch_model.safetensors +3 -0
  29. checkpoint-7000/diffusion_net/config.json +88 -0
  30. checkpoint-7000/diffusion_net/diffusion_pytorch_model.safetensors +3 -0
  31. checkpoint-7000/image_proj/config.json +5 -0
  32. checkpoint-7000/image_proj/diffusion_pytorch_model.safetensors +3 -0
  33. checkpoint-7000/optimizer.bin +3 -0
  34. checkpoint-7000/random_states_0.pkl +3 -0
  35. checkpoint-7000/reference_net/config.json +66 -0
  36. checkpoint-7000/reference_net/diffusion_pytorch_model.safetensors +3 -0
  37. checkpoint-7000/scheduler.bin +3 -0
  38. config.yaml +56 -0
  39. logs/memo/1741022678.2856228/events.out.tfevents.1741022678.3c52c6a06e3b +3 -0
  40. logs/memo/1741022678.2878249/hparams.yml +55 -0
  41. logs/memo/1741046967.6854546/events.out.tfevents.1741046967.369f90aa7051 +3 -0
  42. logs/memo/1741046967.688156/.ipynb_checkpoints/hparams-checkpoint.yml +55 -0
  43. logs/memo/1741046967.688156/hparams.yml +55 -0
  44. logs/memo/1741060237.980531/events.out.tfevents.1741060237.369f90aa7051 +3 -0
  45. logs/memo/1741060237.983221/hparams.yml +55 -0
  46. logs/memo/1741060310.7917793/events.out.tfevents.1741060310.369f90aa7051 +3 -0
  47. logs/memo/1741060310.7946274/hparams.yml +55 -0
  48. logs/memo/1741060338.6906004/events.out.tfevents.1741060338.369f90aa7051 +3 -0
  49. logs/memo/1741060338.6932552/hparams.yml +55 -0
  50. logs/memo/1741145698.900579/events.out.tfevents.1741145698.16e2a27e51cb +3 -0
.ipynb_checkpoints/config-checkpoint.yaml ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tracker_project_name: memo
2
+ output_dir: outputs
3
+ resume_from_checkpoint: null
4
+ model_name_or_path: memoavatar/memo
5
+ vae: stabilityai/sd-vae-ft-mse
6
+ gradient_checkpointing: true
7
+ gradient_accumulation_steps: 1
8
+ train_batch_size: 2
9
+ max_train_steps: 3500
10
+ num_train_epochs: -1
11
+ enable_xformers_memory_efficient_attention: true
12
+ checkpoints_total_limit: 3
13
+ robust_training: true
14
+ learning_rate: 1e-5
15
+ max_grad_norm: 1.0
16
+ scale_lr: false
17
+ lr_scheduler: constant
18
+ lr_warmup_steps: 0
19
+ seed: 42
20
+ mixed_precision: bf16
21
+ use_8bit_adam: false
22
+ allow_tf32: true
23
+ use_ema: false
24
+ adam_beta1: 0.9
25
+ adam_beta2: 0.999
26
+ adam_weight_decay: 0.01
27
+ adam_epsilon: 1e-08
28
+ dataloader_num_workers: 16
29
+ prefetch_factor: 4
30
+ checkpointing_steps: 5000
31
+ data:
32
+ width: 512
33
+ height: 512
34
+ num_past_frames: 16
35
+ dynamic_past_frames: false
36
+ n_sample_frames: 16
37
+ audio_margin: 2
38
+ metadata_paths:
39
+ - assets/embedding/metadata.jsonl
40
+ weighting_scheme: logit_normal
41
+ logit_mean: 0.0
42
+ logit_std: 1.0
43
+ mode_scale: 1.29
44
+ noise_scheduler_kwargs:
45
+ num_train_timesteps: 1000
46
+ train_reference_net: true
47
+ train_diffusion_net: true
48
+ train_image_proj: true
49
+ train_audio_proj: true
50
+ trainable_modules:
51
+ - to_q
52
+ - to_k
53
+ - to_v
54
+ uncond_img_ratio: 0.05
55
+ uncond_audio_ratio: 0.05
56
+ start_ratio: 0.05
checkpoint-1500/audio_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AudioProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-1500/audio_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc1c7d4583571f4648f5bf82b70246671d4576837154be38436327f3e8c0e8cc
3
+ size 72930976
checkpoint-1500/diffusion_net/config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet3DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "audio_attention_dim": 768,
14
+ "block_out_channels": [
15
+ 320,
16
+ 640,
17
+ 1280,
18
+ 1280
19
+ ],
20
+ "center_input_sample": false,
21
+ "class_embed_type": null,
22
+ "class_embeddings_concat": false,
23
+ "conv_in_kernel": 3,
24
+ "cross_attention_dim": 768,
25
+ "down_block_types": [
26
+ "CrossAttnDownBlock3D",
27
+ "CrossAttnDownBlock3D",
28
+ "CrossAttnDownBlock3D",
29
+ "DownBlock3D"
30
+ ],
31
+ "downsample_padding": 1,
32
+ "dropout": 0.0,
33
+ "dual_cross_attention": false,
34
+ "emo_drop_rate": 0.05,
35
+ "encoder_hid_dim": null,
36
+ "encoder_hid_dim_type": null,
37
+ "flip_sin_to_cos": true,
38
+ "freq_shift": 0,
39
+ "in_channels": 4,
40
+ "layers_per_block": 2,
41
+ "mid_block_only_cross_attention": null,
42
+ "mid_block_scale_factor": 1,
43
+ "mid_block_type": "UNetMidBlock3DCrossAttn",
44
+ "motion_module_kwargs": {
45
+ "attention_block_types": [
46
+ "Temporal_Self",
47
+ "Temporal_Self"
48
+ ],
49
+ "num_attention_heads": 8,
50
+ "num_transformer_block": 1,
51
+ "temporal_attention_dim_div": 1,
52
+ "temporal_position_encoding": true,
53
+ "temporal_position_encoding_max_len": 32
54
+ },
55
+ "motion_module_resolutions": [
56
+ 1,
57
+ 2,
58
+ 4,
59
+ 8
60
+ ],
61
+ "norm_eps": 1e-05,
62
+ "norm_num_groups": 32,
63
+ "num_attention_heads": null,
64
+ "num_class_embeds": null,
65
+ "only_cross_attention": false,
66
+ "out_channels": 4,
67
+ "projection_class_embeddings_input_dim": null,
68
+ "resnet_time_scale_shift": "default",
69
+ "reverse_transformer_layers_per_block": null,
70
+ "sample_size": 64,
71
+ "time_cond_proj_dim": null,
72
+ "time_embedding_act_fn": null,
73
+ "time_embedding_dim": null,
74
+ "time_embedding_type": "positional",
75
+ "timestep_post_act": null,
76
+ "transformer_layers_per_block": 1,
77
+ "unet_use_cross_frame_attention": false,
78
+ "unet_use_temporal_attention": false,
79
+ "up_block_types": [
80
+ "UpBlock3D",
81
+ "CrossAttnUpBlock3D",
82
+ "CrossAttnUpBlock3D",
83
+ "CrossAttnUpBlock3D"
84
+ ],
85
+ "upcast_attention": false,
86
+ "use_inflated_groupnorm": true,
87
+ "use_linear_projection": false
88
+ }
checkpoint-3500/audio_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AudioProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-3500/audio_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d58084a9b3eaa79f6748560ca9cce7595931cc6310517f94fec1ae78712cca4
3
+ size 72930976
checkpoint-3500/diffusion_net/config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet3DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "audio_attention_dim": 768,
14
+ "block_out_channels": [
15
+ 320,
16
+ 640,
17
+ 1280,
18
+ 1280
19
+ ],
20
+ "center_input_sample": false,
21
+ "class_embed_type": null,
22
+ "class_embeddings_concat": false,
23
+ "conv_in_kernel": 3,
24
+ "cross_attention_dim": 768,
25
+ "down_block_types": [
26
+ "CrossAttnDownBlock3D",
27
+ "CrossAttnDownBlock3D",
28
+ "CrossAttnDownBlock3D",
29
+ "DownBlock3D"
30
+ ],
31
+ "downsample_padding": 1,
32
+ "dropout": 0.0,
33
+ "dual_cross_attention": false,
34
+ "emo_drop_rate": 0.05,
35
+ "encoder_hid_dim": null,
36
+ "encoder_hid_dim_type": null,
37
+ "flip_sin_to_cos": true,
38
+ "freq_shift": 0,
39
+ "in_channels": 4,
40
+ "layers_per_block": 2,
41
+ "mid_block_only_cross_attention": null,
42
+ "mid_block_scale_factor": 1,
43
+ "mid_block_type": "UNetMidBlock3DCrossAttn",
44
+ "motion_module_kwargs": {
45
+ "attention_block_types": [
46
+ "Temporal_Self",
47
+ "Temporal_Self"
48
+ ],
49
+ "num_attention_heads": 8,
50
+ "num_transformer_block": 1,
51
+ "temporal_attention_dim_div": 1,
52
+ "temporal_position_encoding": true,
53
+ "temporal_position_encoding_max_len": 32
54
+ },
55
+ "motion_module_resolutions": [
56
+ 1,
57
+ 2,
58
+ 4,
59
+ 8
60
+ ],
61
+ "norm_eps": 1e-05,
62
+ "norm_num_groups": 32,
63
+ "num_attention_heads": null,
64
+ "num_class_embeds": null,
65
+ "only_cross_attention": false,
66
+ "out_channels": 4,
67
+ "projection_class_embeddings_input_dim": null,
68
+ "resnet_time_scale_shift": "default",
69
+ "reverse_transformer_layers_per_block": null,
70
+ "sample_size": 64,
71
+ "time_cond_proj_dim": null,
72
+ "time_embedding_act_fn": null,
73
+ "time_embedding_dim": null,
74
+ "time_embedding_type": "positional",
75
+ "timestep_post_act": null,
76
+ "transformer_layers_per_block": 1,
77
+ "unet_use_cross_frame_attention": false,
78
+ "unet_use_temporal_attention": false,
79
+ "up_block_types": [
80
+ "UpBlock3D",
81
+ "CrossAttnUpBlock3D",
82
+ "CrossAttnUpBlock3D",
83
+ "CrossAttnUpBlock3D"
84
+ ],
85
+ "upcast_attention": false,
86
+ "use_inflated_groupnorm": true,
87
+ "use_linear_projection": false
88
+ }
checkpoint-3500/diffusion_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c49282fecedb44b61587af090b411d19c91cf02d1f931f3d09a200dafdf04e4
3
+ size 3356369240
checkpoint-3500/image_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "ImageProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-3500/image_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da65bdfcdf6b4b7fe8fde646092c71fe592741cbcd2fcdb1bbf16ea35982790f
3
+ size 3155280
checkpoint-3500/optimizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b7a90617826926ee0ede2186b38412e05908ef7f83f446e3faa584d32dd37816
3
+ size 1422326990
checkpoint-3500/random_states_0.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8143d967fa3d4e7d3eaa5ed4fb8f80dd746452184ededbfb33cfb2926728d23
3
+ size 14408
checkpoint-3500/reference_net/config.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet2DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "block_out_channels": [
14
+ 320,
15
+ 640,
16
+ 1280,
17
+ 1280
18
+ ],
19
+ "center_input_sample": false,
20
+ "class_embed_type": null,
21
+ "class_embeddings_concat": false,
22
+ "conv_in_kernel": 3,
23
+ "cross_attention_dim": 768,
24
+ "down_block_types": [
25
+ "CrossAttnDownBlock2D",
26
+ "CrossAttnDownBlock2D",
27
+ "CrossAttnDownBlock2D",
28
+ "DownBlock2D"
29
+ ],
30
+ "downsample_padding": 1,
31
+ "dropout": 0.0,
32
+ "dual_cross_attention": false,
33
+ "encoder_hid_dim": null,
34
+ "encoder_hid_dim_type": null,
35
+ "flip_sin_to_cos": true,
36
+ "freq_shift": 0,
37
+ "in_channels": 4,
38
+ "layers_per_block": 2,
39
+ "mid_block_only_cross_attention": null,
40
+ "mid_block_scale_factor": 1,
41
+ "mid_block_type": "UNetMidBlock2DCrossAttn",
42
+ "norm_eps": 1e-05,
43
+ "norm_num_groups": 32,
44
+ "num_attention_heads": null,
45
+ "num_class_embeds": null,
46
+ "only_cross_attention": false,
47
+ "out_channels": 4,
48
+ "projection_class_embeddings_input_dim": null,
49
+ "resnet_time_scale_shift": "default",
50
+ "reverse_transformer_layers_per_block": null,
51
+ "sample_size": 64,
52
+ "time_cond_proj_dim": null,
53
+ "time_embedding_act_fn": null,
54
+ "time_embedding_dim": null,
55
+ "time_embedding_type": "positional",
56
+ "timestep_post_act": null,
57
+ "transformer_layers_per_block": 1,
58
+ "up_block_types": [
59
+ "UpBlock2D",
60
+ "CrossAttnUpBlock2D",
61
+ "CrossAttnUpBlock2D",
62
+ "CrossAttnUpBlock2D"
63
+ ],
64
+ "upcast_attention": false,
65
+ "use_linear_projection": false
66
+ }
checkpoint-3500/reference_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:345b2f774b724493a1f4873560ffd419b81d4324be80028d67625f27ebcab6a1
3
+ size 1714214152
checkpoint-3500/scheduler.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6802221f6347629ca3a6e377a3bbcf1d2a6c0b7ef7e83794854e08587237ffd4
3
+ size 1000
checkpoint-5000/audio_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AudioProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-5000/audio_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc1c7d4583571f4648f5bf82b70246671d4576837154be38436327f3e8c0e8cc
3
+ size 72930976
checkpoint-5000/diffusion_net/config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet3DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "audio_attention_dim": 768,
14
+ "block_out_channels": [
15
+ 320,
16
+ 640,
17
+ 1280,
18
+ 1280
19
+ ],
20
+ "center_input_sample": false,
21
+ "class_embed_type": null,
22
+ "class_embeddings_concat": false,
23
+ "conv_in_kernel": 3,
24
+ "cross_attention_dim": 768,
25
+ "down_block_types": [
26
+ "CrossAttnDownBlock3D",
27
+ "CrossAttnDownBlock3D",
28
+ "CrossAttnDownBlock3D",
29
+ "DownBlock3D"
30
+ ],
31
+ "downsample_padding": 1,
32
+ "dropout": 0.0,
33
+ "dual_cross_attention": false,
34
+ "emo_drop_rate": 0.05,
35
+ "encoder_hid_dim": null,
36
+ "encoder_hid_dim_type": null,
37
+ "flip_sin_to_cos": true,
38
+ "freq_shift": 0,
39
+ "in_channels": 4,
40
+ "layers_per_block": 2,
41
+ "mid_block_only_cross_attention": null,
42
+ "mid_block_scale_factor": 1,
43
+ "mid_block_type": "UNetMidBlock3DCrossAttn",
44
+ "motion_module_kwargs": {
45
+ "attention_block_types": [
46
+ "Temporal_Self",
47
+ "Temporal_Self"
48
+ ],
49
+ "num_attention_heads": 8,
50
+ "num_transformer_block": 1,
51
+ "temporal_attention_dim_div": 1,
52
+ "temporal_position_encoding": true,
53
+ "temporal_position_encoding_max_len": 32
54
+ },
55
+ "motion_module_resolutions": [
56
+ 1,
57
+ 2,
58
+ 4,
59
+ 8
60
+ ],
61
+ "norm_eps": 1e-05,
62
+ "norm_num_groups": 32,
63
+ "num_attention_heads": null,
64
+ "num_class_embeds": null,
65
+ "only_cross_attention": false,
66
+ "out_channels": 4,
67
+ "projection_class_embeddings_input_dim": null,
68
+ "resnet_time_scale_shift": "default",
69
+ "reverse_transformer_layers_per_block": null,
70
+ "sample_size": 64,
71
+ "time_cond_proj_dim": null,
72
+ "time_embedding_act_fn": null,
73
+ "time_embedding_dim": null,
74
+ "time_embedding_type": "positional",
75
+ "timestep_post_act": null,
76
+ "transformer_layers_per_block": 1,
77
+ "unet_use_cross_frame_attention": false,
78
+ "unet_use_temporal_attention": false,
79
+ "up_block_types": [
80
+ "UpBlock3D",
81
+ "CrossAttnUpBlock3D",
82
+ "CrossAttnUpBlock3D",
83
+ "CrossAttnUpBlock3D"
84
+ ],
85
+ "upcast_attention": false,
86
+ "use_inflated_groupnorm": true,
87
+ "use_linear_projection": false
88
+ }
checkpoint-5000/diffusion_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53f04a95f121794e54554f6b8746a239354167a03c70c2d3e6829cabd517754f
3
+ size 3356369240
checkpoint-5000/image_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "ImageProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-5000/image_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9855dabf683d8c17b265567737bc90e5a0b032db7551b78f028cd9b03439abe5
3
+ size 3155280
checkpoint-5000/optimizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5932ca6ad987609976876f0e8473b54ee286a9e62778f4152c1de736a8b98c30
3
+ size 3270385552
checkpoint-5000/random_states_0.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0dc7aef135da1cc848c02bc449b185f15582060d87aa35e527e7d944cd090bd
3
+ size 14344
checkpoint-5000/reference_net/config.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet2DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "block_out_channels": [
14
+ 320,
15
+ 640,
16
+ 1280,
17
+ 1280
18
+ ],
19
+ "center_input_sample": false,
20
+ "class_embed_type": null,
21
+ "class_embeddings_concat": false,
22
+ "conv_in_kernel": 3,
23
+ "cross_attention_dim": 768,
24
+ "down_block_types": [
25
+ "CrossAttnDownBlock2D",
26
+ "CrossAttnDownBlock2D",
27
+ "CrossAttnDownBlock2D",
28
+ "DownBlock2D"
29
+ ],
30
+ "downsample_padding": 1,
31
+ "dropout": 0.0,
32
+ "dual_cross_attention": false,
33
+ "encoder_hid_dim": null,
34
+ "encoder_hid_dim_type": null,
35
+ "flip_sin_to_cos": true,
36
+ "freq_shift": 0,
37
+ "in_channels": 4,
38
+ "layers_per_block": 2,
39
+ "mid_block_only_cross_attention": null,
40
+ "mid_block_scale_factor": 1,
41
+ "mid_block_type": "UNetMidBlock2DCrossAttn",
42
+ "norm_eps": 1e-05,
43
+ "norm_num_groups": 32,
44
+ "num_attention_heads": null,
45
+ "num_class_embeds": null,
46
+ "only_cross_attention": false,
47
+ "out_channels": 4,
48
+ "projection_class_embeddings_input_dim": null,
49
+ "resnet_time_scale_shift": "default",
50
+ "reverse_transformer_layers_per_block": null,
51
+ "sample_size": 64,
52
+ "time_cond_proj_dim": null,
53
+ "time_embedding_act_fn": null,
54
+ "time_embedding_dim": null,
55
+ "time_embedding_type": "positional",
56
+ "timestep_post_act": null,
57
+ "transformer_layers_per_block": 1,
58
+ "up_block_types": [
59
+ "UpBlock2D",
60
+ "CrossAttnUpBlock2D",
61
+ "CrossAttnUpBlock2D",
62
+ "CrossAttnUpBlock2D"
63
+ ],
64
+ "upcast_attention": false,
65
+ "use_linear_projection": false
66
+ }
checkpoint-5000/reference_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0550a76a5df4c295610040f90075b3bbf64f7d6be58cfc93a9722be1377b2486
3
+ size 1714214152
checkpoint-5000/scheduler.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:920333155b08e76154b89a85170c320833074db7b84649a9cfe79fd10b6bf2d9
3
+ size 1000
checkpoint-7000/audio_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AudioProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-7000/audio_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc1c7d4583571f4648f5bf82b70246671d4576837154be38436327f3e8c0e8cc
3
+ size 72930976
checkpoint-7000/diffusion_net/config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet3DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "audio_attention_dim": 768,
14
+ "block_out_channels": [
15
+ 320,
16
+ 640,
17
+ 1280,
18
+ 1280
19
+ ],
20
+ "center_input_sample": false,
21
+ "class_embed_type": null,
22
+ "class_embeddings_concat": false,
23
+ "conv_in_kernel": 3,
24
+ "cross_attention_dim": 768,
25
+ "down_block_types": [
26
+ "CrossAttnDownBlock3D",
27
+ "CrossAttnDownBlock3D",
28
+ "CrossAttnDownBlock3D",
29
+ "DownBlock3D"
30
+ ],
31
+ "downsample_padding": 1,
32
+ "dropout": 0.0,
33
+ "dual_cross_attention": false,
34
+ "emo_drop_rate": 0.05,
35
+ "encoder_hid_dim": null,
36
+ "encoder_hid_dim_type": null,
37
+ "flip_sin_to_cos": true,
38
+ "freq_shift": 0,
39
+ "in_channels": 4,
40
+ "layers_per_block": 2,
41
+ "mid_block_only_cross_attention": null,
42
+ "mid_block_scale_factor": 1,
43
+ "mid_block_type": "UNetMidBlock3DCrossAttn",
44
+ "motion_module_kwargs": {
45
+ "attention_block_types": [
46
+ "Temporal_Self",
47
+ "Temporal_Self"
48
+ ],
49
+ "num_attention_heads": 8,
50
+ "num_transformer_block": 1,
51
+ "temporal_attention_dim_div": 1,
52
+ "temporal_position_encoding": true,
53
+ "temporal_position_encoding_max_len": 32
54
+ },
55
+ "motion_module_resolutions": [
56
+ 1,
57
+ 2,
58
+ 4,
59
+ 8
60
+ ],
61
+ "norm_eps": 1e-05,
62
+ "norm_num_groups": 32,
63
+ "num_attention_heads": null,
64
+ "num_class_embeds": null,
65
+ "only_cross_attention": false,
66
+ "out_channels": 4,
67
+ "projection_class_embeddings_input_dim": null,
68
+ "resnet_time_scale_shift": "default",
69
+ "reverse_transformer_layers_per_block": null,
70
+ "sample_size": 64,
71
+ "time_cond_proj_dim": null,
72
+ "time_embedding_act_fn": null,
73
+ "time_embedding_dim": null,
74
+ "time_embedding_type": "positional",
75
+ "timestep_post_act": null,
76
+ "transformer_layers_per_block": 1,
77
+ "unet_use_cross_frame_attention": false,
78
+ "unet_use_temporal_attention": false,
79
+ "up_block_types": [
80
+ "UpBlock3D",
81
+ "CrossAttnUpBlock3D",
82
+ "CrossAttnUpBlock3D",
83
+ "CrossAttnUpBlock3D"
84
+ ],
85
+ "upcast_attention": false,
86
+ "use_inflated_groupnorm": true,
87
+ "use_linear_projection": false
88
+ }
checkpoint-7000/diffusion_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e24b052422aa8c6dc6b0ed81d03492b83928fc0c25d437cecfda6c65b52348c1
3
+ size 3356369240
checkpoint-7000/image_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "ImageProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-7000/image_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9855dabf683d8c17b265567737bc90e5a0b032db7551b78f028cd9b03439abe5
3
+ size 3155280
checkpoint-7000/optimizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e6bee6eb8cfb1c8bd3314731385820e5ee5dbb0bd57dbb1a55c1ba676f34c2a
3
+ size 3270385552
checkpoint-7000/random_states_0.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bc142465f2d168b5cbd99f93129ea9d61c76a52d868f8d39dbbaed3581a6afd
3
+ size 14344
checkpoint-7000/reference_net/config.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet2DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "block_out_channels": [
14
+ 320,
15
+ 640,
16
+ 1280,
17
+ 1280
18
+ ],
19
+ "center_input_sample": false,
20
+ "class_embed_type": null,
21
+ "class_embeddings_concat": false,
22
+ "conv_in_kernel": 3,
23
+ "cross_attention_dim": 768,
24
+ "down_block_types": [
25
+ "CrossAttnDownBlock2D",
26
+ "CrossAttnDownBlock2D",
27
+ "CrossAttnDownBlock2D",
28
+ "DownBlock2D"
29
+ ],
30
+ "downsample_padding": 1,
31
+ "dropout": 0.0,
32
+ "dual_cross_attention": false,
33
+ "encoder_hid_dim": null,
34
+ "encoder_hid_dim_type": null,
35
+ "flip_sin_to_cos": true,
36
+ "freq_shift": 0,
37
+ "in_channels": 4,
38
+ "layers_per_block": 2,
39
+ "mid_block_only_cross_attention": null,
40
+ "mid_block_scale_factor": 1,
41
+ "mid_block_type": "UNetMidBlock2DCrossAttn",
42
+ "norm_eps": 1e-05,
43
+ "norm_num_groups": 32,
44
+ "num_attention_heads": null,
45
+ "num_class_embeds": null,
46
+ "only_cross_attention": false,
47
+ "out_channels": 4,
48
+ "projection_class_embeddings_input_dim": null,
49
+ "resnet_time_scale_shift": "default",
50
+ "reverse_transformer_layers_per_block": null,
51
+ "sample_size": 64,
52
+ "time_cond_proj_dim": null,
53
+ "time_embedding_act_fn": null,
54
+ "time_embedding_dim": null,
55
+ "time_embedding_type": "positional",
56
+ "timestep_post_act": null,
57
+ "transformer_layers_per_block": 1,
58
+ "up_block_types": [
59
+ "UpBlock2D",
60
+ "CrossAttnUpBlock2D",
61
+ "CrossAttnUpBlock2D",
62
+ "CrossAttnUpBlock2D"
63
+ ],
64
+ "upcast_attention": false,
65
+ "use_linear_projection": false
66
+ }
checkpoint-7000/reference_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0550a76a5df4c295610040f90075b3bbf64f7d6be58cfc93a9722be1377b2486
3
+ size 1714214152
checkpoint-7000/scheduler.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4f80bd4cf789dadaabab339ee4bbcf7677008e876201ba9229aae4c3d8860b1
3
+ size 1000
config.yaml ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tracker_project_name: memo
2
+ output_dir: outputs
3
+ resume_from_checkpoint: null
4
+ model_name_or_path: memoavatar/memo
5
+ vae: stabilityai/sd-vae-ft-mse
6
+ gradient_checkpointing: true
7
+ gradient_accumulation_steps: 1
8
+ train_batch_size: 2
9
+ max_train_steps: 3500
10
+ num_train_epochs: -1
11
+ enable_xformers_memory_efficient_attention: true
12
+ checkpoints_total_limit: 3
13
+ robust_training: true
14
+ learning_rate: 1e-5
15
+ max_grad_norm: 1.0
16
+ scale_lr: false
17
+ lr_scheduler: constant
18
+ lr_warmup_steps: 0
19
+ seed: 42
20
+ mixed_precision: bf16
21
+ use_8bit_adam: false
22
+ allow_tf32: true
23
+ use_ema: false
24
+ adam_beta1: 0.9
25
+ adam_beta2: 0.999
26
+ adam_weight_decay: 0.01
27
+ adam_epsilon: 1e-08
28
+ dataloader_num_workers: 16
29
+ prefetch_factor: 4
30
+ checkpointing_steps: 5000
31
+ data:
32
+ width: 512
33
+ height: 512
34
+ num_past_frames: 16
35
+ dynamic_past_frames: false
36
+ n_sample_frames: 16
37
+ audio_margin: 2
38
+ metadata_paths:
39
+ - assets/embedding/metadata.jsonl
40
+ weighting_scheme: logit_normal
41
+ logit_mean: 0.0
42
+ logit_std: 1.0
43
+ mode_scale: 1.29
44
+ noise_scheduler_kwargs:
45
+ num_train_timesteps: 1000
46
+ train_reference_net: true
47
+ train_diffusion_net: true
48
+ train_image_proj: true
49
+ train_audio_proj: true
50
+ trainable_modules:
51
+ - to_q
52
+ - to_k
53
+ - to_v
54
+ uncond_img_ratio: 0.05
55
+ uncond_audio_ratio: 0.05
56
+ start_ratio: 0.05
logs/memo/1741022678.2856228/events.out.tfevents.1741022678.3c52c6a06e3b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82468bdd6d38c969094750adaf19ce3778a9c50fd26b45c2e643f9852bcd80fd
3
+ size 2329
logs/memo/1741022678.2878249/hparams.yml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ adam_beta1: 0.9
2
+ adam_beta2: 0.999
3
+ adam_epsilon: 1.0e-08
4
+ adam_weight_decay: 0.01
5
+ allow_tf32: true
6
+ checkpointing_steps: 5000
7
+ checkpoints_total_limit: 20
8
+ data:
9
+ audio_margin: 2
10
+ dynamic_past_frames: false
11
+ height: 512
12
+ metadata_paths:
13
+ - data/embedding/metadata.jsonl
14
+ n_sample_frames: 16
15
+ num_past_frames: 16
16
+ width: 512
17
+ dataloader_num_workers: 16
18
+ enable_xformers_memory_efficient_attention: true
19
+ gradient_accumulation_steps: 1
20
+ gradient_checkpointing: true
21
+ learning_rate: 1.0e-05
22
+ logit_mean: 0.0
23
+ logit_std: 1.0
24
+ lr_scheduler: constant
25
+ lr_warmup_steps: 0
26
+ max_grad_norm: 1.0
27
+ max_train_steps: 3500
28
+ mixed_precision: bf16
29
+ mode_scale: 1.29
30
+ model_name_or_path: memoavatar/memo
31
+ noise_scheduler_kwargs:
32
+ num_train_timesteps: 1000
33
+ num_train_epochs: 350
34
+ output_dir: outputs/finetune
35
+ prefetch_factor: 4
36
+ resume_from_checkpoint: null
37
+ robust_training: true
38
+ scale_lr: false
39
+ seed: 42
40
+ start_ratio: 0.05
41
+ tracker_project_name: memo
42
+ train_audio_proj: false
43
+ train_batch_size: 1
44
+ train_diffusion_net: true
45
+ train_image_proj: false
46
+ train_reference_net: false
47
+ trainable_modules:
48
+ - motion_modules
49
+ - audio_modules
50
+ uncond_audio_ratio: 0.05
51
+ uncond_img_ratio: 0.05
52
+ use_8bit_adam: false
53
+ use_ema: false
54
+ vae: stabilityai/sd-vae-ft-mse
55
+ weighting_scheme: logit_normal
logs/memo/1741046967.6854546/events.out.tfevents.1741046967.369f90aa7051 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:516216cf24c6ea13cf020b0b334970a9d7dcaef079db3a8fd0d3cdbc21aa1b58
3
+ size 2329
logs/memo/1741046967.688156/.ipynb_checkpoints/hparams-checkpoint.yml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ adam_beta1: 0.9
2
+ adam_beta2: 0.999
3
+ adam_epsilon: 1.0e-08
4
+ adam_weight_decay: 0.01
5
+ allow_tf32: true
6
+ checkpointing_steps: 5000
7
+ checkpoints_total_limit: 20
8
+ data:
9
+ audio_margin: 2
10
+ dynamic_past_frames: false
11
+ height: 512
12
+ metadata_paths:
13
+ - data/embedding/metadata.jsonl
14
+ n_sample_frames: 16
15
+ num_past_frames: 16
16
+ width: 512
17
+ dataloader_num_workers: 16
18
+ enable_xformers_memory_efficient_attention: true
19
+ gradient_accumulation_steps: 1
20
+ gradient_checkpointing: true
21
+ learning_rate: 1.0e-05
22
+ logit_mean: 0.0
23
+ logit_std: 1.0
24
+ lr_scheduler: constant
25
+ lr_warmup_steps: 0
26
+ max_grad_norm: 1.0
27
+ max_train_steps: 3500
28
+ mixed_precision: bf16
29
+ mode_scale: 1.29
30
+ model_name_or_path: memoavatar/memo
31
+ noise_scheduler_kwargs:
32
+ num_train_timesteps: 1000
33
+ num_train_epochs: 700
34
+ output_dir: outputs/finetune
35
+ prefetch_factor: 4
36
+ resume_from_checkpoint: null
37
+ robust_training: true
38
+ scale_lr: false
39
+ seed: 42
40
+ start_ratio: 0.05
41
+ tracker_project_name: memo
42
+ train_audio_proj: false
43
+ train_batch_size: 2
44
+ train_diffusion_net: true
45
+ train_image_proj: false
46
+ train_reference_net: false
47
+ trainable_modules:
48
+ - motion_modules
49
+ - audio_modules
50
+ uncond_audio_ratio: 0.05
51
+ uncond_img_ratio: 0.05
52
+ use_8bit_adam: true
53
+ use_ema: false
54
+ vae: stabilityai/sd-vae-ft-mse
55
+ weighting_scheme: logit_normal
logs/memo/1741046967.688156/hparams.yml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ adam_beta1: 0.9
2
+ adam_beta2: 0.999
3
+ adam_epsilon: 1.0e-08
4
+ adam_weight_decay: 0.01
5
+ allow_tf32: true
6
+ checkpointing_steps: 5000
7
+ checkpoints_total_limit: 20
8
+ data:
9
+ audio_margin: 2
10
+ dynamic_past_frames: false
11
+ height: 512
12
+ metadata_paths:
13
+ - data/embedding/metadata.jsonl
14
+ n_sample_frames: 16
15
+ num_past_frames: 16
16
+ width: 512
17
+ dataloader_num_workers: 16
18
+ enable_xformers_memory_efficient_attention: true
19
+ gradient_accumulation_steps: 1
20
+ gradient_checkpointing: true
21
+ learning_rate: 1.0e-05
22
+ logit_mean: 0.0
23
+ logit_std: 1.0
24
+ lr_scheduler: constant
25
+ lr_warmup_steps: 0
26
+ max_grad_norm: 1.0
27
+ max_train_steps: 3500
28
+ mixed_precision: bf16
29
+ mode_scale: 1.29
30
+ model_name_or_path: memoavatar/memo
31
+ noise_scheduler_kwargs:
32
+ num_train_timesteps: 1000
33
+ num_train_epochs: 700
34
+ output_dir: outputs/finetune
35
+ prefetch_factor: 4
36
+ resume_from_checkpoint: null
37
+ robust_training: true
38
+ scale_lr: false
39
+ seed: 42
40
+ start_ratio: 0.05
41
+ tracker_project_name: memo
42
+ train_audio_proj: false
43
+ train_batch_size: 2
44
+ train_diffusion_net: true
45
+ train_image_proj: false
46
+ train_reference_net: false
47
+ trainable_modules:
48
+ - motion_modules
49
+ - audio_modules
50
+ uncond_audio_ratio: 0.05
51
+ uncond_img_ratio: 0.05
52
+ use_8bit_adam: true
53
+ use_ema: false
54
+ vae: stabilityai/sd-vae-ft-mse
55
+ weighting_scheme: logit_normal
logs/memo/1741060237.980531/events.out.tfevents.1741060237.369f90aa7051 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:657998ec24fdf97d92613eada6193a47b49c50fb100f3b8f5ec4e25dd8d11249
3
+ size 2329
logs/memo/1741060237.983221/hparams.yml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ adam_beta1: 0.9
2
+ adam_beta2: 0.999
3
+ adam_epsilon: 1.0e-08
4
+ adam_weight_decay: 0.01
5
+ allow_tf32: true
6
+ checkpointing_steps: 5000
7
+ checkpoints_total_limit: 20
8
+ data:
9
+ audio_margin: 2
10
+ dynamic_past_frames: false
11
+ height: 512
12
+ metadata_paths:
13
+ - data/embedding/metadata.jsonl
14
+ n_sample_frames: 16
15
+ num_past_frames: 16
16
+ width: 512
17
+ dataloader_num_workers: 16
18
+ enable_xformers_memory_efficient_attention: true
19
+ gradient_accumulation_steps: 1
20
+ gradient_checkpointing: true
21
+ learning_rate: 1.0e-05
22
+ logit_mean: 0.0
23
+ logit_std: 1.0
24
+ lr_scheduler: constant
25
+ lr_warmup_steps: 0
26
+ max_grad_norm: 1.0
27
+ max_train_steps: 3500
28
+ mixed_precision: bf16
29
+ mode_scale: 1.29
30
+ model_name_or_path: memoavatar/memo
31
+ noise_scheduler_kwargs:
32
+ num_train_timesteps: 1000
33
+ num_train_epochs: 700
34
+ output_dir: outputs/finetune
35
+ prefetch_factor: 4
36
+ resume_from_checkpoint: null
37
+ robust_training: true
38
+ scale_lr: false
39
+ seed: 42
40
+ start_ratio: 0.05
41
+ tracker_project_name: memo
42
+ train_audio_proj: false
43
+ train_batch_size: 2
44
+ train_diffusion_net: true
45
+ train_image_proj: false
46
+ train_reference_net: false
47
+ trainable_modules:
48
+ - motion_modules
49
+ - audio_modules
50
+ uncond_audio_ratio: 0.05
51
+ uncond_img_ratio: 0.05
52
+ use_8bit_adam: true
53
+ use_ema: false
54
+ vae: stabilityai/sd-vae-ft-mse
55
+ weighting_scheme: logit_normal
logs/memo/1741060310.7917793/events.out.tfevents.1741060310.369f90aa7051 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7ab603a92f4ae6f27f318a54ebff660a0c9e707799e389f65f2440b9eb176a2
3
+ size 2329
logs/memo/1741060310.7946274/hparams.yml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ adam_beta1: 0.9
2
+ adam_beta2: 0.999
3
+ adam_epsilon: 1.0e-08
4
+ adam_weight_decay: 0.01
5
+ allow_tf32: true
6
+ checkpointing_steps: 5000
7
+ checkpoints_total_limit: 20
8
+ data:
9
+ audio_margin: 2
10
+ dynamic_past_frames: false
11
+ height: 512
12
+ metadata_paths:
13
+ - data/embedding/metadata.jsonl
14
+ n_sample_frames: 16
15
+ num_past_frames: 16
16
+ width: 512
17
+ dataloader_num_workers: 16
18
+ enable_xformers_memory_efficient_attention: true
19
+ gradient_accumulation_steps: 1
20
+ gradient_checkpointing: true
21
+ learning_rate: 1.0e-05
22
+ logit_mean: 0.0
23
+ logit_std: 1.0
24
+ lr_scheduler: constant
25
+ lr_warmup_steps: 0
26
+ max_grad_norm: 1.0
27
+ max_train_steps: 3500
28
+ mixed_precision: bf16
29
+ mode_scale: 1.29
30
+ model_name_or_path: memoavatar/memo
31
+ noise_scheduler_kwargs:
32
+ num_train_timesteps: 1000
33
+ num_train_epochs: 875
34
+ output_dir: outputs/finetune
35
+ prefetch_factor: 4
36
+ resume_from_checkpoint: null
37
+ robust_training: true
38
+ scale_lr: false
39
+ seed: 42
40
+ start_ratio: 0.05
41
+ tracker_project_name: memo
42
+ train_audio_proj: false
43
+ train_batch_size: 3
44
+ train_diffusion_net: true
45
+ train_image_proj: false
46
+ train_reference_net: false
47
+ trainable_modules:
48
+ - motion_modules
49
+ - audio_modules
50
+ uncond_audio_ratio: 0.05
51
+ uncond_img_ratio: 0.05
52
+ use_8bit_adam: true
53
+ use_ema: false
54
+ vae: stabilityai/sd-vae-ft-mse
55
+ weighting_scheme: logit_normal
logs/memo/1741060338.6906004/events.out.tfevents.1741060338.369f90aa7051 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5e705e83cd16356a16b87991f60c4e05562b9765a3ef319ff245e211feaf078
3
+ size 2329
logs/memo/1741060338.6932552/hparams.yml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ adam_beta1: 0.9
2
+ adam_beta2: 0.999
3
+ adam_epsilon: 1.0e-08
4
+ adam_weight_decay: 0.01
5
+ allow_tf32: true
6
+ checkpointing_steps: 5000
7
+ checkpoints_total_limit: 20
8
+ data:
9
+ audio_margin: 2
10
+ dynamic_past_frames: false
11
+ height: 512
12
+ metadata_paths:
13
+ - data/embedding/metadata.jsonl
14
+ n_sample_frames: 16
15
+ num_past_frames: 16
16
+ width: 512
17
+ dataloader_num_workers: 16
18
+ enable_xformers_memory_efficient_attention: true
19
+ gradient_accumulation_steps: 1
20
+ gradient_checkpointing: true
21
+ learning_rate: 1.0e-05
22
+ logit_mean: 0.0
23
+ logit_std: 1.0
24
+ lr_scheduler: constant
25
+ lr_warmup_steps: 0
26
+ max_grad_norm: 1.0
27
+ max_train_steps: 3500
28
+ mixed_precision: bf16
29
+ mode_scale: 1.29
30
+ model_name_or_path: memoavatar/memo
31
+ noise_scheduler_kwargs:
32
+ num_train_timesteps: 1000
33
+ num_train_epochs: 350
34
+ output_dir: outputs/finetune
35
+ prefetch_factor: 4
36
+ resume_from_checkpoint: null
37
+ robust_training: true
38
+ scale_lr: false
39
+ seed: 42
40
+ start_ratio: 0.05
41
+ tracker_project_name: memo
42
+ train_audio_proj: false
43
+ train_batch_size: 1
44
+ train_diffusion_net: true
45
+ train_image_proj: false
46
+ train_reference_net: false
47
+ trainable_modules:
48
+ - motion_modules
49
+ - audio_modules
50
+ uncond_audio_ratio: 0.05
51
+ uncond_img_ratio: 0.05
52
+ use_8bit_adam: true
53
+ use_ema: false
54
+ vae: stabilityai/sd-vae-ft-mse
55
+ weighting_scheme: logit_normal
logs/memo/1741145698.900579/events.out.tfevents.1741145698.16e2a27e51cb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ce7a3b494bd3470fb3b8c6543ac40d7612c5c71a2c1290c5b5fd711fd79ec93
3
+ size 2329