Hugging Face's logo

Trelis
/
avatar-2epochs

Diffusers
Safetensors
Model card Files Community
rs545837 commited on
Commit
9ffff46
·
verified ·
1 Parent(s): b96b769

Upload folder using huggingface_hub

Browse files
Files changed (13) hide show
  1. .ipynb_checkpoints/config-checkpoint.yaml +54 -0
  2. checkpoint-1500/audio_proj/config.json +5 -0
  3. checkpoint-1500/audio_proj/diffusion_pytorch_model.safetensors +3 -0
  4. checkpoint-1500/diffusion_net/config.json +88 -0
  5. checkpoint-1500/diffusion_net/diffusion_pytorch_model.safetensors +3 -0
  6. checkpoint-1500/image_proj/config.json +5 -0
  7. checkpoint-1500/image_proj/diffusion_pytorch_model.safetensors +3 -0
  8. checkpoint-1500/optimizer.bin +3 -0
  9. checkpoint-1500/random_states_0.pkl +3 -0
  10. checkpoint-1500/reference_net/config.json +66 -0
  11. checkpoint-1500/reference_net/diffusion_pytorch_model.safetensors +3 -0
  12. checkpoint-1500/scheduler.bin +3 -0
  13. config.yaml +54 -0
.ipynb_checkpoints/config-checkpoint.yaml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tracker_project_name: memo
2
+ output_dir: outputs/finetune
3
+ resume_from_checkpoint: null
4
+ model_name_or_path: memoavatar/memo
5
+ vae: stabilityai/sd-vae-ft-mse
6
+ gradient_checkpointing: true
7
+ gradient_accumulation_steps: 1
8
+ train_batch_size: 1
9
+ max_train_steps: 1500
10
+ enable_xformers_memory_efficient_attention: true
11
+ checkpoints_total_limit: 20
12
+ robust_training: true
13
+ learning_rate: 1.0e-05
14
+ max_grad_norm: 1.0
15
+ scale_lr: false
16
+ lr_scheduler: constant
17
+ lr_warmup_steps: 0
18
+ seed: 42
19
+ mixed_precision: bf16
20
+ use_8bit_adam: false
21
+ allow_tf32: true
22
+ use_ema: false
23
+ adam_beta1: 0.9
24
+ adam_beta2: 0.999
25
+ adam_weight_decay: 0.01
26
+ adam_epsilon: 1.0e-08
27
+ dataloader_num_workers: 16
28
+ prefetch_factor: 4
29
+ checkpointing_steps: 5000
30
+ data:
31
+ width: 512
32
+ height: 512
33
+ num_past_frames: 16
34
+ dynamic_past_frames: false
35
+ n_sample_frames: 16
36
+ audio_margin: 2
37
+ metadata_paths:
38
+ - data/embedding/metadata.jsonl
39
+ weighting_scheme: logit_normal
40
+ logit_mean: 0.0
41
+ logit_std: 1.0
42
+ mode_scale: 1.29
43
+ noise_scheduler_kwargs:
44
+ num_train_timesteps: 1000
45
+ train_reference_net: false
46
+ train_diffusion_net: true
47
+ train_image_proj: false
48
+ train_audio_proj: false
49
+ trainable_modules:
50
+ - motion_modules
51
+ - audio_modules
52
+ uncond_img_ratio: 0.05
53
+ uncond_audio_ratio: 0.05
54
+ start_ratio: 0.05
checkpoint-1500/audio_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AudioProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-1500/audio_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc1c7d4583571f4648f5bf82b70246671d4576837154be38436327f3e8c0e8cc
3
+ size 72930976
checkpoint-1500/diffusion_net/config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet3DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "audio_attention_dim": 768,
14
+ "block_out_channels": [
15
+ 320,
16
+ 640,
17
+ 1280,
18
+ 1280
19
+ ],
20
+ "center_input_sample": false,
21
+ "class_embed_type": null,
22
+ "class_embeddings_concat": false,
23
+ "conv_in_kernel": 3,
24
+ "cross_attention_dim": 768,
25
+ "down_block_types": [
26
+ "CrossAttnDownBlock3D",
27
+ "CrossAttnDownBlock3D",
28
+ "CrossAttnDownBlock3D",
29
+ "DownBlock3D"
30
+ ],
31
+ "downsample_padding": 1,
32
+ "dropout": 0.0,
33
+ "dual_cross_attention": false,
34
+ "emo_drop_rate": 0.05,
35
+ "encoder_hid_dim": null,
36
+ "encoder_hid_dim_type": null,
37
+ "flip_sin_to_cos": true,
38
+ "freq_shift": 0,
39
+ "in_channels": 4,
40
+ "layers_per_block": 2,
41
+ "mid_block_only_cross_attention": null,
42
+ "mid_block_scale_factor": 1,
43
+ "mid_block_type": "UNetMidBlock3DCrossAttn",
44
+ "motion_module_kwargs": {
45
+ "attention_block_types": [
46
+ "Temporal_Self",
47
+ "Temporal_Self"
48
+ ],
49
+ "num_attention_heads": 8,
50
+ "num_transformer_block": 1,
51
+ "temporal_attention_dim_div": 1,
52
+ "temporal_position_encoding": true,
53
+ "temporal_position_encoding_max_len": 32
54
+ },
55
+ "motion_module_resolutions": [
56
+ 1,
57
+ 2,
58
+ 4,
59
+ 8
60
+ ],
61
+ "norm_eps": 1e-05,
62
+ "norm_num_groups": 32,
63
+ "num_attention_heads": null,
64
+ "num_class_embeds": null,
65
+ "only_cross_attention": false,
66
+ "out_channels": 4,
67
+ "projection_class_embeddings_input_dim": null,
68
+ "resnet_time_scale_shift": "default",
69
+ "reverse_transformer_layers_per_block": null,
70
+ "sample_size": 64,
71
+ "time_cond_proj_dim": null,
72
+ "time_embedding_act_fn": null,
73
+ "time_embedding_dim": null,
74
+ "time_embedding_type": "positional",
75
+ "timestep_post_act": null,
76
+ "transformer_layers_per_block": 1,
77
+ "unet_use_cross_frame_attention": false,
78
+ "unet_use_temporal_attention": false,
79
+ "up_block_types": [
80
+ "UpBlock3D",
81
+ "CrossAttnUpBlock3D",
82
+ "CrossAttnUpBlock3D",
83
+ "CrossAttnUpBlock3D"
84
+ ],
85
+ "upcast_attention": false,
86
+ "use_inflated_groupnorm": true,
87
+ "use_linear_projection": false
88
+ }
checkpoint-1500/diffusion_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2711322f4317dd996274cac8f555b315b781c58405b4ce9e70ce64d501838f4a
3
+ size 3356369240
checkpoint-1500/image_proj/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "ImageProjModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "_name_or_path": "memoavatar/memo"
5
+ }
checkpoint-1500/image_proj/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9855dabf683d8c17b265567737bc90e5a0b032db7551b78f028cd9b03439abe5
3
+ size 3155280
checkpoint-1500/optimizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b247356a5bec7597bcf105176559d94fa25a02aa24f2bcab6b9bcb00397eab0
3
+ size 3270385552
checkpoint-1500/random_states_0.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f05c5624b7fde6d9291790f7bea93920bbaa08d64c02d2066696ce0990ec07b
3
+ size 14344
checkpoint-1500/reference_net/config.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_center_input_sample": false,
3
+ "_class_name": "UNet2DConditionModel",
4
+ "_diffusers_version": "0.31.0",
5
+ "_name_or_path": "memoavatar/memo",
6
+ "_out_channels": 4,
7
+ "act_fn": "silu",
8
+ "addition_embed_type": null,
9
+ "addition_embed_type_num_heads": 64,
10
+ "addition_time_embed_dim": null,
11
+ "attention_head_dim": 8,
12
+ "attention_type": "default",
13
+ "block_out_channels": [
14
+ 320,
15
+ 640,
16
+ 1280,
17
+ 1280
18
+ ],
19
+ "center_input_sample": false,
20
+ "class_embed_type": null,
21
+ "class_embeddings_concat": false,
22
+ "conv_in_kernel": 3,
23
+ "cross_attention_dim": 768,
24
+ "down_block_types": [
25
+ "CrossAttnDownBlock2D",
26
+ "CrossAttnDownBlock2D",
27
+ "CrossAttnDownBlock2D",
28
+ "DownBlock2D"
29
+ ],
30
+ "downsample_padding": 1,
31
+ "dropout": 0.0,
32
+ "dual_cross_attention": false,
33
+ "encoder_hid_dim": null,
34
+ "encoder_hid_dim_type": null,
35
+ "flip_sin_to_cos": true,
36
+ "freq_shift": 0,
37
+ "in_channels": 4,
38
+ "layers_per_block": 2,
39
+ "mid_block_only_cross_attention": null,
40
+ "mid_block_scale_factor": 1,
41
+ "mid_block_type": "UNetMidBlock2DCrossAttn",
42
+ "norm_eps": 1e-05,
43
+ "norm_num_groups": 32,
44
+ "num_attention_heads": null,
45
+ "num_class_embeds": null,
46
+ "only_cross_attention": false,
47
+ "out_channels": 4,
48
+ "projection_class_embeddings_input_dim": null,
49
+ "resnet_time_scale_shift": "default",
50
+ "reverse_transformer_layers_per_block": null,
51
+ "sample_size": 64,
52
+ "time_cond_proj_dim": null,
53
+ "time_embedding_act_fn": null,
54
+ "time_embedding_dim": null,
55
+ "time_embedding_type": "positional",
56
+ "timestep_post_act": null,
57
+ "transformer_layers_per_block": 1,
58
+ "up_block_types": [
59
+ "UpBlock2D",
60
+ "CrossAttnUpBlock2D",
61
+ "CrossAttnUpBlock2D",
62
+ "CrossAttnUpBlock2D"
63
+ ],
64
+ "upcast_attention": false,
65
+ "use_linear_projection": false
66
+ }
checkpoint-1500/reference_net/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0550a76a5df4c295610040f90075b3bbf64f7d6be58cfc93a9722be1377b2486
3
+ size 1714214152
checkpoint-1500/scheduler.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcdd883061f8f3514731c51041dc7231d10951713a9abbd57d4f968a9c567ce3
3
+ size 1000
config.yaml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tracker_project_name: memo
2
+ output_dir: outputs/finetune
3
+ resume_from_checkpoint: null
4
+ model_name_or_path: memoavatar/memo
5
+ vae: stabilityai/sd-vae-ft-mse
6
+ gradient_checkpointing: true
7
+ gradient_accumulation_steps: 1
8
+ train_batch_size: 1
9
+ max_train_steps: 1500
10
+ enable_xformers_memory_efficient_attention: true
11
+ checkpoints_total_limit: 20
12
+ robust_training: true
13
+ learning_rate: 1.0e-05
14
+ max_grad_norm: 1.0
15
+ scale_lr: false
16
+ lr_scheduler: constant
17
+ lr_warmup_steps: 0
18
+ seed: 42
19
+ mixed_precision: bf16
20
+ use_8bit_adam: false
21
+ allow_tf32: true
22
+ use_ema: false
23
+ adam_beta1: 0.9
24
+ adam_beta2: 0.999
25
+ adam_weight_decay: 0.01
26
+ adam_epsilon: 1.0e-08
27
+ dataloader_num_workers: 16
28
+ prefetch_factor: 4
29
+ checkpointing_steps: 5000
30
+ data:
31
+ width: 512
32
+ height: 512
33
+ num_past_frames: 16
34
+ dynamic_past_frames: false
35
+ n_sample_frames: 16
36
+ audio_margin: 2
37
+ metadata_paths:
38
+ - data/embedding/metadata.jsonl
39
+ weighting_scheme: logit_normal
40
+ logit_mean: 0.0
41
+ logit_std: 1.0
42
+ mode_scale: 1.29
43
+ noise_scheduler_kwargs:
44
+ num_train_timesteps: 1000
45
+ train_reference_net: false
46
+ train_diffusion_net: true
47
+ train_image_proj: false
48
+ train_audio_proj: false
49
+ trainable_modules:
50
+ - motion_modules
51
+ - audio_modules
52
+ uncond_img_ratio: 0.05
53
+ uncond_audio_ratio: 0.05
54
+ start_ratio: 0.05