Hugging Face's logo

htlou
/
backup_0202_sft_tf_cham_1015_5ep_t2i_22.5k-outputs

PyTorch
chameleon
Model card Files Community
htlou commited on
Commit
d13b334
·
verified ·
1 Parent(s): d927116

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +9 -0
  2. arguments.yaml +49 -0
  3. config.json +0 -0
  4. environ.txt +55 -0
  5. preprocessor_config.json +28 -0
  6. processor_config.json +5 -0
  7. pytorch_model.bin +3 -0
  8. script.sh +80 -0
  9. slice_1200/arguments.yaml +49 -0
  10. slice_1200/config.json +0 -0
  11. slice_1200/environ.txt +55 -0
  12. slice_1200/preprocessor_config.json +28 -0
  13. slice_1200/processor_config.json +5 -0
  14. slice_1200/pytorch_model.bin +3 -0
  15. slice_1200/script.sh +80 -0
  16. slice_1200/special_tokens_map.json +37 -0
  17. slice_1200/tokenizer.json +0 -0
  18. slice_1200/tokenizer_config.json +0 -0
  19. slice_1200/wandb/debug-internal.log +20 -0
  20. slice_1200/wandb/debug.log +33 -0
  21. slice_1200/wandb/run-20241112_083557-komthdmh/files/config.yaml +96 -0
  22. slice_1200/wandb/run-20241112_083557-komthdmh/files/output.log +506 -0
  23. slice_1200/wandb/run-20241112_083557-komthdmh/files/requirements.txt +233 -0
  24. slice_1200/wandb/run-20241112_083557-komthdmh/files/wandb-metadata.json +112 -0
  25. slice_1200/wandb/run-20241112_083557-komthdmh/files/wandb-summary.json +1 -0
  26. slice_1200/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log +20 -0
  27. slice_1200/wandb/run-20241112_083557-komthdmh/logs/debug.log +33 -0
  28. slice_1200/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb +3 -0
  29. slice_1600/arguments.yaml +49 -0
  30. slice_1600/config.json +0 -0
  31. slice_1600/environ.txt +55 -0
  32. slice_1600/preprocessor_config.json +28 -0
  33. slice_1600/processor_config.json +5 -0
  34. slice_1600/pytorch_model.bin +3 -0
  35. slice_1600/script.sh +80 -0
  36. slice_1600/special_tokens_map.json +37 -0
  37. slice_1600/tokenizer.json +0 -0
  38. slice_1600/tokenizer_config.json +0 -0
  39. slice_1600/wandb/debug-internal.log +20 -0
  40. slice_1600/wandb/debug.log +33 -0
  41. slice_1600/wandb/run-20241112_083557-komthdmh/files/config.yaml +96 -0
  42. slice_1600/wandb/run-20241112_083557-komthdmh/files/output.log +506 -0
  43. slice_1600/wandb/run-20241112_083557-komthdmh/files/requirements.txt +233 -0
  44. slice_1600/wandb/run-20241112_083557-komthdmh/files/wandb-metadata.json +112 -0
  45. slice_1600/wandb/run-20241112_083557-komthdmh/files/wandb-summary.json +1 -0
  46. slice_1600/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log +20 -0
  47. slice_1600/wandb/run-20241112_083557-komthdmh/logs/debug.log +33 -0
  48. slice_1600/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb +3 -0
  49. slice_2000/arguments.yaml +49 -0
  50. slice_2000/config.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ slice_1200/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
37
+ slice_1600/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
38
+ slice_2000/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
39
+ slice_2400/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
40
+ slice_2800/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
41
+ slice_3200/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
42
+ slice_400/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
43
+ slice_800/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
44
+ wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb filter=lfs diff=lfs merge=lfs -text
arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_t2i_22.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 5.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 5.0e-05
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
config.json ADDED
The diff for this file is too large to render. See raw diff
 
environ.txt ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CONDA_DEFAULT_ENV=hantao_cham
2
+ CONDA_EXE=/home/align-anything/miniconda3/bin/conda
3
+ CONDA_PREFIX=/home/align-anything/miniconda3/envs/hantao_cham
4
+ CONDA_PREFIX_1=/home/align-anything/miniconda3
5
+ CONDA_PROMPT_MODIFIER=(hantao_cham)
6
+ CONDA_PYTHON_EXE=/home/align-anything/miniconda3/bin/python
7
+ CONDA_SHLVL=2
8
+ CROSS_RANK=0
9
+ CROSS_SIZE=1
10
+ CUDA_MODULE_LOADING=LAZY
11
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
12
+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
13
+ HOME=/home/align-anything
14
+ LANG=en_US.UTF-8
15
+ LD_LIBRARY_PATH=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/../../lib64:
16
+ LESSCLOSE=/usr/bin/lesspipe %s %s
17
+ LESSOPEN=| /usr/bin/lesspipe %s
18
+ LOCAL_RANK=0
19
+ LOCAL_SIZE=8
20
+ LOGLEVEL=WARNING
21
+ LOGNAME=align-anything
22
+ LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
23
+ MASTER_ADDR=127.0.0.1
24
+ MASTER_PORT=61417
25
+ MOTD_SHOWN=pam
26
+ OLDPWD=/home/align-anything
27
+ PATH=/home/align-anything/miniconda3/envs/hantao_cham/bin:/home/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
28
+ PWD=/data/align-anything/hantao/align-anything/scripts
29
+ PYGAME_HIDE_SUPPORT_PROMPT=1
30
+ PYTHONHASHSEED=42
31
+ PYTHONPATH=/data/align-anything/hantao/align-anything
32
+ QT_QPA_FONTDIR=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/fonts
33
+ QT_QPA_PLATFORM_PLUGIN_PATH=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/plugins
34
+ RANK=0
35
+ SHELL=/bin/bash
36
+ SHLVL=3
37
+ SSH_CLIENT=111.205.230.212 2817 30600
38
+ SSH_CONNECTION=111.205.230.212 9549 10.10.212.196 30600
39
+ SSH_TTY=/dev/pts/2
40
+ TERM=screen
41
+ TMUX=/tmp//tmux-2000/default,3379,12
42
+ TMUX_PANE=%12
43
+ TRITON_CACHE_DIR=/home/align-anything/cache/triton
44
+ USER=align-anything
45
+ WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
46
+ WANDB_SERVICE=2-1335408-tcp-localhost-40327
47
+ WORLD_SIZE=8
48
+ XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
49
+ XDG_RUNTIME_DIR=/run/user/2000
50
+ XDG_SESSION_CLASS=user
51
+ XDG_SESSION_ID=4
52
+ XDG_SESSION_TYPE=tty
53
+ _=/home/align-anything/miniconda3/envs/hantao_cham/bin/deepspeed
54
+ _CE_CONDA=
55
+ _CE_M=
preprocessor_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 512,
4
+ "width": 512
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 1.0,
13
+ 1.0,
14
+ 1.0
15
+ ],
16
+ "image_processor_type": "ChameleonImageProcessor",
17
+ "image_std": [
18
+ 1.0,
19
+ 1.0,
20
+ 1.0
21
+ ],
22
+ "processor_class": "ChameleonProcessor",
23
+ "resample": 1,
24
+ "rescale_factor": 0.0078,
25
+ "size": {
26
+ "shortest_edge": 512
27
+ }
28
+ }
processor_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "image_seq_length": 1024,
3
+ "image_token": "<image>",
4
+ "processor_class": "ChameleonProcessor"
5
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff9264dd8327f1bea4103851a4cdab7c7641ef06ee520837cde56795b89ebfe9
3
+ size 14165009930
script.sh ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Copyright 2024 PKU-Alignment Team. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ # ==============================================================================
17
+
18
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
19
+ # You can replace it with a local model path
20
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0917_cham_ti2t_sft"
21
+ # You can replace it with a local dataset path
22
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
23
+ TRAIN_DATA_FILES="llf_t2i_22.5k_tokenized.pt"
24
+ # You can replace it with a new path
25
+ OUTPUT_DIR="../outputs/sft_tf_cham_1015_5ep_t2i_22.5k"
26
+ # For wandb online logging
27
+
28
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
29
+ # Source the setup script
30
+ source ./setup.sh
31
+
32
+ # sleep 80m
33
+ # Execute deepspeed command
34
+ deepspeed \
35
+ --master_port ${MASTER_PORT} \
36
+ --module align_anything.trainers.text_image_to_text_image.sft \
37
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
38
+ --train_datasets ${TRAIN_DATASETS} \
39
+ --train_data_files ${TRAIN_DATA_FILES} \
40
+ --output_dir ${OUTPUT_DIR} \
41
+ --train_template AA_textfeedback \
42
+ --train_split train \
43
+ --per_device_train_batch_size 4 \
44
+ --per_device_eval_batch_size 4 \
45
+ --gradient_accumulation_steps 2 \
46
+ --save_interval 400 \
47
+ --learning_rate 5e-5 \
48
+ --epochs 5 \
49
+ --lr_scheduler_type cosine
50
+
51
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
52
+ # You can replace it with a local model path
53
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
54
+ # You can replace it with a local dataset path
55
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
56
+ TRAIN_DATA_FILES="llf_ti2ti_20k_tokenized.pt"
57
+ # You can replace it with a new path
58
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_20k_ti2ti"
59
+ # For wandb online logging
60
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
61
+ # Source the setup script
62
+ source ./setup.sh
63
+
64
+ # Execute deepspeed command
65
+ deepspeed \
66
+ --master_port ${MASTER_PORT} \
67
+ --module align_anything.trainers.text_image_to_text_image.sft \
68
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
69
+ --train_datasets ${TRAIN_DATASETS} \
70
+ --train_data_files ${TRAIN_DATA_FILES} \
71
+ --output_dir ${OUTPUT_DIR} \
72
+ --train_template AA_textfeedback \
73
+ --train_split train \
74
+ --per_device_train_batch_size 4 \
75
+ --per_device_eval_batch_size 4 \
76
+ --gradient_accumulation_steps 2 \
77
+ --save_interval 400 \
78
+ --learning_rate 1e-6 \
79
+ --epochs 3 \
80
+ --lr_scheduler_type cosine
slice_1200/arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_t2i_22.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 5.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 5.0e-05
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
slice_1200/config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1200/environ.txt ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CONDA_DEFAULT_ENV=hantao_cham
2
+ CONDA_EXE=/home/align-anything/miniconda3/bin/conda
3
+ CONDA_PREFIX=/home/align-anything/miniconda3/envs/hantao_cham
4
+ CONDA_PREFIX_1=/home/align-anything/miniconda3
5
+ CONDA_PROMPT_MODIFIER=(hantao_cham)
6
+ CONDA_PYTHON_EXE=/home/align-anything/miniconda3/bin/python
7
+ CONDA_SHLVL=2
8
+ CROSS_RANK=0
9
+ CROSS_SIZE=1
10
+ CUDA_MODULE_LOADING=LAZY
11
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
12
+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
13
+ HOME=/home/align-anything
14
+ LANG=en_US.UTF-8
15
+ LD_LIBRARY_PATH=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/../../lib64:
16
+ LESSCLOSE=/usr/bin/lesspipe %s %s
17
+ LESSOPEN=| /usr/bin/lesspipe %s
18
+ LOCAL_RANK=0
19
+ LOCAL_SIZE=8
20
+ LOGLEVEL=WARNING
21
+ LOGNAME=align-anything
22
+ LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
23
+ MASTER_ADDR=127.0.0.1
24
+ MASTER_PORT=61417
25
+ MOTD_SHOWN=pam
26
+ OLDPWD=/home/align-anything
27
+ PATH=/home/align-anything/miniconda3/envs/hantao_cham/bin:/home/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
28
+ PWD=/data/align-anything/hantao/align-anything/scripts
29
+ PYGAME_HIDE_SUPPORT_PROMPT=1
30
+ PYTHONHASHSEED=42
31
+ PYTHONPATH=/data/align-anything/hantao/align-anything
32
+ QT_QPA_FONTDIR=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/fonts
33
+ QT_QPA_PLATFORM_PLUGIN_PATH=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/plugins
34
+ RANK=0
35
+ SHELL=/bin/bash
36
+ SHLVL=3
37
+ SSH_CLIENT=111.205.230.212 2817 30600
38
+ SSH_CONNECTION=111.205.230.212 9549 10.10.212.196 30600
39
+ SSH_TTY=/dev/pts/2
40
+ TERM=screen
41
+ TMUX=/tmp//tmux-2000/default,3379,12
42
+ TMUX_PANE=%12
43
+ TRITON_CACHE_DIR=/home/align-anything/cache/triton
44
+ USER=align-anything
45
+ WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
46
+ WANDB_SERVICE=2-1335408-tcp-localhost-40327
47
+ WORLD_SIZE=8
48
+ XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
49
+ XDG_RUNTIME_DIR=/run/user/2000
50
+ XDG_SESSION_CLASS=user
51
+ XDG_SESSION_ID=4
52
+ XDG_SESSION_TYPE=tty
53
+ _=/home/align-anything/miniconda3/envs/hantao_cham/bin/deepspeed
54
+ _CE_CONDA=
55
+ _CE_M=
slice_1200/preprocessor_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 512,
4
+ "width": 512
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 1.0,
13
+ 1.0,
14
+ 1.0
15
+ ],
16
+ "image_processor_type": "ChameleonImageProcessor",
17
+ "image_std": [
18
+ 1.0,
19
+ 1.0,
20
+ 1.0
21
+ ],
22
+ "processor_class": "ChameleonProcessor",
23
+ "resample": 1,
24
+ "rescale_factor": 0.0078,
25
+ "size": {
26
+ "shortest_edge": 512
27
+ }
28
+ }
slice_1200/processor_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "image_seq_length": 1024,
3
+ "image_token": "<image>",
4
+ "processor_class": "ChameleonProcessor"
5
+ }
slice_1200/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25098f6f1594c54988a8a2385548acd726905a9fa1fce07aed30b3fd3bddf91f
3
+ size 14165013510
slice_1200/script.sh ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Copyright 2024 PKU-Alignment Team. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ # ==============================================================================
17
+
18
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
19
+ # You can replace it with a local model path
20
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0917_cham_ti2t_sft"
21
+ # You can replace it with a local dataset path
22
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
23
+ TRAIN_DATA_FILES="llf_t2i_22.5k_tokenized.pt"
24
+ # You can replace it with a new path
25
+ OUTPUT_DIR="../outputs/sft_tf_cham_1015_5ep_t2i_22.5k"
26
+ # For wandb online logging
27
+
28
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
29
+ # Source the setup script
30
+ source ./setup.sh
31
+
32
+ # sleep 80m
33
+ # Execute deepspeed command
34
+ deepspeed \
35
+ --master_port ${MASTER_PORT} \
36
+ --module align_anything.trainers.text_image_to_text_image.sft \
37
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
38
+ --train_datasets ${TRAIN_DATASETS} \
39
+ --train_data_files ${TRAIN_DATA_FILES} \
40
+ --output_dir ${OUTPUT_DIR} \
41
+ --train_template AA_textfeedback \
42
+ --train_split train \
43
+ --per_device_train_batch_size 4 \
44
+ --per_device_eval_batch_size 4 \
45
+ --gradient_accumulation_steps 2 \
46
+ --save_interval 400 \
47
+ --learning_rate 5e-5 \
48
+ --epochs 5 \
49
+ --lr_scheduler_type cosine
50
+
51
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
52
+ # You can replace it with a local model path
53
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
54
+ # You can replace it with a local dataset path
55
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
56
+ TRAIN_DATA_FILES="llf_ti2ti_20k_tokenized.pt"
57
+ # You can replace it with a new path
58
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_20k_ti2ti"
59
+ # For wandb online logging
60
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
61
+ # Source the setup script
62
+ source ./setup.sh
63
+
64
+ # Execute deepspeed command
65
+ deepspeed \
66
+ --master_port ${MASTER_PORT} \
67
+ --module align_anything.trainers.text_image_to_text_image.sft \
68
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
69
+ --train_datasets ${TRAIN_DATASETS} \
70
+ --train_data_files ${TRAIN_DATA_FILES} \
71
+ --output_dir ${OUTPUT_DIR} \
72
+ --train_template AA_textfeedback \
73
+ --train_split train \
74
+ --per_device_train_batch_size 4 \
75
+ --per_device_eval_batch_size 4 \
76
+ --gradient_accumulation_steps 2 \
77
+ --save_interval 400 \
78
+ --learning_rate 1e-6 \
79
+ --epochs 3 \
80
+ --lr_scheduler_type cosine
slice_1200/special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<pad>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "<reserved08706>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
slice_1200/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1200/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1200/wandb/debug-internal.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-12T08:35:57.099449248Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-12T08:35:57.099478068Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-core.log"}
3
+ {"time":"2024-11-12T08:35:57.102383132Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-12T08:35:57.129619469Z","level":"INFO","msg":"created new stream","id":"komthdmh"}
5
+ {"time":"2024-11-12T08:35:57.129670069Z","level":"INFO","msg":"stream: started","id":"komthdmh"}
6
+ {"time":"2024-11-12T08:35:57.129709953Z","level":"INFO","msg":"handler: started","stream_id":{"value":"komthdmh"}}
7
+ {"time":"2024-11-12T08:35:57.129693059Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"komthdmh"}}
8
+ {"time":"2024-11-12T08:35:57.129699692Z","level":"INFO","msg":"sender: started","stream_id":{"value":"komthdmh"}}
9
+ {"time":"2024-11-12T08:35:57.832569737Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-12T08:35:57.837905637Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-12T11:16:52.98542237Z","level":"INFO","msg":"Stopping system monitor"}
12
+ {"time":"2024-11-12T11:16:53.007088373Z","level":"INFO","msg":"Stopped system monitor"}
13
+ {"time":"2024-11-12T11:16:53.498790149Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
14
+ {"time":"2024-11-12T11:16:53.49883056Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
15
+ {"time":"2024-11-12T11:16:54.435678748Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
16
+ {"time":"2024-11-12T11:16:55.889045635Z","level":"INFO","msg":"stream: closing","id":"komthdmh"}
17
+ {"time":"2024-11-12T11:16:55.889082565Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"komthdmh"}}
18
+ {"time":"2024-11-12T11:16:55.889144434Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"komthdmh"}}
19
+ {"time":"2024-11-12T11:16:55.889239107Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"komthdmh"}}
20
+ {"time":"2024-11-12T11:16:55.892207197Z","level":"INFO","msg":"stream: closed","id":"komthdmh"}
slice_1200/wandb/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Configure stats pid to 1335408
3
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-12 08:35:57,085 WARNING MainThread:1335408 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug.log
11
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log
12
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 5.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 5e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_t2i_22.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1015_5ep_t2i_22.5k', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0917_cham_ti2t_sft', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():667] starting backend
16
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-12 08:35:57,097 INFO MainThread:1335408 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-12 08:35:57,113 INFO MainThread:1335408 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-12 08:35:57,827 INFO MainThread:1335408 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-12 08:35:57,973 INFO MainThread:1335408 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-12 08:35:57,976 INFO MainThread:1335408 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-12 11:16:52,981 INFO MainThread:1335408 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/komthdmh
28
+ 2024-11-12 11:16:52,983 INFO MainThread:1335408 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-12 11:16:55,871 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-12 11:16:55,873 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-12 11:16:55,884 INFO MainThread:1335408 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_1200/wandb/run-20241112_083557-komthdmh/files/config.yaml ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.18.3
4
+ m: []
5
+ python_version: 3.11.10
6
+ t:
7
+ "1":
8
+ - 1
9
+ - 11
10
+ - 41
11
+ - 49
12
+ - 51
13
+ - 55
14
+ - 71
15
+ - 83
16
+ - 98
17
+ - 105
18
+ "2":
19
+ - 1
20
+ - 11
21
+ - 41
22
+ - 49
23
+ - 51
24
+ - 55
25
+ - 71
26
+ - 83
27
+ - 98
28
+ - 105
29
+ "3":
30
+ - 2
31
+ - 13
32
+ - 16
33
+ - 23
34
+ - 55
35
+ - 61
36
+ "4": 3.11.10
37
+ "5": 0.18.3
38
+ "6": 4.44.0.dev0
39
+ "8":
40
+ - 5
41
+ "12": 0.18.3
42
+ "13": linux-x86_64
43
+ data_cfgs:
44
+ value:
45
+ eval_data_files: null
46
+ eval_datasets: null
47
+ eval_optional_args: []
48
+ eval_size: null
49
+ eval_split: null
50
+ eval_subset: null
51
+ eval_template: null
52
+ train_data_files: llf_t2i_22.5k_tokenized.pt
53
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
54
+ train_optional_args: []
55
+ train_size: null
56
+ train_split: train
57
+ train_subset: null
58
+ train_template: AA_textfeedback
59
+ logger_cfgs:
60
+ value:
61
+ cache_dir: null
62
+ log_project: align-anything
63
+ log_run_name: sft
64
+ log_type: wandb
65
+ output_dir: ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k
66
+ save_interval: 400
67
+ model_cfgs:
68
+ value:
69
+ model_max_length: 4096
70
+ model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
71
+ trust_remote_code: true
72
+ special_tokens:
73
+ value: null
74
+ train_cfgs:
75
+ value:
76
+ adam_betas:
77
+ - 0.9
78
+ - 0.95
79
+ adam_epsilon: 1e-08
80
+ bf16: true
81
+ ds_cfgs: ds_z3_config.json
82
+ epochs: 5
83
+ eval_interval: 1000
84
+ eval_strategy: steps
85
+ fp16: false
86
+ freeze_language_model: false
87
+ gradient_accumulation_steps: 2
88
+ gradient_checkpointing: true
89
+ learning_rate: 5e-05
90
+ lr_scheduler_type: cosine
91
+ lr_warmup_ratio: 0.03
92
+ max_grad_norm: 1
93
+ per_device_eval_batch_size: 4
94
+ per_device_train_batch_size: 4
95
+ seed: 42
96
+ weight_decay: 0
slice_1200/wandb/run-20241112_083557-komthdmh/files/output.log ADDED
@@ -0,0 +1,506 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ***** Running training *****
2
+ Training 1/5.0 epoch: 0%| | 0/3520.0 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
3
+ Training 1/5.0 epoch (loss 1.3799): 9%|█████████████████████▉ | 319/3520.0 [13:44<2:15:00, 2.53s/it]
4
+ [2024-11-12 08:36:54,432] [INFO] [logging.py:96:log_dist] [Rank 0] step=10, skipped=0, lr=[9.615384615384616e-06, 9.615384615384616e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
5
+ [2024-11-12 08:36:54,432] [INFO] [timer.py:264:stop] epoch=0/micro_step=20/global_step=10, RunningAvgSamplesPerSec=14.123909505032028, CurrSamplesPerSec=13.241169083585376, MemAllocated=14.14GB, MaxMemAllocated=20.81GB
6
+ [2024-11-12 08:37:42,564] [INFO] [logging.py:96:log_dist] [Rank 0] step=20, skipped=0, lr=[1.923076923076923e-05, 1.923076923076923e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
7
+ [2024-11-12 08:37:42,565] [INFO] [timer.py:264:stop] epoch=0/micro_step=40/global_step=20, RunningAvgSamplesPerSec=14.240402554293874, CurrSamplesPerSec=15.089060244230348, MemAllocated=14.14GB, MaxMemAllocated=21.06GB
8
+ [2024-11-12 08:38:33,168] [INFO] [logging.py:96:log_dist] [Rank 0] step=30, skipped=0, lr=[2.8846153846153845e-05, 2.8846153846153845e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
9
+ [2024-11-12 08:38:33,169] [INFO] [timer.py:264:stop] epoch=0/micro_step=60/global_step=30, RunningAvgSamplesPerSec=14.022555309940765, CurrSamplesPerSec=10.204952710326344, MemAllocated=14.14GB, MaxMemAllocated=21.06GB
10
+ [2024-11-12 08:39:25,374] [INFO] [logging.py:96:log_dist] [Rank 0] step=40, skipped=0, lr=[3.846153846153846e-05, 3.846153846153846e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
11
+ [2024-11-12 08:39:25,374] [INFO] [timer.py:264:stop] epoch=0/micro_step=80/global_step=40, RunningAvgSamplesPerSec=13.74537082334063, CurrSamplesPerSec=14.73583968737617, MemAllocated=14.14GB, MaxMemAllocated=21.06GB
12
+ [2024-11-12 08:40:15,254] [INFO] [logging.py:96:log_dist] [Rank 0] step=50, skipped=0, lr=[4.8076923076923084e-05, 4.8076923076923084e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
13
+ [2024-11-12 08:40:15,255] [INFO] [timer.py:264:stop] epoch=0/micro_step=100/global_step=50, RunningAvgSamplesPerSec=13.762621202672168, CurrSamplesPerSec=13.81027432650536, MemAllocated=14.14GB, MaxMemAllocated=21.71GB
14
+ [2024-11-12 08:41:04,754] [INFO] [logging.py:96:log_dist] [Rank 0] step=60, skipped=0, lr=[4.9997293511641216e-05, 4.9997293511641216e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
15
+ [2024-11-12 08:41:04,754] [INFO] [timer.py:264:stop] epoch=0/micro_step=120/global_step=60, RunningAvgSamplesPerSec=13.780772643091256, CurrSamplesPerSec=14.821790030364474, MemAllocated=14.14GB, MaxMemAllocated=21.71GB
16
+ [2024-11-12 08:41:56,245] [INFO] [logging.py:96:log_dist] [Rank 0] step=70, skipped=0, lr=[4.9986299407013e-05, 4.9986299407013e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
17
+ [2024-11-12 08:41:56,246] [INFO] [timer.py:264:stop] epoch=0/micro_step=140/global_step=70, RunningAvgSamplesPerSec=13.70794245761943, CurrSamplesPerSec=15.070471709248997, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
18
+ [2024-11-12 08:42:45,692] [INFO] [logging.py:96:log_dist] [Rank 0] step=80, skipped=0, lr=[4.9966852247120764e-05, 4.9966852247120764e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
19
+ [2024-11-12 08:42:45,693] [INFO] [timer.py:264:stop] epoch=0/micro_step=160/global_step=80, RunningAvgSamplesPerSec=13.735055369770809, CurrSamplesPerSec=13.105748516399679, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
20
+ [2024-11-12 08:43:35,397] [INFO] [logging.py:96:log_dist] [Rank 0] step=90, skipped=0, lr=[4.9938958611086705e-05, 4.9938958611086705e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
21
+ [2024-11-12 08:43:35,398] [INFO] [timer.py:264:stop] epoch=0/micro_step=180/global_step=90, RunningAvgSamplesPerSec=13.744955502115493, CurrSamplesPerSec=12.506778047236157, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
22
+ [2024-11-12 08:44:27,939] [INFO] [logging.py:96:log_dist] [Rank 0] step=100, skipped=0, lr=[4.9902627935540205e-05, 4.9902627935540205e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
23
+ [2024-11-12 08:44:27,940] [INFO] [timer.py:264:stop] epoch=0/micro_step=200/global_step=100, RunningAvgSamplesPerSec=13.667787564838962, CurrSamplesPerSec=13.435017581558897, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
24
+ [2024-11-12 08:45:21,475] [INFO] [logging.py:96:log_dist] [Rank 0] step=110, skipped=0, lr=[4.985787251142531e-05, 4.985787251142531e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
25
+ [2024-11-12 08:45:21,476] [INFO] [timer.py:264:stop] epoch=0/micro_step=220/global_step=110, RunningAvgSamplesPerSec=13.568024984812084, CurrSamplesPerSec=11.299057225977563, MemAllocated=14.14GB, MaxMemAllocated=22.23GB
26
+ [2024-11-12 08:46:12,028] [INFO] [logging.py:96:log_dist] [Rank 0] step=120, skipped=0, lr=[4.980470747984265e-05, 4.980470747984265e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
27
+ [2024-11-12 08:46:12,029] [INFO] [timer.py:264:stop] epoch=0/micro_step=240/global_step=120, RunningAvgSamplesPerSec=13.571719220971847, CurrSamplesPerSec=14.505489374647698, MemAllocated=14.14GB, MaxMemAllocated=22.23GB
28
+ [2024-11-12 08:47:06,982] [INFO] [logging.py:96:log_dist] [Rank 0] step=130, skipped=0, lr=[4.974315082692705e-05, 4.974315082692705e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
29
+ [2024-11-12 08:47:06,983] [INFO] [timer.py:264:stop] epoch=0/micro_step=260/global_step=130, RunningAvgSamplesPerSec=13.471915574993211, CurrSamplesPerSec=14.677887159235453, MemAllocated=14.14GB, MaxMemAllocated=22.23GB
30
+ [2024-11-12 08:47:58,851] [INFO] [logging.py:96:log_dist] [Rank 0] step=140, skipped=0, lr=[4.9673223377762715e-05, 4.9673223377762715e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
31
+ [2024-11-12 08:47:58,852] [INFO] [timer.py:264:stop] epoch=0/micro_step=280/global_step=140, RunningAvgSamplesPerSec=13.457893058437664, CurrSamplesPerSec=11.959087221433908, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
32
+ [2024-11-12 08:48:54,301] [INFO] [logging.py:96:log_dist] [Rank 0] step=150, skipped=0, lr=[4.9594948789337914e-05, 4.9594948789337914e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
33
+ [2024-11-12 08:48:54,302] [INFO] [timer.py:264:stop] epoch=0/micro_step=300/global_step=150, RunningAvgSamplesPerSec=13.373347047399044, CurrSamplesPerSec=10.07913456306504, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
34
+ [2024-11-12 08:49:44,210] [INFO] [logging.py:96:log_dist] [Rank 0] step=160, skipped=0, lr=[4.950835354254167e-05, 4.950835354254167e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
35
+ [2024-11-12 08:49:44,210] [INFO] [timer.py:264:stop] epoch=0/micro_step=320/global_step=160, RunningAvgSamplesPerSec=13.393197857147435, CurrSamplesPerSec=14.860385800427661, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
36
+ [2024-11-12 08:50:35,214] [INFO] [logging.py:96:log_dist] [Rank 0] step=170, skipped=0, lr=[4.941346693320509e-05, 4.941346693320509e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
37
+ [2024-11-12 08:50:35,215] [INFO] [timer.py:264:stop] epoch=0/micro_step=340/global_step=170, RunningAvgSamplesPerSec=13.395646345993923, CurrSamplesPerSec=14.323621533076235, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
38
+ [2024-11-12 08:51:28,101] [INFO] [logging.py:96:log_dist] [Rank 0] step=180, skipped=0, lr=[4.931032106219029e-05, 4.931032106219029e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
39
+ [2024-11-12 08:51:28,102] [INFO] [timer.py:264:stop] epoch=0/micro_step=360/global_step=180, RunningAvgSamplesPerSec=13.367218567659057, CurrSamplesPerSec=14.98421830353528, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
40
+ [2024-11-12 08:52:21,270] [INFO] [logging.py:96:log_dist] [Rank 0] step=190, skipped=0, lr=[4.9198950824530485e-05, 4.9198950824530485e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
41
+ [2024-11-12 08:52:21,272] [INFO] [timer.py:264:stop] epoch=0/micro_step=380/global_step=190, RunningAvgSamplesPerSec=13.33740483581657, CurrSamplesPerSec=10.703509865696477, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
42
+ [2024-11-12 08:53:11,495] [INFO] [logging.py:96:log_dist] [Rank 0] step=200, skipped=0, lr=[4.9079393897624745e-05, 4.9079393897624745e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
43
+ [2024-11-12 08:53:11,496] [INFO] [timer.py:264:stop] epoch=0/micro_step=400/global_step=200, RunningAvgSamplesPerSec=13.352653718664833, CurrSamplesPerSec=14.742926067586316, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
44
+ Saving checkpoint at step 400 ...
45
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
46
+ Saving 16-bit model...
47
+ [2024-11-12 08:53:24,811] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
48
+ [2024-11-12 08:53:24,813] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin, tag: global_step200
49
+ [2024-11-12 08:53:24,813] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin...
50
+ [2024-11-12 08:53:43,361] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin.
51
+ [2024-11-12 08:53:43,361] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
52
+ Model saved!
53
+ Saving 16-bit model...
54
+ [2024-11-12 08:53:53,112] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
55
+ [2024-11-12 08:53:53,113] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin, tag: global_step200
56
+ [2024-11-12 08:53:53,113] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin...
57
+ [2024-11-12 08:54:14,787] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin.
58
+ [2024-11-12 08:54:14,789] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
59
+ Model saved!
60
+ Checkpoint saved.
61
+ [2024-11-12 08:55:04,096] [INFO] [logging.py:96:log_dist] [Rank 0] step=210, skipped=0, lr=[4.895169072849142e-05, 4.895169072849142e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
62
+ [2024-11-12 08:55:04,097] [INFO] [timer.py:264:stop] epoch=0/micro_step=420/global_step=210, RunningAvgSamplesPerSec=13.3795627688349, CurrSamplesPerSec=13.546001011776744, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
63
+ [2024-11-12 08:55:57,056] [INFO] [logging.py:96:log_dist] [Rank 0] step=220, skipped=0, lr=[4.881588452008456e-05, 4.881588452008456e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
64
+ [2024-11-12 08:55:57,057] [INFO] [timer.py:264:stop] epoch=0/micro_step=440/global_step=220, RunningAvgSamplesPerSec=13.356234172468818, CurrSamplesPerSec=13.971783082739853, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
65
+ [2024-11-12 08:56:49,663] [INFO] [logging.py:96:log_dist] [Rank 0] step=230, skipped=0, lr=[4.867202121667815e-05, 4.867202121667815e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
66
+ [2024-11-12 08:56:49,664] [INFO] [timer.py:264:stop] epoch=0/micro_step=460/global_step=230, RunningAvgSamplesPerSec=13.346132618764257, CurrSamplesPerSec=14.695504801089417, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
67
+ [2024-11-12 08:57:42,876] [INFO] [logging.py:96:log_dist] [Rank 0] step=240, skipped=0, lr=[4.852014948832268e-05, 4.852014948832268e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
68
+ [2024-11-12 08:57:42,877] [INFO] [timer.py:264:stop] epoch=0/micro_step=480/global_step=240, RunningAvgSamplesPerSec=13.320925710451487, CurrSamplesPerSec=14.158642205669375, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
69
+ [2024-11-12 08:58:35,127] [INFO] [logging.py:96:log_dist] [Rank 0] step=250, skipped=0, lr=[4.836032071437977e-05, 4.836032071437977e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
70
+ [2024-11-12 08:58:35,128] [INFO] [timer.py:264:stop] epoch=0/micro_step=500/global_step=250, RunningAvgSamplesPerSec=13.310236719541589, CurrSamplesPerSec=14.184536914840965, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
71
+ [2024-11-12 08:59:25,936] [INFO] [logging.py:96:log_dist] [Rank 0] step=260, skipped=0, lr=[4.819258896614014e-05, 4.819258896614014e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
72
+ [2024-11-12 08:59:25,936] [INFO] [timer.py:264:stop] epoch=0/micro_step=520/global_step=260, RunningAvgSamplesPerSec=13.314507578988644, CurrSamplesPerSec=13.662068644731564, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
73
+ [2024-11-12 09:00:24,950] [INFO] [logging.py:96:log_dist] [Rank 0] step=270, skipped=0, lr=[4.801701098853083e-05, 4.801701098853083e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
74
+ [2024-11-12 09:00:24,951] [INFO] [timer.py:264:stop] epoch=0/micro_step=540/global_step=270, RunningAvgSamplesPerSec=13.23171381311444, CurrSamplesPerSec=8.591639394767906, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
75
+ [2024-11-12 09:01:15,848] [INFO] [logging.py:96:log_dist] [Rank 0] step=280, skipped=0, lr=[4.783364618091803e-05, 4.783364618091803e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
76
+ [2024-11-12 09:01:15,849] [INFO] [timer.py:264:stop] epoch=0/micro_step=560/global_step=280, RunningAvgSamplesPerSec=13.242062455437987, CurrSamplesPerSec=12.414479790208985, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
77
+ [2024-11-12 09:02:06,734] [INFO] [logging.py:96:log_dist] [Rank 0] step=290, skipped=0, lr=[4.764255657701179e-05, 4.764255657701179e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
78
+ [2024-11-12 09:02:06,735] [INFO] [timer.py:264:stop] epoch=0/micro_step=580/global_step=290, RunningAvgSamplesPerSec=13.247943308956986, CurrSamplesPerSec=10.33724679656727, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
79
+ [2024-11-12 09:02:56,185] [INFO] [logging.py:96:log_dist] [Rank 0] step=300, skipped=0, lr=[4.74438068238795e-05, 4.74438068238795e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
80
+ [2024-11-12 09:02:56,187] [INFO] [timer.py:264:stop] epoch=0/micro_step=600/global_step=300, RunningAvgSamplesPerSec=13.273573937175511, CurrSamplesPerSec=14.875715410485261, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
81
+ [2024-11-12 09:03:46,290] [INFO] [logging.py:96:log_dist] [Rank 0] step=310, skipped=0, lr=[4.723746416007536e-05, 4.723746416007536e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
82
+ [2024-11-12 09:03:46,291] [INFO] [timer.py:264:stop] epoch=0/micro_step=620/global_step=310, RunningAvgSamplesPerSec=13.28483606555138, CurrSamplesPerSec=14.967418276326326, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
83
+ [2024-11-12 09:04:36,313] [INFO] [logging.py:96:log_dist] [Rank 0] step=320, skipped=0, lr=[4.702359839289306e-05, 4.702359839289306e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
84
+ [2024-11-12 09:04:36,314] [INFO] [timer.py:264:stop] epoch=0/micro_step=640/global_step=320, RunningAvgSamplesPerSec=13.29814534561608, CurrSamplesPerSec=13.334811696194658, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
85
+ [2024-11-12 09:05:28,541] [INFO] [logging.py:96:log_dist] [Rank 0] step=330, skipped=0, lr=[4.680228187474939e-05, 4.680228187474939e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
86
+ [2024-11-12 09:05:28,542] [INFO] [timer.py:264:stop] epoch=0/micro_step=660/global_step=330, RunningAvgSamplesPerSec=13.290943667500795, CurrSamplesPerSec=13.2426799914089, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
87
+ [2024-11-12 09:06:17,912] [INFO] [logging.py:96:log_dist] [Rank 0] step=340, skipped=0, lr=[4.657358947870691e-05, 4.657358947870691e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
88
+ [2024-11-12 09:06:17,913] [INFO] [timer.py:264:stop] epoch=0/micro_step=680/global_step=340, RunningAvgSamplesPerSec=13.303275846211843, CurrSamplesPerSec=14.644326425107316, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
89
+ [2024-11-12 09:07:07,262] [INFO] [logging.py:96:log_dist] [Rank 0] step=350, skipped=0, lr=[4.6337598573143823e-05, 4.6337598573143823e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
90
+ [2024-11-12 09:07:07,263] [INFO] [timer.py:264:stop] epoch=0/micro_step=700/global_step=350, RunningAvgSamplesPerSec=13.320609598823237, CurrSamplesPerSec=14.086162727268885, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
91
+ [2024-11-12 09:07:57,713] [INFO] [logging.py:96:log_dist] [Rank 0] step=360, skipped=0, lr=[4.609438899557964e-05, 4.609438899557964e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
92
+ [2024-11-12 09:07:57,714] [INFO] [timer.py:264:stop] epoch=1/micro_step=16/global_step=360, RunningAvgSamplesPerSec=13.325138937093469, CurrSamplesPerSec=13.648595953838816, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
93
+ [2024-11-12 09:08:46,579] [INFO] [logging.py:96:log_dist] [Rank 0] step=370, skipped=0, lr=[4.584404302566556e-05, 4.584404302566556e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
94
+ [2024-11-12 09:08:46,580] [INFO] [timer.py:264:stop] epoch=1/micro_step=36/global_step=370, RunningAvgSamplesPerSec=13.348088837780798, CurrSamplesPerSec=13.251978385199415, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
95
+ [2024-11-12 09:09:35,642] [INFO] [logging.py:96:log_dist] [Rank 0] step=380, skipped=0, lr=[4.5586645357348636e-05, 4.5586645357348636e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
96
+ [2024-11-12 09:09:35,643] [INFO] [timer.py:264:stop] epoch=1/micro_step=56/global_step=380, RunningAvgSamplesPerSec=13.36489338184365, CurrSamplesPerSec=14.590145688968107, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
97
+ [2024-11-12 09:10:29,555] [INFO] [logging.py:96:log_dist] [Rank 0] step=390, skipped=0, lr=[4.532228307021913e-05, 4.532228307021913e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
98
+ [2024-11-12 09:10:29,555] [INFO] [timer.py:264:stop] epoch=1/micro_step=76/global_step=390, RunningAvgSamplesPerSec=13.343550964615849, CurrSamplesPerSec=14.126325253738942, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
99
+ [2024-11-12 09:11:18,926] [INFO] [logging.py:96:log_dist] [Rank 0] step=400, skipped=0, lr=[4.50510456000509e-05, 4.50510456000509e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
100
+ [2024-11-12 09:11:18,926] [INFO] [timer.py:264:stop] epoch=1/micro_step=96/global_step=400, RunningAvgSamplesPerSec=13.358597510292705, CurrSamplesPerSec=14.449643731571005, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
101
+ Saving checkpoint at step 800 ...
102
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
103
+ Saving 16-bit model...
104
+ [2024-11-12 09:11:30,491] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
105
+ [2024-11-12 09:11:30,493] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin, tag: global_step400
106
+ [2024-11-12 09:11:30,493] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin...
107
+ [2024-11-12 09:11:47,011] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin.
108
+ [2024-11-12 09:11:47,013] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
109
+ Model saved!
110
+ Saving 16-bit model...
111
+ [2024-11-12 09:11:56,858] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
112
+ [2024-11-12 09:11:56,859] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin, tag: global_step400
113
+ [2024-11-12 09:11:56,860] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin...
114
+ [2024-11-12 09:12:19,298] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin.
115
+ [2024-11-12 09:12:19,300] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
116
+ Model saved!
117
+ Checkpoint saved.
118
+ [2024-11-12 09:13:09,669] [INFO] [logging.py:96:log_dist] [Rank 0] step=410, skipped=0, lr=[4.477302470854453e-05, 4.477302470854453e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
119
+ [2024-11-12 09:13:09,670] [INFO] [timer.py:264:stop] epoch=1/micro_step=116/global_step=410, RunningAvgSamplesPerSec=13.364449824928494, CurrSamplesPerSec=14.7170956204939, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
120
+ [2024-11-12 09:14:00,777] [INFO] [logging.py:96:log_dist] [Rank 0] step=420, skipped=0, lr=[4.4488314452283675e-05, 4.4488314452283675e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
121
+ [2024-11-12 09:14:00,777] [INFO] [timer.py:264:stop] epoch=1/micro_step=136/global_step=420, RunningAvgSamplesPerSec=13.363891525451283, CurrSamplesPerSec=10.922366588553597, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
122
+ [2024-11-12 09:14:49,428] [INFO] [logging.py:96:log_dist] [Rank 0] step=430, skipped=0, lr=[4.4197011150915e-05, 4.4197011150915e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
123
+ [2024-11-12 09:14:49,429] [INFO] [timer.py:264:stop] epoch=1/micro_step=156/global_step=430, RunningAvgSamplesPerSec=13.383295359926425, CurrSamplesPerSec=14.55651722494042, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
124
+ [2024-11-12 09:15:39,342] [INFO] [logging.py:96:log_dist] [Rank 0] step=440, skipped=0, lr=[4.389921335456253e-05, 4.389921335456253e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
125
+ [2024-11-12 09:15:39,343] [INFO] [timer.py:264:stop] epoch=1/micro_step=176/global_step=440, RunningAvgSamplesPerSec=13.390975239991825, CurrSamplesPerSec=13.200286386506031, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
126
+ [2024-11-12 09:16:30,091] [INFO] [logging.py:96:log_dist] [Rank 0] step=450, skipped=0, lr=[4.359502181048733e-05, 4.359502181048733e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
127
+ [2024-11-12 09:16:30,092] [INFO] [timer.py:264:stop] epoch=1/micro_step=196/global_step=450, RunningAvgSamplesPerSec=13.39418095466066, CurrSamplesPerSec=12.500613123038491, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
128
+ [2024-11-12 09:17:24,651] [INFO] [logging.py:96:log_dist] [Rank 0] step=460, skipped=0, lr=[4.328453942900402e-05, 4.328453942900402e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
129
+ [2024-11-12 09:17:24,651] [INFO] [timer.py:264:stop] epoch=1/micro_step=216/global_step=460, RunningAvgSamplesPerSec=13.373634676462785, CurrSamplesPerSec=10.975064294895416, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
130
+ [2024-11-12 09:18:16,585] [INFO] [logging.py:96:log_dist] [Rank 0] step=470, skipped=0, lr=[4.29678712486654e-05, 4.29678712486654e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
131
+ [2024-11-12 09:18:16,586] [INFO] [timer.py:264:stop] epoch=1/micro_step=236/global_step=470, RunningAvgSamplesPerSec=13.371014296937762, CurrSamplesPerSec=13.500704537918898, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
132
+ [2024-11-12 09:19:12,663] [INFO] [logging.py:96:log_dist] [Rank 0] step=480, skipped=0, lr=[4.2645124400727074e-05, 4.2645124400727074e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
133
+ [2024-11-12 09:19:12,664] [INFO] [timer.py:264:stop] epoch=1/micro_step=256/global_step=480, RunningAvgSamplesPerSec=13.346826564436673, CurrSamplesPerSec=11.446672280190908, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
134
+ [2024-11-12 09:20:02,200] [INFO] [logging.py:96:log_dist] [Rank 0] step=490, skipped=0, lr=[4.231640807290418e-05, 4.231640807290418e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
135
+ [2024-11-12 09:20:02,200] [INFO] [timer.py:264:stop] epoch=1/micro_step=276/global_step=490, RunningAvgSamplesPerSec=13.356418110943588, CurrSamplesPerSec=14.935730763365298, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
136
+ [2024-11-12 09:20:57,703] [INFO] [logging.py:96:log_dist] [Rank 0] step=500, skipped=0, lr=[4.198183347243233e-05, 4.198183347243233e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
137
+ [2024-11-12 09:20:57,704] [INFO] [timer.py:264:stop] epoch=1/micro_step=296/global_step=500, RunningAvgSamplesPerSec=13.334440472618091, CurrSamplesPerSec=14.71236645337987, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
138
+ [2024-11-12 09:21:49,661] [INFO] [logging.py:96:log_dist] [Rank 0] step=510, skipped=0, lr=[4.1641513788445365e-05, 4.1641513788445365e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
139
+ [2024-11-12 09:21:49,662] [INFO] [timer.py:264:stop] epoch=1/micro_step=316/global_step=510, RunningAvgSamplesPerSec=13.33045269258777, CurrSamplesPerSec=12.131904363990161, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
140
+ [2024-11-12 09:22:40,443] [INFO] [logging.py:96:log_dist] [Rank 0] step=520, skipped=0, lr=[4.129556415368261e-05, 4.129556415368261e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
141
+ [2024-11-12 09:22:40,443] [INFO] [timer.py:264:stop] epoch=1/micro_step=336/global_step=520, RunningAvgSamplesPerSec=13.334724290035696, CurrSamplesPerSec=13.00404440542971, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
142
+ [2024-11-12 09:23:33,397] [INFO] [logging.py:96:log_dist] [Rank 0] step=530, skipped=0, lr=[4.0944101605538585e-05, 4.0944101605538585e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
143
+ [2024-11-12 09:23:33,398] [INFO] [timer.py:264:stop] epoch=1/micro_step=356/global_step=530, RunningAvgSamplesPerSec=13.325292224207772, CurrSamplesPerSec=13.89638709040199, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
144
+ [2024-11-12 09:24:25,155] [INFO] [logging.py:96:log_dist] [Rank 0] step=540, skipped=0, lr=[4.058724504646834e-05, 4.058724504646834e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
145
+ [2024-11-12 09:24:25,156] [INFO] [timer.py:264:stop] epoch=1/micro_step=376/global_step=540, RunningAvgSamplesPerSec=13.324965746155074, CurrSamplesPerSec=12.91599583924915, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
146
+ [2024-11-12 09:25:16,862] [INFO] [logging.py:96:log_dist] [Rank 0] step=550, skipped=0, lr=[4.0225115203761924e-05, 4.0225115203761924e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
147
+ [2024-11-12 09:25:16,863] [INFO] [timer.py:264:stop] epoch=1/micro_step=396/global_step=550, RunningAvgSamplesPerSec=13.32296006415578, CurrSamplesPerSec=14.133098580815874, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
148
+ [2024-11-12 09:26:07,277] [INFO] [logging.py:96:log_dist] [Rank 0] step=560, skipped=0, lr=[3.985783458870134e-05, 3.985783458870134e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
149
+ [2024-11-12 09:26:07,278] [INFO] [timer.py:264:stop] epoch=1/micro_step=416/global_step=560, RunningAvgSamplesPerSec=13.328244272608401, CurrSamplesPerSec=11.970295366561219, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
150
+ [2024-11-12 09:27:00,643] [INFO] [logging.py:96:log_dist] [Rank 0] step=570, skipped=0, lr=[3.948552745511409e-05, 3.948552745511409e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
151
+ [2024-11-12 09:27:00,644] [INFO] [timer.py:264:stop] epoch=1/micro_step=436/global_step=570, RunningAvgSamplesPerSec=13.317673249255442, CurrSamplesPerSec=10.870225791334036, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
152
+ [2024-11-12 09:27:51,548] [INFO] [logging.py:96:log_dist] [Rank 0] step=580, skipped=0, lr=[3.910831975733717e-05, 3.910831975733717e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
153
+ [2024-11-12 09:27:51,549] [INFO] [timer.py:264:stop] epoch=1/micro_step=456/global_step=580, RunningAvgSamplesPerSec=13.318033623411376, CurrSamplesPerSec=12.694643791709918, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
154
+ [2024-11-12 09:28:44,874] [INFO] [logging.py:96:log_dist] [Rank 0] step=590, skipped=0, lr=[3.872633910760571e-05, 3.872633910760571e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
155
+ [2024-11-12 09:28:44,875] [INFO] [timer.py:264:stop] epoch=1/micro_step=476/global_step=590, RunningAvgSamplesPerSec=13.30842315273336, CurrSamplesPerSec=12.382377351271012, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
156
+ [2024-11-12 09:29:36,746] [INFO] [logging.py:96:log_dist] [Rank 0] step=600, skipped=0, lr=[3.833971473288084e-05, 3.833971473288084e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
157
+ [2024-11-12 09:29:36,747] [INFO] [timer.py:264:stop] epoch=1/micro_step=496/global_step=600, RunningAvgSamplesPerSec=13.308057817145734, CurrSamplesPerSec=14.699825462285968, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
158
+ Saving checkpoint at step 1200 ...
159
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
160
+ Saving 16-bit model...
161
+ [2024-11-12 09:29:49,497] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
162
+ [2024-11-12 09:29:49,499] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin, tag: global_step600
163
+ [2024-11-12 09:29:49,499] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin...
164
+ [2024-11-12 09:30:10,740] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin.
165
+ [2024-11-12 09:30:10,742] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
166
+ Model saved!
167
+ Saving 16-bit model...
168
+ [2024-11-12 09:30:19,949] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
169
+ [2024-11-12 09:30:19,951] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin, tag: global_step600
170
+ [2024-11-12 09:30:19,951] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin...
171
+ [2024-11-12 09:30:45,036] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin.
172
+ [2024-11-12 09:30:45,038] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
173
+ Model saved!
174
+ Checkpoint saved.
175
+ [2024-11-12 09:31:37,541] [INFO] [logging.py:96:log_dist] [Rank 0] step=610, skipped=0, lr=[3.794857743113117e-05, 3.794857743113117e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
176
+ [2024-11-12 09:31:37,542] [INFO] [timer.py:264:stop] epoch=1/micro_step=516/global_step=610, RunningAvgSamplesPerSec=13.303038238839015, CurrSamplesPerSec=13.471107231584368, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
177
+ [2024-11-12 09:32:33,379] [INFO] [logging.py:96:log_dist] [Rank 0] step=620, skipped=0, lr=[3.7553059527082913e-05, 3.7553059527082913e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
178
+ [2024-11-12 09:32:33,380] [INFO] [timer.py:264:stop] epoch=1/micro_step=536/global_step=620, RunningAvgSamplesPerSec=13.284333701524984, CurrSamplesPerSec=7.212952779012694, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
179
+ [2024-11-12 09:33:26,622] [INFO] [logging.py:96:log_dist] [Rank 0] step=630, skipped=0, lr=[3.715329482745333e-05, 3.715329482745333e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
180
+ [2024-11-12 09:33:26,623] [INFO] [timer.py:264:stop] epoch=1/micro_step=556/global_step=630, RunningAvgSamplesPerSec=13.27712854384248, CurrSamplesPerSec=13.229785334147179, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
181
+ [2024-11-12 09:34:17,583] [INFO] [logging.py:96:log_dist] [Rank 0] step=640, skipped=0, lr=[3.6749418575683e-05, 3.6749418575683e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
182
+ [2024-11-12 09:34:17,584] [INFO] [timer.py:264:stop] epoch=1/micro_step=576/global_step=640, RunningAvgSamplesPerSec=13.279694432663204, CurrSamplesPerSec=11.886098293838835, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
183
+ [2024-11-12 09:35:08,224] [INFO] [logging.py:96:log_dist] [Rank 0] step=650, skipped=0, lr=[3.634156740618186e-05, 3.634156740618186e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
184
+ [2024-11-12 09:35:08,225] [INFO] [timer.py:264:stop] epoch=1/micro_step=596/global_step=650, RunningAvgSamplesPerSec=13.281283354952393, CurrSamplesPerSec=13.977049388035704, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
185
+ [2024-11-12 09:35:58,154] [INFO] [logging.py:96:log_dist] [Rank 0] step=660, skipped=0, lr=[3.592987929810476e-05, 3.592987929810476e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
186
+ [2024-11-12 09:35:58,154] [INFO] [timer.py:264:stop] epoch=1/micro_step=616/global_step=660, RunningAvgSamplesPerSec=13.28894472425476, CurrSamplesPerSec=14.26329493903139, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
187
+ [2024-11-12 09:36:47,989] [INFO] [logging.py:96:log_dist] [Rank 0] step=670, skipped=0, lr=[3.551449352867209e-05, 3.551449352867209e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
188
+ [2024-11-12 09:36:47,990] [INFO] [timer.py:264:stop] epoch=1/micro_step=636/global_step=670, RunningAvgSamplesPerSec=13.295832906227798, CurrSamplesPerSec=12.269052838403406, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
189
+ [2024-11-12 09:37:40,115] [INFO] [logging.py:96:log_dist] [Rank 0] step=680, skipped=0, lr=[3.509555062605121e-05, 3.509555062605121e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
190
+ [2024-11-12 09:37:40,116] [INFO] [timer.py:264:stop] epoch=1/micro_step=656/global_step=680, RunningAvgSamplesPerSec=13.291947948697695, CurrSamplesPerSec=11.483019322453742, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
191
+ [2024-11-12 09:38:31,799] [INFO] [logging.py:96:log_dist] [Rank 0] step=690, skipped=0, lr=[3.467319232181473e-05, 3.467319232181473e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
192
+ [2024-11-12 09:38:31,800] [INFO] [timer.py:264:stop] epoch=1/micro_step=676/global_step=690, RunningAvgSamplesPerSec=13.293062090636276, CurrSamplesPerSec=13.979593389406237, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
193
+ [2024-11-12 09:39:20,875] [INFO] [logging.py:96:log_dist] [Rank 0] step=700, skipped=0, lr=[3.4247561502991604e-05, 3.4247561502991604e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
194
+ [2024-11-12 09:39:20,876] [INFO] [timer.py:264:stop] epoch=1/micro_step=696/global_step=700, RunningAvgSamplesPerSec=13.301456078834374, CurrSamplesPerSec=14.428370061415672, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
195
+ [2024-11-12 09:40:11,651] [INFO] [logging.py:96:log_dist] [Rank 0] step=710, skipped=0, lr=[3.381880216372738e-05, 3.381880216372738e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
196
+ [2024-11-12 09:40:11,652] [INFO] [timer.py:264:stop] epoch=2/micro_step=12/global_step=710, RunningAvgSamplesPerSec=13.303156673817357, CurrSamplesPerSec=14.634354011395214, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
197
+ [2024-11-12 09:41:00,207] [INFO] [logging.py:96:log_dist] [Rank 0] step=720, skipped=0, lr=[3.3387059356569875e-05, 3.3387059356569875e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
198
+ [2024-11-12 09:41:00,208] [INFO] [timer.py:264:stop] epoch=2/micro_step=32/global_step=720, RunningAvgSamplesPerSec=13.314711683634192, CurrSamplesPerSec=13.545782956791136, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
199
+ [2024-11-12 09:41:49,140] [INFO] [logging.py:96:log_dist] [Rank 0] step=730, skipped=0, lr=[3.2952479143396826e-05, 3.2952479143396826e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
200
+ [2024-11-12 09:41:49,141] [INFO] [timer.py:264:stop] epoch=2/micro_step=52/global_step=730, RunningAvgSamplesPerSec=13.325003730032233, CurrSamplesPerSec=14.36111525811834, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
201
+ [2024-11-12 09:42:42,299] [INFO] [logging.py:96:log_dist] [Rank 0] step=740, skipped=0, lr=[3.2515208546002e-05, 3.2515208546002e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
202
+ [2024-11-12 09:42:42,300] [INFO] [timer.py:264:stop] epoch=2/micro_step=72/global_step=740, RunningAvgSamplesPerSec=13.318484161902534, CurrSamplesPerSec=11.156171290123146, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
203
+ [2024-11-12 09:43:33,565] [INFO] [logging.py:96:log_dist] [Rank 0] step=750, skipped=0, lr=[3.207539549635664e-05, 3.207539549635664e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
204
+ [2024-11-12 09:43:33,566] [INFO] [timer.py:264:stop] epoch=2/micro_step=92/global_step=750, RunningAvgSamplesPerSec=13.319682913264439, CurrSamplesPerSec=13.220680176763791, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
205
+ [2024-11-12 09:44:22,447] [INFO] [logging.py:96:log_dist] [Rank 0] step=760, skipped=0, lr=[3.1633188786562914e-05, 3.1633188786562914e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
206
+ [2024-11-12 09:44:22,447] [INFO] [timer.py:264:stop] epoch=2/micro_step=112/global_step=760, RunningAvgSamplesPerSec=13.327851536598244, CurrSamplesPerSec=14.068757540394465, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
207
+ [2024-11-12 09:45:13,278] [INFO] [logging.py:96:log_dist] [Rank 0] step=770, skipped=0, lr=[3.118873801851646e-05, 3.118873801851646e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
208
+ [2024-11-12 09:45:13,279] [INFO] [timer.py:264:stop] epoch=2/micro_step=132/global_step=770, RunningAvgSamplesPerSec=13.330652798672963, CurrSamplesPerSec=11.724133921801615, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
209
+ [2024-11-12 09:46:03,735] [INFO] [logging.py:96:log_dist] [Rank 0] step=780, skipped=0, lr=[3.07421935532949e-05, 3.07421935532949e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
210
+ [2024-11-12 09:46:03,736] [INFO] [timer.py:264:stop] epoch=2/micro_step=152/global_step=780, RunningAvgSamplesPerSec=13.335917143341288, CurrSamplesPerSec=14.432756201254964, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
211
+ [2024-11-12 09:46:53,208] [INFO] [logging.py:96:log_dist] [Rank 0] step=790, skipped=0, lr=[3.0293706460289505e-05, 3.0293706460289505e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
212
+ [2024-11-12 09:46:53,209] [INFO] [timer.py:264:stop] epoch=2/micro_step=172/global_step=790, RunningAvgSamplesPerSec=13.343995330928088, CurrSamplesPerSec=13.47306192122824, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
213
+ [2024-11-12 09:47:43,265] [INFO] [logging.py:96:log_dist] [Rank 0] step=800, skipped=0, lr=[2.9843428466097385e-05, 2.9843428466097385e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
214
+ [2024-11-12 09:47:43,266] [INFO] [timer.py:264:stop] epoch=2/micro_step=192/global_step=800, RunningAvgSamplesPerSec=13.347981134151242, CurrSamplesPerSec=14.23447280515204, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
215
+ Saving checkpoint at step 1600 ...
216
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
217
+ Saving 16-bit model...
218
+ [2024-11-12 09:47:54,616] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step800 is about to be saved!
219
+ [2024-11-12 09:47:54,617] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin, tag: global_step800
220
+ [2024-11-12 09:47:54,617] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin...
221
+ [2024-11-12 09:48:11,335] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin.
222
+ [2024-11-12 09:48:11,336] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step800 is ready now!
223
+ Model saved!
224
+ Saving 16-bit model...
225
+ [2024-11-12 09:48:22,816] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step800 is about to be saved!
226
+ [2024-11-12 09:48:22,817] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin, tag: global_step800
227
+ [2024-11-12 09:48:22,817] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin...
228
+ [2024-11-12 09:48:45,345] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin.
229
+ [2024-11-12 09:48:45,347] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step800 is ready now!
230
+ Model saved!
231
+ Checkpoint saved.
232
+ [2024-11-12 09:49:39,319] [INFO] [logging.py:96:log_dist] [Rank 0] step=810, skipped=0, lr=[2.939151190319108e-05, 2.939151190319108e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
233
+ [2024-11-12 09:49:39,320] [INFO] [timer.py:264:stop] epoch=2/micro_step=212/global_step=810, RunningAvgSamplesPerSec=13.338019751420884, CurrSamplesPerSec=14.761337460745917, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
234
+ [2024-11-12 09:50:31,369] [INFO] [logging.py:96:log_dist] [Rank 0] step=820, skipped=0, lr=[2.8938109658383454e-05, 2.8938109658383454e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
235
+ [2024-11-12 09:50:31,370] [INFO] [timer.py:264:stop] epoch=2/micro_step=232/global_step=820, RunningAvgSamplesPerSec=13.334637882621434, CurrSamplesPerSec=14.700607138327708, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
236
+ [2024-11-12 09:51:26,515] [INFO] [logging.py:96:log_dist] [Rank 0] step=830, skipped=0, lr=[2.8483375121104818e-05, 2.8483375121104818e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
237
+ [2024-11-12 09:51:26,516] [INFO] [timer.py:264:stop] epoch=2/micro_step=252/global_step=830, RunningAvgSamplesPerSec=13.321062900261444, CurrSamplesPerSec=12.880004275047911, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
238
+ [2024-11-12 09:52:16,145] [INFO] [logging.py:96:log_dist] [Rank 0] step=840, skipped=0, lr=[2.8027462131510208e-05, 2.8027462131510208e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
239
+ [2024-11-12 09:52:16,146] [INFO] [timer.py:264:stop] epoch=2/micro_step=272/global_step=840, RunningAvgSamplesPerSec=13.327259566392549, CurrSamplesPerSec=13.7934219369934, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
240
+ [2024-11-12 09:53:12,237] [INFO] [logging.py:96:log_dist] [Rank 0] step=850, skipped=0, lr=[2.757052492843401e-05, 2.757052492843401e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
241
+ [2024-11-12 09:53:12,238] [INFO] [timer.py:264:stop] epoch=2/micro_step=292/global_step=850, RunningAvgSamplesPerSec=13.31062857578834, CurrSamplesPerSec=14.756166118458328, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
242
+ [2024-11-12 09:54:02,303] [INFO] [logging.py:96:log_dist] [Rank 0] step=860, skipped=0, lr=[2.711271809720986e-05, 2.711271809720986e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
243
+ [2024-11-12 09:54:02,304] [INFO] [timer.py:264:stop] epoch=2/micro_step=312/global_step=860, RunningAvgSamplesPerSec=13.313888786479442, CurrSamplesPerSec=14.989720670581756, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
244
+ [2024-11-12 09:54:53,519] [INFO] [logging.py:96:log_dist] [Rank 0] step=870, skipped=0, lr=[2.6654196517373205e-05, 2.6654196517373205e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
245
+ [2024-11-12 09:54:53,520] [INFO] [timer.py:264:stop] epoch=2/micro_step=332/global_step=870, RunningAvgSamplesPerSec=13.314541614255276, CurrSamplesPerSec=13.755073221094372, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
246
+ [2024-11-12 09:55:46,262] [INFO] [logging.py:96:log_dist] [Rank 0] step=880, skipped=0, lr=[2.619511531026436e-05, 2.619511531026436e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
247
+ [2024-11-12 09:55:46,263] [INFO] [timer.py:264:stop] epoch=2/micro_step=352/global_step=880, RunningAvgSamplesPerSec=13.30990443123984, CurrSamplesPerSec=13.64081898485842, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
248
+ [2024-11-12 09:56:38,111] [INFO] [logging.py:96:log_dist] [Rank 0] step=890, skipped=0, lr=[2.573562978654986e-05, 2.573562978654986e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
249
+ [2024-11-12 09:56:38,112] [INFO] [timer.py:264:stop] epoch=2/micro_step=372/global_step=890, RunningAvgSamplesPerSec=13.309379764160232, CurrSamplesPerSec=11.592272177703125, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
250
+ [2024-11-12 09:57:30,442] [INFO] [logging.py:96:log_dist] [Rank 0] step=900, skipped=0, lr=[2.527589539367956e-05, 2.527589539367956e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
251
+ [2024-11-12 09:57:30,443] [INFO] [timer.py:264:stop] epoch=2/micro_step=392/global_step=900, RunningAvgSamplesPerSec=13.306786686847447, CurrSamplesPerSec=13.435805696728409, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
252
+ [2024-11-12 09:58:19,473] [INFO] [logging.py:96:log_dist] [Rank 0] step=910, skipped=0, lr=[2.4816067663297705e-05, 2.4816067663297705e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
253
+ [2024-11-12 09:58:19,474] [INFO] [timer.py:264:stop] epoch=2/micro_step=412/global_step=910, RunningAvgSamplesPerSec=13.313050356453484, CurrSamplesPerSec=13.499937984774144, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
254
+ [2024-11-12 09:59:12,152] [INFO] [logging.py:96:log_dist] [Rank 0] step=920, skipped=0, lr=[2.4356302158625288e-05, 2.4356302158625288e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
255
+ [2024-11-12 09:59:12,153] [INFO] [timer.py:264:stop] epoch=2/micro_step=432/global_step=920, RunningAvgSamplesPerSec=13.310216314626862, CurrSamplesPerSec=14.437108514790491, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
256
+ [2024-11-12 10:00:04,119] [INFO] [logging.py:96:log_dist] [Rank 0] step=930, skipped=0, lr=[2.38967544218319e-05, 2.38967544218319e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
257
+ [2024-11-12 10:00:04,119] [INFO] [timer.py:264:stop] epoch=2/micro_step=452/global_step=930, RunningAvgSamplesPerSec=13.309106357860806, CurrSamplesPerSec=13.867148018326894, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
258
+ [2024-11-12 10:00:57,366] [INFO] [logging.py:96:log_dist] [Rank 0] step=940, skipped=0, lr=[2.3437579921414573e-05, 2.3437579921414573e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
259
+ [2024-11-12 10:00:57,367] [INFO] [timer.py:264:stop] epoch=2/micro_step=472/global_step=940, RunningAvgSamplesPerSec=13.303958047851587, CurrSamplesPerSec=12.214996944223117, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
260
+ [2024-11-12 10:01:49,941] [INFO] [logging.py:96:log_dist] [Rank 0] step=950, skipped=0, lr=[2.2978933999601655e-05, 2.2978933999601655e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
261
+ [2024-11-12 10:01:49,942] [INFO] [timer.py:264:stop] epoch=2/micro_step=492/global_step=950, RunningAvgSamplesPerSec=13.300542723454466, CurrSamplesPerSec=13.28000135254299, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
262
+ [2024-11-12 10:02:41,154] [INFO] [logging.py:96:log_dist] [Rank 0] step=960, skipped=0, lr=[2.2520971819799328e-05, 2.2520971819799328e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
263
+ [2024-11-12 10:02:41,156] [INFO] [timer.py:264:stop] epoch=2/micro_step=512/global_step=960, RunningAvgSamplesPerSec=13.30180341566793, CurrSamplesPerSec=14.012398043896601, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
264
+ [2024-11-12 10:03:33,677] [INFO] [logging.py:96:log_dist] [Rank 0] step=970, skipped=0, lr=[2.206384831409863e-05, 2.206384831409863e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
265
+ [2024-11-12 10:03:33,678] [INFO] [timer.py:264:stop] epoch=2/micro_step=532/global_step=970, RunningAvgSamplesPerSec=13.301177680120674, CurrSamplesPerSec=10.555131044819795, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
266
+ [2024-11-12 10:04:31,856] [INFO] [logging.py:96:log_dist] [Rank 0] step=980, skipped=0, lr=[2.1607718130860782e-05, 2.1607718130860782e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
267
+ [2024-11-12 10:04:31,857] [INFO] [timer.py:264:stop] epoch=2/micro_step=552/global_step=980, RunningAvgSamplesPerSec=13.282526675644393, CurrSamplesPerSec=13.073611291719171, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
268
+ [2024-11-12 10:05:21,746] [INFO] [logging.py:96:log_dist] [Rank 0] step=990, skipped=0, lr=[2.1152735582398452e-05, 2.1152735582398452e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
269
+ [2024-11-12 10:05:21,747] [INFO] [timer.py:264:stop] epoch=2/micro_step=572/global_step=990, RunningAvgSamplesPerSec=13.287191601768065, CurrSamplesPerSec=13.239052563519289, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
270
+ [2024-11-12 10:06:14,126] [INFO] [logging.py:96:log_dist] [Rank 0] step=1000, skipped=0, lr=[2.0699054592770737e-05, 2.0699054592770737e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
271
+ [2024-11-12 10:06:14,127] [INFO] [timer.py:264:stop] epoch=2/micro_step=592/global_step=1000, RunningAvgSamplesPerSec=13.286254771939328, CurrSamplesPerSec=14.674160933553079, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
272
+ Saving checkpoint at step 2000 ...
273
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
274
+ Saving 16-bit model...
275
+ [2024-11-12 10:06:29,485] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is about to be saved!
276
+ [2024-11-12 10:06:29,487] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin, tag: global_step1000
277
+ [2024-11-12 10:06:29,487] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin...
278
+ [2024-11-12 10:06:52,581] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin.
279
+ [2024-11-12 10:06:52,583] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1000 is ready now!
280
+ Model saved!
281
+ Saving 16-bit model...
282
+ [2024-11-12 10:07:01,836] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is about to be saved!
283
+ [2024-11-12 10:07:01,837] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin, tag: global_step1000
284
+ [2024-11-12 10:07:01,838] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin...
285
+ [2024-11-12 10:07:28,200] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin.
286
+ [2024-11-12 10:07:28,202] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1000 is ready now!
287
+ Model saved!
288
+ Checkpoint saved.
289
+ [2024-11-12 10:08:18,416] [INFO] [logging.py:96:log_dist] [Rank 0] step=1010, skipped=0, lr=[2.024682864570945e-05, 2.024682864570945e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
290
+ [2024-11-12 10:08:18,417] [INFO] [timer.py:264:stop] epoch=2/micro_step=612/global_step=1010, RunningAvgSamplesPerSec=13.289360403964261, CurrSamplesPerSec=13.18569467585241, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
291
+ [2024-11-12 10:09:07,466] [INFO] [logging.py:96:log_dist] [Rank 0] step=1020, skipped=0, lr=[1.9796210732694442e-05, 1.9796210732694442e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
292
+ [2024-11-12 10:09:07,467] [INFO] [timer.py:264:stop] epoch=2/micro_step=632/global_step=1020, RunningAvgSamplesPerSec=13.296168396047804, CurrSamplesPerSec=15.105174063576408, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
293
+ [2024-11-12 10:09:59,426] [INFO] [logging.py:96:log_dist] [Rank 0] step=1030, skipped=0, lr=[1.9347353301195424e-05, 1.9347353301195424e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
294
+ [2024-11-12 10:09:59,427] [INFO] [timer.py:264:stop] epoch=2/micro_step=652/global_step=1030, RunningAvgSamplesPerSec=13.294815892948787, CurrSamplesPerSec=13.675949766714641, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
295
+ [2024-11-12 10:10:51,787] [INFO] [logging.py:96:log_dist] [Rank 0] step=1040, skipped=0, lr=[1.8900408203097787e-05, 1.8900408203097787e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
296
+ [2024-11-12 10:10:51,788] [INFO] [timer.py:264:stop] epoch=2/micro_step=672/global_step=1040, RunningAvgSamplesPerSec=13.29338856696237, CurrSamplesPerSec=14.005642697848842, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
297
+ [2024-11-12 10:11:40,380] [INFO] [logging.py:96:log_dist] [Rank 0] step=1050, skipped=0, lr=[1.8455526643329995e-05, 1.8455526643329995e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
298
+ [2024-11-12 10:11:40,381] [INFO] [timer.py:264:stop] epoch=2/micro_step=692/global_step=1050, RunningAvgSamplesPerSec=13.301896593394172, CurrSamplesPerSec=13.401735065616107, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
299
+ [2024-11-12 10:12:32,872] [INFO] [logging.py:96:log_dist] [Rank 0] step=1060, skipped=0, lr=[1.8012859128709766e-05, 1.8012859128709766e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
300
+ [2024-11-12 10:12:32,873] [INFO] [timer.py:264:stop] epoch=3/micro_step=8/global_step=1060, RunningAvgSamplesPerSec=13.30025338311892, CurrSamplesPerSec=13.396459390010323, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
301
+ [2024-11-12 10:13:21,395] [INFO] [logging.py:96:log_dist] [Rank 0] step=1070, skipped=0, lr=[1.7572555417026523e-05, 1.7572555417026523e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
302
+ [2024-11-12 10:13:21,396] [INFO] [timer.py:264:stop] epoch=3/micro_step=28/global_step=1070, RunningAvgSamplesPerSec=13.30848733831013, CurrSamplesPerSec=14.040752938548358, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
303
+ [2024-11-12 10:14:10,532] [INFO] [logging.py:96:log_dist] [Rank 0] step=1080, skipped=0, lr=[1.7134764466377136e-05, 1.7134764466377136e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
304
+ [2024-11-12 10:14:10,533] [INFO] [timer.py:264:stop] epoch=3/micro_step=48/global_step=1080, RunningAvgSamplesPerSec=13.314602105884283, CurrSamplesPerSec=13.125495457445675, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
305
+ [2024-11-12 10:15:01,827] [INFO] [logging.py:96:log_dist] [Rank 0] step=1090, skipped=0, lr=[1.6699634384772317e-05, 1.6699634384772317e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
306
+ [2024-11-12 10:15:01,827] [INFO] [timer.py:264:stop] epoch=3/micro_step=68/global_step=1090, RunningAvgSamplesPerSec=13.314236544377534, CurrSamplesPerSec=14.524569478485214, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
307
+ [2024-11-12 10:15:53,486] [INFO] [logging.py:96:log_dist] [Rank 0] step=1100, skipped=0, lr=[1.6267312380030506e-05, 1.6267312380030506e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
308
+ [2024-11-12 10:15:53,487] [INFO] [timer.py:264:stop] epoch=3/micro_step=88/global_step=1100, RunningAvgSamplesPerSec=13.313312407846855, CurrSamplesPerSec=12.19932859062791, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
309
+ [2024-11-12 10:16:42,803] [INFO] [logging.py:96:log_dist] [Rank 0] step=1110, skipped=0, lr=[1.5837944709976382e-05, 1.5837944709976382e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
310
+ [2024-11-12 10:16:42,804] [INFO] [timer.py:264:stop] epoch=3/micro_step=108/global_step=1110, RunningAvgSamplesPerSec=13.318065107541004, CurrSamplesPerSec=12.966475628687732, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
311
+ [2024-11-12 10:17:32,582] [INFO] [logging.py:96:log_dist] [Rank 0] step=1120, skipped=0, lr=[1.5411676632960713e-05, 1.5411676632960713e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
312
+ [2024-11-12 10:17:32,583] [INFO] [timer.py:264:stop] epoch=3/micro_step=128/global_step=1120, RunningAvgSamplesPerSec=13.322640969741128, CurrSamplesPerSec=13.668152738247908, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
313
+ [2024-11-12 10:18:24,097] [INFO] [logging.py:96:log_dist] [Rank 0] step=1130, skipped=0, lr=[1.4988652358718336e-05, 1.4988652358718336e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
314
+ [2024-11-12 10:18:24,098] [INFO] [timer.py:264:stop] epoch=3/micro_step=148/global_step=1130, RunningAvgSamplesPerSec=13.323294979183146, CurrSamplesPerSec=12.599322897047612, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
315
+ [2024-11-12 10:19:12,704] [INFO] [logging.py:96:log_dist] [Rank 0] step=1140, skipped=0, lr=[1.4569014999580937e-05, 1.4569014999580937e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
316
+ [2024-11-12 10:19:12,705] [INFO] [timer.py:264:stop] epoch=3/micro_step=168/global_step=1140, RunningAvgSamplesPerSec=13.329385073350831, CurrSamplesPerSec=14.52883737710039, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
317
+ [2024-11-12 10:20:02,511] [INFO] [logging.py:96:log_dist] [Rank 0] step=1150, skipped=0, lr=[1.4152906522061048e-05, 1.4152906522061048e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
318
+ [2024-11-12 10:20:02,512] [INFO] [timer.py:264:stop] epoch=3/micro_step=188/global_step=1150, RunningAvgSamplesPerSec=13.333106830069978, CurrSamplesPerSec=14.65574618291602, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
319
+ [2024-11-12 10:20:55,859] [INFO] [logging.py:96:log_dist] [Rank 0] step=1160, skipped=0, lr=[1.3740467698823662e-05, 1.3740467698823662e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
320
+ [2024-11-12 10:20:55,860] [INFO] [timer.py:264:stop] epoch=3/micro_step=208/global_step=1160, RunningAvgSamplesPerSec=13.328603070480009, CurrSamplesPerSec=14.134182082439413, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
321
+ [2024-11-12 10:21:50,334] [INFO] [logging.py:96:log_dist] [Rank 0] step=1170, skipped=0, lr=[1.3331838061061835e-05, 1.3331838061061835e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
322
+ [2024-11-12 10:21:50,336] [INFO] [timer.py:264:stop] epoch=3/micro_step=228/global_step=1170, RunningAvgSamplesPerSec=13.321959058539916, CurrSamplesPerSec=14.914114773186931, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
323
+ [2024-11-12 10:22:44,340] [INFO] [logging.py:96:log_dist] [Rank 0] step=1180, skipped=0, lr=[1.2927155851292145e-05, 1.2927155851292145e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
324
+ [2024-11-12 10:22:44,341] [INFO] [timer.py:264:stop] epoch=3/micro_step=248/global_step=1180, RunningAvgSamplesPerSec=13.315165122600238, CurrSamplesPerSec=12.564047165061556, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
325
+ [2024-11-12 10:23:34,862] [INFO] [logging.py:96:log_dist] [Rank 0] step=1190, skipped=0, lr=[1.2526557976586268e-05, 1.2526557976586268e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
326
+ [2024-11-12 10:23:34,863] [INFO] [timer.py:264:stop] epoch=3/micro_step=268/global_step=1190, RunningAvgSamplesPerSec=13.31844023579684, CurrSamplesPerSec=15.041789137896004, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
327
+ [2024-11-12 10:24:30,091] [INFO] [logging.py:96:log_dist] [Rank 0] step=1200, skipped=0, lr=[1.213017996225424e-05, 1.213017996225424e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
328
+ [2024-11-12 10:24:30,092] [INFO] [timer.py:264:stop] epoch=3/micro_step=288/global_step=1200, RunningAvgSamplesPerSec=13.309001079024005, CurrSamplesPerSec=10.541265033378687, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
329
+ Saving checkpoint at step 2400 ...
330
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
331
+ Saving 16-bit model...
332
+ [2024-11-12 10:24:44,305] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1200 is about to be saved!
333
+ [2024-11-12 10:24:44,307] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin, tag: global_step1200
334
+ [2024-11-12 10:24:44,307] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin...
335
+ [2024-11-12 10:25:00,864] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin.
336
+ [2024-11-12 10:25:00,867] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1200 is ready now!
337
+ Model saved!
338
+ Saving 16-bit model...
339
+ [2024-11-12 10:25:10,219] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1200 is about to be saved!
340
+ [2024-11-12 10:25:10,221] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin, tag: global_step1200
341
+ [2024-11-12 10:25:10,221] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin...
342
+ [2024-11-12 10:25:31,899] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin.
343
+ [2024-11-12 10:25:31,900] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1200 is ready now!
344
+ Model saved!
345
+ Checkpoint saved.
346
+ [2024-11-12 10:26:22,528] [INFO] [logging.py:96:log_dist] [Rank 0] step=1210, skipped=0, lr=[1.1738155905995186e-05, 1.1738155905995186e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
347
+ [2024-11-12 10:26:22,529] [INFO] [timer.py:264:stop] epoch=3/micro_step=308/global_step=1210, RunningAvgSamplesPerSec=13.310976144124597, CurrSamplesPerSec=13.716145845655182, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
348
+ [2024-11-12 10:27:13,156] [INFO] [logging.py:96:log_dist] [Rank 0] step=1220, skipped=0, lr=[1.1350618432531098e-05, 1.1350618432531098e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
349
+ [2024-11-12 10:27:13,157] [INFO] [timer.py:264:stop] epoch=3/micro_step=328/global_step=1220, RunningAvgSamplesPerSec=13.313287037618585, CurrSamplesPerSec=13.249320836208195, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
350
+ [2024-11-12 10:28:06,341] [INFO] [logging.py:96:log_dist] [Rank 0] step=1230, skipped=0, lr=[1.0967698648738866e-05, 1.0967698648738866e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
351
+ [2024-11-12 10:28:06,342] [INFO] [timer.py:264:stop] epoch=3/micro_step=348/global_step=1230, RunningAvgSamplesPerSec=13.309479597244271, CurrSamplesPerSec=13.3808760295492, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
352
+ [2024-11-12 10:28:56,982] [INFO] [logging.py:96:log_dist] [Rank 0] step=1240, skipped=0, lr=[1.0589526099295816e-05, 1.0589526099295816e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
353
+ [2024-11-12 10:28:56,983] [INFO] [timer.py:264:stop] epoch=3/micro_step=368/global_step=1240, RunningAvgSamplesPerSec=13.310696316549159, CurrSamplesPerSec=14.725364240818903, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
354
+ [2024-11-12 10:29:50,064] [INFO] [logging.py:96:log_dist] [Rank 0] step=1250, skipped=0, lr=[1.0216228722853735e-05, 1.0216228722853735e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
355
+ [2024-11-12 10:29:50,065] [INFO] [timer.py:264:stop] epoch=3/micro_step=388/global_step=1250, RunningAvgSamplesPerSec=13.30687912483691, CurrSamplesPerSec=11.146967550500829, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
356
+ [2024-11-12 10:30:39,877] [INFO] [logging.py:96:log_dist] [Rank 0] step=1260, skipped=0, lr=[9.847932808756308e-06, 9.847932808756308e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
357
+ [2024-11-12 10:30:39,878] [INFO] [timer.py:264:stop] epoch=3/micro_step=408/global_step=1260, RunningAvgSamplesPerSec=13.311650776938508, CurrSamplesPerSec=14.829740880556027, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
358
+ [2024-11-12 10:31:32,354] [INFO] [logging.py:96:log_dist] [Rank 0] step=1270, skipped=0, lr=[9.48476295431443e-06, 9.48476295431443e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
359
+ [2024-11-12 10:31:32,355] [INFO] [timer.py:264:stop] epoch=3/micro_step=428/global_step=1270, RunningAvgSamplesPerSec=13.309168729063913, CurrSamplesPerSec=13.571046031877607, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
360
+ [2024-11-12 10:32:24,164] [INFO] [logging.py:96:log_dist] [Rank 0] step=1280, skipped=0, lr=[9.126842022654003e-06, 9.126842022654003e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
361
+ [2024-11-12 10:32:24,165] [INFO] [timer.py:264:stop] epoch=3/micro_step=448/global_step=1280, RunningAvgSamplesPerSec=13.308511944145916, CurrSamplesPerSec=13.94143983438274, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
362
+ [2024-11-12 10:33:16,769] [INFO] [logging.py:96:log_dist] [Rank 0] step=1290, skipped=0, lr=[8.774291101150408e-06, 8.774291101150408e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
363
+ [2024-11-12 10:33:16,770] [INFO] [timer.py:264:stop] epoch=3/micro_step=468/global_step=1290, RunningAvgSamplesPerSec=13.305750646091278, CurrSamplesPerSec=11.319639747356943, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
364
+ [2024-11-12 10:34:10,115] [INFO] [logging.py:96:log_dist] [Rank 0] step=1300, skipped=0, lr=[8.427229460463696e-06, 8.427229460463696e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
365
+ [2024-11-12 10:34:10,116] [INFO] [timer.py:264:stop] epoch=3/micro_step=488/global_step=1300, RunningAvgSamplesPerSec=13.301445208963852, CurrSamplesPerSec=13.849357021669912, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
366
+ [2024-11-12 10:35:01,342] [INFO] [logging.py:96:log_dist] [Rank 0] step=1310, skipped=0, lr=[8.085774514188454e-06, 8.085774514188454e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
367
+ [2024-11-12 10:35:01,344] [INFO] [timer.py:264:stop] epoch=3/micro_step=508/global_step=1310, RunningAvgSamplesPerSec=13.302156872923947, CurrSamplesPerSec=13.639201313267824, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
368
+ [2024-11-12 10:35:52,385] [INFO] [logging.py:96:log_dist] [Rank 0] step=1320, skipped=0, lr=[7.75004177913188e-06, 7.75004177913188e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
369
+ [2024-11-12 10:35:52,386] [INFO] [timer.py:264:stop] epoch=3/micro_step=528/global_step=1320, RunningAvgSamplesPerSec=13.30395516698252, CurrSamplesPerSec=14.59412134465565, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
370
+ [2024-11-12 10:36:51,593] [INFO] [logging.py:96:log_dist] [Rank 0] step=1330, skipped=0, lr=[7.420144836233564e-06, 7.420144836233564e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
371
+ [2024-11-12 10:36:51,594] [INFO] [timer.py:264:stop] epoch=3/micro_step=548/global_step=1330, RunningAvgSamplesPerSec=13.287670368260292, CurrSamplesPerSec=13.454378683308082, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
372
+ [2024-11-12 10:37:41,488] [INFO] [logging.py:96:log_dist] [Rank 0] step=1340, skipped=0, lr=[7.096195292140173e-06, 7.096195292140173e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
373
+ [2024-11-12 10:37:41,489] [INFO] [timer.py:264:stop] epoch=3/micro_step=568/global_step=1340, RunningAvgSamplesPerSec=13.290967903672813, CurrSamplesPerSec=14.066296708353384, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
374
+ [2024-11-12 10:38:33,707] [INFO] [logging.py:96:log_dist] [Rank 0] step=1350, skipped=0, lr=[6.778302741448103e-06, 6.778302741448103e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
375
+ [2024-11-12 10:38:33,708] [INFO] [timer.py:264:stop] epoch=3/micro_step=588/global_step=1350, RunningAvgSamplesPerSec=13.288990599401712, CurrSamplesPerSec=13.881917988904403, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
376
+ [2024-11-12 10:39:22,740] [INFO] [logging.py:96:log_dist] [Rank 0] step=1360, skipped=0, lr=[6.46657472962679e-06, 6.46657472962679e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
377
+ [2024-11-12 10:39:22,741] [INFO] [timer.py:264:stop] epoch=3/micro_step=608/global_step=1360, RunningAvgSamplesPerSec=13.294280380468793, CurrSamplesPerSec=15.113685513207692, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
378
+ [2024-11-12 10:40:12,119] [INFO] [logging.py:96:log_dist] [Rank 0] step=1370, skipped=0, lr=[6.16111671663524e-06, 6.16111671663524e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
379
+ [2024-11-12 10:40:12,120] [INFO] [timer.py:264:stop] epoch=3/micro_step=628/global_step=1370, RunningAvgSamplesPerSec=13.297719201387292, CurrSamplesPerSec=12.79749975952529, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
380
+ [2024-11-12 10:41:02,355] [INFO] [logging.py:96:log_dist] [Rank 0] step=1380, skipped=0, lr=[5.8620320412441475e-06, 5.8620320412441475e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
381
+ [2024-11-12 10:41:02,356] [INFO] [timer.py:264:stop] epoch=3/micro_step=648/global_step=1380, RunningAvgSamplesPerSec=13.300767385841707, CurrSamplesPerSec=13.329218575481406, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
382
+ [2024-11-12 10:41:55,494] [INFO] [logging.py:96:log_dist] [Rank 0] step=1390, skipped=0, lr=[5.5694218860755625e-06, 5.5694218860755625e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
383
+ [2024-11-12 10:41:55,495] [INFO] [timer.py:264:stop] epoch=3/micro_step=668/global_step=1390, RunningAvgSamplesPerSec=13.29664256005591, CurrSamplesPerSec=14.290803571762657, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
384
+ [2024-11-12 10:42:45,379] [INFO] [logging.py:96:log_dist] [Rank 0] step=1400, skipped=0, lr=[5.2833852433720855e-06, 5.2833852433720855e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
385
+ [2024-11-12 10:42:45,380] [INFO] [timer.py:264:stop] epoch=3/micro_step=688/global_step=1400, RunningAvgSamplesPerSec=13.30080907047133, CurrSamplesPerSec=14.279848014465236, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
386
+ Saving checkpoint at step 2800 ...
387
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
388
+ Saving 16-bit model...
389
+ [2024-11-12 10:42:58,030] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1400 is about to be saved!
390
+ [2024-11-12 10:42:58,032] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin, tag: global_step1400
391
+ [2024-11-12 10:42:58,032] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin...
392
+ [2024-11-12 10:43:14,984] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin.
393
+ [2024-11-12 10:43:14,985] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1400 is ready now!
394
+ Model saved!
395
+ Saving 16-bit model...
396
+ [2024-11-12 10:43:25,392] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1400 is about to be saved!
397
+ [2024-11-12 10:43:25,393] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin, tag: global_step1400
398
+ [2024-11-12 10:43:25,393] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin...
399
+ [2024-11-12 10:43:48,363] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin.
400
+ [2024-11-12 10:43:48,364] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1400 is ready now!
401
+ Model saved!
402
+ Checkpoint saved.
403
+ [2024-11-12 10:44:40,329] [INFO] [logging.py:96:log_dist] [Rank 0] step=1410, skipped=0, lr=[5.004018881507016e-06, 5.004018881507016e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
404
+ [2024-11-12 10:44:40,330] [INFO] [timer.py:264:stop] epoch=4/micro_step=4/global_step=1410, RunningAvgSamplesPerSec=13.300082404758356, CurrSamplesPerSec=8.424868847641598, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
405
+ [2024-11-12 10:45:28,697] [INFO] [logging.py:96:log_dist] [Rank 0] step=1420, skipped=0, lr=[4.731417312246877e-06, 4.731417312246877e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
406
+ [2024-11-12 10:45:28,698] [INFO] [timer.py:264:stop] epoch=4/micro_step=24/global_step=1420, RunningAvgSamplesPerSec=13.306174854189013, CurrSamplesPerSec=14.387195950748813, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
407
+ [2024-11-12 10:46:16,858] [INFO] [logging.py:96:log_dist] [Rank 0] step=1430, skipped=0, lr=[4.465672758777351e-06, 4.465672758777351e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
408
+ [2024-11-12 10:46:16,859] [INFO] [timer.py:264:stop] epoch=4/micro_step=44/global_step=1430, RunningAvgSamplesPerSec=13.31298797950946, CurrSamplesPerSec=14.366896812177284, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
409
+ [2024-11-12 10:47:09,377] [INFO] [logging.py:96:log_dist] [Rank 0] step=1440, skipped=0, lr=[4.206875124503506e-06, 4.206875124503506e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
410
+ [2024-11-12 10:47:09,378] [INFO] [timer.py:264:stop] epoch=4/micro_step=64/global_step=1440, RunningAvgSamplesPerSec=13.311559372047915, CurrSamplesPerSec=13.393533075067023, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
411
+ [2024-11-12 10:48:00,280] [INFO] [logging.py:96:log_dist] [Rank 0] step=1450, skipped=0, lr=[3.9551119626347695e-06, 3.9551119626347695e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
412
+ [2024-11-12 10:48:00,280] [INFO] [timer.py:264:stop] epoch=4/micro_step=84/global_step=1450, RunningAvgSamplesPerSec=13.31244648639003, CurrSamplesPerSec=14.058757387182883, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
413
+ [2024-11-12 10:48:49,946] [INFO] [logging.py:96:log_dist] [Rank 0] step=1460, skipped=0, lr=[3.710468446565005e-06, 3.710468446565005e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
414
+ [2024-11-12 10:48:49,946] [INFO] [timer.py:264:stop] epoch=4/micro_step=104/global_step=1460, RunningAvgSamplesPerSec=13.315758552170506, CurrSamplesPerSec=13.839085443416847, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
415
+ [2024-11-12 10:49:40,534] [INFO] [logging.py:96:log_dist] [Rank 0] step=1470, skipped=0, lr=[3.4730273410577274e-06, 3.4730273410577274e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
416
+ [2024-11-12 10:49:40,535] [INFO] [timer.py:264:stop] epoch=4/micro_step=124/global_step=1470, RunningAvgSamplesPerSec=13.318408025641999, CurrSamplesPerSec=12.260271913621812, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
417
+ [2024-11-12 10:50:30,820] [INFO] [logging.py:96:log_dist] [Rank 0] step=1480, skipped=0, lr=[3.2428689742461188e-06, 3.2428689742461188e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
418
+ [2024-11-12 10:50:30,821] [INFO] [timer.py:264:stop] epoch=4/micro_step=144/global_step=1480, RunningAvgSamplesPerSec=13.320489932970325, CurrSamplesPerSec=15.042129663922514, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
419
+ [2024-11-12 10:51:20,889] [INFO] [logging.py:96:log_dist] [Rank 0] step=1490, skipped=0, lr=[3.020071210457451e-06, 3.020071210457451e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
420
+ [2024-11-12 10:51:20,890] [INFO] [timer.py:264:stop] epoch=4/micro_step=164/global_step=1490, RunningAvgSamplesPerSec=13.323469002580365, CurrSamplesPerSec=15.111296440696567, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
421
+ [2024-11-12 10:52:10,423] [INFO] [logging.py:96:log_dist] [Rank 0] step=1500, skipped=0, lr=[2.8047094238709633e-06, 2.8047094238709633e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
422
+ [2024-11-12 10:52:10,424] [INFO] [timer.py:264:stop] epoch=4/micro_step=184/global_step=1500, RunningAvgSamplesPerSec=13.32754448705049, CurrSamplesPerSec=14.979724690127203, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
423
+ [2024-11-12 10:53:03,486] [INFO] [logging.py:96:log_dist] [Rank 0] step=1510, skipped=0, lr=[2.5968564730182083e-06, 2.5968564730182083e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
424
+ [2024-11-12 10:53:03,487] [INFO] [timer.py:264:stop] epoch=4/micro_step=204/global_step=1510, RunningAvgSamplesPerSec=13.324782544632754, CurrSamplesPerSec=14.093607165156875, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
425
+ [2024-11-12 10:53:57,964] [INFO] [logging.py:96:log_dist] [Rank 0] step=1520, skipped=0, lr=[2.396582676134462e-06, 2.396582676134462e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
426
+ [2024-11-12 10:53:57,965] [INFO] [timer.py:264:stop] epoch=4/micro_step=224/global_step=1520, RunningAvgSamplesPerSec=13.318995411927387, CurrSamplesPerSec=13.063804189396569, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
427
+ [2024-11-12 10:54:47,711] [INFO] [logging.py:96:log_dist] [Rank 0] step=1530, skipped=0, lr=[2.2039557873695188e-06, 2.2039557873695188e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
428
+ [2024-11-12 10:54:47,711] [INFO] [timer.py:264:stop] epoch=4/micro_step=244/global_step=1530, RunningAvgSamplesPerSec=13.321911830869968, CurrSamplesPerSec=14.915771367301977, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
429
+ [2024-11-12 10:55:42,069] [INFO] [logging.py:96:log_dist] [Rank 0] step=1540, skipped=0, lr=[2.0190409738659653e-06, 2.0190409738659653e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
430
+ [2024-11-12 10:55:42,070] [INFO] [timer.py:264:stop] epoch=4/micro_step=264/global_step=1540, RunningAvgSamplesPerSec=13.316215176151665, CurrSamplesPerSec=13.612489775121794, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
431
+ [2024-11-12 10:56:33,542] [INFO] [logging.py:96:log_dist] [Rank 0] step=1550, skipped=0, lr=[1.8419007937126255e-06, 1.8419007937126255e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
432
+ [2024-11-12 10:56:33,543] [INFO] [timer.py:264:stop] epoch=4/micro_step=284/global_step=1550, RunningAvgSamplesPerSec=13.31579176262506, CurrSamplesPerSec=11.938031197480003, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
433
+ [2024-11-12 10:57:28,157] [INFO] [logging.py:96:log_dist] [Rank 0] step=1560, skipped=0, lr=[1.6725951747806918e-06, 1.6725951747806918e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
434
+ [2024-11-12 10:57:28,158] [INFO] [timer.py:264:stop] epoch=4/micro_step=304/global_step=1560, RunningAvgSamplesPerSec=13.309479512988279, CurrSamplesPerSec=14.577624486192173, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
435
+ [2024-11-12 10:58:18,721] [INFO] [logging.py:96:log_dist] [Rank 0] step=1570, skipped=0, lr=[1.5111813944496539e-06, 1.5111813944496539e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
436
+ [2024-11-12 10:58:18,722] [INFO] [timer.py:264:stop] epoch=4/micro_step=324/global_step=1570, RunningAvgSamplesPerSec=13.310390185432857, CurrSamplesPerSec=12.173278775829349, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
437
+ [2024-11-12 10:59:11,560] [INFO] [logging.py:96:log_dist] [Rank 0] step=1580, skipped=0, lr=[1.3577140602299448e-06, 1.3577140602299448e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
438
+ [2024-11-12 10:59:11,561] [INFO] [timer.py:264:stop] epoch=4/micro_step=344/global_step=1580, RunningAvgSamplesPerSec=13.307910946447468, CurrSamplesPerSec=11.816768390320345, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
439
+ [2024-11-12 11:00:02,570] [INFO] [logging.py:96:log_dist] [Rank 0] step=1590, skipped=0, lr=[1.212245091288794e-06, 1.212245091288794e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
440
+ [2024-11-12 11:00:02,571] [INFO] [timer.py:264:stop] epoch=4/micro_step=364/global_step=1590, RunningAvgSamplesPerSec=13.309121542768606, CurrSamplesPerSec=12.719856643237279, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
441
+ [2024-11-12 11:00:54,502] [INFO] [logging.py:96:log_dist] [Rank 0] step=1600, skipped=0, lr=[1.0748237008855828e-06, 1.0748237008855828e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
442
+ [2024-11-12 11:00:54,503] [INFO] [timer.py:264:stop] epoch=4/micro_step=384/global_step=1600, RunningAvgSamplesPerSec=13.308074722292908, CurrSamplesPerSec=15.261070434905404, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
443
+ Saving checkpoint at step 3200 ...
444
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
445
+ Saving 16-bit model...
446
+ [2024-11-12 11:01:09,427] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1600 is about to be saved!
447
+ [2024-11-12 11:01:09,429] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin, tag: global_step1600
448
+ [2024-11-12 11:01:09,429] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin...
449
+ [2024-11-12 11:01:30,549] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin.
450
+ [2024-11-12 11:01:30,551] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1600 is ready now!
451
+ Model saved!
452
+ Saving 16-bit model...
453
+ [2024-11-12 11:01:41,122] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1600 is about to be saved!
454
+ [2024-11-12 11:01:41,123] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin, tag: global_step1600
455
+ [2024-11-12 11:01:41,123] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin...
456
+ [2024-11-12 11:02:03,626] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin.
457
+ [2024-11-12 11:02:03,627] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1600 is ready now!
458
+ Model saved!
459
+ Checkpoint saved.
460
+ [2024-11-12 11:02:53,971] [INFO] [logging.py:96:log_dist] [Rank 0] step=1610, skipped=0, lr=[9.45496379722624e-07, 9.45496379722624e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
461
+ [2024-11-12 11:02:53,972] [INFO] [timer.py:264:stop] epoch=4/micro_step=404/global_step=1610, RunningAvgSamplesPerSec=13.309905592491026, CurrSamplesPerSec=14.748584036364806, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
462
+ [2024-11-12 11:03:46,152] [INFO] [logging.py:96:log_dist] [Rank 0] step=1620, skipped=0, lr=[8.243068802169906e-07, 8.243068802169906e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
463
+ [2024-11-12 11:03:46,153] [INFO] [timer.py:264:stop] epoch=4/micro_step=424/global_step=1620, RunningAvgSamplesPerSec=13.308079127511396, CurrSamplesPerSec=8.70323026519259, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
464
+ [2024-11-12 11:04:36,640] [INFO] [logging.py:96:log_dist] [Rank 0] step=1630, skipped=0, lr=[7.112962016987634e-07, 7.112962016987634e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
465
+ [2024-11-12 11:04:36,641] [INFO] [timer.py:264:stop] epoch=4/micro_step=444/global_step=1630, RunningAvgSamplesPerSec=13.309797745853881, CurrSamplesPerSec=13.608449292653019, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
466
+ [2024-11-12 11:05:27,327] [INFO] [logging.py:96:log_dist] [Rank 0] step=1640, skipped=0, lr=[6.065025765406213e-07, 6.065025765406213e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
467
+ [2024-11-12 11:05:27,328] [INFO] [timer.py:264:stop] epoch=4/micro_step=464/global_step=1640, RunningAvgSamplesPerSec=13.310743561152151, CurrSamplesPerSec=14.022836300689034, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
468
+ [2024-11-12 11:06:23,362] [INFO] [logging.py:96:log_dist] [Rank 0] step=1650, skipped=0, lr=[5.099614572235622e-07, 5.099614572235622e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
469
+ [2024-11-12 11:06:23,363] [INFO] [timer.py:264:stop] epoch=4/micro_step=484/global_step=1650, RunningAvgSamplesPerSec=13.303107401732731, CurrSamplesPerSec=11.270466117917143, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
470
+ [2024-11-12 11:07:14,043] [INFO] [logging.py:96:log_dist] [Rank 0] step=1660, skipped=0, lr=[4.217055043430368e-07, 4.217055043430368e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
471
+ [2024-11-12 11:07:14,044] [INFO] [timer.py:264:stop] epoch=4/micro_step=504/global_step=1660, RunningAvgSamplesPerSec=13.303911694244524, CurrSamplesPerSec=12.311078842729271, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
472
+ [2024-11-12 11:08:04,940] [INFO] [logging.py:96:log_dist] [Rank 0] step=1670, skipped=0, lr=[3.417645755596333e-07, 3.417645755596333e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
473
+ [2024-11-12 11:08:04,941] [INFO] [timer.py:264:stop] epoch=4/micro_step=524/global_step=1670, RunningAvgSamplesPerSec=13.30527924084017, CurrSamplesPerSec=12.844344875845103, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
474
+ [2024-11-12 11:09:03,539] [INFO] [logging.py:96:log_dist] [Rank 0] step=1680, skipped=0, lr=[2.7016571549800806e-07, 2.7016571549800806e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
475
+ [2024-11-12 11:09:03,540] [INFO] [timer.py:264:stop] epoch=4/micro_step=544/global_step=1680, RunningAvgSamplesPerSec=13.292778473186681, CurrSamplesPerSec=13.616692212560867, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
476
+ [2024-11-12 11:09:53,417] [INFO] [logging.py:96:log_dist] [Rank 0] step=1690, skipped=0, lr=[2.0693314659746278e-07, 2.0693314659746278e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
477
+ [2024-11-12 11:09:53,417] [INFO] [timer.py:264:stop] epoch=4/micro_step=564/global_step=1690, RunningAvgSamplesPerSec=13.295568640371517, CurrSamplesPerSec=15.061971663296424, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
478
+ [2024-11-12 11:10:46,532] [INFO] [logging.py:96:log_dist] [Rank 0] step=1700, skipped=0, lr=[1.520882609173352e-07, 1.520882609173352e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
479
+ [2024-11-12 11:10:46,533] [INFO] [timer.py:264:stop] epoch=4/micro_step=584/global_step=1700, RunningAvgSamplesPerSec=13.293118088403272, CurrSamplesPerSec=12.895263576244139, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
480
+ [2024-11-12 11:11:35,590] [INFO] [logging.py:96:log_dist] [Rank 0] step=1710, skipped=0, lr=[1.056496128998774e-07, 1.056496128998774e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
481
+ [2024-11-12 11:11:35,591] [INFO] [timer.py:264:stop] epoch=4/micro_step=604/global_step=1710, RunningAvgSamplesPerSec=13.297323115977306, CurrSamplesPerSec=12.737831983944682, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
482
+ [2024-11-12 11:12:24,828] [INFO] [logging.py:96:log_dist] [Rank 0] step=1720, skipped=0, lr=[6.763291309317132e-08, 6.763291309317132e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
483
+ [2024-11-12 11:12:24,829] [INFO] [timer.py:264:stop] epoch=4/micro_step=624/global_step=1720, RunningAvgSamplesPerSec=13.300588368547286, CurrSamplesPerSec=14.138239247081426, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
484
+ [2024-11-12 11:13:15,294] [INFO] [logging.py:96:log_dist] [Rank 0] step=1730, skipped=0, lr=[3.8051022836110836e-08, 3.8051022836110836e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
485
+ [2024-11-12 11:13:15,295] [INFO] [timer.py:264:stop] epoch=4/micro_step=644/global_step=1730, RunningAvgSamplesPerSec=13.302046602956041, CurrSamplesPerSec=13.59455125768058, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
486
+ [2024-11-12 11:14:08,410] [INFO] [logging.py:96:log_dist] [Rank 0] step=1740, skipped=0, lr=[1.6913949907321204e-08, 1.6913949907321204e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
487
+ [2024-11-12 11:14:08,411] [INFO] [timer.py:264:stop] epoch=4/micro_step=664/global_step=1740, RunningAvgSamplesPerSec=13.300620381888876, CurrSamplesPerSec=14.191562012478505, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
488
+ [2024-11-12 11:14:59,106] [INFO] [logging.py:96:log_dist] [Rank 0] step=1750, skipped=0, lr=[4.2288451394562055e-09, 4.2288451394562055e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
489
+ [2024-11-12 11:14:59,107] [INFO] [timer.py:264:stop] epoch=4/micro_step=684/global_step=1750, RunningAvgSamplesPerSec=13.302238401350479, CurrSamplesPerSec=13.361762988342704, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
490
+ [2024-11-12 11:15:48,289] [INFO] [logging.py:96:log_dist] [Rank 0] step=1760, skipped=0, lr=[0.0, 0.0], mom=[[0.9, 0.95], [0.9, 0.95]]
491
+ [2024-11-12 11:15:48,289] [INFO] [timer.py:264:stop] epoch=4/micro_step=704/global_step=1760, RunningAvgSamplesPerSec=13.30676923911034, CurrSamplesPerSec=17.907194447352197, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
492
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
493
+ Saving 16-bit model...
494
+ [2024-11-12 11:16:03,390] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1760 is about to be saved!
495
+ [2024-11-12 11:16:03,391] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin, tag: global_step1760
496
+ [2024-11-12 11:16:03,392] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin...
497
+ [2024-11-12 11:16:19,447] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin.
498
+ [2024-11-12 11:16:19,448] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1760 is ready now!
499
+ Model saved!
500
+ Saving 16-bit model...
501
+ [2024-11-12 11:16:27,719] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1760 is about to be saved!
502
+ [2024-11-12 11:16:27,720] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin, tag: global_step1760
503
+ [2024-11-12 11:16:27,721] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin...
504
+ [2024-11-12 11:16:51,906] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin.
505
+ [2024-11-12 11:16:51,908] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1760 is ready now!
506
+ Model saved!
slice_1200/wandb/run-20241112_083557-komthdmh/files/requirements.txt ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ align-anything==0.0.1.dev0
2
+ nvidia-cusolver-cu12==11.4.5.107
3
+ nvidia-curand-cu12==10.3.2.106
4
+ wcwidth==0.2.13
5
+ nvidia-cuda-nvrtc-cu12==12.1.105
6
+ pycparser==2.22
7
+ tokenizers==0.19.1
8
+ grpcio==1.66.2
9
+ joblib==1.4.2
10
+ virtualenv==20.26.6
11
+ pyzmq==26.2.0
12
+ cffi==1.17.1
13
+ sentencepiece==0.2.0
14
+ nvidia-nvtx-cu12==12.1.105
15
+ docker-pycreds==0.4.0
16
+ outlines==0.1.1.dev4+ga2fd35c
17
+ nvidia-nvjitlink-cu12==12.6.77
18
+ annotated-types==0.7.0
19
+ certifi==2024.8.30
20
+ interegular==0.3.3
21
+ Jinja2==3.1.4
22
+ Brotli==1.1.0
23
+ fairscale==0.4.13
24
+ gradio_client==1.4.0
25
+ opencv-python==4.6.0.66
26
+ pyarrow==17.0.0
27
+ absl-py==2.1.0
28
+ lm-format-enforcer==0.10.6
29
+ pydantic_core==2.23.4
30
+ llvmlite==0.43.0
31
+ accelerate==1.0.1
32
+ pytest-split==0.8.0
33
+ nvidia-cuda-cupti-cu12==12.1.105
34
+ watchfiles==0.24.0
35
+ optree==0.13.0
36
+ py-cpuinfo==9.0.0
37
+ scikit-learn==1.5.2
38
+ ftfy==6.3.0
39
+ fastapi==0.115.0
40
+ psutil==6.0.0
41
+ MarkupSafe==2.1.5
42
+ nvidia-cublas-cu12==12.1.3.1
43
+ pip==24.2
44
+ websockets==12.0
45
+ tomlkit==0.12.0
46
+ torchaudio==2.4.0
47
+ huggingface-hub==0.25.2
48
+ mistral_common==1.4.4
49
+ image-reward==1.5
50
+ pyparsing==3.1.4
51
+ aiohappyeyeballs==2.4.3
52
+ click==8.1.7
53
+ httptools==0.6.1
54
+ decorator==4.4.2
55
+ tqdm==4.66.5
56
+ fonttools==4.54.1
57
+ kiwisolver==1.4.7
58
+ ruff==0.6.9
59
+ openai==1.51.2
60
+ partial-json-parser==0.2.1.1.post4
61
+ xformers==0.0.27.post2
62
+ distlib==0.3.9
63
+ GitPython==3.1.43
64
+ pytest==7.2.0
65
+ imageio==2.35.1
66
+ msgspec==0.18.6
67
+ proglog==0.1.10
68
+ yarl==1.15.0
69
+ markdown-it-py==3.0.0
70
+ PyYAML==6.0.2
71
+ xxhash==3.5.0
72
+ braceexpand==0.1.7
73
+ datasets==3.0.1
74
+ mpmath==1.3.0
75
+ distro==1.9.0
76
+ term-image==0.7.2
77
+ python-dotenv==1.0.1
78
+ semantic-version==2.10.0
79
+ multidict==6.1.0
80
+ vllm==0.6.2
81
+ sentry-sdk==2.16.0
82
+ idna==3.10
83
+ starlette==0.38.6
84
+ args==0.1.0
85
+ peft==0.13.2
86
+ librosa==0.10.2.post1
87
+ urllib3==2.2.3
88
+ python-dateutil==2.9.0.post0
89
+ pycountry==24.6.1
90
+ six==1.16.0
91
+ ffmpy==0.4.0
92
+ nvitop==1.3.2
93
+ termcolor==2.5.0
94
+ multiprocess==0.70.16
95
+ cycler==0.12.1
96
+ charset-normalizer==3.4.0
97
+ aiofiles==23.2.1
98
+ shellingham==1.5.4
99
+ propcache==0.2.0
100
+ lark==1.2.2
101
+ torch==2.4.0
102
+ Werkzeug==3.0.4
103
+ nvidia-cusparse-cu12==12.1.0.106
104
+ clip==0.2.0
105
+ hjson==3.1.0
106
+ diffusers==0.30.3
107
+ attrs==24.2.0
108
+ lazy_loader==0.4
109
+ numpy==1.26.4
110
+ rpds-py==0.20.0
111
+ pytz==2024.2
112
+ audioread==3.0.1
113
+ platformdirs==4.3.6
114
+ deepspeed==0.15.2
115
+ gguf==0.10.0
116
+ wandb==0.18.3
117
+ prometheus_client==0.21.0
118
+ gitdb==4.0.11
119
+ packaging==24.1
120
+ cachetools==5.5.0
121
+ sympy==1.13.3
122
+ mutagen==1.47.0
123
+ contourpy==1.3.0
124
+ pluggy==1.5.0
125
+ python-multipart==0.0.12
126
+ soundfile==0.12.1
127
+ typer==0.12.5
128
+ timm==0.6.13
129
+ frozenlist==1.4.1
130
+ httpx==0.27.2
131
+ mmsg==0.1.dev20+g585c63a.d20241012
132
+ tiktoken==0.7.0
133
+ pydub==0.25.1
134
+ diskcache==5.6.3
135
+ einops==0.8.0
136
+ setproctitle==1.3.3
137
+ scipy==1.14.1
138
+ typing_extensions==4.12.2
139
+ httpcore==1.0.6
140
+ cfgv==3.4.0
141
+ requests==2.32.3
142
+ torchlibrosa==0.1.0
143
+ pydantic==2.9.2
144
+ torchvision==0.19.0
145
+ sniffio==1.3.1
146
+ pyairports==2.1.1
147
+ hpsv2==1.2.0
148
+ protobuf==3.20.3
149
+ wheel==0.44.0
150
+ smmap==5.0.1
151
+ zipp==3.20.2
152
+ iniconfig==2.0.0
153
+ airportsdata==20241001
154
+ clint==0.5.1
155
+ pooch==1.8.2
156
+ shortuuid==1.0.13
157
+ pycryptodomex==3.21.0
158
+ cloudpickle==3.1.0
159
+ transformers==4.44.0.dev0
160
+ regex==2024.9.11
161
+ numba==0.60.0
162
+ tzdata==2024.2
163
+ orjson==3.10.7
164
+ jsonschema-specifications==2024.10.1
165
+ safetensors==0.4.5
166
+ outlines_core==0.1.0
167
+ filelock==3.16.1
168
+ threadpoolctl==3.5.0
169
+ soxr==0.5.0.post1
170
+ nvidia-cufft-cu12==11.0.2.54
171
+ networkx==3.4.1
172
+ msgpack==1.1.0
173
+ pandas==2.2.3
174
+ align-anything==0.0.1.dev0
175
+ anyio==4.6.0
176
+ nvidia-cuda-runtime-cu12==12.1.105
177
+ bitsandbytes==0.44.1
178
+ aiohttp==3.10.10
179
+ matplotlib==3.9.2
180
+ triton==3.0.0
181
+ tensorboard==2.18.0
182
+ nodeenv==1.9.1
183
+ fsspec==2024.6.1
184
+ webdataset==0.2.100
185
+ imageio-ffmpeg==0.5.1
186
+ mdurl==0.1.2
187
+ identify==2.6.1
188
+ h11==0.14.0
189
+ uvloop==0.20.0
190
+ rich==13.9.2
191
+ frechet-audio-distance==0.1.2
192
+ uvicorn==0.31.1
193
+ pytorch-fid==0.3.0
194
+ yt-dlp==2024.8.6
195
+ jiter==0.6.1
196
+ nest-asyncio==1.6.0
197
+ pre_commit==4.0.1
198
+ referencing==0.35.1
199
+ resampy==0.4.3
200
+ tensorboard-data-server==0.7.2
201
+ importlib_metadata==8.5.0
202
+ aiosignal==1.3.1
203
+ dill==0.3.8
204
+ prometheus-fastapi-instrumentator==7.0.0
205
+ nvidia-ml-py==12.535.161
206
+ ninja==1.11.1.1
207
+ moviepy==1.0.3
208
+ nvidia-cudnn-cu12==9.1.0.70
209
+ Markdown==3.7
210
+ ray==2.37.0
211
+ gradio==5.0.2
212
+ jsonschema==4.23.0
213
+ Pygments==2.18.0
214
+ nvidia-nccl-cu12==2.20.5
215
+ pillow==10.4.0
216
+ setuptools==75.1.0
217
+ jaraco.text==3.12.1
218
+ inflect==7.3.1
219
+ jaraco.collections==5.1.0
220
+ autocommand==2.2.2
221
+ tomli==2.0.1
222
+ jaraco.context==5.3.0
223
+ jaraco.functools==4.0.1
224
+ importlib_resources==6.4.0
225
+ wheel==0.43.0
226
+ packaging==24.1
227
+ backports.tarfile==1.2.0
228
+ importlib_metadata==8.0.0
229
+ typing_extensions==4.12.2
230
+ zipp==3.19.2
231
+ typeguard==4.3.0
232
+ more-itertools==10.3.0
233
+ platformdirs==4.2.2
slice_1200/wandb/run-20241112_083557-komthdmh/files/wandb-metadata.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-198-generic-x86_64-with-glibc2.31",
3
+ "python": "3.11.10",
4
+ "startedAt": "2024-11-12T08:35:57.093246Z",
5
+ "args": [
6
+ "--local_rank=0",
7
+ "--model_name_or_path",
8
+ "/data/align-anything/hantao/models/0917_cham_ti2t_sft",
9
+ "--train_datasets",
10
+ "/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs",
11
+ "--train_data_files",
12
+ "llf_t2i_22.5k_tokenized.pt",
13
+ "--output_dir",
14
+ "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k",
15
+ "--train_template",
16
+ "AA_textfeedback",
17
+ "--train_split",
18
+ "train",
19
+ "--per_device_train_batch_size",
20
+ "4",
21
+ "--per_device_eval_batch_size",
22
+ "4",
23
+ "--gradient_accumulation_steps",
24
+ "2",
25
+ "--save_interval",
26
+ "400",
27
+ "--learning_rate",
28
+ "5e-5",
29
+ "--epochs",
30
+ "5",
31
+ "--lr_scheduler_type",
32
+ "cosine"
33
+ ],
34
+ "program": "-m align_anything.trainers.text_image_to_text_image.sft",
35
+ "git": {
36
+ "remote": "https://github.com/PKU-Alignment/align-anything.git",
37
+ "commit": "6fde660afc9985323f147930eedf188a5699adc7"
38
+ },
39
+ "email": "[email protected]",
40
+ "root": "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k",
41
+ "host": "lyg0196",
42
+ "username": "align-anything",
43
+ "executable": "/home/align-anything/miniconda3/envs/hantao_cham/bin/python",
44
+ "cpu_count": 64,
45
+ "cpu_count_logical": 128,
46
+ "gpu": "[NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB]",
47
+ "gpu_count": 8,
48
+ "disk": {
49
+ "/": {
50
+ "total": "940744544256",
51
+ "used": "333241483264"
52
+ }
53
+ },
54
+ "memory": {
55
+ "total": "540647530496"
56
+ },
57
+ "cpu": {
58
+ "count": 64,
59
+ "countLogical": 128
60
+ },
61
+ "gpu_nvidia": [
62
+ {
63
+ "name": "NVIDIA A100-SXM4-80GB",
64
+ "memoryTotal": "85899345920",
65
+ "cudaCores": 6912,
66
+ "architecture": "Ampere"
67
+ },
68
+ {
69
+ "name": "NVIDIA A100-SXM4-80GB",
70
+ "memoryTotal": "85899345920",
71
+ "cudaCores": 6912,
72
+ "architecture": "Ampere"
73
+ },
74
+ {
75
+ "name": "NVIDIA A100-SXM4-80GB",
76
+ "memoryTotal": "85899345920",
77
+ "cudaCores": 6912,
78
+ "architecture": "Ampere"
79
+ },
80
+ {
81
+ "name": "NVIDIA A100-SXM4-80GB",
82
+ "memoryTotal": "85899345920",
83
+ "cudaCores": 6912,
84
+ "architecture": "Ampere"
85
+ },
86
+ {
87
+ "name": "NVIDIA A100-SXM4-80GB",
88
+ "memoryTotal": "85899345920",
89
+ "cudaCores": 6912,
90
+ "architecture": "Ampere"
91
+ },
92
+ {
93
+ "name": "NVIDIA A100-SXM4-80GB",
94
+ "memoryTotal": "85899345920",
95
+ "cudaCores": 6912,
96
+ "architecture": "Ampere"
97
+ },
98
+ {
99
+ "name": "NVIDIA A100-SXM4-80GB",
100
+ "memoryTotal": "85899345920",
101
+ "cudaCores": 6912,
102
+ "architecture": "Ampere"
103
+ },
104
+ {
105
+ "name": "NVIDIA A100-SXM4-80GB",
106
+ "memoryTotal": "85899345920",
107
+ "cudaCores": 6912,
108
+ "architecture": "Ampere"
109
+ }
110
+ ],
111
+ "cudaVersion": "12.4"
112
+ }
slice_1200/wandb/run-20241112_083557-komthdmh/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_step":3520,"train/step":3520,"train/loss":0.007134987972676754,"train/lr":0,"train/epoch":5,"_timestamp":1.731410148381941e+09,"_runtime":9655.892149045,"_wandb":{"runtime":9655}}
slice_1200/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-12T08:35:57.099449248Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-12T08:35:57.099478068Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-core.log"}
3
+ {"time":"2024-11-12T08:35:57.102383132Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-12T08:35:57.129619469Z","level":"INFO","msg":"created new stream","id":"komthdmh"}
5
+ {"time":"2024-11-12T08:35:57.129670069Z","level":"INFO","msg":"stream: started","id":"komthdmh"}
6
+ {"time":"2024-11-12T08:35:57.129709953Z","level":"INFO","msg":"handler: started","stream_id":{"value":"komthdmh"}}
7
+ {"time":"2024-11-12T08:35:57.129693059Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"komthdmh"}}
8
+ {"time":"2024-11-12T08:35:57.129699692Z","level":"INFO","msg":"sender: started","stream_id":{"value":"komthdmh"}}
9
+ {"time":"2024-11-12T08:35:57.832569737Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-12T08:35:57.837905637Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-12T11:16:52.98542237Z","level":"INFO","msg":"Stopping system monitor"}
12
+ {"time":"2024-11-12T11:16:53.007088373Z","level":"INFO","msg":"Stopped system monitor"}
13
+ {"time":"2024-11-12T11:16:53.498790149Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
14
+ {"time":"2024-11-12T11:16:53.49883056Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
15
+ {"time":"2024-11-12T11:16:54.435678748Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
16
+ {"time":"2024-11-12T11:16:55.889045635Z","level":"INFO","msg":"stream: closing","id":"komthdmh"}
17
+ {"time":"2024-11-12T11:16:55.889082565Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"komthdmh"}}
18
+ {"time":"2024-11-12T11:16:55.889144434Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"komthdmh"}}
19
+ {"time":"2024-11-12T11:16:55.889239107Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"komthdmh"}}
20
+ {"time":"2024-11-12T11:16:55.892207197Z","level":"INFO","msg":"stream: closed","id":"komthdmh"}
slice_1200/wandb/run-20241112_083557-komthdmh/logs/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Configure stats pid to 1335408
3
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-12 08:35:57,085 WARNING MainThread:1335408 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug.log
11
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log
12
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 5.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 5e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_t2i_22.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1015_5ep_t2i_22.5k', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0917_cham_ti2t_sft', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():667] starting backend
16
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-12 08:35:57,097 INFO MainThread:1335408 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-12 08:35:57,113 INFO MainThread:1335408 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-12 08:35:57,827 INFO MainThread:1335408 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-12 08:35:57,973 INFO MainThread:1335408 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-12 08:35:57,976 INFO MainThread:1335408 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-12 11:16:52,981 INFO MainThread:1335408 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/komthdmh
28
+ 2024-11-12 11:16:52,983 INFO MainThread:1335408 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-12 11:16:55,871 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-12 11:16:55,873 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-12 11:16:55,884 INFO MainThread:1335408 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_1200/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:561218f9100c1d380fac9397893e89b2567c6f999883a5cac5456da1d5c76d97
3
+ size 13892810
slice_1600/arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_t2i_22.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 5.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 5.0e-05
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
slice_1600/config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1600/environ.txt ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CONDA_DEFAULT_ENV=hantao_cham
2
+ CONDA_EXE=/home/align-anything/miniconda3/bin/conda
3
+ CONDA_PREFIX=/home/align-anything/miniconda3/envs/hantao_cham
4
+ CONDA_PREFIX_1=/home/align-anything/miniconda3
5
+ CONDA_PROMPT_MODIFIER=(hantao_cham)
6
+ CONDA_PYTHON_EXE=/home/align-anything/miniconda3/bin/python
7
+ CONDA_SHLVL=2
8
+ CROSS_RANK=0
9
+ CROSS_SIZE=1
10
+ CUDA_MODULE_LOADING=LAZY
11
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
12
+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
13
+ HOME=/home/align-anything
14
+ LANG=en_US.UTF-8
15
+ LD_LIBRARY_PATH=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/../../lib64:
16
+ LESSCLOSE=/usr/bin/lesspipe %s %s
17
+ LESSOPEN=| /usr/bin/lesspipe %s
18
+ LOCAL_RANK=0
19
+ LOCAL_SIZE=8
20
+ LOGLEVEL=WARNING
21
+ LOGNAME=align-anything
22
+ LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
23
+ MASTER_ADDR=127.0.0.1
24
+ MASTER_PORT=61417
25
+ MOTD_SHOWN=pam
26
+ OLDPWD=/home/align-anything
27
+ PATH=/home/align-anything/miniconda3/envs/hantao_cham/bin:/home/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
28
+ PWD=/data/align-anything/hantao/align-anything/scripts
29
+ PYGAME_HIDE_SUPPORT_PROMPT=1
30
+ PYTHONHASHSEED=42
31
+ PYTHONPATH=/data/align-anything/hantao/align-anything
32
+ QT_QPA_FONTDIR=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/fonts
33
+ QT_QPA_PLATFORM_PLUGIN_PATH=/home/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/plugins
34
+ RANK=0
35
+ SHELL=/bin/bash
36
+ SHLVL=3
37
+ SSH_CLIENT=111.205.230.212 2817 30600
38
+ SSH_CONNECTION=111.205.230.212 9549 10.10.212.196 30600
39
+ SSH_TTY=/dev/pts/2
40
+ TERM=screen
41
+ TMUX=/tmp//tmux-2000/default,3379,12
42
+ TMUX_PANE=%12
43
+ TRITON_CACHE_DIR=/home/align-anything/cache/triton
44
+ USER=align-anything
45
+ WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
46
+ WANDB_SERVICE=2-1335408-tcp-localhost-40327
47
+ WORLD_SIZE=8
48
+ XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
49
+ XDG_RUNTIME_DIR=/run/user/2000
50
+ XDG_SESSION_CLASS=user
51
+ XDG_SESSION_ID=4
52
+ XDG_SESSION_TYPE=tty
53
+ _=/home/align-anything/miniconda3/envs/hantao_cham/bin/deepspeed
54
+ _CE_CONDA=
55
+ _CE_M=
slice_1600/preprocessor_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 512,
4
+ "width": 512
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 1.0,
13
+ 1.0,
14
+ 1.0
15
+ ],
16
+ "image_processor_type": "ChameleonImageProcessor",
17
+ "image_std": [
18
+ 1.0,
19
+ 1.0,
20
+ 1.0
21
+ ],
22
+ "processor_class": "ChameleonProcessor",
23
+ "resample": 1,
24
+ "rescale_factor": 0.0078,
25
+ "size": {
26
+ "shortest_edge": 512
27
+ }
28
+ }
slice_1600/processor_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "image_seq_length": 1024,
3
+ "image_token": "<image>",
4
+ "processor_class": "ChameleonProcessor"
5
+ }
slice_1600/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b411b0855f26644f160f55524e35f48ebcc86cb380b0520ea7c9ce49d06f408
3
+ size 14165013510
slice_1600/script.sh ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Copyright 2024 PKU-Alignment Team. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ # ==============================================================================
17
+
18
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
19
+ # You can replace it with a local model path
20
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0917_cham_ti2t_sft"
21
+ # You can replace it with a local dataset path
22
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
23
+ TRAIN_DATA_FILES="llf_t2i_22.5k_tokenized.pt"
24
+ # You can replace it with a new path
25
+ OUTPUT_DIR="../outputs/sft_tf_cham_1015_5ep_t2i_22.5k"
26
+ # For wandb online logging
27
+
28
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
29
+ # Source the setup script
30
+ source ./setup.sh
31
+
32
+ # sleep 80m
33
+ # Execute deepspeed command
34
+ deepspeed \
35
+ --master_port ${MASTER_PORT} \
36
+ --module align_anything.trainers.text_image_to_text_image.sft \
37
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
38
+ --train_datasets ${TRAIN_DATASETS} \
39
+ --train_data_files ${TRAIN_DATA_FILES} \
40
+ --output_dir ${OUTPUT_DIR} \
41
+ --train_template AA_textfeedback \
42
+ --train_split train \
43
+ --per_device_train_batch_size 4 \
44
+ --per_device_eval_batch_size 4 \
45
+ --gradient_accumulation_steps 2 \
46
+ --save_interval 400 \
47
+ --learning_rate 5e-5 \
48
+ --epochs 5 \
49
+ --lr_scheduler_type cosine
50
+
51
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
52
+ # You can replace it with a local model path
53
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
54
+ # You can replace it with a local dataset path
55
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
56
+ TRAIN_DATA_FILES="llf_ti2ti_20k_tokenized.pt"
57
+ # You can replace it with a new path
58
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_20k_ti2ti"
59
+ # For wandb online logging
60
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
61
+ # Source the setup script
62
+ source ./setup.sh
63
+
64
+ # Execute deepspeed command
65
+ deepspeed \
66
+ --master_port ${MASTER_PORT} \
67
+ --module align_anything.trainers.text_image_to_text_image.sft \
68
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
69
+ --train_datasets ${TRAIN_DATASETS} \
70
+ --train_data_files ${TRAIN_DATA_FILES} \
71
+ --output_dir ${OUTPUT_DIR} \
72
+ --train_template AA_textfeedback \
73
+ --train_split train \
74
+ --per_device_train_batch_size 4 \
75
+ --per_device_eval_batch_size 4 \
76
+ --gradient_accumulation_steps 2 \
77
+ --save_interval 400 \
78
+ --learning_rate 1e-6 \
79
+ --epochs 3 \
80
+ --lr_scheduler_type cosine
slice_1600/special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<pad>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "<reserved08706>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
slice_1600/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1600/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1600/wandb/debug-internal.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-12T08:35:57.099449248Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-12T08:35:57.099478068Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-core.log"}
3
+ {"time":"2024-11-12T08:35:57.102383132Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-12T08:35:57.129619469Z","level":"INFO","msg":"created new stream","id":"komthdmh"}
5
+ {"time":"2024-11-12T08:35:57.129670069Z","level":"INFO","msg":"stream: started","id":"komthdmh"}
6
+ {"time":"2024-11-12T08:35:57.129709953Z","level":"INFO","msg":"handler: started","stream_id":{"value":"komthdmh"}}
7
+ {"time":"2024-11-12T08:35:57.129693059Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"komthdmh"}}
8
+ {"time":"2024-11-12T08:35:57.129699692Z","level":"INFO","msg":"sender: started","stream_id":{"value":"komthdmh"}}
9
+ {"time":"2024-11-12T08:35:57.832569737Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-12T08:35:57.837905637Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-12T11:16:52.98542237Z","level":"INFO","msg":"Stopping system monitor"}
12
+ {"time":"2024-11-12T11:16:53.007088373Z","level":"INFO","msg":"Stopped system monitor"}
13
+ {"time":"2024-11-12T11:16:53.498790149Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
14
+ {"time":"2024-11-12T11:16:53.49883056Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
15
+ {"time":"2024-11-12T11:16:54.435678748Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
16
+ {"time":"2024-11-12T11:16:55.889045635Z","level":"INFO","msg":"stream: closing","id":"komthdmh"}
17
+ {"time":"2024-11-12T11:16:55.889082565Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"komthdmh"}}
18
+ {"time":"2024-11-12T11:16:55.889144434Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"komthdmh"}}
19
+ {"time":"2024-11-12T11:16:55.889239107Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"komthdmh"}}
20
+ {"time":"2024-11-12T11:16:55.892207197Z","level":"INFO","msg":"stream: closed","id":"komthdmh"}
slice_1600/wandb/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Configure stats pid to 1335408
3
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-12 08:35:57,085 WARNING MainThread:1335408 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug.log
11
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log
12
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 5.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 5e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_t2i_22.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1015_5ep_t2i_22.5k', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0917_cham_ti2t_sft', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():667] starting backend
16
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-12 08:35:57,097 INFO MainThread:1335408 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-12 08:35:57,113 INFO MainThread:1335408 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-12 08:35:57,827 INFO MainThread:1335408 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-12 08:35:57,973 INFO MainThread:1335408 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-12 08:35:57,976 INFO MainThread:1335408 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-12 11:16:52,981 INFO MainThread:1335408 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/komthdmh
28
+ 2024-11-12 11:16:52,983 INFO MainThread:1335408 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-12 11:16:55,871 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-12 11:16:55,873 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-12 11:16:55,884 INFO MainThread:1335408 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_1600/wandb/run-20241112_083557-komthdmh/files/config.yaml ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.18.3
4
+ m: []
5
+ python_version: 3.11.10
6
+ t:
7
+ "1":
8
+ - 1
9
+ - 11
10
+ - 41
11
+ - 49
12
+ - 51
13
+ - 55
14
+ - 71
15
+ - 83
16
+ - 98
17
+ - 105
18
+ "2":
19
+ - 1
20
+ - 11
21
+ - 41
22
+ - 49
23
+ - 51
24
+ - 55
25
+ - 71
26
+ - 83
27
+ - 98
28
+ - 105
29
+ "3":
30
+ - 2
31
+ - 13
32
+ - 16
33
+ - 23
34
+ - 55
35
+ - 61
36
+ "4": 3.11.10
37
+ "5": 0.18.3
38
+ "6": 4.44.0.dev0
39
+ "8":
40
+ - 5
41
+ "12": 0.18.3
42
+ "13": linux-x86_64
43
+ data_cfgs:
44
+ value:
45
+ eval_data_files: null
46
+ eval_datasets: null
47
+ eval_optional_args: []
48
+ eval_size: null
49
+ eval_split: null
50
+ eval_subset: null
51
+ eval_template: null
52
+ train_data_files: llf_t2i_22.5k_tokenized.pt
53
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
54
+ train_optional_args: []
55
+ train_size: null
56
+ train_split: train
57
+ train_subset: null
58
+ train_template: AA_textfeedback
59
+ logger_cfgs:
60
+ value:
61
+ cache_dir: null
62
+ log_project: align-anything
63
+ log_run_name: sft
64
+ log_type: wandb
65
+ output_dir: ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k
66
+ save_interval: 400
67
+ model_cfgs:
68
+ value:
69
+ model_max_length: 4096
70
+ model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
71
+ trust_remote_code: true
72
+ special_tokens:
73
+ value: null
74
+ train_cfgs:
75
+ value:
76
+ adam_betas:
77
+ - 0.9
78
+ - 0.95
79
+ adam_epsilon: 1e-08
80
+ bf16: true
81
+ ds_cfgs: ds_z3_config.json
82
+ epochs: 5
83
+ eval_interval: 1000
84
+ eval_strategy: steps
85
+ fp16: false
86
+ freeze_language_model: false
87
+ gradient_accumulation_steps: 2
88
+ gradient_checkpointing: true
89
+ learning_rate: 5e-05
90
+ lr_scheduler_type: cosine
91
+ lr_warmup_ratio: 0.03
92
+ max_grad_norm: 1
93
+ per_device_eval_batch_size: 4
94
+ per_device_train_batch_size: 4
95
+ seed: 42
96
+ weight_decay: 0
slice_1600/wandb/run-20241112_083557-komthdmh/files/output.log ADDED
@@ -0,0 +1,506 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ***** Running training *****
2
+ Training 1/5.0 epoch: 0%| | 0/3520.0 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
3
+ Training 1/5.0 epoch (loss 1.3799): 9%|█████████████████████▉ | 319/3520.0 [13:44<2:15:00, 2.53s/it]
4
+ [2024-11-12 08:36:54,432] [INFO] [logging.py:96:log_dist] [Rank 0] step=10, skipped=0, lr=[9.615384615384616e-06, 9.615384615384616e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
5
+ [2024-11-12 08:36:54,432] [INFO] [timer.py:264:stop] epoch=0/micro_step=20/global_step=10, RunningAvgSamplesPerSec=14.123909505032028, CurrSamplesPerSec=13.241169083585376, MemAllocated=14.14GB, MaxMemAllocated=20.81GB
6
+ [2024-11-12 08:37:42,564] [INFO] [logging.py:96:log_dist] [Rank 0] step=20, skipped=0, lr=[1.923076923076923e-05, 1.923076923076923e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
7
+ [2024-11-12 08:37:42,565] [INFO] [timer.py:264:stop] epoch=0/micro_step=40/global_step=20, RunningAvgSamplesPerSec=14.240402554293874, CurrSamplesPerSec=15.089060244230348, MemAllocated=14.14GB, MaxMemAllocated=21.06GB
8
+ [2024-11-12 08:38:33,168] [INFO] [logging.py:96:log_dist] [Rank 0] step=30, skipped=0, lr=[2.8846153846153845e-05, 2.8846153846153845e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
9
+ [2024-11-12 08:38:33,169] [INFO] [timer.py:264:stop] epoch=0/micro_step=60/global_step=30, RunningAvgSamplesPerSec=14.022555309940765, CurrSamplesPerSec=10.204952710326344, MemAllocated=14.14GB, MaxMemAllocated=21.06GB
10
+ [2024-11-12 08:39:25,374] [INFO] [logging.py:96:log_dist] [Rank 0] step=40, skipped=0, lr=[3.846153846153846e-05, 3.846153846153846e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
11
+ [2024-11-12 08:39:25,374] [INFO] [timer.py:264:stop] epoch=0/micro_step=80/global_step=40, RunningAvgSamplesPerSec=13.74537082334063, CurrSamplesPerSec=14.73583968737617, MemAllocated=14.14GB, MaxMemAllocated=21.06GB
12
+ [2024-11-12 08:40:15,254] [INFO] [logging.py:96:log_dist] [Rank 0] step=50, skipped=0, lr=[4.8076923076923084e-05, 4.8076923076923084e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
13
+ [2024-11-12 08:40:15,255] [INFO] [timer.py:264:stop] epoch=0/micro_step=100/global_step=50, RunningAvgSamplesPerSec=13.762621202672168, CurrSamplesPerSec=13.81027432650536, MemAllocated=14.14GB, MaxMemAllocated=21.71GB
14
+ [2024-11-12 08:41:04,754] [INFO] [logging.py:96:log_dist] [Rank 0] step=60, skipped=0, lr=[4.9997293511641216e-05, 4.9997293511641216e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
15
+ [2024-11-12 08:41:04,754] [INFO] [timer.py:264:stop] epoch=0/micro_step=120/global_step=60, RunningAvgSamplesPerSec=13.780772643091256, CurrSamplesPerSec=14.821790030364474, MemAllocated=14.14GB, MaxMemAllocated=21.71GB
16
+ [2024-11-12 08:41:56,245] [INFO] [logging.py:96:log_dist] [Rank 0] step=70, skipped=0, lr=[4.9986299407013e-05, 4.9986299407013e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
17
+ [2024-11-12 08:41:56,246] [INFO] [timer.py:264:stop] epoch=0/micro_step=140/global_step=70, RunningAvgSamplesPerSec=13.70794245761943, CurrSamplesPerSec=15.070471709248997, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
18
+ [2024-11-12 08:42:45,692] [INFO] [logging.py:96:log_dist] [Rank 0] step=80, skipped=0, lr=[4.9966852247120764e-05, 4.9966852247120764e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
19
+ [2024-11-12 08:42:45,693] [INFO] [timer.py:264:stop] epoch=0/micro_step=160/global_step=80, RunningAvgSamplesPerSec=13.735055369770809, CurrSamplesPerSec=13.105748516399679, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
20
+ [2024-11-12 08:43:35,397] [INFO] [logging.py:96:log_dist] [Rank 0] step=90, skipped=0, lr=[4.9938958611086705e-05, 4.9938958611086705e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
21
+ [2024-11-12 08:43:35,398] [INFO] [timer.py:264:stop] epoch=0/micro_step=180/global_step=90, RunningAvgSamplesPerSec=13.744955502115493, CurrSamplesPerSec=12.506778047236157, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
22
+ [2024-11-12 08:44:27,939] [INFO] [logging.py:96:log_dist] [Rank 0] step=100, skipped=0, lr=[4.9902627935540205e-05, 4.9902627935540205e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
23
+ [2024-11-12 08:44:27,940] [INFO] [timer.py:264:stop] epoch=0/micro_step=200/global_step=100, RunningAvgSamplesPerSec=13.667787564838962, CurrSamplesPerSec=13.435017581558897, MemAllocated=14.14GB, MaxMemAllocated=22.0GB
24
+ [2024-11-12 08:45:21,475] [INFO] [logging.py:96:log_dist] [Rank 0] step=110, skipped=0, lr=[4.985787251142531e-05, 4.985787251142531e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
25
+ [2024-11-12 08:45:21,476] [INFO] [timer.py:264:stop] epoch=0/micro_step=220/global_step=110, RunningAvgSamplesPerSec=13.568024984812084, CurrSamplesPerSec=11.299057225977563, MemAllocated=14.14GB, MaxMemAllocated=22.23GB
26
+ [2024-11-12 08:46:12,028] [INFO] [logging.py:96:log_dist] [Rank 0] step=120, skipped=0, lr=[4.980470747984265e-05, 4.980470747984265e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
27
+ [2024-11-12 08:46:12,029] [INFO] [timer.py:264:stop] epoch=0/micro_step=240/global_step=120, RunningAvgSamplesPerSec=13.571719220971847, CurrSamplesPerSec=14.505489374647698, MemAllocated=14.14GB, MaxMemAllocated=22.23GB
28
+ [2024-11-12 08:47:06,982] [INFO] [logging.py:96:log_dist] [Rank 0] step=130, skipped=0, lr=[4.974315082692705e-05, 4.974315082692705e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
29
+ [2024-11-12 08:47:06,983] [INFO] [timer.py:264:stop] epoch=0/micro_step=260/global_step=130, RunningAvgSamplesPerSec=13.471915574993211, CurrSamplesPerSec=14.677887159235453, MemAllocated=14.14GB, MaxMemAllocated=22.23GB
30
+ [2024-11-12 08:47:58,851] [INFO] [logging.py:96:log_dist] [Rank 0] step=140, skipped=0, lr=[4.9673223377762715e-05, 4.9673223377762715e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
31
+ [2024-11-12 08:47:58,852] [INFO] [timer.py:264:stop] epoch=0/micro_step=280/global_step=140, RunningAvgSamplesPerSec=13.457893058437664, CurrSamplesPerSec=11.959087221433908, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
32
+ [2024-11-12 08:48:54,301] [INFO] [logging.py:96:log_dist] [Rank 0] step=150, skipped=0, lr=[4.9594948789337914e-05, 4.9594948789337914e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
33
+ [2024-11-12 08:48:54,302] [INFO] [timer.py:264:stop] epoch=0/micro_step=300/global_step=150, RunningAvgSamplesPerSec=13.373347047399044, CurrSamplesPerSec=10.07913456306504, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
34
+ [2024-11-12 08:49:44,210] [INFO] [logging.py:96:log_dist] [Rank 0] step=160, skipped=0, lr=[4.950835354254167e-05, 4.950835354254167e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
35
+ [2024-11-12 08:49:44,210] [INFO] [timer.py:264:stop] epoch=0/micro_step=320/global_step=160, RunningAvgSamplesPerSec=13.393197857147435, CurrSamplesPerSec=14.860385800427661, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
36
+ [2024-11-12 08:50:35,214] [INFO] [logging.py:96:log_dist] [Rank 0] step=170, skipped=0, lr=[4.941346693320509e-05, 4.941346693320509e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
37
+ [2024-11-12 08:50:35,215] [INFO] [timer.py:264:stop] epoch=0/micro_step=340/global_step=170, RunningAvgSamplesPerSec=13.395646345993923, CurrSamplesPerSec=14.323621533076235, MemAllocated=14.14GB, MaxMemAllocated=22.83GB
38
+ [2024-11-12 08:51:28,101] [INFO] [logging.py:96:log_dist] [Rank 0] step=180, skipped=0, lr=[4.931032106219029e-05, 4.931032106219029e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
39
+ [2024-11-12 08:51:28,102] [INFO] [timer.py:264:stop] epoch=0/micro_step=360/global_step=180, RunningAvgSamplesPerSec=13.367218567659057, CurrSamplesPerSec=14.98421830353528, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
40
+ [2024-11-12 08:52:21,270] [INFO] [logging.py:96:log_dist] [Rank 0] step=190, skipped=0, lr=[4.9198950824530485e-05, 4.9198950824530485e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
41
+ [2024-11-12 08:52:21,272] [INFO] [timer.py:264:stop] epoch=0/micro_step=380/global_step=190, RunningAvgSamplesPerSec=13.33740483581657, CurrSamplesPerSec=10.703509865696477, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
42
+ [2024-11-12 08:53:11,495] [INFO] [logging.py:96:log_dist] [Rank 0] step=200, skipped=0, lr=[4.9079393897624745e-05, 4.9079393897624745e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
43
+ [2024-11-12 08:53:11,496] [INFO] [timer.py:264:stop] epoch=0/micro_step=400/global_step=200, RunningAvgSamplesPerSec=13.352653718664833, CurrSamplesPerSec=14.742926067586316, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
44
+ Saving checkpoint at step 400 ...
45
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
46
+ Saving 16-bit model...
47
+ [2024-11-12 08:53:24,811] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
48
+ [2024-11-12 08:53:24,813] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin, tag: global_step200
49
+ [2024-11-12 08:53:24,813] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin...
50
+ [2024-11-12 08:53:43,361] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin.
51
+ [2024-11-12 08:53:43,361] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
52
+ Model saved!
53
+ Saving 16-bit model...
54
+ [2024-11-12 08:53:53,112] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
55
+ [2024-11-12 08:53:53,113] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin, tag: global_step200
56
+ [2024-11-12 08:53:53,113] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin...
57
+ [2024-11-12 08:54:14,787] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_400.bin.
58
+ [2024-11-12 08:54:14,789] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
59
+ Model saved!
60
+ Checkpoint saved.
61
+ [2024-11-12 08:55:04,096] [INFO] [logging.py:96:log_dist] [Rank 0] step=210, skipped=0, lr=[4.895169072849142e-05, 4.895169072849142e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
62
+ [2024-11-12 08:55:04,097] [INFO] [timer.py:264:stop] epoch=0/micro_step=420/global_step=210, RunningAvgSamplesPerSec=13.3795627688349, CurrSamplesPerSec=13.546001011776744, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
63
+ [2024-11-12 08:55:57,056] [INFO] [logging.py:96:log_dist] [Rank 0] step=220, skipped=0, lr=[4.881588452008456e-05, 4.881588452008456e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
64
+ [2024-11-12 08:55:57,057] [INFO] [timer.py:264:stop] epoch=0/micro_step=440/global_step=220, RunningAvgSamplesPerSec=13.356234172468818, CurrSamplesPerSec=13.971783082739853, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
65
+ [2024-11-12 08:56:49,663] [INFO] [logging.py:96:log_dist] [Rank 0] step=230, skipped=0, lr=[4.867202121667815e-05, 4.867202121667815e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
66
+ [2024-11-12 08:56:49,664] [INFO] [timer.py:264:stop] epoch=0/micro_step=460/global_step=230, RunningAvgSamplesPerSec=13.346132618764257, CurrSamplesPerSec=14.695504801089417, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
67
+ [2024-11-12 08:57:42,876] [INFO] [logging.py:96:log_dist] [Rank 0] step=240, skipped=0, lr=[4.852014948832268e-05, 4.852014948832268e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
68
+ [2024-11-12 08:57:42,877] [INFO] [timer.py:264:stop] epoch=0/micro_step=480/global_step=240, RunningAvgSamplesPerSec=13.320925710451487, CurrSamplesPerSec=14.158642205669375, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
69
+ [2024-11-12 08:58:35,127] [INFO] [logging.py:96:log_dist] [Rank 0] step=250, skipped=0, lr=[4.836032071437977e-05, 4.836032071437977e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
70
+ [2024-11-12 08:58:35,128] [INFO] [timer.py:264:stop] epoch=0/micro_step=500/global_step=250, RunningAvgSamplesPerSec=13.310236719541589, CurrSamplesPerSec=14.184536914840965, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
71
+ [2024-11-12 08:59:25,936] [INFO] [logging.py:96:log_dist] [Rank 0] step=260, skipped=0, lr=[4.819258896614014e-05, 4.819258896614014e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
72
+ [2024-11-12 08:59:25,936] [INFO] [timer.py:264:stop] epoch=0/micro_step=520/global_step=260, RunningAvgSamplesPerSec=13.314507578988644, CurrSamplesPerSec=13.662068644731564, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
73
+ [2024-11-12 09:00:24,950] [INFO] [logging.py:96:log_dist] [Rank 0] step=270, skipped=0, lr=[4.801701098853083e-05, 4.801701098853083e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
74
+ [2024-11-12 09:00:24,951] [INFO] [timer.py:264:stop] epoch=0/micro_step=540/global_step=270, RunningAvgSamplesPerSec=13.23171381311444, CurrSamplesPerSec=8.591639394767906, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
75
+ [2024-11-12 09:01:15,848] [INFO] [logging.py:96:log_dist] [Rank 0] step=280, skipped=0, lr=[4.783364618091803e-05, 4.783364618091803e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
76
+ [2024-11-12 09:01:15,849] [INFO] [timer.py:264:stop] epoch=0/micro_step=560/global_step=280, RunningAvgSamplesPerSec=13.242062455437987, CurrSamplesPerSec=12.414479790208985, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
77
+ [2024-11-12 09:02:06,734] [INFO] [logging.py:96:log_dist] [Rank 0] step=290, skipped=0, lr=[4.764255657701179e-05, 4.764255657701179e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
78
+ [2024-11-12 09:02:06,735] [INFO] [timer.py:264:stop] epoch=0/micro_step=580/global_step=290, RunningAvgSamplesPerSec=13.247943308956986, CurrSamplesPerSec=10.33724679656727, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
79
+ [2024-11-12 09:02:56,185] [INFO] [logging.py:96:log_dist] [Rank 0] step=300, skipped=0, lr=[4.74438068238795e-05, 4.74438068238795e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
80
+ [2024-11-12 09:02:56,187] [INFO] [timer.py:264:stop] epoch=0/micro_step=600/global_step=300, RunningAvgSamplesPerSec=13.273573937175511, CurrSamplesPerSec=14.875715410485261, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
81
+ [2024-11-12 09:03:46,290] [INFO] [logging.py:96:log_dist] [Rank 0] step=310, skipped=0, lr=[4.723746416007536e-05, 4.723746416007536e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
82
+ [2024-11-12 09:03:46,291] [INFO] [timer.py:264:stop] epoch=0/micro_step=620/global_step=310, RunningAvgSamplesPerSec=13.28483606555138, CurrSamplesPerSec=14.967418276326326, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
83
+ [2024-11-12 09:04:36,313] [INFO] [logging.py:96:log_dist] [Rank 0] step=320, skipped=0, lr=[4.702359839289306e-05, 4.702359839289306e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
84
+ [2024-11-12 09:04:36,314] [INFO] [timer.py:264:stop] epoch=0/micro_step=640/global_step=320, RunningAvgSamplesPerSec=13.29814534561608, CurrSamplesPerSec=13.334811696194658, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
85
+ [2024-11-12 09:05:28,541] [INFO] [logging.py:96:log_dist] [Rank 0] step=330, skipped=0, lr=[4.680228187474939e-05, 4.680228187474939e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
86
+ [2024-11-12 09:05:28,542] [INFO] [timer.py:264:stop] epoch=0/micro_step=660/global_step=330, RunningAvgSamplesPerSec=13.290943667500795, CurrSamplesPerSec=13.2426799914089, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
87
+ [2024-11-12 09:06:17,912] [INFO] [logging.py:96:log_dist] [Rank 0] step=340, skipped=0, lr=[4.657358947870691e-05, 4.657358947870691e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
88
+ [2024-11-12 09:06:17,913] [INFO] [timer.py:264:stop] epoch=0/micro_step=680/global_step=340, RunningAvgSamplesPerSec=13.303275846211843, CurrSamplesPerSec=14.644326425107316, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
89
+ [2024-11-12 09:07:07,262] [INFO] [logging.py:96:log_dist] [Rank 0] step=350, skipped=0, lr=[4.6337598573143823e-05, 4.6337598573143823e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
90
+ [2024-11-12 09:07:07,263] [INFO] [timer.py:264:stop] epoch=0/micro_step=700/global_step=350, RunningAvgSamplesPerSec=13.320609598823237, CurrSamplesPerSec=14.086162727268885, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
91
+ [2024-11-12 09:07:57,713] [INFO] [logging.py:96:log_dist] [Rank 0] step=360, skipped=0, lr=[4.609438899557964e-05, 4.609438899557964e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
92
+ [2024-11-12 09:07:57,714] [INFO] [timer.py:264:stop] epoch=1/micro_step=16/global_step=360, RunningAvgSamplesPerSec=13.325138937093469, CurrSamplesPerSec=13.648595953838816, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
93
+ [2024-11-12 09:08:46,579] [INFO] [logging.py:96:log_dist] [Rank 0] step=370, skipped=0, lr=[4.584404302566556e-05, 4.584404302566556e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
94
+ [2024-11-12 09:08:46,580] [INFO] [timer.py:264:stop] epoch=1/micro_step=36/global_step=370, RunningAvgSamplesPerSec=13.348088837780798, CurrSamplesPerSec=13.251978385199415, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
95
+ [2024-11-12 09:09:35,642] [INFO] [logging.py:96:log_dist] [Rank 0] step=380, skipped=0, lr=[4.5586645357348636e-05, 4.5586645357348636e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
96
+ [2024-11-12 09:09:35,643] [INFO] [timer.py:264:stop] epoch=1/micro_step=56/global_step=380, RunningAvgSamplesPerSec=13.36489338184365, CurrSamplesPerSec=14.590145688968107, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
97
+ [2024-11-12 09:10:29,555] [INFO] [logging.py:96:log_dist] [Rank 0] step=390, skipped=0, lr=[4.532228307021913e-05, 4.532228307021913e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
98
+ [2024-11-12 09:10:29,555] [INFO] [timer.py:264:stop] epoch=1/micro_step=76/global_step=390, RunningAvgSamplesPerSec=13.343550964615849, CurrSamplesPerSec=14.126325253738942, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
99
+ [2024-11-12 09:11:18,926] [INFO] [logging.py:96:log_dist] [Rank 0] step=400, skipped=0, lr=[4.50510456000509e-05, 4.50510456000509e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
100
+ [2024-11-12 09:11:18,926] [INFO] [timer.py:264:stop] epoch=1/micro_step=96/global_step=400, RunningAvgSamplesPerSec=13.358597510292705, CurrSamplesPerSec=14.449643731571005, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
101
+ Saving checkpoint at step 800 ...
102
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
103
+ Saving 16-bit model...
104
+ [2024-11-12 09:11:30,491] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
105
+ [2024-11-12 09:11:30,493] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin, tag: global_step400
106
+ [2024-11-12 09:11:30,493] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin...
107
+ [2024-11-12 09:11:47,011] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin.
108
+ [2024-11-12 09:11:47,013] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
109
+ Model saved!
110
+ Saving 16-bit model...
111
+ [2024-11-12 09:11:56,858] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
112
+ [2024-11-12 09:11:56,859] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin, tag: global_step400
113
+ [2024-11-12 09:11:56,860] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin...
114
+ [2024-11-12 09:12:19,298] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_800.bin.
115
+ [2024-11-12 09:12:19,300] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
116
+ Model saved!
117
+ Checkpoint saved.
118
+ [2024-11-12 09:13:09,669] [INFO] [logging.py:96:log_dist] [Rank 0] step=410, skipped=0, lr=[4.477302470854453e-05, 4.477302470854453e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
119
+ [2024-11-12 09:13:09,670] [INFO] [timer.py:264:stop] epoch=1/micro_step=116/global_step=410, RunningAvgSamplesPerSec=13.364449824928494, CurrSamplesPerSec=14.7170956204939, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
120
+ [2024-11-12 09:14:00,777] [INFO] [logging.py:96:log_dist] [Rank 0] step=420, skipped=0, lr=[4.4488314452283675e-05, 4.4488314452283675e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
121
+ [2024-11-12 09:14:00,777] [INFO] [timer.py:264:stop] epoch=1/micro_step=136/global_step=420, RunningAvgSamplesPerSec=13.363891525451283, CurrSamplesPerSec=10.922366588553597, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
122
+ [2024-11-12 09:14:49,428] [INFO] [logging.py:96:log_dist] [Rank 0] step=430, skipped=0, lr=[4.4197011150915e-05, 4.4197011150915e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
123
+ [2024-11-12 09:14:49,429] [INFO] [timer.py:264:stop] epoch=1/micro_step=156/global_step=430, RunningAvgSamplesPerSec=13.383295359926425, CurrSamplesPerSec=14.55651722494042, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
124
+ [2024-11-12 09:15:39,342] [INFO] [logging.py:96:log_dist] [Rank 0] step=440, skipped=0, lr=[4.389921335456253e-05, 4.389921335456253e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
125
+ [2024-11-12 09:15:39,343] [INFO] [timer.py:264:stop] epoch=1/micro_step=176/global_step=440, RunningAvgSamplesPerSec=13.390975239991825, CurrSamplesPerSec=13.200286386506031, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
126
+ [2024-11-12 09:16:30,091] [INFO] [logging.py:96:log_dist] [Rank 0] step=450, skipped=0, lr=[4.359502181048733e-05, 4.359502181048733e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
127
+ [2024-11-12 09:16:30,092] [INFO] [timer.py:264:stop] epoch=1/micro_step=196/global_step=450, RunningAvgSamplesPerSec=13.39418095466066, CurrSamplesPerSec=12.500613123038491, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
128
+ [2024-11-12 09:17:24,651] [INFO] [logging.py:96:log_dist] [Rank 0] step=460, skipped=0, lr=[4.328453942900402e-05, 4.328453942900402e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
129
+ [2024-11-12 09:17:24,651] [INFO] [timer.py:264:stop] epoch=1/micro_step=216/global_step=460, RunningAvgSamplesPerSec=13.373634676462785, CurrSamplesPerSec=10.975064294895416, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
130
+ [2024-11-12 09:18:16,585] [INFO] [logging.py:96:log_dist] [Rank 0] step=470, skipped=0, lr=[4.29678712486654e-05, 4.29678712486654e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
131
+ [2024-11-12 09:18:16,586] [INFO] [timer.py:264:stop] epoch=1/micro_step=236/global_step=470, RunningAvgSamplesPerSec=13.371014296937762, CurrSamplesPerSec=13.500704537918898, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
132
+ [2024-11-12 09:19:12,663] [INFO] [logging.py:96:log_dist] [Rank 0] step=480, skipped=0, lr=[4.2645124400727074e-05, 4.2645124400727074e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
133
+ [2024-11-12 09:19:12,664] [INFO] [timer.py:264:stop] epoch=1/micro_step=256/global_step=480, RunningAvgSamplesPerSec=13.346826564436673, CurrSamplesPerSec=11.446672280190908, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
134
+ [2024-11-12 09:20:02,200] [INFO] [logging.py:96:log_dist] [Rank 0] step=490, skipped=0, lr=[4.231640807290418e-05, 4.231640807290418e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
135
+ [2024-11-12 09:20:02,200] [INFO] [timer.py:264:stop] epoch=1/micro_step=276/global_step=490, RunningAvgSamplesPerSec=13.356418110943588, CurrSamplesPerSec=14.935730763365298, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
136
+ [2024-11-12 09:20:57,703] [INFO] [logging.py:96:log_dist] [Rank 0] step=500, skipped=0, lr=[4.198183347243233e-05, 4.198183347243233e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
137
+ [2024-11-12 09:20:57,704] [INFO] [timer.py:264:stop] epoch=1/micro_step=296/global_step=500, RunningAvgSamplesPerSec=13.334440472618091, CurrSamplesPerSec=14.71236645337987, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
138
+ [2024-11-12 09:21:49,661] [INFO] [logging.py:96:log_dist] [Rank 0] step=510, skipped=0, lr=[4.1641513788445365e-05, 4.1641513788445365e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
139
+ [2024-11-12 09:21:49,662] [INFO] [timer.py:264:stop] epoch=1/micro_step=316/global_step=510, RunningAvgSamplesPerSec=13.33045269258777, CurrSamplesPerSec=12.131904363990161, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
140
+ [2024-11-12 09:22:40,443] [INFO] [logging.py:96:log_dist] [Rank 0] step=520, skipped=0, lr=[4.129556415368261e-05, 4.129556415368261e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
141
+ [2024-11-12 09:22:40,443] [INFO] [timer.py:264:stop] epoch=1/micro_step=336/global_step=520, RunningAvgSamplesPerSec=13.334724290035696, CurrSamplesPerSec=13.00404440542971, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
142
+ [2024-11-12 09:23:33,397] [INFO] [logging.py:96:log_dist] [Rank 0] step=530, skipped=0, lr=[4.0944101605538585e-05, 4.0944101605538585e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
143
+ [2024-11-12 09:23:33,398] [INFO] [timer.py:264:stop] epoch=1/micro_step=356/global_step=530, RunningAvgSamplesPerSec=13.325292224207772, CurrSamplesPerSec=13.89638709040199, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
144
+ [2024-11-12 09:24:25,155] [INFO] [logging.py:96:log_dist] [Rank 0] step=540, skipped=0, lr=[4.058724504646834e-05, 4.058724504646834e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
145
+ [2024-11-12 09:24:25,156] [INFO] [timer.py:264:stop] epoch=1/micro_step=376/global_step=540, RunningAvgSamplesPerSec=13.324965746155074, CurrSamplesPerSec=12.91599583924915, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
146
+ [2024-11-12 09:25:16,862] [INFO] [logging.py:96:log_dist] [Rank 0] step=550, skipped=0, lr=[4.0225115203761924e-05, 4.0225115203761924e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
147
+ [2024-11-12 09:25:16,863] [INFO] [timer.py:264:stop] epoch=1/micro_step=396/global_step=550, RunningAvgSamplesPerSec=13.32296006415578, CurrSamplesPerSec=14.133098580815874, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
148
+ [2024-11-12 09:26:07,277] [INFO] [logging.py:96:log_dist] [Rank 0] step=560, skipped=0, lr=[3.985783458870134e-05, 3.985783458870134e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
149
+ [2024-11-12 09:26:07,278] [INFO] [timer.py:264:stop] epoch=1/micro_step=416/global_step=560, RunningAvgSamplesPerSec=13.328244272608401, CurrSamplesPerSec=11.970295366561219, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
150
+ [2024-11-12 09:27:00,643] [INFO] [logging.py:96:log_dist] [Rank 0] step=570, skipped=0, lr=[3.948552745511409e-05, 3.948552745511409e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
151
+ [2024-11-12 09:27:00,644] [INFO] [timer.py:264:stop] epoch=1/micro_step=436/global_step=570, RunningAvgSamplesPerSec=13.317673249255442, CurrSamplesPerSec=10.870225791334036, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
152
+ [2024-11-12 09:27:51,548] [INFO] [logging.py:96:log_dist] [Rank 0] step=580, skipped=0, lr=[3.910831975733717e-05, 3.910831975733717e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
153
+ [2024-11-12 09:27:51,549] [INFO] [timer.py:264:stop] epoch=1/micro_step=456/global_step=580, RunningAvgSamplesPerSec=13.318033623411376, CurrSamplesPerSec=12.694643791709918, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
154
+ [2024-11-12 09:28:44,874] [INFO] [logging.py:96:log_dist] [Rank 0] step=590, skipped=0, lr=[3.872633910760571e-05, 3.872633910760571e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
155
+ [2024-11-12 09:28:44,875] [INFO] [timer.py:264:stop] epoch=1/micro_step=476/global_step=590, RunningAvgSamplesPerSec=13.30842315273336, CurrSamplesPerSec=12.382377351271012, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
156
+ [2024-11-12 09:29:36,746] [INFO] [logging.py:96:log_dist] [Rank 0] step=600, skipped=0, lr=[3.833971473288084e-05, 3.833971473288084e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
157
+ [2024-11-12 09:29:36,747] [INFO] [timer.py:264:stop] epoch=1/micro_step=496/global_step=600, RunningAvgSamplesPerSec=13.308057817145734, CurrSamplesPerSec=14.699825462285968, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
158
+ Saving checkpoint at step 1200 ...
159
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
160
+ Saving 16-bit model...
161
+ [2024-11-12 09:29:49,497] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
162
+ [2024-11-12 09:29:49,499] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin, tag: global_step600
163
+ [2024-11-12 09:29:49,499] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin...
164
+ [2024-11-12 09:30:10,740] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin.
165
+ [2024-11-12 09:30:10,742] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
166
+ Model saved!
167
+ Saving 16-bit model...
168
+ [2024-11-12 09:30:19,949] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
169
+ [2024-11-12 09:30:19,951] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin, tag: global_step600
170
+ [2024-11-12 09:30:19,951] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin...
171
+ [2024-11-12 09:30:45,036] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1200.bin.
172
+ [2024-11-12 09:30:45,038] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
173
+ Model saved!
174
+ Checkpoint saved.
175
+ [2024-11-12 09:31:37,541] [INFO] [logging.py:96:log_dist] [Rank 0] step=610, skipped=0, lr=[3.794857743113117e-05, 3.794857743113117e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
176
+ [2024-11-12 09:31:37,542] [INFO] [timer.py:264:stop] epoch=1/micro_step=516/global_step=610, RunningAvgSamplesPerSec=13.303038238839015, CurrSamplesPerSec=13.471107231584368, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
177
+ [2024-11-12 09:32:33,379] [INFO] [logging.py:96:log_dist] [Rank 0] step=620, skipped=0, lr=[3.7553059527082913e-05, 3.7553059527082913e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
178
+ [2024-11-12 09:32:33,380] [INFO] [timer.py:264:stop] epoch=1/micro_step=536/global_step=620, RunningAvgSamplesPerSec=13.284333701524984, CurrSamplesPerSec=7.212952779012694, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
179
+ [2024-11-12 09:33:26,622] [INFO] [logging.py:96:log_dist] [Rank 0] step=630, skipped=0, lr=[3.715329482745333e-05, 3.715329482745333e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
180
+ [2024-11-12 09:33:26,623] [INFO] [timer.py:264:stop] epoch=1/micro_step=556/global_step=630, RunningAvgSamplesPerSec=13.27712854384248, CurrSamplesPerSec=13.229785334147179, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
181
+ [2024-11-12 09:34:17,583] [INFO] [logging.py:96:log_dist] [Rank 0] step=640, skipped=0, lr=[3.6749418575683e-05, 3.6749418575683e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
182
+ [2024-11-12 09:34:17,584] [INFO] [timer.py:264:stop] epoch=1/micro_step=576/global_step=640, RunningAvgSamplesPerSec=13.279694432663204, CurrSamplesPerSec=11.886098293838835, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
183
+ [2024-11-12 09:35:08,224] [INFO] [logging.py:96:log_dist] [Rank 0] step=650, skipped=0, lr=[3.634156740618186e-05, 3.634156740618186e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
184
+ [2024-11-12 09:35:08,225] [INFO] [timer.py:264:stop] epoch=1/micro_step=596/global_step=650, RunningAvgSamplesPerSec=13.281283354952393, CurrSamplesPerSec=13.977049388035704, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
185
+ [2024-11-12 09:35:58,154] [INFO] [logging.py:96:log_dist] [Rank 0] step=660, skipped=0, lr=[3.592987929810476e-05, 3.592987929810476e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
186
+ [2024-11-12 09:35:58,154] [INFO] [timer.py:264:stop] epoch=1/micro_step=616/global_step=660, RunningAvgSamplesPerSec=13.28894472425476, CurrSamplesPerSec=14.26329493903139, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
187
+ [2024-11-12 09:36:47,989] [INFO] [logging.py:96:log_dist] [Rank 0] step=670, skipped=0, lr=[3.551449352867209e-05, 3.551449352867209e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
188
+ [2024-11-12 09:36:47,990] [INFO] [timer.py:264:stop] epoch=1/micro_step=636/global_step=670, RunningAvgSamplesPerSec=13.295832906227798, CurrSamplesPerSec=12.269052838403406, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
189
+ [2024-11-12 09:37:40,115] [INFO] [logging.py:96:log_dist] [Rank 0] step=680, skipped=0, lr=[3.509555062605121e-05, 3.509555062605121e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
190
+ [2024-11-12 09:37:40,116] [INFO] [timer.py:264:stop] epoch=1/micro_step=656/global_step=680, RunningAvgSamplesPerSec=13.291947948697695, CurrSamplesPerSec=11.483019322453742, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
191
+ [2024-11-12 09:38:31,799] [INFO] [logging.py:96:log_dist] [Rank 0] step=690, skipped=0, lr=[3.467319232181473e-05, 3.467319232181473e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
192
+ [2024-11-12 09:38:31,800] [INFO] [timer.py:264:stop] epoch=1/micro_step=676/global_step=690, RunningAvgSamplesPerSec=13.293062090636276, CurrSamplesPerSec=13.979593389406237, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
193
+ [2024-11-12 09:39:20,875] [INFO] [logging.py:96:log_dist] [Rank 0] step=700, skipped=0, lr=[3.4247561502991604e-05, 3.4247561502991604e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
194
+ [2024-11-12 09:39:20,876] [INFO] [timer.py:264:stop] epoch=1/micro_step=696/global_step=700, RunningAvgSamplesPerSec=13.301456078834374, CurrSamplesPerSec=14.428370061415672, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
195
+ [2024-11-12 09:40:11,651] [INFO] [logging.py:96:log_dist] [Rank 0] step=710, skipped=0, lr=[3.381880216372738e-05, 3.381880216372738e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
196
+ [2024-11-12 09:40:11,652] [INFO] [timer.py:264:stop] epoch=2/micro_step=12/global_step=710, RunningAvgSamplesPerSec=13.303156673817357, CurrSamplesPerSec=14.634354011395214, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
197
+ [2024-11-12 09:41:00,207] [INFO] [logging.py:96:log_dist] [Rank 0] step=720, skipped=0, lr=[3.3387059356569875e-05, 3.3387059356569875e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
198
+ [2024-11-12 09:41:00,208] [INFO] [timer.py:264:stop] epoch=2/micro_step=32/global_step=720, RunningAvgSamplesPerSec=13.314711683634192, CurrSamplesPerSec=13.545782956791136, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
199
+ [2024-11-12 09:41:49,140] [INFO] [logging.py:96:log_dist] [Rank 0] step=730, skipped=0, lr=[3.2952479143396826e-05, 3.2952479143396826e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
200
+ [2024-11-12 09:41:49,141] [INFO] [timer.py:264:stop] epoch=2/micro_step=52/global_step=730, RunningAvgSamplesPerSec=13.325003730032233, CurrSamplesPerSec=14.36111525811834, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
201
+ [2024-11-12 09:42:42,299] [INFO] [logging.py:96:log_dist] [Rank 0] step=740, skipped=0, lr=[3.2515208546002e-05, 3.2515208546002e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
202
+ [2024-11-12 09:42:42,300] [INFO] [timer.py:264:stop] epoch=2/micro_step=72/global_step=740, RunningAvgSamplesPerSec=13.318484161902534, CurrSamplesPerSec=11.156171290123146, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
203
+ [2024-11-12 09:43:33,565] [INFO] [logging.py:96:log_dist] [Rank 0] step=750, skipped=0, lr=[3.207539549635664e-05, 3.207539549635664e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
204
+ [2024-11-12 09:43:33,566] [INFO] [timer.py:264:stop] epoch=2/micro_step=92/global_step=750, RunningAvgSamplesPerSec=13.319682913264439, CurrSamplesPerSec=13.220680176763791, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
205
+ [2024-11-12 09:44:22,447] [INFO] [logging.py:96:log_dist] [Rank 0] step=760, skipped=0, lr=[3.1633188786562914e-05, 3.1633188786562914e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
206
+ [2024-11-12 09:44:22,447] [INFO] [timer.py:264:stop] epoch=2/micro_step=112/global_step=760, RunningAvgSamplesPerSec=13.327851536598244, CurrSamplesPerSec=14.068757540394465, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
207
+ [2024-11-12 09:45:13,278] [INFO] [logging.py:96:log_dist] [Rank 0] step=770, skipped=0, lr=[3.118873801851646e-05, 3.118873801851646e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
208
+ [2024-11-12 09:45:13,279] [INFO] [timer.py:264:stop] epoch=2/micro_step=132/global_step=770, RunningAvgSamplesPerSec=13.330652798672963, CurrSamplesPerSec=11.724133921801615, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
209
+ [2024-11-12 09:46:03,735] [INFO] [logging.py:96:log_dist] [Rank 0] step=780, skipped=0, lr=[3.07421935532949e-05, 3.07421935532949e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
210
+ [2024-11-12 09:46:03,736] [INFO] [timer.py:264:stop] epoch=2/micro_step=152/global_step=780, RunningAvgSamplesPerSec=13.335917143341288, CurrSamplesPerSec=14.432756201254964, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
211
+ [2024-11-12 09:46:53,208] [INFO] [logging.py:96:log_dist] [Rank 0] step=790, skipped=0, lr=[3.0293706460289505e-05, 3.0293706460289505e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
212
+ [2024-11-12 09:46:53,209] [INFO] [timer.py:264:stop] epoch=2/micro_step=172/global_step=790, RunningAvgSamplesPerSec=13.343995330928088, CurrSamplesPerSec=13.47306192122824, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
213
+ [2024-11-12 09:47:43,265] [INFO] [logging.py:96:log_dist] [Rank 0] step=800, skipped=0, lr=[2.9843428466097385e-05, 2.9843428466097385e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
214
+ [2024-11-12 09:47:43,266] [INFO] [timer.py:264:stop] epoch=2/micro_step=192/global_step=800, RunningAvgSamplesPerSec=13.347981134151242, CurrSamplesPerSec=14.23447280515204, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
215
+ Saving checkpoint at step 1600 ...
216
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
217
+ Saving 16-bit model...
218
+ [2024-11-12 09:47:54,616] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step800 is about to be saved!
219
+ [2024-11-12 09:47:54,617] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin, tag: global_step800
220
+ [2024-11-12 09:47:54,617] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin...
221
+ [2024-11-12 09:48:11,335] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin.
222
+ [2024-11-12 09:48:11,336] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step800 is ready now!
223
+ Model saved!
224
+ Saving 16-bit model...
225
+ [2024-11-12 09:48:22,816] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step800 is about to be saved!
226
+ [2024-11-12 09:48:22,817] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin, tag: global_step800
227
+ [2024-11-12 09:48:22,817] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin...
228
+ [2024-11-12 09:48:45,345] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_1600.bin.
229
+ [2024-11-12 09:48:45,347] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step800 is ready now!
230
+ Model saved!
231
+ Checkpoint saved.
232
+ [2024-11-12 09:49:39,319] [INFO] [logging.py:96:log_dist] [Rank 0] step=810, skipped=0, lr=[2.939151190319108e-05, 2.939151190319108e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
233
+ [2024-11-12 09:49:39,320] [INFO] [timer.py:264:stop] epoch=2/micro_step=212/global_step=810, RunningAvgSamplesPerSec=13.338019751420884, CurrSamplesPerSec=14.761337460745917, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
234
+ [2024-11-12 09:50:31,369] [INFO] [logging.py:96:log_dist] [Rank 0] step=820, skipped=0, lr=[2.8938109658383454e-05, 2.8938109658383454e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
235
+ [2024-11-12 09:50:31,370] [INFO] [timer.py:264:stop] epoch=2/micro_step=232/global_step=820, RunningAvgSamplesPerSec=13.334637882621434, CurrSamplesPerSec=14.700607138327708, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
236
+ [2024-11-12 09:51:26,515] [INFO] [logging.py:96:log_dist] [Rank 0] step=830, skipped=0, lr=[2.8483375121104818e-05, 2.8483375121104818e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
237
+ [2024-11-12 09:51:26,516] [INFO] [timer.py:264:stop] epoch=2/micro_step=252/global_step=830, RunningAvgSamplesPerSec=13.321062900261444, CurrSamplesPerSec=12.880004275047911, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
238
+ [2024-11-12 09:52:16,145] [INFO] [logging.py:96:log_dist] [Rank 0] step=840, skipped=0, lr=[2.8027462131510208e-05, 2.8027462131510208e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
239
+ [2024-11-12 09:52:16,146] [INFO] [timer.py:264:stop] epoch=2/micro_step=272/global_step=840, RunningAvgSamplesPerSec=13.327259566392549, CurrSamplesPerSec=13.7934219369934, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
240
+ [2024-11-12 09:53:12,237] [INFO] [logging.py:96:log_dist] [Rank 0] step=850, skipped=0, lr=[2.757052492843401e-05, 2.757052492843401e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
241
+ [2024-11-12 09:53:12,238] [INFO] [timer.py:264:stop] epoch=2/micro_step=292/global_step=850, RunningAvgSamplesPerSec=13.31062857578834, CurrSamplesPerSec=14.756166118458328, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
242
+ [2024-11-12 09:54:02,303] [INFO] [logging.py:96:log_dist] [Rank 0] step=860, skipped=0, lr=[2.711271809720986e-05, 2.711271809720986e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
243
+ [2024-11-12 09:54:02,304] [INFO] [timer.py:264:stop] epoch=2/micro_step=312/global_step=860, RunningAvgSamplesPerSec=13.313888786479442, CurrSamplesPerSec=14.989720670581756, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
244
+ [2024-11-12 09:54:53,519] [INFO] [logging.py:96:log_dist] [Rank 0] step=870, skipped=0, lr=[2.6654196517373205e-05, 2.6654196517373205e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
245
+ [2024-11-12 09:54:53,520] [INFO] [timer.py:264:stop] epoch=2/micro_step=332/global_step=870, RunningAvgSamplesPerSec=13.314541614255276, CurrSamplesPerSec=13.755073221094372, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
246
+ [2024-11-12 09:55:46,262] [INFO] [logging.py:96:log_dist] [Rank 0] step=880, skipped=0, lr=[2.619511531026436e-05, 2.619511531026436e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
247
+ [2024-11-12 09:55:46,263] [INFO] [timer.py:264:stop] epoch=2/micro_step=352/global_step=880, RunningAvgSamplesPerSec=13.30990443123984, CurrSamplesPerSec=13.64081898485842, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
248
+ [2024-11-12 09:56:38,111] [INFO] [logging.py:96:log_dist] [Rank 0] step=890, skipped=0, lr=[2.573562978654986e-05, 2.573562978654986e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
249
+ [2024-11-12 09:56:38,112] [INFO] [timer.py:264:stop] epoch=2/micro_step=372/global_step=890, RunningAvgSamplesPerSec=13.309379764160232, CurrSamplesPerSec=11.592272177703125, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
250
+ [2024-11-12 09:57:30,442] [INFO] [logging.py:96:log_dist] [Rank 0] step=900, skipped=0, lr=[2.527589539367956e-05, 2.527589539367956e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
251
+ [2024-11-12 09:57:30,443] [INFO] [timer.py:264:stop] epoch=2/micro_step=392/global_step=900, RunningAvgSamplesPerSec=13.306786686847447, CurrSamplesPerSec=13.435805696728409, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
252
+ [2024-11-12 09:58:19,473] [INFO] [logging.py:96:log_dist] [Rank 0] step=910, skipped=0, lr=[2.4816067663297705e-05, 2.4816067663297705e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
253
+ [2024-11-12 09:58:19,474] [INFO] [timer.py:264:stop] epoch=2/micro_step=412/global_step=910, RunningAvgSamplesPerSec=13.313050356453484, CurrSamplesPerSec=13.499937984774144, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
254
+ [2024-11-12 09:59:12,152] [INFO] [logging.py:96:log_dist] [Rank 0] step=920, skipped=0, lr=[2.4356302158625288e-05, 2.4356302158625288e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
255
+ [2024-11-12 09:59:12,153] [INFO] [timer.py:264:stop] epoch=2/micro_step=432/global_step=920, RunningAvgSamplesPerSec=13.310216314626862, CurrSamplesPerSec=14.437108514790491, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
256
+ [2024-11-12 10:00:04,119] [INFO] [logging.py:96:log_dist] [Rank 0] step=930, skipped=0, lr=[2.38967544218319e-05, 2.38967544218319e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
257
+ [2024-11-12 10:00:04,119] [INFO] [timer.py:264:stop] epoch=2/micro_step=452/global_step=930, RunningAvgSamplesPerSec=13.309106357860806, CurrSamplesPerSec=13.867148018326894, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
258
+ [2024-11-12 10:00:57,366] [INFO] [logging.py:96:log_dist] [Rank 0] step=940, skipped=0, lr=[2.3437579921414573e-05, 2.3437579921414573e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
259
+ [2024-11-12 10:00:57,367] [INFO] [timer.py:264:stop] epoch=2/micro_step=472/global_step=940, RunningAvgSamplesPerSec=13.303958047851587, CurrSamplesPerSec=12.214996944223117, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
260
+ [2024-11-12 10:01:49,941] [INFO] [logging.py:96:log_dist] [Rank 0] step=950, skipped=0, lr=[2.2978933999601655e-05, 2.2978933999601655e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
261
+ [2024-11-12 10:01:49,942] [INFO] [timer.py:264:stop] epoch=2/micro_step=492/global_step=950, RunningAvgSamplesPerSec=13.300542723454466, CurrSamplesPerSec=13.28000135254299, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
262
+ [2024-11-12 10:02:41,154] [INFO] [logging.py:96:log_dist] [Rank 0] step=960, skipped=0, lr=[2.2520971819799328e-05, 2.2520971819799328e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
263
+ [2024-11-12 10:02:41,156] [INFO] [timer.py:264:stop] epoch=2/micro_step=512/global_step=960, RunningAvgSamplesPerSec=13.30180341566793, CurrSamplesPerSec=14.012398043896601, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
264
+ [2024-11-12 10:03:33,677] [INFO] [logging.py:96:log_dist] [Rank 0] step=970, skipped=0, lr=[2.206384831409863e-05, 2.206384831409863e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
265
+ [2024-11-12 10:03:33,678] [INFO] [timer.py:264:stop] epoch=2/micro_step=532/global_step=970, RunningAvgSamplesPerSec=13.301177680120674, CurrSamplesPerSec=10.555131044819795, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
266
+ [2024-11-12 10:04:31,856] [INFO] [logging.py:96:log_dist] [Rank 0] step=980, skipped=0, lr=[2.1607718130860782e-05, 2.1607718130860782e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
267
+ [2024-11-12 10:04:31,857] [INFO] [timer.py:264:stop] epoch=2/micro_step=552/global_step=980, RunningAvgSamplesPerSec=13.282526675644393, CurrSamplesPerSec=13.073611291719171, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
268
+ [2024-11-12 10:05:21,746] [INFO] [logging.py:96:log_dist] [Rank 0] step=990, skipped=0, lr=[2.1152735582398452e-05, 2.1152735582398452e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
269
+ [2024-11-12 10:05:21,747] [INFO] [timer.py:264:stop] epoch=2/micro_step=572/global_step=990, RunningAvgSamplesPerSec=13.287191601768065, CurrSamplesPerSec=13.239052563519289, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
270
+ [2024-11-12 10:06:14,126] [INFO] [logging.py:96:log_dist] [Rank 0] step=1000, skipped=0, lr=[2.0699054592770737e-05, 2.0699054592770737e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
271
+ [2024-11-12 10:06:14,127] [INFO] [timer.py:264:stop] epoch=2/micro_step=592/global_step=1000, RunningAvgSamplesPerSec=13.286254771939328, CurrSamplesPerSec=14.674160933553079, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
272
+ Saving checkpoint at step 2000 ...
273
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
274
+ Saving 16-bit model...
275
+ [2024-11-12 10:06:29,485] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is about to be saved!
276
+ [2024-11-12 10:06:29,487] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin, tag: global_step1000
277
+ [2024-11-12 10:06:29,487] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin...
278
+ [2024-11-12 10:06:52,581] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin.
279
+ [2024-11-12 10:06:52,583] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1000 is ready now!
280
+ Model saved!
281
+ Saving 16-bit model...
282
+ [2024-11-12 10:07:01,836] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is about to be saved!
283
+ [2024-11-12 10:07:01,837] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin, tag: global_step1000
284
+ [2024-11-12 10:07:01,838] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin...
285
+ [2024-11-12 10:07:28,200] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2000.bin.
286
+ [2024-11-12 10:07:28,202] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1000 is ready now!
287
+ Model saved!
288
+ Checkpoint saved.
289
+ [2024-11-12 10:08:18,416] [INFO] [logging.py:96:log_dist] [Rank 0] step=1010, skipped=0, lr=[2.024682864570945e-05, 2.024682864570945e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
290
+ [2024-11-12 10:08:18,417] [INFO] [timer.py:264:stop] epoch=2/micro_step=612/global_step=1010, RunningAvgSamplesPerSec=13.289360403964261, CurrSamplesPerSec=13.18569467585241, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
291
+ [2024-11-12 10:09:07,466] [INFO] [logging.py:96:log_dist] [Rank 0] step=1020, skipped=0, lr=[1.9796210732694442e-05, 1.9796210732694442e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
292
+ [2024-11-12 10:09:07,467] [INFO] [timer.py:264:stop] epoch=2/micro_step=632/global_step=1020, RunningAvgSamplesPerSec=13.296168396047804, CurrSamplesPerSec=15.105174063576408, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
293
+ [2024-11-12 10:09:59,426] [INFO] [logging.py:96:log_dist] [Rank 0] step=1030, skipped=0, lr=[1.9347353301195424e-05, 1.9347353301195424e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
294
+ [2024-11-12 10:09:59,427] [INFO] [timer.py:264:stop] epoch=2/micro_step=652/global_step=1030, RunningAvgSamplesPerSec=13.294815892948787, CurrSamplesPerSec=13.675949766714641, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
295
+ [2024-11-12 10:10:51,787] [INFO] [logging.py:96:log_dist] [Rank 0] step=1040, skipped=0, lr=[1.8900408203097787e-05, 1.8900408203097787e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
296
+ [2024-11-12 10:10:51,788] [INFO] [timer.py:264:stop] epoch=2/micro_step=672/global_step=1040, RunningAvgSamplesPerSec=13.29338856696237, CurrSamplesPerSec=14.005642697848842, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
297
+ [2024-11-12 10:11:40,380] [INFO] [logging.py:96:log_dist] [Rank 0] step=1050, skipped=0, lr=[1.8455526643329995e-05, 1.8455526643329995e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
298
+ [2024-11-12 10:11:40,381] [INFO] [timer.py:264:stop] epoch=2/micro_step=692/global_step=1050, RunningAvgSamplesPerSec=13.301896593394172, CurrSamplesPerSec=13.401735065616107, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
299
+ [2024-11-12 10:12:32,872] [INFO] [logging.py:96:log_dist] [Rank 0] step=1060, skipped=0, lr=[1.8012859128709766e-05, 1.8012859128709766e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
300
+ [2024-11-12 10:12:32,873] [INFO] [timer.py:264:stop] epoch=3/micro_step=8/global_step=1060, RunningAvgSamplesPerSec=13.30025338311892, CurrSamplesPerSec=13.396459390010323, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
301
+ [2024-11-12 10:13:21,395] [INFO] [logging.py:96:log_dist] [Rank 0] step=1070, skipped=0, lr=[1.7572555417026523e-05, 1.7572555417026523e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
302
+ [2024-11-12 10:13:21,396] [INFO] [timer.py:264:stop] epoch=3/micro_step=28/global_step=1070, RunningAvgSamplesPerSec=13.30848733831013, CurrSamplesPerSec=14.040752938548358, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
303
+ [2024-11-12 10:14:10,532] [INFO] [logging.py:96:log_dist] [Rank 0] step=1080, skipped=0, lr=[1.7134764466377136e-05, 1.7134764466377136e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
304
+ [2024-11-12 10:14:10,533] [INFO] [timer.py:264:stop] epoch=3/micro_step=48/global_step=1080, RunningAvgSamplesPerSec=13.314602105884283, CurrSamplesPerSec=13.125495457445675, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
305
+ [2024-11-12 10:15:01,827] [INFO] [logging.py:96:log_dist] [Rank 0] step=1090, skipped=0, lr=[1.6699634384772317e-05, 1.6699634384772317e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
306
+ [2024-11-12 10:15:01,827] [INFO] [timer.py:264:stop] epoch=3/micro_step=68/global_step=1090, RunningAvgSamplesPerSec=13.314236544377534, CurrSamplesPerSec=14.524569478485214, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
307
+ [2024-11-12 10:15:53,486] [INFO] [logging.py:96:log_dist] [Rank 0] step=1100, skipped=0, lr=[1.6267312380030506e-05, 1.6267312380030506e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
308
+ [2024-11-12 10:15:53,487] [INFO] [timer.py:264:stop] epoch=3/micro_step=88/global_step=1100, RunningAvgSamplesPerSec=13.313312407846855, CurrSamplesPerSec=12.19932859062791, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
309
+ [2024-11-12 10:16:42,803] [INFO] [logging.py:96:log_dist] [Rank 0] step=1110, skipped=0, lr=[1.5837944709976382e-05, 1.5837944709976382e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
310
+ [2024-11-12 10:16:42,804] [INFO] [timer.py:264:stop] epoch=3/micro_step=108/global_step=1110, RunningAvgSamplesPerSec=13.318065107541004, CurrSamplesPerSec=12.966475628687732, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
311
+ [2024-11-12 10:17:32,582] [INFO] [logging.py:96:log_dist] [Rank 0] step=1120, skipped=0, lr=[1.5411676632960713e-05, 1.5411676632960713e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
312
+ [2024-11-12 10:17:32,583] [INFO] [timer.py:264:stop] epoch=3/micro_step=128/global_step=1120, RunningAvgSamplesPerSec=13.322640969741128, CurrSamplesPerSec=13.668152738247908, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
313
+ [2024-11-12 10:18:24,097] [INFO] [logging.py:96:log_dist] [Rank 0] step=1130, skipped=0, lr=[1.4988652358718336e-05, 1.4988652358718336e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
314
+ [2024-11-12 10:18:24,098] [INFO] [timer.py:264:stop] epoch=3/micro_step=148/global_step=1130, RunningAvgSamplesPerSec=13.323294979183146, CurrSamplesPerSec=12.599322897047612, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
315
+ [2024-11-12 10:19:12,704] [INFO] [logging.py:96:log_dist] [Rank 0] step=1140, skipped=0, lr=[1.4569014999580937e-05, 1.4569014999580937e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
316
+ [2024-11-12 10:19:12,705] [INFO] [timer.py:264:stop] epoch=3/micro_step=168/global_step=1140, RunningAvgSamplesPerSec=13.329385073350831, CurrSamplesPerSec=14.52883737710039, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
317
+ [2024-11-12 10:20:02,511] [INFO] [logging.py:96:log_dist] [Rank 0] step=1150, skipped=0, lr=[1.4152906522061048e-05, 1.4152906522061048e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
318
+ [2024-11-12 10:20:02,512] [INFO] [timer.py:264:stop] epoch=3/micro_step=188/global_step=1150, RunningAvgSamplesPerSec=13.333106830069978, CurrSamplesPerSec=14.65574618291602, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
319
+ [2024-11-12 10:20:55,859] [INFO] [logging.py:96:log_dist] [Rank 0] step=1160, skipped=0, lr=[1.3740467698823662e-05, 1.3740467698823662e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
320
+ [2024-11-12 10:20:55,860] [INFO] [timer.py:264:stop] epoch=3/micro_step=208/global_step=1160, RunningAvgSamplesPerSec=13.328603070480009, CurrSamplesPerSec=14.134182082439413, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
321
+ [2024-11-12 10:21:50,334] [INFO] [logging.py:96:log_dist] [Rank 0] step=1170, skipped=0, lr=[1.3331838061061835e-05, 1.3331838061061835e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
322
+ [2024-11-12 10:21:50,336] [INFO] [timer.py:264:stop] epoch=3/micro_step=228/global_step=1170, RunningAvgSamplesPerSec=13.321959058539916, CurrSamplesPerSec=14.914114773186931, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
323
+ [2024-11-12 10:22:44,340] [INFO] [logging.py:96:log_dist] [Rank 0] step=1180, skipped=0, lr=[1.2927155851292145e-05, 1.2927155851292145e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
324
+ [2024-11-12 10:22:44,341] [INFO] [timer.py:264:stop] epoch=3/micro_step=248/global_step=1180, RunningAvgSamplesPerSec=13.315165122600238, CurrSamplesPerSec=12.564047165061556, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
325
+ [2024-11-12 10:23:34,862] [INFO] [logging.py:96:log_dist] [Rank 0] step=1190, skipped=0, lr=[1.2526557976586268e-05, 1.2526557976586268e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
326
+ [2024-11-12 10:23:34,863] [INFO] [timer.py:264:stop] epoch=3/micro_step=268/global_step=1190, RunningAvgSamplesPerSec=13.31844023579684, CurrSamplesPerSec=15.041789137896004, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
327
+ [2024-11-12 10:24:30,091] [INFO] [logging.py:96:log_dist] [Rank 0] step=1200, skipped=0, lr=[1.213017996225424e-05, 1.213017996225424e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
328
+ [2024-11-12 10:24:30,092] [INFO] [timer.py:264:stop] epoch=3/micro_step=288/global_step=1200, RunningAvgSamplesPerSec=13.309001079024005, CurrSamplesPerSec=10.541265033378687, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
329
+ Saving checkpoint at step 2400 ...
330
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
331
+ Saving 16-bit model...
332
+ [2024-11-12 10:24:44,305] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1200 is about to be saved!
333
+ [2024-11-12 10:24:44,307] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin, tag: global_step1200
334
+ [2024-11-12 10:24:44,307] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin...
335
+ [2024-11-12 10:25:00,864] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin.
336
+ [2024-11-12 10:25:00,867] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1200 is ready now!
337
+ Model saved!
338
+ Saving 16-bit model...
339
+ [2024-11-12 10:25:10,219] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1200 is about to be saved!
340
+ [2024-11-12 10:25:10,221] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin, tag: global_step1200
341
+ [2024-11-12 10:25:10,221] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin...
342
+ [2024-11-12 10:25:31,899] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2400.bin.
343
+ [2024-11-12 10:25:31,900] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1200 is ready now!
344
+ Model saved!
345
+ Checkpoint saved.
346
+ [2024-11-12 10:26:22,528] [INFO] [logging.py:96:log_dist] [Rank 0] step=1210, skipped=0, lr=[1.1738155905995186e-05, 1.1738155905995186e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
347
+ [2024-11-12 10:26:22,529] [INFO] [timer.py:264:stop] epoch=3/micro_step=308/global_step=1210, RunningAvgSamplesPerSec=13.310976144124597, CurrSamplesPerSec=13.716145845655182, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
348
+ [2024-11-12 10:27:13,156] [INFO] [logging.py:96:log_dist] [Rank 0] step=1220, skipped=0, lr=[1.1350618432531098e-05, 1.1350618432531098e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
349
+ [2024-11-12 10:27:13,157] [INFO] [timer.py:264:stop] epoch=3/micro_step=328/global_step=1220, RunningAvgSamplesPerSec=13.313287037618585, CurrSamplesPerSec=13.249320836208195, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
350
+ [2024-11-12 10:28:06,341] [INFO] [logging.py:96:log_dist] [Rank 0] step=1230, skipped=0, lr=[1.0967698648738866e-05, 1.0967698648738866e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
351
+ [2024-11-12 10:28:06,342] [INFO] [timer.py:264:stop] epoch=3/micro_step=348/global_step=1230, RunningAvgSamplesPerSec=13.309479597244271, CurrSamplesPerSec=13.3808760295492, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
352
+ [2024-11-12 10:28:56,982] [INFO] [logging.py:96:log_dist] [Rank 0] step=1240, skipped=0, lr=[1.0589526099295816e-05, 1.0589526099295816e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
353
+ [2024-11-12 10:28:56,983] [INFO] [timer.py:264:stop] epoch=3/micro_step=368/global_step=1240, RunningAvgSamplesPerSec=13.310696316549159, CurrSamplesPerSec=14.725364240818903, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
354
+ [2024-11-12 10:29:50,064] [INFO] [logging.py:96:log_dist] [Rank 0] step=1250, skipped=0, lr=[1.0216228722853735e-05, 1.0216228722853735e-05], mom=[[0.9, 0.95], [0.9, 0.95]]
355
+ [2024-11-12 10:29:50,065] [INFO] [timer.py:264:stop] epoch=3/micro_step=388/global_step=1250, RunningAvgSamplesPerSec=13.30687912483691, CurrSamplesPerSec=11.146967550500829, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
356
+ [2024-11-12 10:30:39,877] [INFO] [logging.py:96:log_dist] [Rank 0] step=1260, skipped=0, lr=[9.847932808756308e-06, 9.847932808756308e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
357
+ [2024-11-12 10:30:39,878] [INFO] [timer.py:264:stop] epoch=3/micro_step=408/global_step=1260, RunningAvgSamplesPerSec=13.311650776938508, CurrSamplesPerSec=14.829740880556027, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
358
+ [2024-11-12 10:31:32,354] [INFO] [logging.py:96:log_dist] [Rank 0] step=1270, skipped=0, lr=[9.48476295431443e-06, 9.48476295431443e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
359
+ [2024-11-12 10:31:32,355] [INFO] [timer.py:264:stop] epoch=3/micro_step=428/global_step=1270, RunningAvgSamplesPerSec=13.309168729063913, CurrSamplesPerSec=13.571046031877607, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
360
+ [2024-11-12 10:32:24,164] [INFO] [logging.py:96:log_dist] [Rank 0] step=1280, skipped=0, lr=[9.126842022654003e-06, 9.126842022654003e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
361
+ [2024-11-12 10:32:24,165] [INFO] [timer.py:264:stop] epoch=3/micro_step=448/global_step=1280, RunningAvgSamplesPerSec=13.308511944145916, CurrSamplesPerSec=13.94143983438274, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
362
+ [2024-11-12 10:33:16,769] [INFO] [logging.py:96:log_dist] [Rank 0] step=1290, skipped=0, lr=[8.774291101150408e-06, 8.774291101150408e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
363
+ [2024-11-12 10:33:16,770] [INFO] [timer.py:264:stop] epoch=3/micro_step=468/global_step=1290, RunningAvgSamplesPerSec=13.305750646091278, CurrSamplesPerSec=11.319639747356943, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
364
+ [2024-11-12 10:34:10,115] [INFO] [logging.py:96:log_dist] [Rank 0] step=1300, skipped=0, lr=[8.427229460463696e-06, 8.427229460463696e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
365
+ [2024-11-12 10:34:10,116] [INFO] [timer.py:264:stop] epoch=3/micro_step=488/global_step=1300, RunningAvgSamplesPerSec=13.301445208963852, CurrSamplesPerSec=13.849357021669912, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
366
+ [2024-11-12 10:35:01,342] [INFO] [logging.py:96:log_dist] [Rank 0] step=1310, skipped=0, lr=[8.085774514188454e-06, 8.085774514188454e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
367
+ [2024-11-12 10:35:01,344] [INFO] [timer.py:264:stop] epoch=3/micro_step=508/global_step=1310, RunningAvgSamplesPerSec=13.302156872923947, CurrSamplesPerSec=13.639201313267824, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
368
+ [2024-11-12 10:35:52,385] [INFO] [logging.py:96:log_dist] [Rank 0] step=1320, skipped=0, lr=[7.75004177913188e-06, 7.75004177913188e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
369
+ [2024-11-12 10:35:52,386] [INFO] [timer.py:264:stop] epoch=3/micro_step=528/global_step=1320, RunningAvgSamplesPerSec=13.30395516698252, CurrSamplesPerSec=14.59412134465565, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
370
+ [2024-11-12 10:36:51,593] [INFO] [logging.py:96:log_dist] [Rank 0] step=1330, skipped=0, lr=[7.420144836233564e-06, 7.420144836233564e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
371
+ [2024-11-12 10:36:51,594] [INFO] [timer.py:264:stop] epoch=3/micro_step=548/global_step=1330, RunningAvgSamplesPerSec=13.287670368260292, CurrSamplesPerSec=13.454378683308082, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
372
+ [2024-11-12 10:37:41,488] [INFO] [logging.py:96:log_dist] [Rank 0] step=1340, skipped=0, lr=[7.096195292140173e-06, 7.096195292140173e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
373
+ [2024-11-12 10:37:41,489] [INFO] [timer.py:264:stop] epoch=3/micro_step=568/global_step=1340, RunningAvgSamplesPerSec=13.290967903672813, CurrSamplesPerSec=14.066296708353384, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
374
+ [2024-11-12 10:38:33,707] [INFO] [logging.py:96:log_dist] [Rank 0] step=1350, skipped=0, lr=[6.778302741448103e-06, 6.778302741448103e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
375
+ [2024-11-12 10:38:33,708] [INFO] [timer.py:264:stop] epoch=3/micro_step=588/global_step=1350, RunningAvgSamplesPerSec=13.288990599401712, CurrSamplesPerSec=13.881917988904403, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
376
+ [2024-11-12 10:39:22,740] [INFO] [logging.py:96:log_dist] [Rank 0] step=1360, skipped=0, lr=[6.46657472962679e-06, 6.46657472962679e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
377
+ [2024-11-12 10:39:22,741] [INFO] [timer.py:264:stop] epoch=3/micro_step=608/global_step=1360, RunningAvgSamplesPerSec=13.294280380468793, CurrSamplesPerSec=15.113685513207692, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
378
+ [2024-11-12 10:40:12,119] [INFO] [logging.py:96:log_dist] [Rank 0] step=1370, skipped=0, lr=[6.16111671663524e-06, 6.16111671663524e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
379
+ [2024-11-12 10:40:12,120] [INFO] [timer.py:264:stop] epoch=3/micro_step=628/global_step=1370, RunningAvgSamplesPerSec=13.297719201387292, CurrSamplesPerSec=12.79749975952529, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
380
+ [2024-11-12 10:41:02,355] [INFO] [logging.py:96:log_dist] [Rank 0] step=1380, skipped=0, lr=[5.8620320412441475e-06, 5.8620320412441475e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
381
+ [2024-11-12 10:41:02,356] [INFO] [timer.py:264:stop] epoch=3/micro_step=648/global_step=1380, RunningAvgSamplesPerSec=13.300767385841707, CurrSamplesPerSec=13.329218575481406, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
382
+ [2024-11-12 10:41:55,494] [INFO] [logging.py:96:log_dist] [Rank 0] step=1390, skipped=0, lr=[5.5694218860755625e-06, 5.5694218860755625e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
383
+ [2024-11-12 10:41:55,495] [INFO] [timer.py:264:stop] epoch=3/micro_step=668/global_step=1390, RunningAvgSamplesPerSec=13.29664256005591, CurrSamplesPerSec=14.290803571762657, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
384
+ [2024-11-12 10:42:45,379] [INFO] [logging.py:96:log_dist] [Rank 0] step=1400, skipped=0, lr=[5.2833852433720855e-06, 5.2833852433720855e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
385
+ [2024-11-12 10:42:45,380] [INFO] [timer.py:264:stop] epoch=3/micro_step=688/global_step=1400, RunningAvgSamplesPerSec=13.30080907047133, CurrSamplesPerSec=14.279848014465236, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
386
+ Saving checkpoint at step 2800 ...
387
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
388
+ Saving 16-bit model...
389
+ [2024-11-12 10:42:58,030] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1400 is about to be saved!
390
+ [2024-11-12 10:42:58,032] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin, tag: global_step1400
391
+ [2024-11-12 10:42:58,032] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin...
392
+ [2024-11-12 10:43:14,984] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin.
393
+ [2024-11-12 10:43:14,985] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1400 is ready now!
394
+ Model saved!
395
+ Saving 16-bit model...
396
+ [2024-11-12 10:43:25,392] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1400 is about to be saved!
397
+ [2024-11-12 10:43:25,393] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin, tag: global_step1400
398
+ [2024-11-12 10:43:25,393] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin...
399
+ [2024-11-12 10:43:48,363] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_2800.bin.
400
+ [2024-11-12 10:43:48,364] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1400 is ready now!
401
+ Model saved!
402
+ Checkpoint saved.
403
+ [2024-11-12 10:44:40,329] [INFO] [logging.py:96:log_dist] [Rank 0] step=1410, skipped=0, lr=[5.004018881507016e-06, 5.004018881507016e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
404
+ [2024-11-12 10:44:40,330] [INFO] [timer.py:264:stop] epoch=4/micro_step=4/global_step=1410, RunningAvgSamplesPerSec=13.300082404758356, CurrSamplesPerSec=8.424868847641598, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
405
+ [2024-11-12 10:45:28,697] [INFO] [logging.py:96:log_dist] [Rank 0] step=1420, skipped=0, lr=[4.731417312246877e-06, 4.731417312246877e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
406
+ [2024-11-12 10:45:28,698] [INFO] [timer.py:264:stop] epoch=4/micro_step=24/global_step=1420, RunningAvgSamplesPerSec=13.306174854189013, CurrSamplesPerSec=14.387195950748813, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
407
+ [2024-11-12 10:46:16,858] [INFO] [logging.py:96:log_dist] [Rank 0] step=1430, skipped=0, lr=[4.465672758777351e-06, 4.465672758777351e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
408
+ [2024-11-12 10:46:16,859] [INFO] [timer.py:264:stop] epoch=4/micro_step=44/global_step=1430, RunningAvgSamplesPerSec=13.31298797950946, CurrSamplesPerSec=14.366896812177284, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
409
+ [2024-11-12 10:47:09,377] [INFO] [logging.py:96:log_dist] [Rank 0] step=1440, skipped=0, lr=[4.206875124503506e-06, 4.206875124503506e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
410
+ [2024-11-12 10:47:09,378] [INFO] [timer.py:264:stop] epoch=4/micro_step=64/global_step=1440, RunningAvgSamplesPerSec=13.311559372047915, CurrSamplesPerSec=13.393533075067023, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
411
+ [2024-11-12 10:48:00,280] [INFO] [logging.py:96:log_dist] [Rank 0] step=1450, skipped=0, lr=[3.9551119626347695e-06, 3.9551119626347695e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
412
+ [2024-11-12 10:48:00,280] [INFO] [timer.py:264:stop] epoch=4/micro_step=84/global_step=1450, RunningAvgSamplesPerSec=13.31244648639003, CurrSamplesPerSec=14.058757387182883, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
413
+ [2024-11-12 10:48:49,946] [INFO] [logging.py:96:log_dist] [Rank 0] step=1460, skipped=0, lr=[3.710468446565005e-06, 3.710468446565005e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
414
+ [2024-11-12 10:48:49,946] [INFO] [timer.py:264:stop] epoch=4/micro_step=104/global_step=1460, RunningAvgSamplesPerSec=13.315758552170506, CurrSamplesPerSec=13.839085443416847, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
415
+ [2024-11-12 10:49:40,534] [INFO] [logging.py:96:log_dist] [Rank 0] step=1470, skipped=0, lr=[3.4730273410577274e-06, 3.4730273410577274e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
416
+ [2024-11-12 10:49:40,535] [INFO] [timer.py:264:stop] epoch=4/micro_step=124/global_step=1470, RunningAvgSamplesPerSec=13.318408025641999, CurrSamplesPerSec=12.260271913621812, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
417
+ [2024-11-12 10:50:30,820] [INFO] [logging.py:96:log_dist] [Rank 0] step=1480, skipped=0, lr=[3.2428689742461188e-06, 3.2428689742461188e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
418
+ [2024-11-12 10:50:30,821] [INFO] [timer.py:264:stop] epoch=4/micro_step=144/global_step=1480, RunningAvgSamplesPerSec=13.320489932970325, CurrSamplesPerSec=15.042129663922514, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
419
+ [2024-11-12 10:51:20,889] [INFO] [logging.py:96:log_dist] [Rank 0] step=1490, skipped=0, lr=[3.020071210457451e-06, 3.020071210457451e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
420
+ [2024-11-12 10:51:20,890] [INFO] [timer.py:264:stop] epoch=4/micro_step=164/global_step=1490, RunningAvgSamplesPerSec=13.323469002580365, CurrSamplesPerSec=15.111296440696567, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
421
+ [2024-11-12 10:52:10,423] [INFO] [logging.py:96:log_dist] [Rank 0] step=1500, skipped=0, lr=[2.8047094238709633e-06, 2.8047094238709633e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
422
+ [2024-11-12 10:52:10,424] [INFO] [timer.py:264:stop] epoch=4/micro_step=184/global_step=1500, RunningAvgSamplesPerSec=13.32754448705049, CurrSamplesPerSec=14.979724690127203, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
423
+ [2024-11-12 10:53:03,486] [INFO] [logging.py:96:log_dist] [Rank 0] step=1510, skipped=0, lr=[2.5968564730182083e-06, 2.5968564730182083e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
424
+ [2024-11-12 10:53:03,487] [INFO] [timer.py:264:stop] epoch=4/micro_step=204/global_step=1510, RunningAvgSamplesPerSec=13.324782544632754, CurrSamplesPerSec=14.093607165156875, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
425
+ [2024-11-12 10:53:57,964] [INFO] [logging.py:96:log_dist] [Rank 0] step=1520, skipped=0, lr=[2.396582676134462e-06, 2.396582676134462e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
426
+ [2024-11-12 10:53:57,965] [INFO] [timer.py:264:stop] epoch=4/micro_step=224/global_step=1520, RunningAvgSamplesPerSec=13.318995411927387, CurrSamplesPerSec=13.063804189396569, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
427
+ [2024-11-12 10:54:47,711] [INFO] [logging.py:96:log_dist] [Rank 0] step=1530, skipped=0, lr=[2.2039557873695188e-06, 2.2039557873695188e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
428
+ [2024-11-12 10:54:47,711] [INFO] [timer.py:264:stop] epoch=4/micro_step=244/global_step=1530, RunningAvgSamplesPerSec=13.321911830869968, CurrSamplesPerSec=14.915771367301977, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
429
+ [2024-11-12 10:55:42,069] [INFO] [logging.py:96:log_dist] [Rank 0] step=1540, skipped=0, lr=[2.0190409738659653e-06, 2.0190409738659653e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
430
+ [2024-11-12 10:55:42,070] [INFO] [timer.py:264:stop] epoch=4/micro_step=264/global_step=1540, RunningAvgSamplesPerSec=13.316215176151665, CurrSamplesPerSec=13.612489775121794, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
431
+ [2024-11-12 10:56:33,542] [INFO] [logging.py:96:log_dist] [Rank 0] step=1550, skipped=0, lr=[1.8419007937126255e-06, 1.8419007937126255e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
432
+ [2024-11-12 10:56:33,543] [INFO] [timer.py:264:stop] epoch=4/micro_step=284/global_step=1550, RunningAvgSamplesPerSec=13.31579176262506, CurrSamplesPerSec=11.938031197480003, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
433
+ [2024-11-12 10:57:28,157] [INFO] [logging.py:96:log_dist] [Rank 0] step=1560, skipped=0, lr=[1.6725951747806918e-06, 1.6725951747806918e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
434
+ [2024-11-12 10:57:28,158] [INFO] [timer.py:264:stop] epoch=4/micro_step=304/global_step=1560, RunningAvgSamplesPerSec=13.309479512988279, CurrSamplesPerSec=14.577624486192173, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
435
+ [2024-11-12 10:58:18,721] [INFO] [logging.py:96:log_dist] [Rank 0] step=1570, skipped=0, lr=[1.5111813944496539e-06, 1.5111813944496539e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
436
+ [2024-11-12 10:58:18,722] [INFO] [timer.py:264:stop] epoch=4/micro_step=324/global_step=1570, RunningAvgSamplesPerSec=13.310390185432857, CurrSamplesPerSec=12.173278775829349, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
437
+ [2024-11-12 10:59:11,560] [INFO] [logging.py:96:log_dist] [Rank 0] step=1580, skipped=0, lr=[1.3577140602299448e-06, 1.3577140602299448e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
438
+ [2024-11-12 10:59:11,561] [INFO] [timer.py:264:stop] epoch=4/micro_step=344/global_step=1580, RunningAvgSamplesPerSec=13.307910946447468, CurrSamplesPerSec=11.816768390320345, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
439
+ [2024-11-12 11:00:02,570] [INFO] [logging.py:96:log_dist] [Rank 0] step=1590, skipped=0, lr=[1.212245091288794e-06, 1.212245091288794e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
440
+ [2024-11-12 11:00:02,571] [INFO] [timer.py:264:stop] epoch=4/micro_step=364/global_step=1590, RunningAvgSamplesPerSec=13.309121542768606, CurrSamplesPerSec=12.719856643237279, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
441
+ [2024-11-12 11:00:54,502] [INFO] [logging.py:96:log_dist] [Rank 0] step=1600, skipped=0, lr=[1.0748237008855828e-06, 1.0748237008855828e-06], mom=[[0.9, 0.95], [0.9, 0.95]]
442
+ [2024-11-12 11:00:54,503] [INFO] [timer.py:264:stop] epoch=4/micro_step=384/global_step=1600, RunningAvgSamplesPerSec=13.308074722292908, CurrSamplesPerSec=15.261070434905404, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
443
+ Saving checkpoint at step 3200 ...
444
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
445
+ Saving 16-bit model...
446
+ [2024-11-12 11:01:09,427] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1600 is about to be saved!
447
+ [2024-11-12 11:01:09,429] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin, tag: global_step1600
448
+ [2024-11-12 11:01:09,429] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin...
449
+ [2024-11-12 11:01:30,549] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin.
450
+ [2024-11-12 11:01:30,551] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1600 is ready now!
451
+ Model saved!
452
+ Saving 16-bit model...
453
+ [2024-11-12 11:01:41,122] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1600 is about to be saved!
454
+ [2024-11-12 11:01:41,123] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin, tag: global_step1600
455
+ [2024-11-12 11:01:41,123] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin...
456
+ [2024-11-12 11:02:03,626] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model_3200.bin.
457
+ [2024-11-12 11:02:03,627] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1600 is ready now!
458
+ Model saved!
459
+ Checkpoint saved.
460
+ [2024-11-12 11:02:53,971] [INFO] [logging.py:96:log_dist] [Rank 0] step=1610, skipped=0, lr=[9.45496379722624e-07, 9.45496379722624e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
461
+ [2024-11-12 11:02:53,972] [INFO] [timer.py:264:stop] epoch=4/micro_step=404/global_step=1610, RunningAvgSamplesPerSec=13.309905592491026, CurrSamplesPerSec=14.748584036364806, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
462
+ [2024-11-12 11:03:46,152] [INFO] [logging.py:96:log_dist] [Rank 0] step=1620, skipped=0, lr=[8.243068802169906e-07, 8.243068802169906e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
463
+ [2024-11-12 11:03:46,153] [INFO] [timer.py:264:stop] epoch=4/micro_step=424/global_step=1620, RunningAvgSamplesPerSec=13.308079127511396, CurrSamplesPerSec=8.70323026519259, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
464
+ [2024-11-12 11:04:36,640] [INFO] [logging.py:96:log_dist] [Rank 0] step=1630, skipped=0, lr=[7.112962016987634e-07, 7.112962016987634e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
465
+ [2024-11-12 11:04:36,641] [INFO] [timer.py:264:stop] epoch=4/micro_step=444/global_step=1630, RunningAvgSamplesPerSec=13.309797745853881, CurrSamplesPerSec=13.608449292653019, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
466
+ [2024-11-12 11:05:27,327] [INFO] [logging.py:96:log_dist] [Rank 0] step=1640, skipped=0, lr=[6.065025765406213e-07, 6.065025765406213e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
467
+ [2024-11-12 11:05:27,328] [INFO] [timer.py:264:stop] epoch=4/micro_step=464/global_step=1640, RunningAvgSamplesPerSec=13.310743561152151, CurrSamplesPerSec=14.022836300689034, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
468
+ [2024-11-12 11:06:23,362] [INFO] [logging.py:96:log_dist] [Rank 0] step=1650, skipped=0, lr=[5.099614572235622e-07, 5.099614572235622e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
469
+ [2024-11-12 11:06:23,363] [INFO] [timer.py:264:stop] epoch=4/micro_step=484/global_step=1650, RunningAvgSamplesPerSec=13.303107401732731, CurrSamplesPerSec=11.270466117917143, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
470
+ [2024-11-12 11:07:14,043] [INFO] [logging.py:96:log_dist] [Rank 0] step=1660, skipped=0, lr=[4.217055043430368e-07, 4.217055043430368e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
471
+ [2024-11-12 11:07:14,044] [INFO] [timer.py:264:stop] epoch=4/micro_step=504/global_step=1660, RunningAvgSamplesPerSec=13.303911694244524, CurrSamplesPerSec=12.311078842729271, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
472
+ [2024-11-12 11:08:04,940] [INFO] [logging.py:96:log_dist] [Rank 0] step=1670, skipped=0, lr=[3.417645755596333e-07, 3.417645755596333e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
473
+ [2024-11-12 11:08:04,941] [INFO] [timer.py:264:stop] epoch=4/micro_step=524/global_step=1670, RunningAvgSamplesPerSec=13.30527924084017, CurrSamplesPerSec=12.844344875845103, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
474
+ [2024-11-12 11:09:03,539] [INFO] [logging.py:96:log_dist] [Rank 0] step=1680, skipped=0, lr=[2.7016571549800806e-07, 2.7016571549800806e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
475
+ [2024-11-12 11:09:03,540] [INFO] [timer.py:264:stop] epoch=4/micro_step=544/global_step=1680, RunningAvgSamplesPerSec=13.292778473186681, CurrSamplesPerSec=13.616692212560867, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
476
+ [2024-11-12 11:09:53,417] [INFO] [logging.py:96:log_dist] [Rank 0] step=1690, skipped=0, lr=[2.0693314659746278e-07, 2.0693314659746278e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
477
+ [2024-11-12 11:09:53,417] [INFO] [timer.py:264:stop] epoch=4/micro_step=564/global_step=1690, RunningAvgSamplesPerSec=13.295568640371517, CurrSamplesPerSec=15.061971663296424, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
478
+ [2024-11-12 11:10:46,532] [INFO] [logging.py:96:log_dist] [Rank 0] step=1700, skipped=0, lr=[1.520882609173352e-07, 1.520882609173352e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
479
+ [2024-11-12 11:10:46,533] [INFO] [timer.py:264:stop] epoch=4/micro_step=584/global_step=1700, RunningAvgSamplesPerSec=13.293118088403272, CurrSamplesPerSec=12.895263576244139, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
480
+ [2024-11-12 11:11:35,590] [INFO] [logging.py:96:log_dist] [Rank 0] step=1710, skipped=0, lr=[1.056496128998774e-07, 1.056496128998774e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
481
+ [2024-11-12 11:11:35,591] [INFO] [timer.py:264:stop] epoch=4/micro_step=604/global_step=1710, RunningAvgSamplesPerSec=13.297323115977306, CurrSamplesPerSec=12.737831983944682, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
482
+ [2024-11-12 11:12:24,828] [INFO] [logging.py:96:log_dist] [Rank 0] step=1720, skipped=0, lr=[6.763291309317132e-08, 6.763291309317132e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
483
+ [2024-11-12 11:12:24,829] [INFO] [timer.py:264:stop] epoch=4/micro_step=624/global_step=1720, RunningAvgSamplesPerSec=13.300588368547286, CurrSamplesPerSec=14.138239247081426, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
484
+ [2024-11-12 11:13:15,294] [INFO] [logging.py:96:log_dist] [Rank 0] step=1730, skipped=0, lr=[3.8051022836110836e-08, 3.8051022836110836e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
485
+ [2024-11-12 11:13:15,295] [INFO] [timer.py:264:stop] epoch=4/micro_step=644/global_step=1730, RunningAvgSamplesPerSec=13.302046602956041, CurrSamplesPerSec=13.59455125768058, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
486
+ [2024-11-12 11:14:08,410] [INFO] [logging.py:96:log_dist] [Rank 0] step=1740, skipped=0, lr=[1.6913949907321204e-08, 1.6913949907321204e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
487
+ [2024-11-12 11:14:08,411] [INFO] [timer.py:264:stop] epoch=4/micro_step=664/global_step=1740, RunningAvgSamplesPerSec=13.300620381888876, CurrSamplesPerSec=14.191562012478505, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
488
+ [2024-11-12 11:14:59,106] [INFO] [logging.py:96:log_dist] [Rank 0] step=1750, skipped=0, lr=[4.2288451394562055e-09, 4.2288451394562055e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
489
+ [2024-11-12 11:14:59,107] [INFO] [timer.py:264:stop] epoch=4/micro_step=684/global_step=1750, RunningAvgSamplesPerSec=13.302238401350479, CurrSamplesPerSec=13.361762988342704, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
490
+ [2024-11-12 11:15:48,289] [INFO] [logging.py:96:log_dist] [Rank 0] step=1760, skipped=0, lr=[0.0, 0.0], mom=[[0.9, 0.95], [0.9, 0.95]]
491
+ [2024-11-12 11:15:48,289] [INFO] [timer.py:264:stop] epoch=4/micro_step=704/global_step=1760, RunningAvgSamplesPerSec=13.30676923911034, CurrSamplesPerSec=17.907194447352197, MemAllocated=14.14GB, MaxMemAllocated=25.49GB
492
+ Saving model to "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k" ...
493
+ Saving 16-bit model...
494
+ [2024-11-12 11:16:03,390] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1760 is about to be saved!
495
+ [2024-11-12 11:16:03,391] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin, tag: global_step1760
496
+ [2024-11-12 11:16:03,392] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin...
497
+ [2024-11-12 11:16:19,447] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin.
498
+ [2024-11-12 11:16:19,448] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1760 is ready now!
499
+ Model saved!
500
+ Saving 16-bit model...
501
+ [2024-11-12 11:16:27,719] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1760 is about to be saved!
502
+ [2024-11-12 11:16:27,720] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin, tag: global_step1760
503
+ [2024-11-12 11:16:27,721] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin...
504
+ [2024-11-12 11:16:51,906] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/pytorch_model.bin.
505
+ [2024-11-12 11:16:51,908] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1760 is ready now!
506
+ Model saved!
slice_1600/wandb/run-20241112_083557-komthdmh/files/requirements.txt ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ align-anything==0.0.1.dev0
2
+ nvidia-cusolver-cu12==11.4.5.107
3
+ nvidia-curand-cu12==10.3.2.106
4
+ wcwidth==0.2.13
5
+ nvidia-cuda-nvrtc-cu12==12.1.105
6
+ pycparser==2.22
7
+ tokenizers==0.19.1
8
+ grpcio==1.66.2
9
+ joblib==1.4.2
10
+ virtualenv==20.26.6
11
+ pyzmq==26.2.0
12
+ cffi==1.17.1
13
+ sentencepiece==0.2.0
14
+ nvidia-nvtx-cu12==12.1.105
15
+ docker-pycreds==0.4.0
16
+ outlines==0.1.1.dev4+ga2fd35c
17
+ nvidia-nvjitlink-cu12==12.6.77
18
+ annotated-types==0.7.0
19
+ certifi==2024.8.30
20
+ interegular==0.3.3
21
+ Jinja2==3.1.4
22
+ Brotli==1.1.0
23
+ fairscale==0.4.13
24
+ gradio_client==1.4.0
25
+ opencv-python==4.6.0.66
26
+ pyarrow==17.0.0
27
+ absl-py==2.1.0
28
+ lm-format-enforcer==0.10.6
29
+ pydantic_core==2.23.4
30
+ llvmlite==0.43.0
31
+ accelerate==1.0.1
32
+ pytest-split==0.8.0
33
+ nvidia-cuda-cupti-cu12==12.1.105
34
+ watchfiles==0.24.0
35
+ optree==0.13.0
36
+ py-cpuinfo==9.0.0
37
+ scikit-learn==1.5.2
38
+ ftfy==6.3.0
39
+ fastapi==0.115.0
40
+ psutil==6.0.0
41
+ MarkupSafe==2.1.5
42
+ nvidia-cublas-cu12==12.1.3.1
43
+ pip==24.2
44
+ websockets==12.0
45
+ tomlkit==0.12.0
46
+ torchaudio==2.4.0
47
+ huggingface-hub==0.25.2
48
+ mistral_common==1.4.4
49
+ image-reward==1.5
50
+ pyparsing==3.1.4
51
+ aiohappyeyeballs==2.4.3
52
+ click==8.1.7
53
+ httptools==0.6.1
54
+ decorator==4.4.2
55
+ tqdm==4.66.5
56
+ fonttools==4.54.1
57
+ kiwisolver==1.4.7
58
+ ruff==0.6.9
59
+ openai==1.51.2
60
+ partial-json-parser==0.2.1.1.post4
61
+ xformers==0.0.27.post2
62
+ distlib==0.3.9
63
+ GitPython==3.1.43
64
+ pytest==7.2.0
65
+ imageio==2.35.1
66
+ msgspec==0.18.6
67
+ proglog==0.1.10
68
+ yarl==1.15.0
69
+ markdown-it-py==3.0.0
70
+ PyYAML==6.0.2
71
+ xxhash==3.5.0
72
+ braceexpand==0.1.7
73
+ datasets==3.0.1
74
+ mpmath==1.3.0
75
+ distro==1.9.0
76
+ term-image==0.7.2
77
+ python-dotenv==1.0.1
78
+ semantic-version==2.10.0
79
+ multidict==6.1.0
80
+ vllm==0.6.2
81
+ sentry-sdk==2.16.0
82
+ idna==3.10
83
+ starlette==0.38.6
84
+ args==0.1.0
85
+ peft==0.13.2
86
+ librosa==0.10.2.post1
87
+ urllib3==2.2.3
88
+ python-dateutil==2.9.0.post0
89
+ pycountry==24.6.1
90
+ six==1.16.0
91
+ ffmpy==0.4.0
92
+ nvitop==1.3.2
93
+ termcolor==2.5.0
94
+ multiprocess==0.70.16
95
+ cycler==0.12.1
96
+ charset-normalizer==3.4.0
97
+ aiofiles==23.2.1
98
+ shellingham==1.5.4
99
+ propcache==0.2.0
100
+ lark==1.2.2
101
+ torch==2.4.0
102
+ Werkzeug==3.0.4
103
+ nvidia-cusparse-cu12==12.1.0.106
104
+ clip==0.2.0
105
+ hjson==3.1.0
106
+ diffusers==0.30.3
107
+ attrs==24.2.0
108
+ lazy_loader==0.4
109
+ numpy==1.26.4
110
+ rpds-py==0.20.0
111
+ pytz==2024.2
112
+ audioread==3.0.1
113
+ platformdirs==4.3.6
114
+ deepspeed==0.15.2
115
+ gguf==0.10.0
116
+ wandb==0.18.3
117
+ prometheus_client==0.21.0
118
+ gitdb==4.0.11
119
+ packaging==24.1
120
+ cachetools==5.5.0
121
+ sympy==1.13.3
122
+ mutagen==1.47.0
123
+ contourpy==1.3.0
124
+ pluggy==1.5.0
125
+ python-multipart==0.0.12
126
+ soundfile==0.12.1
127
+ typer==0.12.5
128
+ timm==0.6.13
129
+ frozenlist==1.4.1
130
+ httpx==0.27.2
131
+ mmsg==0.1.dev20+g585c63a.d20241012
132
+ tiktoken==0.7.0
133
+ pydub==0.25.1
134
+ diskcache==5.6.3
135
+ einops==0.8.0
136
+ setproctitle==1.3.3
137
+ scipy==1.14.1
138
+ typing_extensions==4.12.2
139
+ httpcore==1.0.6
140
+ cfgv==3.4.0
141
+ requests==2.32.3
142
+ torchlibrosa==0.1.0
143
+ pydantic==2.9.2
144
+ torchvision==0.19.0
145
+ sniffio==1.3.1
146
+ pyairports==2.1.1
147
+ hpsv2==1.2.0
148
+ protobuf==3.20.3
149
+ wheel==0.44.0
150
+ smmap==5.0.1
151
+ zipp==3.20.2
152
+ iniconfig==2.0.0
153
+ airportsdata==20241001
154
+ clint==0.5.1
155
+ pooch==1.8.2
156
+ shortuuid==1.0.13
157
+ pycryptodomex==3.21.0
158
+ cloudpickle==3.1.0
159
+ transformers==4.44.0.dev0
160
+ regex==2024.9.11
161
+ numba==0.60.0
162
+ tzdata==2024.2
163
+ orjson==3.10.7
164
+ jsonschema-specifications==2024.10.1
165
+ safetensors==0.4.5
166
+ outlines_core==0.1.0
167
+ filelock==3.16.1
168
+ threadpoolctl==3.5.0
169
+ soxr==0.5.0.post1
170
+ nvidia-cufft-cu12==11.0.2.54
171
+ networkx==3.4.1
172
+ msgpack==1.1.0
173
+ pandas==2.2.3
174
+ align-anything==0.0.1.dev0
175
+ anyio==4.6.0
176
+ nvidia-cuda-runtime-cu12==12.1.105
177
+ bitsandbytes==0.44.1
178
+ aiohttp==3.10.10
179
+ matplotlib==3.9.2
180
+ triton==3.0.0
181
+ tensorboard==2.18.0
182
+ nodeenv==1.9.1
183
+ fsspec==2024.6.1
184
+ webdataset==0.2.100
185
+ imageio-ffmpeg==0.5.1
186
+ mdurl==0.1.2
187
+ identify==2.6.1
188
+ h11==0.14.0
189
+ uvloop==0.20.0
190
+ rich==13.9.2
191
+ frechet-audio-distance==0.1.2
192
+ uvicorn==0.31.1
193
+ pytorch-fid==0.3.0
194
+ yt-dlp==2024.8.6
195
+ jiter==0.6.1
196
+ nest-asyncio==1.6.0
197
+ pre_commit==4.0.1
198
+ referencing==0.35.1
199
+ resampy==0.4.3
200
+ tensorboard-data-server==0.7.2
201
+ importlib_metadata==8.5.0
202
+ aiosignal==1.3.1
203
+ dill==0.3.8
204
+ prometheus-fastapi-instrumentator==7.0.0
205
+ nvidia-ml-py==12.535.161
206
+ ninja==1.11.1.1
207
+ moviepy==1.0.3
208
+ nvidia-cudnn-cu12==9.1.0.70
209
+ Markdown==3.7
210
+ ray==2.37.0
211
+ gradio==5.0.2
212
+ jsonschema==4.23.0
213
+ Pygments==2.18.0
214
+ nvidia-nccl-cu12==2.20.5
215
+ pillow==10.4.0
216
+ setuptools==75.1.0
217
+ jaraco.text==3.12.1
218
+ inflect==7.3.1
219
+ jaraco.collections==5.1.0
220
+ autocommand==2.2.2
221
+ tomli==2.0.1
222
+ jaraco.context==5.3.0
223
+ jaraco.functools==4.0.1
224
+ importlib_resources==6.4.0
225
+ wheel==0.43.0
226
+ packaging==24.1
227
+ backports.tarfile==1.2.0
228
+ importlib_metadata==8.0.0
229
+ typing_extensions==4.12.2
230
+ zipp==3.19.2
231
+ typeguard==4.3.0
232
+ more-itertools==10.3.0
233
+ platformdirs==4.2.2
slice_1600/wandb/run-20241112_083557-komthdmh/files/wandb-metadata.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-198-generic-x86_64-with-glibc2.31",
3
+ "python": "3.11.10",
4
+ "startedAt": "2024-11-12T08:35:57.093246Z",
5
+ "args": [
6
+ "--local_rank=0",
7
+ "--model_name_or_path",
8
+ "/data/align-anything/hantao/models/0917_cham_ti2t_sft",
9
+ "--train_datasets",
10
+ "/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs",
11
+ "--train_data_files",
12
+ "llf_t2i_22.5k_tokenized.pt",
13
+ "--output_dir",
14
+ "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k",
15
+ "--train_template",
16
+ "AA_textfeedback",
17
+ "--train_split",
18
+ "train",
19
+ "--per_device_train_batch_size",
20
+ "4",
21
+ "--per_device_eval_batch_size",
22
+ "4",
23
+ "--gradient_accumulation_steps",
24
+ "2",
25
+ "--save_interval",
26
+ "400",
27
+ "--learning_rate",
28
+ "5e-5",
29
+ "--epochs",
30
+ "5",
31
+ "--lr_scheduler_type",
32
+ "cosine"
33
+ ],
34
+ "program": "-m align_anything.trainers.text_image_to_text_image.sft",
35
+ "git": {
36
+ "remote": "https://github.com/PKU-Alignment/align-anything.git",
37
+ "commit": "6fde660afc9985323f147930eedf188a5699adc7"
38
+ },
39
+ "email": "[email protected]",
40
+ "root": "../outputs/sft_tf_cham_1015_5ep_t2i_22.5k",
41
+ "host": "lyg0196",
42
+ "username": "align-anything",
43
+ "executable": "/home/align-anything/miniconda3/envs/hantao_cham/bin/python",
44
+ "cpu_count": 64,
45
+ "cpu_count_logical": 128,
46
+ "gpu": "[NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB]",
47
+ "gpu_count": 8,
48
+ "disk": {
49
+ "/": {
50
+ "total": "940744544256",
51
+ "used": "333241483264"
52
+ }
53
+ },
54
+ "memory": {
55
+ "total": "540647530496"
56
+ },
57
+ "cpu": {
58
+ "count": 64,
59
+ "countLogical": 128
60
+ },
61
+ "gpu_nvidia": [
62
+ {
63
+ "name": "NVIDIA A100-SXM4-80GB",
64
+ "memoryTotal": "85899345920",
65
+ "cudaCores": 6912,
66
+ "architecture": "Ampere"
67
+ },
68
+ {
69
+ "name": "NVIDIA A100-SXM4-80GB",
70
+ "memoryTotal": "85899345920",
71
+ "cudaCores": 6912,
72
+ "architecture": "Ampere"
73
+ },
74
+ {
75
+ "name": "NVIDIA A100-SXM4-80GB",
76
+ "memoryTotal": "85899345920",
77
+ "cudaCores": 6912,
78
+ "architecture": "Ampere"
79
+ },
80
+ {
81
+ "name": "NVIDIA A100-SXM4-80GB",
82
+ "memoryTotal": "85899345920",
83
+ "cudaCores": 6912,
84
+ "architecture": "Ampere"
85
+ },
86
+ {
87
+ "name": "NVIDIA A100-SXM4-80GB",
88
+ "memoryTotal": "85899345920",
89
+ "cudaCores": 6912,
90
+ "architecture": "Ampere"
91
+ },
92
+ {
93
+ "name": "NVIDIA A100-SXM4-80GB",
94
+ "memoryTotal": "85899345920",
95
+ "cudaCores": 6912,
96
+ "architecture": "Ampere"
97
+ },
98
+ {
99
+ "name": "NVIDIA A100-SXM4-80GB",
100
+ "memoryTotal": "85899345920",
101
+ "cudaCores": 6912,
102
+ "architecture": "Ampere"
103
+ },
104
+ {
105
+ "name": "NVIDIA A100-SXM4-80GB",
106
+ "memoryTotal": "85899345920",
107
+ "cudaCores": 6912,
108
+ "architecture": "Ampere"
109
+ }
110
+ ],
111
+ "cudaVersion": "12.4"
112
+ }
slice_1600/wandb/run-20241112_083557-komthdmh/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_step":3520,"train/step":3520,"train/loss":0.007134987972676754,"train/lr":0,"train/epoch":5,"_timestamp":1.731410148381941e+09,"_runtime":9655.892149045,"_wandb":{"runtime":9655}}
slice_1600/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-12T08:35:57.099449248Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-12T08:35:57.099478068Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-core.log"}
3
+ {"time":"2024-11-12T08:35:57.102383132Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-12T08:35:57.129619469Z","level":"INFO","msg":"created new stream","id":"komthdmh"}
5
+ {"time":"2024-11-12T08:35:57.129670069Z","level":"INFO","msg":"stream: started","id":"komthdmh"}
6
+ {"time":"2024-11-12T08:35:57.129709953Z","level":"INFO","msg":"handler: started","stream_id":{"value":"komthdmh"}}
7
+ {"time":"2024-11-12T08:35:57.129693059Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"komthdmh"}}
8
+ {"time":"2024-11-12T08:35:57.129699692Z","level":"INFO","msg":"sender: started","stream_id":{"value":"komthdmh"}}
9
+ {"time":"2024-11-12T08:35:57.832569737Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-12T08:35:57.837905637Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-12T11:16:52.98542237Z","level":"INFO","msg":"Stopping system monitor"}
12
+ {"time":"2024-11-12T11:16:53.007088373Z","level":"INFO","msg":"Stopped system monitor"}
13
+ {"time":"2024-11-12T11:16:53.498790149Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
14
+ {"time":"2024-11-12T11:16:53.49883056Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
15
+ {"time":"2024-11-12T11:16:54.435678748Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
16
+ {"time":"2024-11-12T11:16:55.889045635Z","level":"INFO","msg":"stream: closing","id":"komthdmh"}
17
+ {"time":"2024-11-12T11:16:55.889082565Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"komthdmh"}}
18
+ {"time":"2024-11-12T11:16:55.889144434Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"komthdmh"}}
19
+ {"time":"2024-11-12T11:16:55.889239107Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"komthdmh"}}
20
+ {"time":"2024-11-12T11:16:55.892207197Z","level":"INFO","msg":"stream: closed","id":"komthdmh"}
slice_1600/wandb/run-20241112_083557-komthdmh/logs/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Configure stats pid to 1335408
3
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-12 08:35:57,085 WARNING MainThread:1335408 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug.log
11
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k/wandb/run-20241112_083557-komthdmh/logs/debug-internal.log
12
+ 2024-11-12 08:35:57,085 INFO MainThread:1335408 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 5.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 5e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_t2i_22.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1015_5ep_t2i_22.5k', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0917_cham_ti2t_sft', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():667] starting backend
16
+ 2024-11-12 08:35:57,086 INFO MainThread:1335408 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-12 08:35:57,092 INFO MainThread:1335408 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-12 08:35:57,097 INFO MainThread:1335408 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-12 08:35:57,113 INFO MainThread:1335408 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-12 08:35:57,827 INFO MainThread:1335408 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-12 08:35:57,973 INFO MainThread:1335408 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-12 08:35:57,974 INFO MainThread:1335408 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-12 08:35:57,976 INFO MainThread:1335408 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-12 11:16:52,981 INFO MainThread:1335408 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/komthdmh
28
+ 2024-11-12 11:16:52,983 INFO MainThread:1335408 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-12 11:16:52,984 INFO MainThread:1335408 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-12 11:16:55,871 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-12 11:16:55,873 INFO MainThread:1335408 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-12 11:16:55,884 INFO MainThread:1335408 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_1600/wandb/run-20241112_083557-komthdmh/run-komthdmh.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:561218f9100c1d380fac9397893e89b2567c6f999883a5cac5456da1d5c76d97
3
+ size 13892810
slice_2000/arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_t2i_22.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1015_5ep_t2i_22.5k
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 5.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 5.0e-05
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
slice_2000/config.json ADDED
The diff for this file is too large to render. See raw diff