Hugging Face's logo

htlou
/
backup_0202_sft_tf_cham_1111_13.5k_ti2ti-outputs

PyTorch
Safetensors
chameleon
Model card Files Community
htlou commited on
Commit
07865ff
·
verified ·
1 Parent(s): 1530c1c

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +4 -0
  2. arguments.yaml +49 -0
  3. config.json +0 -0
  4. environ.txt +149 -0
  5. preprocessor_config.json +28 -0
  6. processor_config.json +5 -0
  7. pytorch_model.bin +3 -0
  8. script.sh +81 -0
  9. slice_1200/arguments.yaml +49 -0
  10. slice_1200/config.json +0 -0
  11. slice_1200/environ.txt +149 -0
  12. slice_1200/preprocessor_config.json +28 -0
  13. slice_1200/processor_config.json +5 -0
  14. slice_1200/pytorch_model.bin +3 -0
  15. slice_1200/script.sh +81 -0
  16. slice_1200/special_tokens_map.json +37 -0
  17. slice_1200/tokenizer.json +0 -0
  18. slice_1200/tokenizer_config.json +0 -0
  19. slice_1200/wandb/debug-internal.log +21 -0
  20. slice_1200/wandb/debug.log +33 -0
  21. slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/config.yaml +96 -0
  22. slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/output.log +195 -0
  23. slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/requirements.txt +226 -0
  24. slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/wandb-metadata.json +112 -0
  25. slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/wandb-summary.json +1 -0
  26. slice_1200/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log +21 -0
  27. slice_1200/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log +33 -0
  28. slice_1200/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb +3 -0
  29. slice_400/arguments.yaml +49 -0
  30. slice_400/config.json +0 -0
  31. slice_400/environ.txt +149 -0
  32. slice_400/preprocessor_config.json +28 -0
  33. slice_400/processor_config.json +5 -0
  34. slice_400/pytorch_model.bin +3 -0
  35. slice_400/script.sh +81 -0
  36. slice_400/special_tokens_map.json +37 -0
  37. slice_400/tokenizer.json +0 -0
  38. slice_400/tokenizer_config.json +0 -0
  39. slice_400/wandb/debug-internal.log +21 -0
  40. slice_400/wandb/debug.log +33 -0
  41. slice_400/wandb/run-20241111_095028-sq0dhwvm/files/config.yaml +96 -0
  42. slice_400/wandb/run-20241111_095028-sq0dhwvm/files/output.log +195 -0
  43. slice_400/wandb/run-20241111_095028-sq0dhwvm/files/requirements.txt +226 -0
  44. slice_400/wandb/run-20241111_095028-sq0dhwvm/files/wandb-metadata.json +112 -0
  45. slice_400/wandb/run-20241111_095028-sq0dhwvm/files/wandb-summary.json +1 -0
  46. slice_400/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log +21 -0
  47. slice_400/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log +33 -0
  48. slice_400/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb +3 -0
  49. slice_800/arguments.yaml +49 -0
  50. slice_800/config.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ slice_1200/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb filter=lfs diff=lfs merge=lfs -text
37
+ slice_400/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb filter=lfs diff=lfs merge=lfs -text
38
+ slice_800/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb filter=lfs diff=lfs merge=lfs -text
39
+ wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb filter=lfs diff=lfs merge=lfs -text
arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_ti2ti_13.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1111_13.5k_ti2ti
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0916_ti_to_ti_sft/
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 3.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 1.0e-06
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
config.json ADDED
The diff for this file is too large to render. See raw diff
 
environ.txt ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ADDR2LINE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-addr2line
2
+ AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ar
3
+ AS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-as
4
+ BROWSER=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/bin/helpers/browser.sh
5
+ BUILD=x86_64-conda-linux-gnu
6
+ CC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
7
+ CC_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
8
+ CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
9
+ CMAKE_PREFIX_PATH=/data/align-anything/miniconda3/envs/jy-s:/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot/usr
10
+ COLORTERM=truecolor
11
+ CONDA_BACKUP_ADDR2LINE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-addr2line
12
+ CONDA_BACKUP_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ar
13
+ CONDA_BACKUP_AS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-as
14
+ CONDA_BACKUP_BUILD=x86_64-conda-linux-gnu
15
+ CONDA_BACKUP_CC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
16
+ CONDA_BACKUP_CC_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
17
+ CONDA_BACKUP_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
18
+ CONDA_BACKUP_CMAKE_PREFIX_PATH=/data/align-anything/miniconda3/envs/jy-s:/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot/usr
19
+ CONDA_BACKUP_CONDA_BUILD_SYSROOT=/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot
20
+ CONDA_BACKUP_CPP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cpp
21
+ CONDA_BACKUP_CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
22
+ CONDA_BACKUP_CXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
23
+ CONDA_BACKUP_CXXFILT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++filt
24
+ CONDA_BACKUP_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
25
+ CONDA_BACKUP_CXX_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
26
+ CONDA_BACKUP_DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
27
+ CONDA_BACKUP_DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /data/align-anything/miniconda3/envs/jy-s/include
28
+ CONDA_BACKUP_DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
29
+ CONDA_BACKUP_ELFEDIT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-elfedit
30
+ CONDA_BACKUP_GCC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc
31
+ CONDA_BACKUP_GCC_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ar
32
+ CONDA_BACKUP_GCC_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-nm
33
+ CONDA_BACKUP_GCC_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ranlib
34
+ CONDA_BACKUP_GPROF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gprof
35
+ CONDA_BACKUP_GXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-g++
36
+ CONDA_BACKUP_HOST=x86_64-conda-linux-gnu
37
+ CONDA_BACKUP_LD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld
38
+ CONDA_BACKUP_LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/data/align-anything/miniconda3/envs/jy-s/lib -Wl,-rpath-link,/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
39
+ CONDA_BACKUP_LD_GOLD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld.gold
40
+ CONDA_BACKUP_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-nm
41
+ CONDA_BACKUP_OBJCOPY=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objcopy
42
+ CONDA_BACKUP_OBJDUMP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objdump
43
+ CONDA_BACKUP_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ranlib
44
+ CONDA_BACKUP_READELF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-readelf
45
+ CONDA_BACKUP_SIZE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-size
46
+ CONDA_BACKUP_STRINGS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strings
47
+ CONDA_BACKUP_STRIP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strip
48
+ CONDA_BACKUP_build_alias=x86_64-conda-linux-gnu
49
+ CONDA_BACKUP_host_alias=x86_64-conda-linux-gnu
50
+ CONDA_BUILD_SYSROOT=/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot
51
+ CONDA_DEFAULT_ENV=hantao_cham
52
+ CONDA_EXE=/data/align-anything/miniconda3/bin/conda
53
+ CONDA_PREFIX=/data/align-anything/miniconda3/envs/hantao_cham
54
+ CONDA_PREFIX_1=/home/align-anything/miniconda3
55
+ CONDA_PREFIX_2=/data/align-anything/miniconda3/envs/jy-s
56
+ CONDA_PREFIX_3=/home/align-anything/miniconda3
57
+ CONDA_PREFIX_4=/data/align-anything/miniconda3
58
+ CONDA_PROMPT_MODIFIER=(hantao_cham)
59
+ CONDA_PYTHON_EXE=/data/align-anything/miniconda3/bin/python
60
+ CONDA_ROOT=/home/align-anything/miniconda3
61
+ CONDA_SHLVL=5
62
+ CPP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cpp
63
+ CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
64
+ CROSS_RANK=0
65
+ CROSS_SIZE=1
66
+ CUDA_MODULE_LOADING=LAZY
67
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
68
+ CXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
69
+ CXXFILT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++filt
70
+ CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
71
+ CXX_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
72
+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
73
+ DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
74
+ DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /data/align-anything/miniconda3/envs/jy-s/include
75
+ DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
76
+ ELFEDIT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-elfedit
77
+ GCC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc
78
+ GCC_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ar
79
+ GCC_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-nm
80
+ GCC_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ranlib
81
+ GIT_ASKPASS=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/extensions/git/dist/askpass.sh
82
+ GPROF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gprof
83
+ GXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-g++
84
+ HOME=/home/align-anything
85
+ HOST=x86_64-conda-linux-gnu
86
+ LANG=en_US.UTF-8
87
+ LD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld
88
+ LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/data/align-anything/miniconda3/envs/jy-s/lib -Wl,-rpath-link,/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
89
+ LD_GOLD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld.gold
90
+ LD_LIBRARY_PATH=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/../../lib64:
91
+ LESSCLOSE=/usr/bin/lesspipe %s %s
92
+ LESSOPEN=| /usr/bin/lesspipe %s
93
+ LOCAL_RANK=0
94
+ LOCAL_SIZE=8
95
+ LOGLEVEL=WARNING
96
+ LOGNAME=align-anything
97
+ LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
98
+ MASTER_ADDR=127.0.0.1
99
+ MASTER_PORT=19698
100
+ MOTD_SHOWN=pam
101
+ NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-nm
102
+ NVCC_PREPEND_FLAGS= -ccbin=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
103
+ OBJCOPY=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objcopy
104
+ OBJDUMP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objdump
105
+ OLDPWD=/data/align-anything/hantao/align-anything/projects/text_image_to_text_image
106
+ PATH=/data/align-anything/miniconda3/envs/hantao_cham/bin:/home/align-anything/miniconda3/bin:/home/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
107
+ PWD=/data/align-anything/hantao/align-anything/scripts
108
+ PYGAME_HIDE_SUPPORT_PROMPT=1
109
+ PYTHONHASHSEED=42
110
+ PYTHONPATH=/data/align-anything/hantao/align-anything
111
+ QT_QPA_FONTDIR=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/fonts
112
+ QT_QPA_PLATFORM_PLUGIN_PATH=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/plugins
113
+ RANK=0
114
+ RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ranlib
115
+ READELF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-readelf
116
+ SHELL=/bin/bash
117
+ SHLVL=3
118
+ SIZE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-size
119
+ SSH_CLIENT=203.93.11.8 7409 30800
120
+ SSH_CONNECTION=111.205.230.212 11374 10.10.212.198 30800
121
+ SSL_CERT_DIR=/usr/lib/ssl/certs
122
+ SSL_CERT_FILE=/usr/lib/ssl/certs/ca-certificates.crt
123
+ STRINGS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strings
124
+ STRIP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strip
125
+ TERM=screen
126
+ TERM_PROGRAM=tmux
127
+ TERM_PROGRAM_VERSION=3.2a
128
+ TMUX=/tmp/tmux-2000/default,2557274,4
129
+ TMUX_PANE=%4
130
+ TRITON_CACHE_DIR=/home/align-anything/cache/triton
131
+ USER=align-anything
132
+ VSCODE_GIT_ASKPASS_EXTRA_ARGS=
133
+ VSCODE_GIT_ASKPASS_MAIN=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/extensions/git/dist/askpass-main.js
134
+ VSCODE_GIT_ASKPASS_NODE=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/node
135
+ VSCODE_GIT_IPC_HANDLE=/run/user/2000/vscode-git-d8ae57dc60.sock
136
+ VSCODE_IPC_HOOK_CLI=/run/user/2000/vscode-ipc-45219155-ac87-45dc-978b-d20ac490d167.sock
137
+ WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
138
+ WANDB_SERVICE=2-3212652-tcp-localhost-41187
139
+ WORLD_SIZE=8
140
+ XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
141
+ XDG_RUNTIME_DIR=/run/user/2000
142
+ XDG_SESSION_CLASS=user
143
+ XDG_SESSION_ID=446
144
+ XDG_SESSION_TYPE=tty
145
+ _=/data/align-anything/miniconda3/envs/hantao_cham/bin/deepspeed
146
+ _CE_CONDA=
147
+ _CE_M=
148
+ build_alias=x86_64-conda-linux-gnu
149
+ host_alias=x86_64-conda-linux-gnu
preprocessor_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 512,
4
+ "width": 512
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 1.0,
13
+ 1.0,
14
+ 1.0
15
+ ],
16
+ "image_processor_type": "ChameleonImageProcessor",
17
+ "image_std": [
18
+ 1.0,
19
+ 1.0,
20
+ 1.0
21
+ ],
22
+ "processor_class": "ChameleonProcessor",
23
+ "resample": 1,
24
+ "rescale_factor": 0.0078,
25
+ "size": {
26
+ "shortest_edge": 512
27
+ }
28
+ }
processor_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "image_seq_length": 1024,
3
+ "image_token": "<image>",
4
+ "processor_class": "ChameleonProcessor"
5
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89974fc34f54d06cc78aa835c56cc3529eb55161b7b6808142528fe532282d48
3
+ size 14086364170
script.sh ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Copyright 2024 PKU-Alignment Team. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ # ==============================================================================
17
+
18
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
19
+ # You can replace it with a local model path
20
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
21
+ # You can replace it with a local dataset path
22
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
23
+ TRAIN_DATA_FILES="llf_ti2ti_13.5k_tokenized.pt"
24
+ # You can replace it with a new path
25
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_13.5k_ti2ti"
26
+ # For wandb online logging
27
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
28
+ # Source the setup script
29
+ source ./setup.sh
30
+
31
+ # sleep 30m
32
+ # Execute deepspeed command
33
+ deepspeed \
34
+ --master_port ${MASTER_PORT} \
35
+ --module align_anything.trainers.text_image_to_text_image.sft \
36
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
37
+ --train_datasets ${TRAIN_DATASETS} \
38
+ --train_data_files ${TRAIN_DATA_FILES} \
39
+ --output_dir ${OUTPUT_DIR} \
40
+ --train_template AA_textfeedback \
41
+ --train_split train \
42
+ --per_device_train_batch_size 4 \
43
+ --per_device_eval_batch_size 4 \
44
+ --gradient_accumulation_steps 2 \
45
+ --save_interval 400 \
46
+ --learning_rate 1e-6 \
47
+ --epochs 3 \
48
+ --lr_scheduler_type cosine
49
+
50
+
51
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
52
+ # You can replace it with a local model path
53
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
54
+ # You can replace it with a local dataset path
55
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
56
+ TRAIN_DATA_FILES="llf_ti2ti_6.75k_tokenized.pt"
57
+ # You can replace it with a new path
58
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_6.75k_ti2ti"
59
+ # For wandb online logging
60
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
61
+ # Source the setup script
62
+ source ./setup.sh
63
+
64
+ # sleep 30m
65
+ # Execute deepspeed command
66
+ deepspeed \
67
+ --master_port ${MASTER_PORT} \
68
+ --module align_anything.trainers.text_image_to_text_image.sft \
69
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
70
+ --train_datasets ${TRAIN_DATASETS} \
71
+ --train_data_files ${TRAIN_DATA_FILES} \
72
+ --output_dir ${OUTPUT_DIR} \
73
+ --train_template AA_textfeedback \
74
+ --train_split train \
75
+ --per_device_train_batch_size 4 \
76
+ --per_device_eval_batch_size 4 \
77
+ --gradient_accumulation_steps 2 \
78
+ --save_interval 400 \
79
+ --learning_rate 1e-6 \
80
+ --epochs 3 \
81
+ --lr_scheduler_type cosine
slice_1200/arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_ti2ti_13.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1111_13.5k_ti2ti
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0916_ti_to_ti_sft/
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 3.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 1.0e-06
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
slice_1200/config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1200/environ.txt ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ADDR2LINE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-addr2line
2
+ AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ar
3
+ AS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-as
4
+ BROWSER=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/bin/helpers/browser.sh
5
+ BUILD=x86_64-conda-linux-gnu
6
+ CC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
7
+ CC_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
8
+ CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
9
+ CMAKE_PREFIX_PATH=/data/align-anything/miniconda3/envs/jy-s:/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot/usr
10
+ COLORTERM=truecolor
11
+ CONDA_BACKUP_ADDR2LINE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-addr2line
12
+ CONDA_BACKUP_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ar
13
+ CONDA_BACKUP_AS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-as
14
+ CONDA_BACKUP_BUILD=x86_64-conda-linux-gnu
15
+ CONDA_BACKUP_CC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
16
+ CONDA_BACKUP_CC_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
17
+ CONDA_BACKUP_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
18
+ CONDA_BACKUP_CMAKE_PREFIX_PATH=/data/align-anything/miniconda3/envs/jy-s:/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot/usr
19
+ CONDA_BACKUP_CONDA_BUILD_SYSROOT=/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot
20
+ CONDA_BACKUP_CPP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cpp
21
+ CONDA_BACKUP_CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
22
+ CONDA_BACKUP_CXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
23
+ CONDA_BACKUP_CXXFILT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++filt
24
+ CONDA_BACKUP_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
25
+ CONDA_BACKUP_CXX_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
26
+ CONDA_BACKUP_DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
27
+ CONDA_BACKUP_DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /data/align-anything/miniconda3/envs/jy-s/include
28
+ CONDA_BACKUP_DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
29
+ CONDA_BACKUP_ELFEDIT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-elfedit
30
+ CONDA_BACKUP_GCC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc
31
+ CONDA_BACKUP_GCC_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ar
32
+ CONDA_BACKUP_GCC_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-nm
33
+ CONDA_BACKUP_GCC_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ranlib
34
+ CONDA_BACKUP_GPROF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gprof
35
+ CONDA_BACKUP_GXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-g++
36
+ CONDA_BACKUP_HOST=x86_64-conda-linux-gnu
37
+ CONDA_BACKUP_LD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld
38
+ CONDA_BACKUP_LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/data/align-anything/miniconda3/envs/jy-s/lib -Wl,-rpath-link,/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
39
+ CONDA_BACKUP_LD_GOLD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld.gold
40
+ CONDA_BACKUP_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-nm
41
+ CONDA_BACKUP_OBJCOPY=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objcopy
42
+ CONDA_BACKUP_OBJDUMP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objdump
43
+ CONDA_BACKUP_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ranlib
44
+ CONDA_BACKUP_READELF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-readelf
45
+ CONDA_BACKUP_SIZE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-size
46
+ CONDA_BACKUP_STRINGS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strings
47
+ CONDA_BACKUP_STRIP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strip
48
+ CONDA_BACKUP_build_alias=x86_64-conda-linux-gnu
49
+ CONDA_BACKUP_host_alias=x86_64-conda-linux-gnu
50
+ CONDA_BUILD_SYSROOT=/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot
51
+ CONDA_DEFAULT_ENV=hantao_cham
52
+ CONDA_EXE=/data/align-anything/miniconda3/bin/conda
53
+ CONDA_PREFIX=/data/align-anything/miniconda3/envs/hantao_cham
54
+ CONDA_PREFIX_1=/home/align-anything/miniconda3
55
+ CONDA_PREFIX_2=/data/align-anything/miniconda3/envs/jy-s
56
+ CONDA_PREFIX_3=/home/align-anything/miniconda3
57
+ CONDA_PREFIX_4=/data/align-anything/miniconda3
58
+ CONDA_PROMPT_MODIFIER=(hantao_cham)
59
+ CONDA_PYTHON_EXE=/data/align-anything/miniconda3/bin/python
60
+ CONDA_ROOT=/home/align-anything/miniconda3
61
+ CONDA_SHLVL=5
62
+ CPP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cpp
63
+ CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
64
+ CROSS_RANK=0
65
+ CROSS_SIZE=1
66
+ CUDA_MODULE_LOADING=LAZY
67
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
68
+ CXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
69
+ CXXFILT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++filt
70
+ CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
71
+ CXX_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
72
+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
73
+ DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
74
+ DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /data/align-anything/miniconda3/envs/jy-s/include
75
+ DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
76
+ ELFEDIT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-elfedit
77
+ GCC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc
78
+ GCC_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ar
79
+ GCC_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-nm
80
+ GCC_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ranlib
81
+ GIT_ASKPASS=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/extensions/git/dist/askpass.sh
82
+ GPROF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gprof
83
+ GXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-g++
84
+ HOME=/home/align-anything
85
+ HOST=x86_64-conda-linux-gnu
86
+ LANG=en_US.UTF-8
87
+ LD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld
88
+ LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/data/align-anything/miniconda3/envs/jy-s/lib -Wl,-rpath-link,/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
89
+ LD_GOLD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld.gold
90
+ LD_LIBRARY_PATH=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/../../lib64:
91
+ LESSCLOSE=/usr/bin/lesspipe %s %s
92
+ LESSOPEN=| /usr/bin/lesspipe %s
93
+ LOCAL_RANK=0
94
+ LOCAL_SIZE=8
95
+ LOGLEVEL=WARNING
96
+ LOGNAME=align-anything
97
+ LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
98
+ MASTER_ADDR=127.0.0.1
99
+ MASTER_PORT=19698
100
+ MOTD_SHOWN=pam
101
+ NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-nm
102
+ NVCC_PREPEND_FLAGS= -ccbin=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
103
+ OBJCOPY=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objcopy
104
+ OBJDUMP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objdump
105
+ OLDPWD=/data/align-anything/hantao/align-anything/projects/text_image_to_text_image
106
+ PATH=/data/align-anything/miniconda3/envs/hantao_cham/bin:/home/align-anything/miniconda3/bin:/home/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
107
+ PWD=/data/align-anything/hantao/align-anything/scripts
108
+ PYGAME_HIDE_SUPPORT_PROMPT=1
109
+ PYTHONHASHSEED=42
110
+ PYTHONPATH=/data/align-anything/hantao/align-anything
111
+ QT_QPA_FONTDIR=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/fonts
112
+ QT_QPA_PLATFORM_PLUGIN_PATH=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/plugins
113
+ RANK=0
114
+ RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ranlib
115
+ READELF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-readelf
116
+ SHELL=/bin/bash
117
+ SHLVL=3
118
+ SIZE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-size
119
+ SSH_CLIENT=203.93.11.8 7409 30800
120
+ SSH_CONNECTION=111.205.230.212 11374 10.10.212.198 30800
121
+ SSL_CERT_DIR=/usr/lib/ssl/certs
122
+ SSL_CERT_FILE=/usr/lib/ssl/certs/ca-certificates.crt
123
+ STRINGS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strings
124
+ STRIP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strip
125
+ TERM=screen
126
+ TERM_PROGRAM=tmux
127
+ TERM_PROGRAM_VERSION=3.2a
128
+ TMUX=/tmp/tmux-2000/default,2557274,4
129
+ TMUX_PANE=%4
130
+ TRITON_CACHE_DIR=/home/align-anything/cache/triton
131
+ USER=align-anything
132
+ VSCODE_GIT_ASKPASS_EXTRA_ARGS=
133
+ VSCODE_GIT_ASKPASS_MAIN=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/extensions/git/dist/askpass-main.js
134
+ VSCODE_GIT_ASKPASS_NODE=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/node
135
+ VSCODE_GIT_IPC_HANDLE=/run/user/2000/vscode-git-d8ae57dc60.sock
136
+ VSCODE_IPC_HOOK_CLI=/run/user/2000/vscode-ipc-45219155-ac87-45dc-978b-d20ac490d167.sock
137
+ WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
138
+ WANDB_SERVICE=2-3212652-tcp-localhost-41187
139
+ WORLD_SIZE=8
140
+ XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
141
+ XDG_RUNTIME_DIR=/run/user/2000
142
+ XDG_SESSION_CLASS=user
143
+ XDG_SESSION_ID=446
144
+ XDG_SESSION_TYPE=tty
145
+ _=/data/align-anything/miniconda3/envs/hantao_cham/bin/deepspeed
146
+ _CE_CONDA=
147
+ _CE_M=
148
+ build_alias=x86_64-conda-linux-gnu
149
+ host_alias=x86_64-conda-linux-gnu
slice_1200/preprocessor_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 512,
4
+ "width": 512
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 1.0,
13
+ 1.0,
14
+ 1.0
15
+ ],
16
+ "image_processor_type": "ChameleonImageProcessor",
17
+ "image_std": [
18
+ 1.0,
19
+ 1.0,
20
+ 1.0
21
+ ],
22
+ "processor_class": "ChameleonProcessor",
23
+ "resample": 1,
24
+ "rescale_factor": 0.0078,
25
+ "size": {
26
+ "shortest_edge": 512
27
+ }
28
+ }
slice_1200/processor_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "image_seq_length": 1024,
3
+ "image_token": "<image>",
4
+ "processor_class": "ChameleonProcessor"
5
+ }
slice_1200/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ff0d39a7919fdd9e8cb64d7b6717ceef60f5ef14432e42c38ce344895fca353
3
+ size 14086366930
slice_1200/script.sh ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Copyright 2024 PKU-Alignment Team. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ # ==============================================================================
17
+
18
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
19
+ # You can replace it with a local model path
20
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
21
+ # You can replace it with a local dataset path
22
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
23
+ TRAIN_DATA_FILES="llf_ti2ti_13.5k_tokenized.pt"
24
+ # You can replace it with a new path
25
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_13.5k_ti2ti"
26
+ # For wandb online logging
27
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
28
+ # Source the setup script
29
+ source ./setup.sh
30
+
31
+ # sleep 30m
32
+ # Execute deepspeed command
33
+ deepspeed \
34
+ --master_port ${MASTER_PORT} \
35
+ --module align_anything.trainers.text_image_to_text_image.sft \
36
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
37
+ --train_datasets ${TRAIN_DATASETS} \
38
+ --train_data_files ${TRAIN_DATA_FILES} \
39
+ --output_dir ${OUTPUT_DIR} \
40
+ --train_template AA_textfeedback \
41
+ --train_split train \
42
+ --per_device_train_batch_size 4 \
43
+ --per_device_eval_batch_size 4 \
44
+ --gradient_accumulation_steps 2 \
45
+ --save_interval 400 \
46
+ --learning_rate 1e-6 \
47
+ --epochs 3 \
48
+ --lr_scheduler_type cosine
49
+
50
+
51
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
52
+ # You can replace it with a local model path
53
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
54
+ # You can replace it with a local dataset path
55
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
56
+ TRAIN_DATA_FILES="llf_ti2ti_6.75k_tokenized.pt"
57
+ # You can replace it with a new path
58
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_6.75k_ti2ti"
59
+ # For wandb online logging
60
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
61
+ # Source the setup script
62
+ source ./setup.sh
63
+
64
+ # sleep 30m
65
+ # Execute deepspeed command
66
+ deepspeed \
67
+ --master_port ${MASTER_PORT} \
68
+ --module align_anything.trainers.text_image_to_text_image.sft \
69
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
70
+ --train_datasets ${TRAIN_DATASETS} \
71
+ --train_data_files ${TRAIN_DATA_FILES} \
72
+ --output_dir ${OUTPUT_DIR} \
73
+ --train_template AA_textfeedback \
74
+ --train_split train \
75
+ --per_device_train_batch_size 4 \
76
+ --per_device_eval_batch_size 4 \
77
+ --gradient_accumulation_steps 2 \
78
+ --save_interval 400 \
79
+ --learning_rate 1e-6 \
80
+ --epochs 3 \
81
+ --lr_scheduler_type cosine
slice_1200/special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<pad>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "<reserved08706>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
slice_1200/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1200/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_1200/wandb/debug-internal.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-11T09:50:28.93257347Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-11T09:50:28.932602745Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-core.log"}
3
+ {"time":"2024-11-11T09:50:28.93714017Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-11T09:50:28.968182627Z","level":"INFO","msg":"created new stream","id":"sq0dhwvm"}
5
+ {"time":"2024-11-11T09:50:28.968263994Z","level":"INFO","msg":"stream: started","id":"sq0dhwvm"}
6
+ {"time":"2024-11-11T09:50:28.968288809Z","level":"INFO","msg":"sender: started","stream_id":{"value":"sq0dhwvm"}}
7
+ {"time":"2024-11-11T09:50:28.968279236Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"sq0dhwvm"}}
8
+ {"time":"2024-11-11T09:50:28.968330962Z","level":"INFO","msg":"handler: started","stream_id":{"value":"sq0dhwvm"}}
9
+ {"time":"2024-11-11T09:50:34.632278843Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-11T09:50:34.636084019Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-11T11:20:45.19395353Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/htlou/align-anything/sq0dhwvm/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
12
+ {"time":"2024-11-11T11:45:29.821937086Z","level":"INFO","msg":"Stopping system monitor"}
13
+ {"time":"2024-11-11T11:45:29.846984762Z","level":"INFO","msg":"Stopped system monitor"}
14
+ {"time":"2024-11-11T11:45:30.253659937Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
15
+ {"time":"2024-11-11T11:45:30.253692046Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
16
+ {"time":"2024-11-11T11:45:30.882510953Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
17
+ {"time":"2024-11-11T11:45:32.320976723Z","level":"INFO","msg":"stream: closing","id":"sq0dhwvm"}
18
+ {"time":"2024-11-11T11:45:32.321006242Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"sq0dhwvm"}}
19
+ {"time":"2024-11-11T11:45:32.321051996Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"sq0dhwvm"}}
20
+ {"time":"2024-11-11T11:45:32.321127635Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"sq0dhwvm"}}
21
+ {"time":"2024-11-11T11:45:32.323850284Z","level":"INFO","msg":"stream: closed","id":"sq0dhwvm"}
slice_1200/wandb/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Configure stats pid to 3212652
3
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-11 09:50:28,918 WARNING MainThread:3212652 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log
11
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log
12
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_ti2ti_13.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1111_13.5k_ti2ti', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0916_ti_to_ti_sft/', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():667] starting backend
16
+ 2024-11-11 09:50:28,919 INFO MainThread:3212652 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-11 09:50:28,924 INFO MainThread:3212652 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-11 09:50:28,925 INFO MainThread:3212652 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-11 09:50:28,929 INFO MainThread:3212652 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-11 09:50:28,983 INFO MainThread:3212652 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-11 09:50:34,620 INFO MainThread:3212652 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-11 09:50:35,184 INFO MainThread:3212652 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-11 11:45:29,778 INFO MainThread:3212652 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/sq0dhwvm
28
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-11 11:45:32,305 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-11 11:45:32,307 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-11 11:45:32,318 INFO MainThread:3212652 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/config.yaml ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.18.3
4
+ m: []
5
+ python_version: 3.11.10
6
+ t:
7
+ "1":
8
+ - 1
9
+ - 11
10
+ - 41
11
+ - 49
12
+ - 51
13
+ - 55
14
+ - 71
15
+ - 83
16
+ - 98
17
+ - 105
18
+ "2":
19
+ - 1
20
+ - 11
21
+ - 41
22
+ - 49
23
+ - 51
24
+ - 55
25
+ - 71
26
+ - 83
27
+ - 98
28
+ - 105
29
+ "3":
30
+ - 2
31
+ - 13
32
+ - 16
33
+ - 23
34
+ - 55
35
+ - 61
36
+ "4": 3.11.10
37
+ "5": 0.18.3
38
+ "6": 4.46.0.dev0
39
+ "8":
40
+ - 5
41
+ "12": 0.18.3
42
+ "13": linux-x86_64
43
+ data_cfgs:
44
+ value:
45
+ eval_data_files: null
46
+ eval_datasets: null
47
+ eval_optional_args: []
48
+ eval_size: null
49
+ eval_split: null
50
+ eval_subset: null
51
+ eval_template: null
52
+ train_data_files: llf_ti2ti_13.5k_tokenized.pt
53
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
54
+ train_optional_args: []
55
+ train_size: null
56
+ train_split: train
57
+ train_subset: null
58
+ train_template: AA_textfeedback
59
+ logger_cfgs:
60
+ value:
61
+ cache_dir: null
62
+ log_project: align-anything
63
+ log_run_name: sft
64
+ log_type: wandb
65
+ output_dir: ../outputs/sft_tf_cham_1111_13.5k_ti2ti
66
+ save_interval: 400
67
+ model_cfgs:
68
+ value:
69
+ model_max_length: 4096
70
+ model_name_or_path: /data/align-anything/hantao/models/0916_ti_to_ti_sft/
71
+ trust_remote_code: true
72
+ special_tokens:
73
+ value: null
74
+ train_cfgs:
75
+ value:
76
+ adam_betas:
77
+ - 0.9
78
+ - 0.95
79
+ adam_epsilon: 1e-08
80
+ bf16: true
81
+ ds_cfgs: ds_z3_config.json
82
+ epochs: 3
83
+ eval_interval: 1000
84
+ eval_strategy: steps
85
+ fp16: false
86
+ freeze_language_model: false
87
+ gradient_accumulation_steps: 2
88
+ gradient_checkpointing: true
89
+ learning_rate: 1e-06
90
+ lr_scheduler_type: cosine
91
+ lr_warmup_ratio: 0.03
92
+ max_grad_norm: 1
93
+ per_device_eval_batch_size: 4
94
+ per_device_train_batch_size: 4
95
+ seed: 42
96
+ weight_decay: 0
slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/output.log ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ***** Running training *****
2
+ Training 1/3.0 epoch: 0%| | 0/1266.0 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
3
+ Training 1/3.0 epoch (loss 0.6685): 25%|███████████████████████████▋ | 319/1266.0 [28:07<1:20:30, 5.10s/it]
4
+ [2024-11-11 09:52:24,616] [INFO] [logging.py:96:log_dist] [Rank 0] step=10, skipped=0, lr=[5.555555555555555e-07, 5.555555555555555e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
5
+ [2024-11-11 09:52:24,617] [INFO] [timer.py:264:stop] epoch=0/micro_step=20/global_step=10, RunningAvgSamplesPerSec=6.195061818187147, CurrSamplesPerSec=6.335947040765943, MemAllocated=14.07GB, MaxMemAllocated=25.28GB
6
+ [2024-11-11 09:54:06,853] [INFO] [logging.py:96:log_dist] [Rank 0] step=20, skipped=0, lr=[9.999739056948805e-07, 9.999739056948805e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
7
+ [2024-11-11 09:54:06,854] [INFO] [timer.py:264:stop] epoch=0/micro_step=40/global_step=20, RunningAvgSamplesPerSec=6.316656927899698, CurrSamplesPerSec=6.164126358699434, MemAllocated=14.07GB, MaxMemAllocated=25.28GB
8
+ [2024-11-11 09:55:53,507] [INFO] [logging.py:96:log_dist] [Rank 0] step=30, skipped=0, lr=[9.99060890967219e-07, 9.99060890967219e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
9
+ [2024-11-11 09:55:53,508] [INFO] [timer.py:264:stop] epoch=0/micro_step=60/global_step=30, RunningAvgSamplesPerSec=6.248466505115529, CurrSamplesPerSec=7.491092861751436, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
10
+ [2024-11-11 09:57:41,293] [INFO] [logging.py:96:log_dist] [Rank 0] step=40, skipped=0, lr=[9.968458833570276e-07, 9.968458833570276e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
11
+ [2024-11-11 09:57:41,294] [INFO] [timer.py:264:stop] epoch=0/micro_step=80/global_step=40, RunningAvgSamplesPerSec=6.196372346586397, CurrSamplesPerSec=6.409875261403376, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
12
+ [2024-11-11 09:59:23,446] [INFO] [logging.py:96:log_dist] [Rank 0] step=50, skipped=0, lr=[9.933346615662559e-07, 9.933346615662559e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
13
+ [2024-11-11 09:59:23,447] [INFO] [timer.py:264:stop] epoch=0/micro_step=100/global_step=50, RunningAvgSamplesPerSec=6.246806387061766, CurrSamplesPerSec=7.260616697354314, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
14
+ [2024-11-11 10:01:09,165] [INFO] [logging.py:96:log_dist] [Rank 0] step=60, skipped=0, lr=[9.885363859716496e-07, 9.885363859716496e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
15
+ [2024-11-11 10:01:09,166] [INFO] [timer.py:264:stop] epoch=0/micro_step=120/global_step=60, RunningAvgSamplesPerSec=6.2467760579358185, CurrSamplesPerSec=5.904287617244474, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
16
+ [2024-11-11 10:02:58,559] [INFO] [logging.py:96:log_dist] [Rank 0] step=70, skipped=0, lr=[9.824635747263753e-07, 9.824635747263753e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
17
+ [2024-11-11 10:02:58,560] [INFO] [timer.py:264:stop] epoch=0/micro_step=140/global_step=70, RunningAvgSamplesPerSec=6.205922498836564, CurrSamplesPerSec=7.320670588968733, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
18
+ [2024-11-11 10:04:44,012] [INFO] [logging.py:96:log_dist] [Rank 0] step=80, skipped=0, lr=[9.75132071101588e-07, 9.75132071101588e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
19
+ [2024-11-11 10:04:44,013] [INFO] [timer.py:264:stop] epoch=0/micro_step=160/global_step=80, RunningAvgSamplesPerSec=6.20796558015961, CurrSamplesPerSec=6.143883631130097, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
20
+ [2024-11-11 10:06:26,906] [INFO] [logging.py:96:log_dist] [Rank 0] step=90, skipped=0, lr=[9.665610021531446e-07, 9.665610021531446e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
21
+ [2024-11-11 10:06:26,907] [INFO] [timer.py:264:stop] epoch=0/micro_step=180/global_step=90, RunningAvgSamplesPerSec=6.223837975387682, CurrSamplesPerSec=7.466477040673676, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
22
+ [2024-11-11 10:08:15,344] [INFO] [logging.py:96:log_dist] [Rank 0] step=100, skipped=0, lr=[9.567727288213004e-07, 9.567727288213004e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
23
+ [2024-11-11 10:08:15,345] [INFO] [timer.py:264:stop] epoch=0/micro_step=200/global_step=100, RunningAvgSamplesPerSec=6.209057201170874, CurrSamplesPerSec=5.302689344409979, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
24
+ [2024-11-11 10:10:01,328] [INFO] [logging.py:96:log_dist] [Rank 0] step=110, skipped=0, lr=[9.457927875935681e-07, 9.457927875935681e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
25
+ [2024-11-11 10:10:01,329] [INFO] [timer.py:264:stop] epoch=0/micro_step=220/global_step=110, RunningAvgSamplesPerSec=6.208473867040452, CurrSamplesPerSec=7.698975362834649, MemAllocated=14.07GB, MaxMemAllocated=29.74GB
26
+ [2024-11-11 10:11:47,804] [INFO] [logging.py:96:log_dist] [Rank 0] step=120, skipped=0, lr=[9.336498238829382e-07, 9.336498238829382e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
27
+ [2024-11-11 10:11:47,804] [INFO] [timer.py:264:stop] epoch=0/micro_step=240/global_step=120, RunningAvgSamplesPerSec=6.202644364760127, CurrSamplesPerSec=7.252698675359632, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
28
+ [2024-11-11 10:13:25,206] [INFO] [logging.py:96:log_dist] [Rank 0] step=130, skipped=0, lr=[9.203755172952665e-07, 9.203755172952665e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
29
+ [2024-11-11 10:13:25,207] [INFO] [timer.py:264:stop] epoch=0/micro_step=260/global_step=130, RunningAvgSamplesPerSec=6.240439626002477, CurrSamplesPerSec=6.299954067945365, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
30
+ [2024-11-11 10:15:07,236] [INFO] [logging.py:96:log_dist] [Rank 0] step=140, skipped=0, lr=[9.060044989808009e-07, 9.060044989808009e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
31
+ [2024-11-11 10:15:07,237] [INFO] [timer.py:264:stop] epoch=0/micro_step=280/global_step=140, RunningAvgSamplesPerSec=6.253829602651008, CurrSamplesPerSec=6.320296322752446, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
32
+ [2024-11-11 10:16:59,755] [INFO] [logging.py:96:log_dist] [Rank 0] step=150, skipped=0, lr=[8.905742612854626e-07, 8.905742612854626e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
33
+ [2024-11-11 10:16:59,756] [INFO] [timer.py:264:stop] epoch=0/micro_step=300/global_step=150, RunningAvgSamplesPerSec=6.220856964798348, CurrSamplesPerSec=6.140029977114886, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
34
+ [2024-11-11 10:18:48,289] [INFO] [logging.py:96:log_dist] [Rank 0] step=160, skipped=0, lr=[8.741250599375982e-07, 8.741250599375982e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
35
+ [2024-11-11 10:18:48,289] [INFO] [timer.py:264:stop] epoch=0/micro_step=320/global_step=160, RunningAvgSamplesPerSec=6.20887491385521, CurrSamplesPerSec=6.3240117862507015, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
36
+ [2024-11-11 10:20:33,500] [INFO] [logging.py:96:log_dist] [Rank 0] step=170, skipped=0, lr=[8.566998090253799e-07, 8.566998090253799e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
37
+ [2024-11-11 10:20:33,500] [INFO] [timer.py:264:stop] epoch=0/micro_step=340/global_step=170, RunningAvgSamplesPerSec=6.211329294057541, CurrSamplesPerSec=6.262193329135546, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
38
+ [2024-11-11 10:22:18,917] [INFO] [logging.py:96:log_dist] [Rank 0] step=180, skipped=0, lr=[8.38343969038849e-07, 8.38343969038849e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
39
+ [2024-11-11 10:22:18,917] [INFO] [timer.py:264:stop] epoch=0/micro_step=360/global_step=180, RunningAvgSamplesPerSec=6.211341226400399, CurrSamplesPerSec=6.3439491768216785, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
40
+ [2024-11-11 10:24:04,535] [INFO] [logging.py:96:log_dist] [Rank 0] step=190, skipped=0, lr=[8.191054282686891e-07, 8.191054282686891e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
41
+ [2024-11-11 10:24:04,536] [INFO] [timer.py:264:stop] epoch=0/micro_step=380/global_step=190, RunningAvgSamplesPerSec=6.210682927569051, CurrSamplesPerSec=6.298895458953062, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
42
+ [2024-11-11 10:25:42,250] [INFO] [logging.py:96:log_dist] [Rank 0] step=200, skipped=0, lr=[7.990343778711425e-07, 7.990343778711425e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
43
+ [2024-11-11 10:25:42,251] [INFO] [timer.py:264:stop] epoch=0/micro_step=400/global_step=200, RunningAvgSamplesPerSec=6.233484883515772, CurrSamplesPerSec=7.014839355184877, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
44
+ Saving checkpoint at step 400 ...
45
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
46
+ Saving 16-bit model...
47
+ [2024-11-11 10:25:51,218] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
48
+ [2024-11-11 10:25:51,219] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin, tag: global_step200
49
+ [2024-11-11 10:25:51,219] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin...
50
+ [2024-11-11 10:26:14,887] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin.
51
+ [2024-11-11 10:26:14,889] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
52
+ Model saved!
53
+ Saving 16-bit model...
54
+ [2024-11-11 10:26:23,191] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
55
+ [2024-11-11 10:26:23,192] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin, tag: global_step200
56
+ [2024-11-11 10:26:23,193] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin...
57
+ [2024-11-11 10:26:45,365] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin.
58
+ [2024-11-11 10:26:45,366] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
59
+ Model saved!
60
+ Checkpoint saved.
61
+ [2024-11-11 10:28:30,912] [INFO] [logging.py:96:log_dist] [Rank 0] step=210, skipped=0, lr=[7.78183180925015e-07, 7.78183180925015e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
62
+ [2024-11-11 10:28:30,912] [INFO] [timer.py:264:stop] epoch=0/micro_step=420/global_step=210, RunningAvgSamplesPerSec=6.231559886416094, CurrSamplesPerSec=7.658031726766873, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
63
+ [2024-11-11 10:30:17,140] [INFO] [logging.py:96:log_dist] [Rank 0] step=220, skipped=0, lr=[7.566062358223813e-07, 7.566062358223813e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
64
+ [2024-11-11 10:30:17,141] [INFO] [timer.py:264:stop] epoch=1/micro_step=18/global_step=220, RunningAvgSamplesPerSec=6.227392476536316, CurrSamplesPerSec=6.2512248951017435, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
65
+ [2024-11-11 10:31:58,864] [INFO] [logging.py:96:log_dist] [Rank 0] step=230, skipped=0, lr=[7.343598343493916e-07, 7.343598343493916e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
66
+ [2024-11-11 10:31:58,865] [INFO] [timer.py:264:stop] epoch=1/micro_step=38/global_step=230, RunningAvgSamplesPerSec=6.236228746087023, CurrSamplesPerSec=6.314374734308299, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
67
+ [2024-11-11 10:33:47,346] [INFO] [logging.py:96:log_dist] [Rank 0] step=240, skipped=0, lr=[7.115020148274293e-07, 7.115020148274293e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
68
+ [2024-11-11 10:33:47,347] [INFO] [timer.py:264:stop] epoch=1/micro_step=58/global_step=240, RunningAvgSamplesPerSec=6.227453579084282, CurrSamplesPerSec=4.989636440920279, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
69
+ [2024-11-11 10:35:34,343] [INFO] [logging.py:96:log_dist] [Rank 0] step=250, skipped=0, lr=[6.880924106977566e-07, 6.880924106977566e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
70
+ [2024-11-11 10:35:34,343] [INFO] [timer.py:264:stop] epoch=1/micro_step=78/global_step=250, RunningAvgSamplesPerSec=6.22403220340251, CurrSamplesPerSec=6.249119262604737, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
71
+ [2024-11-11 10:37:16,755] [INFO] [logging.py:96:log_dist] [Rank 0] step=260, skipped=0, lr=[6.641920949446755e-07, 6.641920949446755e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
72
+ [2024-11-11 10:37:16,755] [INFO] [timer.py:264:stop] epoch=1/micro_step=98/global_step=260, RunningAvgSamplesPerSec=6.229929565927251, CurrSamplesPerSec=6.314551641200963, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
73
+ [2024-11-11 10:39:00,686] [INFO] [logging.py:96:log_dist] [Rank 0] step=270, skipped=0, lr=[6.39863420763084e-07, 6.39863420763084e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
74
+ [2024-11-11 10:39:00,687] [INFO] [timer.py:264:stop] epoch=1/micro_step=118/global_step=270, RunningAvgSamplesPerSec=6.234223067872221, CurrSamplesPerSec=5.347220757882646, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
75
+ [2024-11-11 10:40:52,162] [INFO] [logging.py:96:log_dist] [Rank 0] step=280, skipped=0, lr=[6.151698588861116e-07, 6.151698588861116e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
76
+ [2024-11-11 10:40:52,163] [INFO] [timer.py:264:stop] epoch=1/micro_step=138/global_step=280, RunningAvgSamplesPerSec=6.22032747774778, CurrSamplesPerSec=5.386764780463646, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
77
+ [2024-11-11 10:42:36,292] [INFO] [logging.py:96:log_dist] [Rank 0] step=290, skipped=0, lr=[5.901758319972226e-07, 5.901758319972226e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
78
+ [2024-11-11 10:42:36,293] [INFO] [timer.py:264:stop] epoch=1/micro_step=158/global_step=290, RunningAvgSamplesPerSec=6.223707028785581, CurrSamplesPerSec=6.004583836826392, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
79
+ [2024-11-11 10:44:21,117] [INFO] [logging.py:96:log_dist] [Rank 0] step=300, skipped=0, lr=[5.649465466587902e-07, 5.649465466587902e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
80
+ [2024-11-11 10:44:21,117] [INFO] [timer.py:264:stop] epoch=1/micro_step=178/global_step=300, RunningAvgSamplesPerSec=6.22408930425845, CurrSamplesPerSec=6.223819151883097, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
81
+ [2024-11-11 10:46:05,125] [INFO] [logging.py:96:log_dist] [Rank 0] step=310, skipped=0, lr=[5.395478231956219e-07, 5.395478231956219e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
82
+ [2024-11-11 10:46:05,126] [INFO] [timer.py:264:stop] epoch=1/micro_step=198/global_step=310, RunningAvgSamplesPerSec=6.226128614306769, CurrSamplesPerSec=6.262398734901036, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
83
+ [2024-11-11 10:47:53,756] [INFO] [logging.py:96:log_dist] [Rank 0] step=320, skipped=0, lr=[5.140459239772456e-07, 5.140459239772456e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
84
+ [2024-11-11 10:47:53,757] [INFO] [timer.py:264:stop] epoch=1/micro_step=218/global_step=320, RunningAvgSamplesPerSec=6.219037182578649, CurrSamplesPerSec=5.474378062298524, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
85
+ [2024-11-11 10:49:39,475] [INFO] [logging.py:96:log_dist] [Rank 0] step=330, skipped=0, lr=[4.885073805469548e-07, 4.885073805469548e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
86
+ [2024-11-11 10:49:39,476] [INFO] [timer.py:264:stop] epoch=1/micro_step=238/global_step=330, RunningAvgSamplesPerSec=6.218320029955548, CurrSamplesPerSec=6.184670912366926, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
87
+ [2024-11-11 10:51:15,751] [INFO] [logging.py:96:log_dist] [Rank 0] step=340, skipped=0, lr=[4.629988200486073e-07, 4.629988200486073e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
88
+ [2024-11-11 10:51:15,751] [INFO] [timer.py:264:stop] epoch=1/micro_step=258/global_step=340, RunningAvgSamplesPerSec=6.234659527486788, CurrSamplesPerSec=6.166076513611049, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
89
+ [2024-11-11 10:52:57,448] [INFO] [logging.py:96:log_dist] [Rank 0] step=350, skipped=0, lr=[4.3758679140401534e-07, 4.3758679140401534e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
90
+ [2024-11-11 10:52:57,448] [INFO] [timer.py:264:stop] epoch=1/micro_step=278/global_step=350, RunningAvgSamplesPerSec=6.2408272898718575, CurrSamplesPerSec=5.479743497267358, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
91
+ [2024-11-11 10:54:49,320] [INFO] [logging.py:96:log_dist] [Rank 0] step=360, skipped=0, lr=[4.123375916944061e-07, 4.123375916944061e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
92
+ [2024-11-11 10:54:49,321] [INFO] [timer.py:264:stop] epoch=1/micro_step=298/global_step=360, RunningAvgSamplesPerSec=6.228467713430319, CurrSamplesPerSec=5.247047078296396, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
93
+ [2024-11-11 10:56:37,584] [INFO] [logging.py:96:log_dist] [Rank 0] step=370, skipped=0, lr=[3.8731709319890747e-07, 3.8731709319890747e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
94
+ [2024-11-11 10:56:37,585] [INFO] [timer.py:264:stop] epoch=1/micro_step=318/global_step=370, RunningAvgSamplesPerSec=6.223037095561094, CurrSamplesPerSec=6.317240925196704, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
95
+ [2024-11-11 10:58:22,183] [INFO] [logging.py:96:log_dist] [Rank 0] step=380, skipped=0, lr=[3.6259057154129225e-07, 3.6259057154129225e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
96
+ [2024-11-11 10:58:22,183] [INFO] [timer.py:264:stop] epoch=1/micro_step=338/global_step=380, RunningAvgSamplesPerSec=6.224228924255615, CurrSamplesPerSec=5.1994408858020025, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
97
+ [2024-11-11 11:00:07,320] [INFO] [logging.py:96:log_dist] [Rank 0] step=390, skipped=0, lr=[3.382225353933288e-07, 3.382225353933288e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
98
+ [2024-11-11 11:00:07,321] [INFO] [timer.py:264:stop] epoch=1/micro_step=358/global_step=390, RunningAvgSamplesPerSec=6.224158604390081, CurrSamplesPerSec=6.24098657587071, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
99
+ [2024-11-11 11:01:52,825] [INFO] [logging.py:96:log_dist] [Rank 0] step=400, skipped=0, lr=[3.142765581790224e-07, 3.142765581790224e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
100
+ [2024-11-11 11:01:52,826] [INFO] [timer.py:264:stop] epoch=1/micro_step=378/global_step=400, RunningAvgSamplesPerSec=6.224258305845644, CurrSamplesPerSec=5.508992960001429, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
101
+ Saving checkpoint at step 800 ...
102
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
103
+ Saving 16-bit model...
104
+ [2024-11-11 11:02:01,814] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
105
+ [2024-11-11 11:02:01,815] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin, tag: global_step400
106
+ [2024-11-11 11:02:01,815] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin...
107
+ [2024-11-11 11:02:24,168] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin.
108
+ [2024-11-11 11:02:24,170] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
109
+ Model saved!
110
+ Saving 16-bit model...
111
+ [2024-11-11 11:02:32,111] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
112
+ [2024-11-11 11:02:32,112] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin, tag: global_step400
113
+ [2024-11-11 11:02:32,112] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin...
114
+ [2024-11-11 11:02:53,588] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin.
115
+ [2024-11-11 11:02:53,590] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
116
+ Model saved!
117
+ Checkpoint saved.
118
+ [2024-11-11 11:04:32,610] [INFO] [logging.py:96:log_dist] [Rank 0] step=410, skipped=0, lr=[2.9081511221881027e-07, 2.9081511221881027e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
119
+ [2024-11-11 11:04:32,611] [INFO] [timer.py:264:stop] epoch=1/micro_step=398/global_step=410, RunningAvgSamplesPerSec=6.233501350397177, CurrSamplesPerSec=7.624818018964297, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
120
+ [2024-11-11 11:06:18,893] [INFO] [logging.py:96:log_dist] [Rank 0] step=420, skipped=0, lr=[2.67899405746411e-07, 2.67899405746411e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
121
+ [2024-11-11 11:06:18,894] [INFO] [timer.py:264:stop] epoch=1/micro_step=418/global_step=420, RunningAvgSamplesPerSec=6.2314715200711, CurrSamplesPerSec=5.346454162982587, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
122
+ [2024-11-11 11:08:03,330] [INFO] [logging.py:96:log_dist] [Rank 0] step=430, skipped=0, lr=[2.4558922322353115e-07, 2.4558922322353115e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
123
+ [2024-11-11 11:08:03,330] [INFO] [timer.py:264:stop] epoch=2/micro_step=16/global_step=430, RunningAvgSamplesPerSec=6.2324828606092995, CurrSamplesPerSec=5.855626964991123, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
124
+ [2024-11-11 11:09:45,387] [INFO] [logging.py:96:log_dist] [Rank 0] step=440, skipped=0, lr=[2.2394276936903022e-07, 2.2394276936903022e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
125
+ [2024-11-11 11:09:45,387] [INFO] [timer.py:264:stop] epoch=2/micro_step=36/global_step=440, RunningAvgSamplesPerSec=6.2365997777478235, CurrSamplesPerSec=7.074766361276321, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
126
+ [2024-11-11 11:11:31,116] [INFO] [logging.py:96:log_dist] [Rank 0] step=450, skipped=0, lr=[2.0301651730945623e-07, 2.0301651730945623e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
127
+ [2024-11-11 11:11:31,117] [INFO] [timer.py:264:stop] epoch=2/micro_step=56/global_step=450, RunningAvgSamplesPerSec=6.235535040398496, CurrSamplesPerSec=6.263325127446985, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
128
+ [2024-11-11 11:13:19,722] [INFO] [logging.py:96:log_dist] [Rank 0] step=460, skipped=0, lr=[1.8286506124710487e-07, 1.8286506124710487e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
129
+ [2024-11-11 11:13:19,723] [INFO] [timer.py:264:stop] epoch=2/micro_step=76/global_step=460, RunningAvgSamplesPerSec=6.230500323816332, CurrSamplesPerSec=6.036847858086253, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
130
+ [2024-11-11 11:15:02,353] [INFO] [logging.py:96:log_dist] [Rank 0] step=470, skipped=0, lr=[1.6354097402998124e-07, 1.6354097402998124e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
131
+ [2024-11-11 11:15:02,354] [INFO] [timer.py:264:stop] epoch=2/micro_step=96/global_step=470, RunningAvgSamplesPerSec=6.233932103977919, CurrSamplesPerSec=6.2977051152249395, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
132
+ [2024-11-11 11:16:42,997] [INFO] [logging.py:96:log_dist] [Rank 0] step=480, skipped=0, lr=[1.4509466999523984e-07, 1.4509466999523984e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
133
+ [2024-11-11 11:16:42,998] [INFO] [timer.py:264:stop] epoch=2/micro_step=116/global_step=480, RunningAvgSamplesPerSec=6.239483803482549, CurrSamplesPerSec=6.033187064976004, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
134
+ [2024-11-11 11:18:34,535] [INFO] [logging.py:96:log_dist] [Rank 0] step=490, skipped=0, lr=[1.2757427344393702e-07, 1.2757427344393702e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
135
+ [2024-11-11 11:18:34,535] [INFO] [timer.py:264:stop] epoch=2/micro_step=136/global_step=490, RunningAvgSamplesPerSec=6.231258151619256, CurrSamplesPerSec=5.3911203096440845, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
136
+ [2024-11-11 11:20:18,891] [INFO] [logging.py:96:log_dist] [Rank 0] step=500, skipped=0, lr=[1.1102549309022291e-07, 1.1102549309022291e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
137
+ [2024-11-11 11:20:18,892] [INFO] [timer.py:264:stop] epoch=2/micro_step=156/global_step=500, RunningAvgSamplesPerSec=6.231896275894786, CurrSamplesPerSec=5.431348295652274, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
138
+ [2024-11-11 11:22:03,897] [INFO] [logging.py:96:log_dist] [Rank 0] step=510, skipped=0, lr=[9.549150281252632e-08, 9.549150281252632e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
139
+ [2024-11-11 11:22:03,898] [INFO] [timer.py:264:stop] epoch=2/micro_step=176/global_step=510, RunningAvgSamplesPerSec=6.231784979733288, CurrSamplesPerSec=6.118001686313543, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
140
+ [2024-11-11 11:23:47,595] [INFO] [logging.py:96:log_dist] [Rank 0] step=520, skipped=0, lr=[8.101282901783457e-08, 8.101282901783457e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
141
+ [2024-11-11 11:23:47,596] [INFO] [timer.py:264:stop] epoch=2/micro_step=196/global_step=520, RunningAvgSamplesPerSec=6.233175716519829, CurrSamplesPerSec=6.4252008944871175, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
142
+ [2024-11-11 11:25:34,694] [INFO] [logging.py:96:log_dist] [Rank 0] step=530, skipped=0, lr=[6.762724491292555e-08, 6.762724491292555e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
143
+ [2024-11-11 11:25:34,694] [INFO] [timer.py:264:stop] epoch=2/micro_step=216/global_step=530, RunningAvgSamplesPerSec=6.230586016932243, CurrSamplesPerSec=6.31995020734794, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
144
+ [2024-11-11 11:27:21,825] [INFO] [logging.py:96:log_dist] [Rank 0] step=540, skipped=0, lr=[5.5369671958383326e-08, 5.5369671958383326e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
145
+ [2024-11-11 11:27:21,826] [INFO] [timer.py:264:stop] epoch=2/micro_step=236/global_step=540, RunningAvgSamplesPerSec=6.228137586678738, CurrSamplesPerSec=6.294527897652772, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
146
+ [2024-11-11 11:28:57,980] [INFO] [logging.py:96:log_dist] [Rank 0] step=550, skipped=0, lr=[4.4272088762495554e-08, 4.4272088762495554e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
147
+ [2024-11-11 11:28:57,981] [INFO] [timer.py:264:stop] epoch=2/micro_step=256/global_step=550, RunningAvgSamplesPerSec=6.237979411534363, CurrSamplesPerSec=7.440681418583277, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
148
+ [2024-11-11 11:30:37,963] [INFO] [logging.py:96:log_dist] [Rank 0] step=560, skipped=0, lr=[3.4363447652715425e-08, 3.4363447652715425e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
149
+ [2024-11-11 11:30:37,963] [INFO] [timer.py:264:stop] epoch=2/micro_step=276/global_step=560, RunningAvgSamplesPerSec=6.2432383070279265, CurrSamplesPerSec=6.325613791478562, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
150
+ [2024-11-11 11:32:29,640] [INFO] [logging.py:96:log_dist] [Rank 0] step=570, skipped=0, lr=[2.5669599142344954e-08, 2.5669599142344954e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
151
+ [2024-11-11 11:32:29,641] [INFO] [timer.py:264:stop] epoch=2/micro_step=296/global_step=570, RunningAvgSamplesPerSec=6.235729072836607, CurrSamplesPerSec=6.363454933868642, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
152
+ [2024-11-11 11:34:19,919] [INFO] [logging.py:96:log_dist] [Rank 0] step=580, skipped=0, lr=[1.8213224489497525e-08, 1.8213224489497525e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
153
+ [2024-11-11 11:34:19,920] [INFO] [timer.py:264:stop] epoch=2/micro_step=316/global_step=580, RunningAvgSamplesPerSec=6.229848630704788, CurrSamplesPerSec=5.9735175986720614, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
154
+ [2024-11-11 11:36:02,300] [INFO] [logging.py:96:log_dist] [Rank 0] step=590, skipped=0, lr=[1.2013776524284869e-08, 1.2013776524284869e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
155
+ [2024-11-11 11:36:02,300] [INFO] [timer.py:264:stop] epoch=2/micro_step=336/global_step=590, RunningAvgSamplesPerSec=6.232535497944164, CurrSamplesPerSec=7.393825768980957, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
156
+ [2024-11-11 11:37:49,415] [INFO] [logging.py:96:log_dist] [Rank 0] step=600, skipped=0, lr=[7.087428898604974e-09, 7.087428898604974e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
157
+ [2024-11-11 11:37:49,415] [INFO] [timer.py:264:stop] epoch=2/micro_step=356/global_step=600, RunningAvgSamplesPerSec=6.230310889018549, CurrSamplesPerSec=7.70988597085285, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
158
+ Saving checkpoint at step 1200 ...
159
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
160
+ Saving 16-bit model...
161
+ [2024-11-11 11:37:58,346] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
162
+ [2024-11-11 11:37:58,346] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin, tag: global_step600
163
+ [2024-11-11 11:37:58,346] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin...
164
+ [2024-11-11 11:38:17,278] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin.
165
+ [2024-11-11 11:38:17,279] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
166
+ Model saved!
167
+ Saving 16-bit model...
168
+ [2024-11-11 11:38:24,358] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
169
+ [2024-11-11 11:38:24,359] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin, tag: global_step600
170
+ [2024-11-11 11:38:24,359] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin...
171
+ [2024-11-11 11:38:45,808] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin.
172
+ [2024-11-11 11:38:45,809] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
173
+ Model saved!
174
+ Checkpoint saved.
175
+ [2024-11-11 11:40:29,433] [INFO] [logging.py:96:log_dist] [Rank 0] step=610, skipped=0, lr=[3.4470338909303176e-09, 3.4470338909303176e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
176
+ [2024-11-11 11:40:29,434] [INFO] [timer.py:264:stop] epoch=2/micro_step=376/global_step=610, RunningAvgSamplesPerSec=6.2316845073786, CurrSamplesPerSec=6.228612676006549, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
177
+ [2024-11-11 11:42:11,651] [INFO] [logging.py:96:log_dist] [Rank 0] step=620, skipped=0, lr=[1.1020888761813507e-09, 1.1020888761813507e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
178
+ [2024-11-11 11:42:11,652] [INFO] [timer.py:264:stop] epoch=2/micro_step=396/global_step=620, RunningAvgSamplesPerSec=6.234524546765965, CurrSamplesPerSec=7.421455063273693, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
179
+ [2024-11-11 11:43:54,536] [INFO] [logging.py:96:log_dist] [Rank 0] step=630, skipped=0, lr=[5.87115481592293e-11, 5.87115481592293e-11], mom=[[0.9, 0.95], [0.9, 0.95]]
180
+ [2024-11-11 11:43:54,537] [INFO] [timer.py:264:stop] epoch=2/micro_step=416/global_step=630, RunningAvgSamplesPerSec=6.2363677298557585, CurrSamplesPerSec=5.864153736679168, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
181
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
182
+ Saving 16-bit model...
183
+ [2024-11-11 11:44:37,766] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step633 is about to be saved!
184
+ [2024-11-11 11:44:37,767] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin, tag: global_step633
185
+ [2024-11-11 11:44:37,767] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin...
186
+ [2024-11-11 11:44:57,052] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin.
187
+ [2024-11-11 11:44:57,053] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step633 is ready now!
188
+ Model saved!
189
+ Saving 16-bit model...
190
+ [2024-11-11 11:45:06,835] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step633 is about to be saved!
191
+ [2024-11-11 11:45:06,836] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin, tag: global_step633
192
+ [2024-11-11 11:45:06,836] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin...
193
+ [2024-11-11 11:45:29,691] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin.
194
+ [2024-11-11 11:45:29,692] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step633 is ready now!
195
+ Model saved!
slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/requirements.txt ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ align-anything==0.0.1.dev0
2
+ uvloop==0.20.0
3
+ absl-py==2.1.0
4
+ distro==1.9.0
5
+ bitsandbytes==0.44.1
6
+ uvicorn==0.31.1
7
+ safetensors==0.4.5
8
+ gradio_client==1.4.0
9
+ propcache==0.2.0
10
+ GitPython==3.1.43
11
+ pyzmq==26.2.0
12
+ nvidia-nvtx-cu12==12.1.105
13
+ einops==0.8.0
14
+ nvidia-cusolver-cu12==11.4.5.107
15
+ six==1.16.0
16
+ smmap==5.0.1
17
+ python-dotenv==1.0.1
18
+ multiprocess==0.70.16
19
+ pyarrow==17.0.0
20
+ pytest==7.2.0
21
+ lazy_loader==0.4
22
+ contourpy==1.3.0
23
+ aiofiles==23.2.1
24
+ torchlibrosa==0.1.0
25
+ protobuf==3.20.3
26
+ nvidia-cuda-runtime-cu12==12.1.105
27
+ pycparser==2.22
28
+ PyYAML==6.0.2
29
+ hjson==3.1.0
30
+ xxhash==3.5.0
31
+ platformdirs==4.3.6
32
+ diskcache==5.6.3
33
+ fairscale==0.4.13
34
+ certifi==2024.8.30
35
+ docker-pycreds==0.4.0
36
+ braceexpand==0.1.7
37
+ virtualenv==20.26.6
38
+ tokenizers==0.20.1
39
+ nvidia-cuda-nvrtc-cu12==12.1.105
40
+ grpcio==1.66.2
41
+ outlines==0.0.46
42
+ yarl==1.15.0
43
+ aiohttp==3.10.10
44
+ referencing==0.35.1
45
+ fsspec==2024.6.1
46
+ nvidia-nccl-cu12==2.20.5
47
+ Jinja2==3.1.4
48
+ timm==0.6.13
49
+ opencv-python==4.6.0.66
50
+ tomlkit==0.12.0
51
+ pandas==2.2.3
52
+ wcwidth==0.2.13
53
+ identify==2.6.1
54
+ deepspeed==0.15.2
55
+ datasets==3.0.1
56
+ ray==2.37.0
57
+ align-anything==0.0.1.dev0
58
+ nvidia-cufft-cu12==11.0.2.54
59
+ mistral_common==1.4.4
60
+ httptools==0.6.1
61
+ scipy==1.14.1
62
+ mdurl==0.1.2
63
+ clip==0.2.0
64
+ cycler==0.12.1
65
+ pyairports==2.1.1
66
+ charset-normalizer==3.4.0
67
+ torch==2.4.0
68
+ mpmath==1.3.0
69
+ tzdata==2024.2
70
+ nest-asyncio==1.6.0
71
+ sentencepiece==0.2.0
72
+ aiohappyeyeballs==2.4.3
73
+ ffmpy==0.4.0
74
+ tiktoken==0.7.0
75
+ nvidia-curand-cu12==10.3.2.106
76
+ kiwisolver==1.4.7
77
+ audioread==3.0.1
78
+ cffi==1.17.1
79
+ clint==0.5.1
80
+ partial-json-parser==0.2.1.1.post4
81
+ dill==0.3.8
82
+ ninja==1.11.1.1
83
+ tqdm==4.66.5
84
+ gitdb==4.0.11
85
+ regex==2024.9.11
86
+ nvidia-cusparse-cu12==12.1.0.106
87
+ aiosignal==1.3.1
88
+ jsonschema-specifications==2024.10.1
89
+ yt-dlp==2024.8.6
90
+ triton==3.0.0
91
+ pydub==0.25.1
92
+ nodeenv==1.9.1
93
+ pooch==1.8.2
94
+ MarkupSafe==2.1.5
95
+ fastapi==0.115.0
96
+ setproctitle==1.3.3
97
+ pycountry==24.6.1
98
+ anyio==4.6.0
99
+ matplotlib==3.9.2
100
+ diffusers==0.30.3
101
+ librosa==0.10.2.post1
102
+ filelock==3.16.1
103
+ jiter==0.6.1
104
+ sentry-sdk==2.16.0
105
+ starlette==0.38.6
106
+ py-cpuinfo==9.0.0
107
+ typer==0.12.5
108
+ zipp==3.20.2
109
+ args==0.1.0
110
+ jsonschema==4.23.0
111
+ llvmlite==0.43.0
112
+ interegular==0.3.3
113
+ wheel==0.44.0
114
+ frechet-audio-distance==0.1.2
115
+ pytz==2024.2
116
+ pytorch-fid==0.3.0
117
+ optree==0.13.0
118
+ lark==1.2.2
119
+ msgpack==1.1.0
120
+ prometheus_client==0.21.0
121
+ typing_extensions==4.12.2
122
+ mutagen==1.47.0
123
+ pycryptodomex==3.21.0
124
+ fonttools==4.54.1
125
+ ftfy==6.3.0
126
+ orjson==3.10.7
127
+ vllm==0.6.2
128
+ watchfiles==0.24.0
129
+ iniconfig==2.0.0
130
+ idna==3.10
131
+ proglog==0.1.10
132
+ sniffio==1.3.1
133
+ pyparsing==3.1.4
134
+ h11==0.14.0
135
+ networkx==3.4.1
136
+ xformers==0.0.27.post2
137
+ hpsv2==1.2.0
138
+ pluggy==1.5.0
139
+ gguf==0.10.0
140
+ imageio==2.35.1
141
+ importlib_metadata==8.5.0
142
+ urllib3==2.2.3
143
+ nvidia-nvjitlink-cu12==12.6.77
144
+ semantic-version==2.10.0
145
+ decorator==4.4.2
146
+ attrs==24.2.0
147
+ Brotli==1.1.0
148
+ numpy==1.26.4
149
+ soxr==0.5.0.post1
150
+ requests==2.32.3
151
+ pytest-split==0.8.0
152
+ httpcore==1.0.6
153
+ webdataset==0.2.100
154
+ rpds-py==0.20.0
155
+ shellingham==1.5.4
156
+ annotated-types==0.7.0
157
+ pillow==10.4.0
158
+ nvidia-ml-py==12.560.30
159
+ packaging==24.1
160
+ peft==0.13.2
161
+ imageio-ffmpeg==0.5.1
162
+ setuptools==75.1.0
163
+ pydantic==2.9.2
164
+ Pygments==2.18.0
165
+ cloudpickle==3.1.0
166
+ torchvision==0.19.0
167
+ threadpoolctl==3.5.0
168
+ cfgv==3.4.0
169
+ tensorboard==2.18.0
170
+ moviepy==1.0.3
171
+ nvidia-cuda-cupti-cu12==12.1.105
172
+ msgspec==0.18.6
173
+ Markdown==3.7
174
+ huggingface-hub==0.25.2
175
+ scikit-learn==1.5.2
176
+ distlib==0.3.9
177
+ shortuuid==1.0.13
178
+ nvidia-cublas-cu12==12.1.3.1
179
+ pip==24.2
180
+ image-reward==1.5
181
+ click==8.1.7
182
+ lm-format-enforcer==0.10.6
183
+ joblib==1.4.2
184
+ torchaudio==2.4.0
185
+ rich==13.9.2
186
+ resampy==0.4.3
187
+ numba==0.60.0
188
+ gradio==5.0.2
189
+ transformers==4.46.0.dev0
190
+ tensorboard-data-server==0.7.2
191
+ soundfile==0.12.1
192
+ multidict==6.1.0
193
+ wandb==0.18.3
194
+ openai==1.51.2
195
+ nvidia-cudnn-cu12==9.1.0.70
196
+ python-dateutil==2.9.0.post0
197
+ psutil==6.0.0
198
+ frozenlist==1.4.1
199
+ python-multipart==0.0.12
200
+ Werkzeug==3.0.4
201
+ markdown-it-py==3.0.0
202
+ pydantic_core==2.23.4
203
+ prometheus-fastapi-instrumentator==7.0.0
204
+ sympy==1.13.3
205
+ accelerate==1.0.1
206
+ httpx==0.27.2
207
+ ruff==0.6.9
208
+ pre_commit==4.0.1
209
+ websockets==12.0
210
+ importlib_resources==6.4.0
211
+ packaging==24.1
212
+ typing_extensions==4.12.2
213
+ wheel==0.43.0
214
+ zipp==3.19.2
215
+ jaraco.text==3.12.1
216
+ inflect==7.3.1
217
+ more-itertools==10.3.0
218
+ autocommand==2.2.2
219
+ platformdirs==4.2.2
220
+ typeguard==4.3.0
221
+ jaraco.functools==4.0.1
222
+ jaraco.context==5.3.0
223
+ tomli==2.0.1
224
+ jaraco.collections==5.1.0
225
+ importlib_metadata==8.0.0
226
+ backports.tarfile==1.2.0
slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/wandb-metadata.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "3.11.10",
4
+ "startedAt": "2024-11-11T09:50:28.925585Z",
5
+ "args": [
6
+ "--local_rank=0",
7
+ "--model_name_or_path",
8
+ "/data/align-anything/hantao/models/0916_ti_to_ti_sft/",
9
+ "--train_datasets",
10
+ "/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs",
11
+ "--train_data_files",
12
+ "llf_ti2ti_13.5k_tokenized.pt",
13
+ "--output_dir",
14
+ "../outputs/sft_tf_cham_1111_13.5k_ti2ti",
15
+ "--train_template",
16
+ "AA_textfeedback",
17
+ "--train_split",
18
+ "train",
19
+ "--per_device_train_batch_size",
20
+ "4",
21
+ "--per_device_eval_batch_size",
22
+ "4",
23
+ "--gradient_accumulation_steps",
24
+ "2",
25
+ "--save_interval",
26
+ "400",
27
+ "--learning_rate",
28
+ "1e-6",
29
+ "--epochs",
30
+ "3",
31
+ "--lr_scheduler_type",
32
+ "cosine"
33
+ ],
34
+ "program": "-m align_anything.trainers.text_image_to_text_image.sft",
35
+ "git": {
36
+ "remote": "https://github.com/PKU-Alignment/align-anything.git",
37
+ "commit": "6fde660afc9985323f147930eedf188a5699adc7"
38
+ },
39
+ "email": "[email protected]",
40
+ "root": "../outputs/sft_tf_cham_1111_13.5k_ti2ti",
41
+ "host": "lyg0198",
42
+ "username": "align-anything",
43
+ "executable": "/data/align-anything/miniconda3/envs/hantao_cham/bin/python",
44
+ "cpu_count": 64,
45
+ "cpu_count_logical": 128,
46
+ "gpu": "[NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB]",
47
+ "gpu_count": 8,
48
+ "disk": {
49
+ "/": {
50
+ "total": "938421047296",
51
+ "used": "460055379968"
52
+ }
53
+ },
54
+ "memory": {
55
+ "total": "540633423872"
56
+ },
57
+ "cpu": {
58
+ "count": 64,
59
+ "countLogical": 128
60
+ },
61
+ "gpu_nvidia": [
62
+ {
63
+ "name": "NVIDIA A100-SXM4-80GB",
64
+ "memoryTotal": "85899345920",
65
+ "cudaCores": 6912,
66
+ "architecture": "Ampere"
67
+ },
68
+ {
69
+ "name": "NVIDIA A100-SXM4-80GB",
70
+ "memoryTotal": "85899345920",
71
+ "cudaCores": 6912,
72
+ "architecture": "Ampere"
73
+ },
74
+ {
75
+ "name": "NVIDIA A100-SXM4-80GB",
76
+ "memoryTotal": "85899345920",
77
+ "cudaCores": 6912,
78
+ "architecture": "Ampere"
79
+ },
80
+ {
81
+ "name": "NVIDIA A100-SXM4-80GB",
82
+ "memoryTotal": "85899345920",
83
+ "cudaCores": 6912,
84
+ "architecture": "Ampere"
85
+ },
86
+ {
87
+ "name": "NVIDIA A100-SXM4-80GB",
88
+ "memoryTotal": "85899345920",
89
+ "cudaCores": 6912,
90
+ "architecture": "Ampere"
91
+ },
92
+ {
93
+ "name": "NVIDIA A100-SXM4-80GB",
94
+ "memoryTotal": "85899345920",
95
+ "cudaCores": 6912,
96
+ "architecture": "Ampere"
97
+ },
98
+ {
99
+ "name": "NVIDIA A100-SXM4-80GB",
100
+ "memoryTotal": "85899345920",
101
+ "cudaCores": 6912,
102
+ "architecture": "Ampere"
103
+ },
104
+ {
105
+ "name": "NVIDIA A100-SXM4-80GB",
106
+ "memoryTotal": "85899345920",
107
+ "cudaCores": 6912,
108
+ "architecture": "Ampere"
109
+ }
110
+ ],
111
+ "cudaVersion": "12.4"
112
+ }
slice_1200/wandb/run-20241111_095028-sq0dhwvm/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_timestamp":1.731325467962379e+09,"_runtime":6900.89633248,"_step":1266,"train/step":1266,"train/loss":0.6548570990562439,"train/lr":0,"train/epoch":3,"_wandb":{"runtime":6900}}
slice_1200/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-11T09:50:28.93257347Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-11T09:50:28.932602745Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-core.log"}
3
+ {"time":"2024-11-11T09:50:28.93714017Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-11T09:50:28.968182627Z","level":"INFO","msg":"created new stream","id":"sq0dhwvm"}
5
+ {"time":"2024-11-11T09:50:28.968263994Z","level":"INFO","msg":"stream: started","id":"sq0dhwvm"}
6
+ {"time":"2024-11-11T09:50:28.968288809Z","level":"INFO","msg":"sender: started","stream_id":{"value":"sq0dhwvm"}}
7
+ {"time":"2024-11-11T09:50:28.968279236Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"sq0dhwvm"}}
8
+ {"time":"2024-11-11T09:50:28.968330962Z","level":"INFO","msg":"handler: started","stream_id":{"value":"sq0dhwvm"}}
9
+ {"time":"2024-11-11T09:50:34.632278843Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-11T09:50:34.636084019Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-11T11:20:45.19395353Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/htlou/align-anything/sq0dhwvm/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
12
+ {"time":"2024-11-11T11:45:29.821937086Z","level":"INFO","msg":"Stopping system monitor"}
13
+ {"time":"2024-11-11T11:45:29.846984762Z","level":"INFO","msg":"Stopped system monitor"}
14
+ {"time":"2024-11-11T11:45:30.253659937Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
15
+ {"time":"2024-11-11T11:45:30.253692046Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
16
+ {"time":"2024-11-11T11:45:30.882510953Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
17
+ {"time":"2024-11-11T11:45:32.320976723Z","level":"INFO","msg":"stream: closing","id":"sq0dhwvm"}
18
+ {"time":"2024-11-11T11:45:32.321006242Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"sq0dhwvm"}}
19
+ {"time":"2024-11-11T11:45:32.321051996Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"sq0dhwvm"}}
20
+ {"time":"2024-11-11T11:45:32.321127635Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"sq0dhwvm"}}
21
+ {"time":"2024-11-11T11:45:32.323850284Z","level":"INFO","msg":"stream: closed","id":"sq0dhwvm"}
slice_1200/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Configure stats pid to 3212652
3
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-11 09:50:28,918 WARNING MainThread:3212652 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log
11
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log
12
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_ti2ti_13.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1111_13.5k_ti2ti', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0916_ti_to_ti_sft/', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():667] starting backend
16
+ 2024-11-11 09:50:28,919 INFO MainThread:3212652 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-11 09:50:28,924 INFO MainThread:3212652 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-11 09:50:28,925 INFO MainThread:3212652 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-11 09:50:28,929 INFO MainThread:3212652 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-11 09:50:28,983 INFO MainThread:3212652 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-11 09:50:34,620 INFO MainThread:3212652 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-11 09:50:35,184 INFO MainThread:3212652 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-11 11:45:29,778 INFO MainThread:3212652 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/sq0dhwvm
28
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-11 11:45:32,305 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-11 11:45:32,307 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-11 11:45:32,318 INFO MainThread:3212652 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_1200/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d94ac37632f72a3f8cd135073052c80433a6d2851df145837d3165fe8e647a8d
3
+ size 6923036
slice_400/arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_ti2ti_13.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1111_13.5k_ti2ti
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0916_ti_to_ti_sft/
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 3.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 1.0e-06
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
slice_400/config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_400/environ.txt ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ADDR2LINE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-addr2line
2
+ AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ar
3
+ AS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-as
4
+ BROWSER=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/bin/helpers/browser.sh
5
+ BUILD=x86_64-conda-linux-gnu
6
+ CC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
7
+ CC_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
8
+ CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
9
+ CMAKE_PREFIX_PATH=/data/align-anything/miniconda3/envs/jy-s:/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot/usr
10
+ COLORTERM=truecolor
11
+ CONDA_BACKUP_ADDR2LINE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-addr2line
12
+ CONDA_BACKUP_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ar
13
+ CONDA_BACKUP_AS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-as
14
+ CONDA_BACKUP_BUILD=x86_64-conda-linux-gnu
15
+ CONDA_BACKUP_CC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
16
+ CONDA_BACKUP_CC_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cc
17
+ CONDA_BACKUP_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
18
+ CONDA_BACKUP_CMAKE_PREFIX_PATH=/data/align-anything/miniconda3/envs/jy-s:/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot/usr
19
+ CONDA_BACKUP_CONDA_BUILD_SYSROOT=/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot
20
+ CONDA_BACKUP_CPP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cpp
21
+ CONDA_BACKUP_CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
22
+ CONDA_BACKUP_CXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
23
+ CONDA_BACKUP_CXXFILT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++filt
24
+ CONDA_BACKUP_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
25
+ CONDA_BACKUP_CXX_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
26
+ CONDA_BACKUP_DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
27
+ CONDA_BACKUP_DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /data/align-anything/miniconda3/envs/jy-s/include
28
+ CONDA_BACKUP_DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
29
+ CONDA_BACKUP_ELFEDIT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-elfedit
30
+ CONDA_BACKUP_GCC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc
31
+ CONDA_BACKUP_GCC_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ar
32
+ CONDA_BACKUP_GCC_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-nm
33
+ CONDA_BACKUP_GCC_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ranlib
34
+ CONDA_BACKUP_GPROF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gprof
35
+ CONDA_BACKUP_GXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-g++
36
+ CONDA_BACKUP_HOST=x86_64-conda-linux-gnu
37
+ CONDA_BACKUP_LD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld
38
+ CONDA_BACKUP_LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/data/align-anything/miniconda3/envs/jy-s/lib -Wl,-rpath-link,/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
39
+ CONDA_BACKUP_LD_GOLD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld.gold
40
+ CONDA_BACKUP_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-nm
41
+ CONDA_BACKUP_OBJCOPY=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objcopy
42
+ CONDA_BACKUP_OBJDUMP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objdump
43
+ CONDA_BACKUP_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ranlib
44
+ CONDA_BACKUP_READELF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-readelf
45
+ CONDA_BACKUP_SIZE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-size
46
+ CONDA_BACKUP_STRINGS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strings
47
+ CONDA_BACKUP_STRIP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strip
48
+ CONDA_BACKUP_build_alias=x86_64-conda-linux-gnu
49
+ CONDA_BACKUP_host_alias=x86_64-conda-linux-gnu
50
+ CONDA_BUILD_SYSROOT=/data/align-anything/miniconda3/envs/jy-s/x86_64-conda-linux-gnu/sysroot
51
+ CONDA_DEFAULT_ENV=hantao_cham
52
+ CONDA_EXE=/data/align-anything/miniconda3/bin/conda
53
+ CONDA_PREFIX=/data/align-anything/miniconda3/envs/hantao_cham
54
+ CONDA_PREFIX_1=/home/align-anything/miniconda3
55
+ CONDA_PREFIX_2=/data/align-anything/miniconda3/envs/jy-s
56
+ CONDA_PREFIX_3=/home/align-anything/miniconda3
57
+ CONDA_PREFIX_4=/data/align-anything/miniconda3
58
+ CONDA_PROMPT_MODIFIER=(hantao_cham)
59
+ CONDA_PYTHON_EXE=/data/align-anything/miniconda3/bin/python
60
+ CONDA_ROOT=/home/align-anything/miniconda3
61
+ CONDA_SHLVL=5
62
+ CPP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-cpp
63
+ CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
64
+ CROSS_RANK=0
65
+ CROSS_SIZE=1
66
+ CUDA_MODULE_LOADING=LAZY
67
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
68
+ CXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
69
+ CXXFILT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++filt
70
+ CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include -I/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/include -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
71
+ CXX_FOR_BUILD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
72
+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
73
+ DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
74
+ DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /data/align-anything/miniconda3/envs/jy-s/include
75
+ DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /data/align-anything/miniconda3/envs/jy-s/include
76
+ ELFEDIT=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-elfedit
77
+ GCC=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc
78
+ GCC_AR=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ar
79
+ GCC_NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-nm
80
+ GCC_RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gcc-ranlib
81
+ GIT_ASKPASS=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/extensions/git/dist/askpass.sh
82
+ GPROF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-gprof
83
+ GXX=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-g++
84
+ HOME=/home/align-anything
85
+ HOST=x86_64-conda-linux-gnu
86
+ LANG=en_US.UTF-8
87
+ LD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld
88
+ LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/data/align-anything/miniconda3/envs/jy-s/lib -Wl,-rpath-link,/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib -L/data/align-anything/miniconda3/envs/jy-s/targets/x86_64-linux/lib/stubs
89
+ LD_GOLD=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ld.gold
90
+ LD_LIBRARY_PATH=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/../../lib64:
91
+ LESSCLOSE=/usr/bin/lesspipe %s %s
92
+ LESSOPEN=| /usr/bin/lesspipe %s
93
+ LOCAL_RANK=0
94
+ LOCAL_SIZE=8
95
+ LOGLEVEL=WARNING
96
+ LOGNAME=align-anything
97
+ LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
98
+ MASTER_ADDR=127.0.0.1
99
+ MASTER_PORT=19698
100
+ MOTD_SHOWN=pam
101
+ NM=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-nm
102
+ NVCC_PREPEND_FLAGS= -ccbin=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-c++
103
+ OBJCOPY=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objcopy
104
+ OBJDUMP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-objdump
105
+ OLDPWD=/data/align-anything/hantao/align-anything/projects/text_image_to_text_image
106
+ PATH=/data/align-anything/miniconda3/envs/hantao_cham/bin:/home/align-anything/miniconda3/bin:/home/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
107
+ PWD=/data/align-anything/hantao/align-anything/scripts
108
+ PYGAME_HIDE_SUPPORT_PROMPT=1
109
+ PYTHONHASHSEED=42
110
+ PYTHONPATH=/data/align-anything/hantao/align-anything
111
+ QT_QPA_FONTDIR=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/fonts
112
+ QT_QPA_PLATFORM_PLUGIN_PATH=/data/align-anything/miniconda3/envs/hantao_cham/lib/python3.11/site-packages/cv2/qt/plugins
113
+ RANK=0
114
+ RANLIB=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-ranlib
115
+ READELF=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-readelf
116
+ SHELL=/bin/bash
117
+ SHLVL=3
118
+ SIZE=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-size
119
+ SSH_CLIENT=203.93.11.8 7409 30800
120
+ SSH_CONNECTION=111.205.230.212 11374 10.10.212.198 30800
121
+ SSL_CERT_DIR=/usr/lib/ssl/certs
122
+ SSL_CERT_FILE=/usr/lib/ssl/certs/ca-certificates.crt
123
+ STRINGS=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strings
124
+ STRIP=/data/align-anything/miniconda3/envs/jy-s/bin/x86_64-conda-linux-gnu-strip
125
+ TERM=screen
126
+ TERM_PROGRAM=tmux
127
+ TERM_PROGRAM_VERSION=3.2a
128
+ TMUX=/tmp/tmux-2000/default,2557274,4
129
+ TMUX_PANE=%4
130
+ TRITON_CACHE_DIR=/home/align-anything/cache/triton
131
+ USER=align-anything
132
+ VSCODE_GIT_ASKPASS_EXTRA_ARGS=
133
+ VSCODE_GIT_ASKPASS_MAIN=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/extensions/git/dist/askpass-main.js
134
+ VSCODE_GIT_ASKPASS_NODE=/home/align-anything/.cursor-server/cli/servers/Stable-b1e87884330fc271d5eb589e368c35f14e76dec0/server/node
135
+ VSCODE_GIT_IPC_HANDLE=/run/user/2000/vscode-git-d8ae57dc60.sock
136
+ VSCODE_IPC_HOOK_CLI=/run/user/2000/vscode-ipc-45219155-ac87-45dc-978b-d20ac490d167.sock
137
+ WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
138
+ WANDB_SERVICE=2-3212652-tcp-localhost-41187
139
+ WORLD_SIZE=8
140
+ XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
141
+ XDG_RUNTIME_DIR=/run/user/2000
142
+ XDG_SESSION_CLASS=user
143
+ XDG_SESSION_ID=446
144
+ XDG_SESSION_TYPE=tty
145
+ _=/data/align-anything/miniconda3/envs/hantao_cham/bin/deepspeed
146
+ _CE_CONDA=
147
+ _CE_M=
148
+ build_alias=x86_64-conda-linux-gnu
149
+ host_alias=x86_64-conda-linux-gnu
slice_400/preprocessor_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 512,
4
+ "width": 512
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 1.0,
13
+ 1.0,
14
+ 1.0
15
+ ],
16
+ "image_processor_type": "ChameleonImageProcessor",
17
+ "image_std": [
18
+ 1.0,
19
+ 1.0,
20
+ 1.0
21
+ ],
22
+ "processor_class": "ChameleonProcessor",
23
+ "resample": 1,
24
+ "rescale_factor": 0.0078,
25
+ "size": {
26
+ "shortest_edge": 512
27
+ }
28
+ }
slice_400/processor_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "image_seq_length": 1024,
3
+ "image_token": "<image>",
4
+ "processor_class": "ChameleonProcessor"
5
+ }
slice_400/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea2bb6e36bc2e40d03fefac96e2a5f8abebc25924edf5d554f0333b803942dc0
3
+ size 14086366378
slice_400/script.sh ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Copyright 2024 PKU-Alignment Team. All Rights Reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ # ==============================================================================
17
+
18
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
19
+ # You can replace it with a local model path
20
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
21
+ # You can replace it with a local dataset path
22
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
23
+ TRAIN_DATA_FILES="llf_ti2ti_13.5k_tokenized.pt"
24
+ # You can replace it with a new path
25
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_13.5k_ti2ti"
26
+ # For wandb online logging
27
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
28
+ # Source the setup script
29
+ source ./setup.sh
30
+
31
+ # sleep 30m
32
+ # Execute deepspeed command
33
+ deepspeed \
34
+ --master_port ${MASTER_PORT} \
35
+ --module align_anything.trainers.text_image_to_text_image.sft \
36
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
37
+ --train_datasets ${TRAIN_DATASETS} \
38
+ --train_data_files ${TRAIN_DATA_FILES} \
39
+ --output_dir ${OUTPUT_DIR} \
40
+ --train_template AA_textfeedback \
41
+ --train_split train \
42
+ --per_device_train_batch_size 4 \
43
+ --per_device_eval_batch_size 4 \
44
+ --gradient_accumulation_steps 2 \
45
+ --save_interval 400 \
46
+ --learning_rate 1e-6 \
47
+ --epochs 3 \
48
+ --lr_scheduler_type cosine
49
+
50
+
51
+ export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
52
+ # You can replace it with a local model path
53
+ MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0916_ti_to_ti_sft/"
54
+ # You can replace it with a local dataset path
55
+ TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
56
+ TRAIN_DATA_FILES="llf_ti2ti_6.75k_tokenized.pt"
57
+ # You can replace it with a new path
58
+ OUTPUT_DIR="../outputs/sft_tf_cham_1111_6.75k_ti2ti"
59
+ # For wandb online logging
60
+ export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
61
+ # Source the setup script
62
+ source ./setup.sh
63
+
64
+ # sleep 30m
65
+ # Execute deepspeed command
66
+ deepspeed \
67
+ --master_port ${MASTER_PORT} \
68
+ --module align_anything.trainers.text_image_to_text_image.sft \
69
+ --model_name_or_path ${MODEL_NAME_OR_PATH} \
70
+ --train_datasets ${TRAIN_DATASETS} \
71
+ --train_data_files ${TRAIN_DATA_FILES} \
72
+ --output_dir ${OUTPUT_DIR} \
73
+ --train_template AA_textfeedback \
74
+ --train_split train \
75
+ --per_device_train_batch_size 4 \
76
+ --per_device_eval_batch_size 4 \
77
+ --gradient_accumulation_steps 2 \
78
+ --save_interval 400 \
79
+ --learning_rate 1e-6 \
80
+ --epochs 3 \
81
+ --lr_scheduler_type cosine
slice_400/special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<pad>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "<reserved08706>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
slice_400/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_400/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
slice_400/wandb/debug-internal.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-11T09:50:28.93257347Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-11T09:50:28.932602745Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-core.log"}
3
+ {"time":"2024-11-11T09:50:28.93714017Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-11T09:50:28.968182627Z","level":"INFO","msg":"created new stream","id":"sq0dhwvm"}
5
+ {"time":"2024-11-11T09:50:28.968263994Z","level":"INFO","msg":"stream: started","id":"sq0dhwvm"}
6
+ {"time":"2024-11-11T09:50:28.968288809Z","level":"INFO","msg":"sender: started","stream_id":{"value":"sq0dhwvm"}}
7
+ {"time":"2024-11-11T09:50:28.968279236Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"sq0dhwvm"}}
8
+ {"time":"2024-11-11T09:50:28.968330962Z","level":"INFO","msg":"handler: started","stream_id":{"value":"sq0dhwvm"}}
9
+ {"time":"2024-11-11T09:50:34.632278843Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-11T09:50:34.636084019Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-11T11:20:45.19395353Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/htlou/align-anything/sq0dhwvm/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
12
+ {"time":"2024-11-11T11:45:29.821937086Z","level":"INFO","msg":"Stopping system monitor"}
13
+ {"time":"2024-11-11T11:45:29.846984762Z","level":"INFO","msg":"Stopped system monitor"}
14
+ {"time":"2024-11-11T11:45:30.253659937Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
15
+ {"time":"2024-11-11T11:45:30.253692046Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
16
+ {"time":"2024-11-11T11:45:30.882510953Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
17
+ {"time":"2024-11-11T11:45:32.320976723Z","level":"INFO","msg":"stream: closing","id":"sq0dhwvm"}
18
+ {"time":"2024-11-11T11:45:32.321006242Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"sq0dhwvm"}}
19
+ {"time":"2024-11-11T11:45:32.321051996Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"sq0dhwvm"}}
20
+ {"time":"2024-11-11T11:45:32.321127635Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"sq0dhwvm"}}
21
+ {"time":"2024-11-11T11:45:32.323850284Z","level":"INFO","msg":"stream: closed","id":"sq0dhwvm"}
slice_400/wandb/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Configure stats pid to 3212652
3
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-11 09:50:28,918 WARNING MainThread:3212652 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log
11
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log
12
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_ti2ti_13.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1111_13.5k_ti2ti', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0916_ti_to_ti_sft/', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():667] starting backend
16
+ 2024-11-11 09:50:28,919 INFO MainThread:3212652 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-11 09:50:28,924 INFO MainThread:3212652 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-11 09:50:28,925 INFO MainThread:3212652 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-11 09:50:28,929 INFO MainThread:3212652 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-11 09:50:28,983 INFO MainThread:3212652 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-11 09:50:34,620 INFO MainThread:3212652 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-11 09:50:35,184 INFO MainThread:3212652 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-11 11:45:29,778 INFO MainThread:3212652 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/sq0dhwvm
28
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-11 11:45:32,305 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-11 11:45:32,307 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-11 11:45:32,318 INFO MainThread:3212652 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_400/wandb/run-20241111_095028-sq0dhwvm/files/config.yaml ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.18.3
4
+ m: []
5
+ python_version: 3.11.10
6
+ t:
7
+ "1":
8
+ - 1
9
+ - 11
10
+ - 41
11
+ - 49
12
+ - 51
13
+ - 55
14
+ - 71
15
+ - 83
16
+ - 98
17
+ - 105
18
+ "2":
19
+ - 1
20
+ - 11
21
+ - 41
22
+ - 49
23
+ - 51
24
+ - 55
25
+ - 71
26
+ - 83
27
+ - 98
28
+ - 105
29
+ "3":
30
+ - 2
31
+ - 13
32
+ - 16
33
+ - 23
34
+ - 55
35
+ - 61
36
+ "4": 3.11.10
37
+ "5": 0.18.3
38
+ "6": 4.46.0.dev0
39
+ "8":
40
+ - 5
41
+ "12": 0.18.3
42
+ "13": linux-x86_64
43
+ data_cfgs:
44
+ value:
45
+ eval_data_files: null
46
+ eval_datasets: null
47
+ eval_optional_args: []
48
+ eval_size: null
49
+ eval_split: null
50
+ eval_subset: null
51
+ eval_template: null
52
+ train_data_files: llf_ti2ti_13.5k_tokenized.pt
53
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
54
+ train_optional_args: []
55
+ train_size: null
56
+ train_split: train
57
+ train_subset: null
58
+ train_template: AA_textfeedback
59
+ logger_cfgs:
60
+ value:
61
+ cache_dir: null
62
+ log_project: align-anything
63
+ log_run_name: sft
64
+ log_type: wandb
65
+ output_dir: ../outputs/sft_tf_cham_1111_13.5k_ti2ti
66
+ save_interval: 400
67
+ model_cfgs:
68
+ value:
69
+ model_max_length: 4096
70
+ model_name_or_path: /data/align-anything/hantao/models/0916_ti_to_ti_sft/
71
+ trust_remote_code: true
72
+ special_tokens:
73
+ value: null
74
+ train_cfgs:
75
+ value:
76
+ adam_betas:
77
+ - 0.9
78
+ - 0.95
79
+ adam_epsilon: 1e-08
80
+ bf16: true
81
+ ds_cfgs: ds_z3_config.json
82
+ epochs: 3
83
+ eval_interval: 1000
84
+ eval_strategy: steps
85
+ fp16: false
86
+ freeze_language_model: false
87
+ gradient_accumulation_steps: 2
88
+ gradient_checkpointing: true
89
+ learning_rate: 1e-06
90
+ lr_scheduler_type: cosine
91
+ lr_warmup_ratio: 0.03
92
+ max_grad_norm: 1
93
+ per_device_eval_batch_size: 4
94
+ per_device_train_batch_size: 4
95
+ seed: 42
96
+ weight_decay: 0
slice_400/wandb/run-20241111_095028-sq0dhwvm/files/output.log ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ***** Running training *****
2
+ Training 1/3.0 epoch: 0%| | 0/1266.0 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
3
+ Training 1/3.0 epoch (loss 0.6685): 25%|███████████████████████████▋ | 319/1266.0 [28:07<1:20:30, 5.10s/it]
4
+ [2024-11-11 09:52:24,616] [INFO] [logging.py:96:log_dist] [Rank 0] step=10, skipped=0, lr=[5.555555555555555e-07, 5.555555555555555e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
5
+ [2024-11-11 09:52:24,617] [INFO] [timer.py:264:stop] epoch=0/micro_step=20/global_step=10, RunningAvgSamplesPerSec=6.195061818187147, CurrSamplesPerSec=6.335947040765943, MemAllocated=14.07GB, MaxMemAllocated=25.28GB
6
+ [2024-11-11 09:54:06,853] [INFO] [logging.py:96:log_dist] [Rank 0] step=20, skipped=0, lr=[9.999739056948805e-07, 9.999739056948805e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
7
+ [2024-11-11 09:54:06,854] [INFO] [timer.py:264:stop] epoch=0/micro_step=40/global_step=20, RunningAvgSamplesPerSec=6.316656927899698, CurrSamplesPerSec=6.164126358699434, MemAllocated=14.07GB, MaxMemAllocated=25.28GB
8
+ [2024-11-11 09:55:53,507] [INFO] [logging.py:96:log_dist] [Rank 0] step=30, skipped=0, lr=[9.99060890967219e-07, 9.99060890967219e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
9
+ [2024-11-11 09:55:53,508] [INFO] [timer.py:264:stop] epoch=0/micro_step=60/global_step=30, RunningAvgSamplesPerSec=6.248466505115529, CurrSamplesPerSec=7.491092861751436, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
10
+ [2024-11-11 09:57:41,293] [INFO] [logging.py:96:log_dist] [Rank 0] step=40, skipped=0, lr=[9.968458833570276e-07, 9.968458833570276e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
11
+ [2024-11-11 09:57:41,294] [INFO] [timer.py:264:stop] epoch=0/micro_step=80/global_step=40, RunningAvgSamplesPerSec=6.196372346586397, CurrSamplesPerSec=6.409875261403376, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
12
+ [2024-11-11 09:59:23,446] [INFO] [logging.py:96:log_dist] [Rank 0] step=50, skipped=0, lr=[9.933346615662559e-07, 9.933346615662559e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
13
+ [2024-11-11 09:59:23,447] [INFO] [timer.py:264:stop] epoch=0/micro_step=100/global_step=50, RunningAvgSamplesPerSec=6.246806387061766, CurrSamplesPerSec=7.260616697354314, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
14
+ [2024-11-11 10:01:09,165] [INFO] [logging.py:96:log_dist] [Rank 0] step=60, skipped=0, lr=[9.885363859716496e-07, 9.885363859716496e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
15
+ [2024-11-11 10:01:09,166] [INFO] [timer.py:264:stop] epoch=0/micro_step=120/global_step=60, RunningAvgSamplesPerSec=6.2467760579358185, CurrSamplesPerSec=5.904287617244474, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
16
+ [2024-11-11 10:02:58,559] [INFO] [logging.py:96:log_dist] [Rank 0] step=70, skipped=0, lr=[9.824635747263753e-07, 9.824635747263753e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
17
+ [2024-11-11 10:02:58,560] [INFO] [timer.py:264:stop] epoch=0/micro_step=140/global_step=70, RunningAvgSamplesPerSec=6.205922498836564, CurrSamplesPerSec=7.320670588968733, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
18
+ [2024-11-11 10:04:44,012] [INFO] [logging.py:96:log_dist] [Rank 0] step=80, skipped=0, lr=[9.75132071101588e-07, 9.75132071101588e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
19
+ [2024-11-11 10:04:44,013] [INFO] [timer.py:264:stop] epoch=0/micro_step=160/global_step=80, RunningAvgSamplesPerSec=6.20796558015961, CurrSamplesPerSec=6.143883631130097, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
20
+ [2024-11-11 10:06:26,906] [INFO] [logging.py:96:log_dist] [Rank 0] step=90, skipped=0, lr=[9.665610021531446e-07, 9.665610021531446e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
21
+ [2024-11-11 10:06:26,907] [INFO] [timer.py:264:stop] epoch=0/micro_step=180/global_step=90, RunningAvgSamplesPerSec=6.223837975387682, CurrSamplesPerSec=7.466477040673676, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
22
+ [2024-11-11 10:08:15,344] [INFO] [logging.py:96:log_dist] [Rank 0] step=100, skipped=0, lr=[9.567727288213004e-07, 9.567727288213004e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
23
+ [2024-11-11 10:08:15,345] [INFO] [timer.py:264:stop] epoch=0/micro_step=200/global_step=100, RunningAvgSamplesPerSec=6.209057201170874, CurrSamplesPerSec=5.302689344409979, MemAllocated=14.07GB, MaxMemAllocated=29.07GB
24
+ [2024-11-11 10:10:01,328] [INFO] [logging.py:96:log_dist] [Rank 0] step=110, skipped=0, lr=[9.457927875935681e-07, 9.457927875935681e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
25
+ [2024-11-11 10:10:01,329] [INFO] [timer.py:264:stop] epoch=0/micro_step=220/global_step=110, RunningAvgSamplesPerSec=6.208473867040452, CurrSamplesPerSec=7.698975362834649, MemAllocated=14.07GB, MaxMemAllocated=29.74GB
26
+ [2024-11-11 10:11:47,804] [INFO] [logging.py:96:log_dist] [Rank 0] step=120, skipped=0, lr=[9.336498238829382e-07, 9.336498238829382e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
27
+ [2024-11-11 10:11:47,804] [INFO] [timer.py:264:stop] epoch=0/micro_step=240/global_step=120, RunningAvgSamplesPerSec=6.202644364760127, CurrSamplesPerSec=7.252698675359632, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
28
+ [2024-11-11 10:13:25,206] [INFO] [logging.py:96:log_dist] [Rank 0] step=130, skipped=0, lr=[9.203755172952665e-07, 9.203755172952665e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
29
+ [2024-11-11 10:13:25,207] [INFO] [timer.py:264:stop] epoch=0/micro_step=260/global_step=130, RunningAvgSamplesPerSec=6.240439626002477, CurrSamplesPerSec=6.299954067945365, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
30
+ [2024-11-11 10:15:07,236] [INFO] [logging.py:96:log_dist] [Rank 0] step=140, skipped=0, lr=[9.060044989808009e-07, 9.060044989808009e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
31
+ [2024-11-11 10:15:07,237] [INFO] [timer.py:264:stop] epoch=0/micro_step=280/global_step=140, RunningAvgSamplesPerSec=6.253829602651008, CurrSamplesPerSec=6.320296322752446, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
32
+ [2024-11-11 10:16:59,755] [INFO] [logging.py:96:log_dist] [Rank 0] step=150, skipped=0, lr=[8.905742612854626e-07, 8.905742612854626e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
33
+ [2024-11-11 10:16:59,756] [INFO] [timer.py:264:stop] epoch=0/micro_step=300/global_step=150, RunningAvgSamplesPerSec=6.220856964798348, CurrSamplesPerSec=6.140029977114886, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
34
+ [2024-11-11 10:18:48,289] [INFO] [logging.py:96:log_dist] [Rank 0] step=160, skipped=0, lr=[8.741250599375982e-07, 8.741250599375982e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
35
+ [2024-11-11 10:18:48,289] [INFO] [timer.py:264:stop] epoch=0/micro_step=320/global_step=160, RunningAvgSamplesPerSec=6.20887491385521, CurrSamplesPerSec=6.3240117862507015, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
36
+ [2024-11-11 10:20:33,500] [INFO] [logging.py:96:log_dist] [Rank 0] step=170, skipped=0, lr=[8.566998090253799e-07, 8.566998090253799e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
37
+ [2024-11-11 10:20:33,500] [INFO] [timer.py:264:stop] epoch=0/micro_step=340/global_step=170, RunningAvgSamplesPerSec=6.211329294057541, CurrSamplesPerSec=6.262193329135546, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
38
+ [2024-11-11 10:22:18,917] [INFO] [logging.py:96:log_dist] [Rank 0] step=180, skipped=0, lr=[8.38343969038849e-07, 8.38343969038849e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
39
+ [2024-11-11 10:22:18,917] [INFO] [timer.py:264:stop] epoch=0/micro_step=360/global_step=180, RunningAvgSamplesPerSec=6.211341226400399, CurrSamplesPerSec=6.3439491768216785, MemAllocated=14.07GB, MaxMemAllocated=30.02GB
40
+ [2024-11-11 10:24:04,535] [INFO] [logging.py:96:log_dist] [Rank 0] step=190, skipped=0, lr=[8.191054282686891e-07, 8.191054282686891e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
41
+ [2024-11-11 10:24:04,536] [INFO] [timer.py:264:stop] epoch=0/micro_step=380/global_step=190, RunningAvgSamplesPerSec=6.210682927569051, CurrSamplesPerSec=6.298895458953062, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
42
+ [2024-11-11 10:25:42,250] [INFO] [logging.py:96:log_dist] [Rank 0] step=200, skipped=0, lr=[7.990343778711425e-07, 7.990343778711425e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
43
+ [2024-11-11 10:25:42,251] [INFO] [timer.py:264:stop] epoch=0/micro_step=400/global_step=200, RunningAvgSamplesPerSec=6.233484883515772, CurrSamplesPerSec=7.014839355184877, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
44
+ Saving checkpoint at step 400 ...
45
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
46
+ Saving 16-bit model...
47
+ [2024-11-11 10:25:51,218] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
48
+ [2024-11-11 10:25:51,219] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin, tag: global_step200
49
+ [2024-11-11 10:25:51,219] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin...
50
+ [2024-11-11 10:26:14,887] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin.
51
+ [2024-11-11 10:26:14,889] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
52
+ Model saved!
53
+ Saving 16-bit model...
54
+ [2024-11-11 10:26:23,191] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!
55
+ [2024-11-11 10:26:23,192] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin, tag: global_step200
56
+ [2024-11-11 10:26:23,193] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin...
57
+ [2024-11-11 10:26:45,365] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_400.bin.
58
+ [2024-11-11 10:26:45,366] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!
59
+ Model saved!
60
+ Checkpoint saved.
61
+ [2024-11-11 10:28:30,912] [INFO] [logging.py:96:log_dist] [Rank 0] step=210, skipped=0, lr=[7.78183180925015e-07, 7.78183180925015e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
62
+ [2024-11-11 10:28:30,912] [INFO] [timer.py:264:stop] epoch=0/micro_step=420/global_step=210, RunningAvgSamplesPerSec=6.231559886416094, CurrSamplesPerSec=7.658031726766873, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
63
+ [2024-11-11 10:30:17,140] [INFO] [logging.py:96:log_dist] [Rank 0] step=220, skipped=0, lr=[7.566062358223813e-07, 7.566062358223813e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
64
+ [2024-11-11 10:30:17,141] [INFO] [timer.py:264:stop] epoch=1/micro_step=18/global_step=220, RunningAvgSamplesPerSec=6.227392476536316, CurrSamplesPerSec=6.2512248951017435, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
65
+ [2024-11-11 10:31:58,864] [INFO] [logging.py:96:log_dist] [Rank 0] step=230, skipped=0, lr=[7.343598343493916e-07, 7.343598343493916e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
66
+ [2024-11-11 10:31:58,865] [INFO] [timer.py:264:stop] epoch=1/micro_step=38/global_step=230, RunningAvgSamplesPerSec=6.236228746087023, CurrSamplesPerSec=6.314374734308299, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
67
+ [2024-11-11 10:33:47,346] [INFO] [logging.py:96:log_dist] [Rank 0] step=240, skipped=0, lr=[7.115020148274293e-07, 7.115020148274293e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
68
+ [2024-11-11 10:33:47,347] [INFO] [timer.py:264:stop] epoch=1/micro_step=58/global_step=240, RunningAvgSamplesPerSec=6.227453579084282, CurrSamplesPerSec=4.989636440920279, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
69
+ [2024-11-11 10:35:34,343] [INFO] [logging.py:96:log_dist] [Rank 0] step=250, skipped=0, lr=[6.880924106977566e-07, 6.880924106977566e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
70
+ [2024-11-11 10:35:34,343] [INFO] [timer.py:264:stop] epoch=1/micro_step=78/global_step=250, RunningAvgSamplesPerSec=6.22403220340251, CurrSamplesPerSec=6.249119262604737, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
71
+ [2024-11-11 10:37:16,755] [INFO] [logging.py:96:log_dist] [Rank 0] step=260, skipped=0, lr=[6.641920949446755e-07, 6.641920949446755e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
72
+ [2024-11-11 10:37:16,755] [INFO] [timer.py:264:stop] epoch=1/micro_step=98/global_step=260, RunningAvgSamplesPerSec=6.229929565927251, CurrSamplesPerSec=6.314551641200963, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
73
+ [2024-11-11 10:39:00,686] [INFO] [logging.py:96:log_dist] [Rank 0] step=270, skipped=0, lr=[6.39863420763084e-07, 6.39863420763084e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
74
+ [2024-11-11 10:39:00,687] [INFO] [timer.py:264:stop] epoch=1/micro_step=118/global_step=270, RunningAvgSamplesPerSec=6.234223067872221, CurrSamplesPerSec=5.347220757882646, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
75
+ [2024-11-11 10:40:52,162] [INFO] [logging.py:96:log_dist] [Rank 0] step=280, skipped=0, lr=[6.151698588861116e-07, 6.151698588861116e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
76
+ [2024-11-11 10:40:52,163] [INFO] [timer.py:264:stop] epoch=1/micro_step=138/global_step=280, RunningAvgSamplesPerSec=6.22032747774778, CurrSamplesPerSec=5.386764780463646, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
77
+ [2024-11-11 10:42:36,292] [INFO] [logging.py:96:log_dist] [Rank 0] step=290, skipped=0, lr=[5.901758319972226e-07, 5.901758319972226e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
78
+ [2024-11-11 10:42:36,293] [INFO] [timer.py:264:stop] epoch=1/micro_step=158/global_step=290, RunningAvgSamplesPerSec=6.223707028785581, CurrSamplesPerSec=6.004583836826392, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
79
+ [2024-11-11 10:44:21,117] [INFO] [logging.py:96:log_dist] [Rank 0] step=300, skipped=0, lr=[5.649465466587902e-07, 5.649465466587902e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
80
+ [2024-11-11 10:44:21,117] [INFO] [timer.py:264:stop] epoch=1/micro_step=178/global_step=300, RunningAvgSamplesPerSec=6.22408930425845, CurrSamplesPerSec=6.223819151883097, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
81
+ [2024-11-11 10:46:05,125] [INFO] [logging.py:96:log_dist] [Rank 0] step=310, skipped=0, lr=[5.395478231956219e-07, 5.395478231956219e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
82
+ [2024-11-11 10:46:05,126] [INFO] [timer.py:264:stop] epoch=1/micro_step=198/global_step=310, RunningAvgSamplesPerSec=6.226128614306769, CurrSamplesPerSec=6.262398734901036, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
83
+ [2024-11-11 10:47:53,756] [INFO] [logging.py:96:log_dist] [Rank 0] step=320, skipped=0, lr=[5.140459239772456e-07, 5.140459239772456e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
84
+ [2024-11-11 10:47:53,757] [INFO] [timer.py:264:stop] epoch=1/micro_step=218/global_step=320, RunningAvgSamplesPerSec=6.219037182578649, CurrSamplesPerSec=5.474378062298524, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
85
+ [2024-11-11 10:49:39,475] [INFO] [logging.py:96:log_dist] [Rank 0] step=330, skipped=0, lr=[4.885073805469548e-07, 4.885073805469548e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
86
+ [2024-11-11 10:49:39,476] [INFO] [timer.py:264:stop] epoch=1/micro_step=238/global_step=330, RunningAvgSamplesPerSec=6.218320029955548, CurrSamplesPerSec=6.184670912366926, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
87
+ [2024-11-11 10:51:15,751] [INFO] [logging.py:96:log_dist] [Rank 0] step=340, skipped=0, lr=[4.629988200486073e-07, 4.629988200486073e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
88
+ [2024-11-11 10:51:15,751] [INFO] [timer.py:264:stop] epoch=1/micro_step=258/global_step=340, RunningAvgSamplesPerSec=6.234659527486788, CurrSamplesPerSec=6.166076513611049, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
89
+ [2024-11-11 10:52:57,448] [INFO] [logging.py:96:log_dist] [Rank 0] step=350, skipped=0, lr=[4.3758679140401534e-07, 4.3758679140401534e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
90
+ [2024-11-11 10:52:57,448] [INFO] [timer.py:264:stop] epoch=1/micro_step=278/global_step=350, RunningAvgSamplesPerSec=6.2408272898718575, CurrSamplesPerSec=5.479743497267358, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
91
+ [2024-11-11 10:54:49,320] [INFO] [logging.py:96:log_dist] [Rank 0] step=360, skipped=0, lr=[4.123375916944061e-07, 4.123375916944061e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
92
+ [2024-11-11 10:54:49,321] [INFO] [timer.py:264:stop] epoch=1/micro_step=298/global_step=360, RunningAvgSamplesPerSec=6.228467713430319, CurrSamplesPerSec=5.247047078296396, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
93
+ [2024-11-11 10:56:37,584] [INFO] [logging.py:96:log_dist] [Rank 0] step=370, skipped=0, lr=[3.8731709319890747e-07, 3.8731709319890747e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
94
+ [2024-11-11 10:56:37,585] [INFO] [timer.py:264:stop] epoch=1/micro_step=318/global_step=370, RunningAvgSamplesPerSec=6.223037095561094, CurrSamplesPerSec=6.317240925196704, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
95
+ [2024-11-11 10:58:22,183] [INFO] [logging.py:96:log_dist] [Rank 0] step=380, skipped=0, lr=[3.6259057154129225e-07, 3.6259057154129225e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
96
+ [2024-11-11 10:58:22,183] [INFO] [timer.py:264:stop] epoch=1/micro_step=338/global_step=380, RunningAvgSamplesPerSec=6.224228924255615, CurrSamplesPerSec=5.1994408858020025, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
97
+ [2024-11-11 11:00:07,320] [INFO] [logging.py:96:log_dist] [Rank 0] step=390, skipped=0, lr=[3.382225353933288e-07, 3.382225353933288e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
98
+ [2024-11-11 11:00:07,321] [INFO] [timer.py:264:stop] epoch=1/micro_step=358/global_step=390, RunningAvgSamplesPerSec=6.224158604390081, CurrSamplesPerSec=6.24098657587071, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
99
+ [2024-11-11 11:01:52,825] [INFO] [logging.py:96:log_dist] [Rank 0] step=400, skipped=0, lr=[3.142765581790224e-07, 3.142765581790224e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
100
+ [2024-11-11 11:01:52,826] [INFO] [timer.py:264:stop] epoch=1/micro_step=378/global_step=400, RunningAvgSamplesPerSec=6.224258305845644, CurrSamplesPerSec=5.508992960001429, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
101
+ Saving checkpoint at step 800 ...
102
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
103
+ Saving 16-bit model...
104
+ [2024-11-11 11:02:01,814] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
105
+ [2024-11-11 11:02:01,815] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin, tag: global_step400
106
+ [2024-11-11 11:02:01,815] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin...
107
+ [2024-11-11 11:02:24,168] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin.
108
+ [2024-11-11 11:02:24,170] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
109
+ Model saved!
110
+ Saving 16-bit model...
111
+ [2024-11-11 11:02:32,111] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step400 is about to be saved!
112
+ [2024-11-11 11:02:32,112] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin, tag: global_step400
113
+ [2024-11-11 11:02:32,112] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin...
114
+ [2024-11-11 11:02:53,588] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_800.bin.
115
+ [2024-11-11 11:02:53,590] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step400 is ready now!
116
+ Model saved!
117
+ Checkpoint saved.
118
+ [2024-11-11 11:04:32,610] [INFO] [logging.py:96:log_dist] [Rank 0] step=410, skipped=0, lr=[2.9081511221881027e-07, 2.9081511221881027e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
119
+ [2024-11-11 11:04:32,611] [INFO] [timer.py:264:stop] epoch=1/micro_step=398/global_step=410, RunningAvgSamplesPerSec=6.233501350397177, CurrSamplesPerSec=7.624818018964297, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
120
+ [2024-11-11 11:06:18,893] [INFO] [logging.py:96:log_dist] [Rank 0] step=420, skipped=0, lr=[2.67899405746411e-07, 2.67899405746411e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
121
+ [2024-11-11 11:06:18,894] [INFO] [timer.py:264:stop] epoch=1/micro_step=418/global_step=420, RunningAvgSamplesPerSec=6.2314715200711, CurrSamplesPerSec=5.346454162982587, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
122
+ [2024-11-11 11:08:03,330] [INFO] [logging.py:96:log_dist] [Rank 0] step=430, skipped=0, lr=[2.4558922322353115e-07, 2.4558922322353115e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
123
+ [2024-11-11 11:08:03,330] [INFO] [timer.py:264:stop] epoch=2/micro_step=16/global_step=430, RunningAvgSamplesPerSec=6.2324828606092995, CurrSamplesPerSec=5.855626964991123, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
124
+ [2024-11-11 11:09:45,387] [INFO] [logging.py:96:log_dist] [Rank 0] step=440, skipped=0, lr=[2.2394276936903022e-07, 2.2394276936903022e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
125
+ [2024-11-11 11:09:45,387] [INFO] [timer.py:264:stop] epoch=2/micro_step=36/global_step=440, RunningAvgSamplesPerSec=6.2365997777478235, CurrSamplesPerSec=7.074766361276321, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
126
+ [2024-11-11 11:11:31,116] [INFO] [logging.py:96:log_dist] [Rank 0] step=450, skipped=0, lr=[2.0301651730945623e-07, 2.0301651730945623e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
127
+ [2024-11-11 11:11:31,117] [INFO] [timer.py:264:stop] epoch=2/micro_step=56/global_step=450, RunningAvgSamplesPerSec=6.235535040398496, CurrSamplesPerSec=6.263325127446985, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
128
+ [2024-11-11 11:13:19,722] [INFO] [logging.py:96:log_dist] [Rank 0] step=460, skipped=0, lr=[1.8286506124710487e-07, 1.8286506124710487e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
129
+ [2024-11-11 11:13:19,723] [INFO] [timer.py:264:stop] epoch=2/micro_step=76/global_step=460, RunningAvgSamplesPerSec=6.230500323816332, CurrSamplesPerSec=6.036847858086253, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
130
+ [2024-11-11 11:15:02,353] [INFO] [logging.py:96:log_dist] [Rank 0] step=470, skipped=0, lr=[1.6354097402998124e-07, 1.6354097402998124e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
131
+ [2024-11-11 11:15:02,354] [INFO] [timer.py:264:stop] epoch=2/micro_step=96/global_step=470, RunningAvgSamplesPerSec=6.233932103977919, CurrSamplesPerSec=6.2977051152249395, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
132
+ [2024-11-11 11:16:42,997] [INFO] [logging.py:96:log_dist] [Rank 0] step=480, skipped=0, lr=[1.4509466999523984e-07, 1.4509466999523984e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
133
+ [2024-11-11 11:16:42,998] [INFO] [timer.py:264:stop] epoch=2/micro_step=116/global_step=480, RunningAvgSamplesPerSec=6.239483803482549, CurrSamplesPerSec=6.033187064976004, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
134
+ [2024-11-11 11:18:34,535] [INFO] [logging.py:96:log_dist] [Rank 0] step=490, skipped=0, lr=[1.2757427344393702e-07, 1.2757427344393702e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
135
+ [2024-11-11 11:18:34,535] [INFO] [timer.py:264:stop] epoch=2/micro_step=136/global_step=490, RunningAvgSamplesPerSec=6.231258151619256, CurrSamplesPerSec=5.3911203096440845, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
136
+ [2024-11-11 11:20:18,891] [INFO] [logging.py:96:log_dist] [Rank 0] step=500, skipped=0, lr=[1.1102549309022291e-07, 1.1102549309022291e-07], mom=[[0.9, 0.95], [0.9, 0.95]]
137
+ [2024-11-11 11:20:18,892] [INFO] [timer.py:264:stop] epoch=2/micro_step=156/global_step=500, RunningAvgSamplesPerSec=6.231896275894786, CurrSamplesPerSec=5.431348295652274, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
138
+ [2024-11-11 11:22:03,897] [INFO] [logging.py:96:log_dist] [Rank 0] step=510, skipped=0, lr=[9.549150281252632e-08, 9.549150281252632e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
139
+ [2024-11-11 11:22:03,898] [INFO] [timer.py:264:stop] epoch=2/micro_step=176/global_step=510, RunningAvgSamplesPerSec=6.231784979733288, CurrSamplesPerSec=6.118001686313543, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
140
+ [2024-11-11 11:23:47,595] [INFO] [logging.py:96:log_dist] [Rank 0] step=520, skipped=0, lr=[8.101282901783457e-08, 8.101282901783457e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
141
+ [2024-11-11 11:23:47,596] [INFO] [timer.py:264:stop] epoch=2/micro_step=196/global_step=520, RunningAvgSamplesPerSec=6.233175716519829, CurrSamplesPerSec=6.4252008944871175, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
142
+ [2024-11-11 11:25:34,694] [INFO] [logging.py:96:log_dist] [Rank 0] step=530, skipped=0, lr=[6.762724491292555e-08, 6.762724491292555e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
143
+ [2024-11-11 11:25:34,694] [INFO] [timer.py:264:stop] epoch=2/micro_step=216/global_step=530, RunningAvgSamplesPerSec=6.230586016932243, CurrSamplesPerSec=6.31995020734794, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
144
+ [2024-11-11 11:27:21,825] [INFO] [logging.py:96:log_dist] [Rank 0] step=540, skipped=0, lr=[5.5369671958383326e-08, 5.5369671958383326e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
145
+ [2024-11-11 11:27:21,826] [INFO] [timer.py:264:stop] epoch=2/micro_step=236/global_step=540, RunningAvgSamplesPerSec=6.228137586678738, CurrSamplesPerSec=6.294527897652772, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
146
+ [2024-11-11 11:28:57,980] [INFO] [logging.py:96:log_dist] [Rank 0] step=550, skipped=0, lr=[4.4272088762495554e-08, 4.4272088762495554e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
147
+ [2024-11-11 11:28:57,981] [INFO] [timer.py:264:stop] epoch=2/micro_step=256/global_step=550, RunningAvgSamplesPerSec=6.237979411534363, CurrSamplesPerSec=7.440681418583277, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
148
+ [2024-11-11 11:30:37,963] [INFO] [logging.py:96:log_dist] [Rank 0] step=560, skipped=0, lr=[3.4363447652715425e-08, 3.4363447652715425e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
149
+ [2024-11-11 11:30:37,963] [INFO] [timer.py:264:stop] epoch=2/micro_step=276/global_step=560, RunningAvgSamplesPerSec=6.2432383070279265, CurrSamplesPerSec=6.325613791478562, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
150
+ [2024-11-11 11:32:29,640] [INFO] [logging.py:96:log_dist] [Rank 0] step=570, skipped=0, lr=[2.5669599142344954e-08, 2.5669599142344954e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
151
+ [2024-11-11 11:32:29,641] [INFO] [timer.py:264:stop] epoch=2/micro_step=296/global_step=570, RunningAvgSamplesPerSec=6.235729072836607, CurrSamplesPerSec=6.363454933868642, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
152
+ [2024-11-11 11:34:19,919] [INFO] [logging.py:96:log_dist] [Rank 0] step=580, skipped=0, lr=[1.8213224489497525e-08, 1.8213224489497525e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
153
+ [2024-11-11 11:34:19,920] [INFO] [timer.py:264:stop] epoch=2/micro_step=316/global_step=580, RunningAvgSamplesPerSec=6.229848630704788, CurrSamplesPerSec=5.9735175986720614, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
154
+ [2024-11-11 11:36:02,300] [INFO] [logging.py:96:log_dist] [Rank 0] step=590, skipped=0, lr=[1.2013776524284869e-08, 1.2013776524284869e-08], mom=[[0.9, 0.95], [0.9, 0.95]]
155
+ [2024-11-11 11:36:02,300] [INFO] [timer.py:264:stop] epoch=2/micro_step=336/global_step=590, RunningAvgSamplesPerSec=6.232535497944164, CurrSamplesPerSec=7.393825768980957, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
156
+ [2024-11-11 11:37:49,415] [INFO] [logging.py:96:log_dist] [Rank 0] step=600, skipped=0, lr=[7.087428898604974e-09, 7.087428898604974e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
157
+ [2024-11-11 11:37:49,415] [INFO] [timer.py:264:stop] epoch=2/micro_step=356/global_step=600, RunningAvgSamplesPerSec=6.230310889018549, CurrSamplesPerSec=7.70988597085285, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
158
+ Saving checkpoint at step 1200 ...
159
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
160
+ Saving 16-bit model...
161
+ [2024-11-11 11:37:58,346] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
162
+ [2024-11-11 11:37:58,346] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin, tag: global_step600
163
+ [2024-11-11 11:37:58,346] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin...
164
+ [2024-11-11 11:38:17,278] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin.
165
+ [2024-11-11 11:38:17,279] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
166
+ Model saved!
167
+ Saving 16-bit model...
168
+ [2024-11-11 11:38:24,358] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step600 is about to be saved!
169
+ [2024-11-11 11:38:24,359] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin, tag: global_step600
170
+ [2024-11-11 11:38:24,359] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin...
171
+ [2024-11-11 11:38:45,808] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model_1200.bin.
172
+ [2024-11-11 11:38:45,809] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step600 is ready now!
173
+ Model saved!
174
+ Checkpoint saved.
175
+ [2024-11-11 11:40:29,433] [INFO] [logging.py:96:log_dist] [Rank 0] step=610, skipped=0, lr=[3.4470338909303176e-09, 3.4470338909303176e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
176
+ [2024-11-11 11:40:29,434] [INFO] [timer.py:264:stop] epoch=2/micro_step=376/global_step=610, RunningAvgSamplesPerSec=6.2316845073786, CurrSamplesPerSec=6.228612676006549, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
177
+ [2024-11-11 11:42:11,651] [INFO] [logging.py:96:log_dist] [Rank 0] step=620, skipped=0, lr=[1.1020888761813507e-09, 1.1020888761813507e-09], mom=[[0.9, 0.95], [0.9, 0.95]]
178
+ [2024-11-11 11:42:11,652] [INFO] [timer.py:264:stop] epoch=2/micro_step=396/global_step=620, RunningAvgSamplesPerSec=6.234524546765965, CurrSamplesPerSec=7.421455063273693, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
179
+ [2024-11-11 11:43:54,536] [INFO] [logging.py:96:log_dist] [Rank 0] step=630, skipped=0, lr=[5.87115481592293e-11, 5.87115481592293e-11], mom=[[0.9, 0.95], [0.9, 0.95]]
180
+ [2024-11-11 11:43:54,537] [INFO] [timer.py:264:stop] epoch=2/micro_step=416/global_step=630, RunningAvgSamplesPerSec=6.2363677298557585, CurrSamplesPerSec=5.864153736679168, MemAllocated=14.07GB, MaxMemAllocated=30.44GB
181
+ Saving model to "../outputs/sft_tf_cham_1111_13.5k_ti2ti" ...
182
+ Saving 16-bit model...
183
+ [2024-11-11 11:44:37,766] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step633 is about to be saved!
184
+ [2024-11-11 11:44:37,767] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin, tag: global_step633
185
+ [2024-11-11 11:44:37,767] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin...
186
+ [2024-11-11 11:44:57,052] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin.
187
+ [2024-11-11 11:44:57,053] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step633 is ready now!
188
+ Model saved!
189
+ Saving 16-bit model...
190
+ [2024-11-11 11:45:06,835] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step633 is about to be saved!
191
+ [2024-11-11 11:45:06,836] [INFO] [engine.py:3649:save_16bit_model] Saving model weights to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin, tag: global_step633
192
+ [2024-11-11 11:45:06,836] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin...
193
+ [2024-11-11 11:45:29,691] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/sft_tf_cham_1111_13.5k_ti2ti/pytorch_model.bin.
194
+ [2024-11-11 11:45:29,692] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step633 is ready now!
195
+ Model saved!
slice_400/wandb/run-20241111_095028-sq0dhwvm/files/requirements.txt ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ align-anything==0.0.1.dev0
2
+ uvloop==0.20.0
3
+ absl-py==2.1.0
4
+ distro==1.9.0
5
+ bitsandbytes==0.44.1
6
+ uvicorn==0.31.1
7
+ safetensors==0.4.5
8
+ gradio_client==1.4.0
9
+ propcache==0.2.0
10
+ GitPython==3.1.43
11
+ pyzmq==26.2.0
12
+ nvidia-nvtx-cu12==12.1.105
13
+ einops==0.8.0
14
+ nvidia-cusolver-cu12==11.4.5.107
15
+ six==1.16.0
16
+ smmap==5.0.1
17
+ python-dotenv==1.0.1
18
+ multiprocess==0.70.16
19
+ pyarrow==17.0.0
20
+ pytest==7.2.0
21
+ lazy_loader==0.4
22
+ contourpy==1.3.0
23
+ aiofiles==23.2.1
24
+ torchlibrosa==0.1.0
25
+ protobuf==3.20.3
26
+ nvidia-cuda-runtime-cu12==12.1.105
27
+ pycparser==2.22
28
+ PyYAML==6.0.2
29
+ hjson==3.1.0
30
+ xxhash==3.5.0
31
+ platformdirs==4.3.6
32
+ diskcache==5.6.3
33
+ fairscale==0.4.13
34
+ certifi==2024.8.30
35
+ docker-pycreds==0.4.0
36
+ braceexpand==0.1.7
37
+ virtualenv==20.26.6
38
+ tokenizers==0.20.1
39
+ nvidia-cuda-nvrtc-cu12==12.1.105
40
+ grpcio==1.66.2
41
+ outlines==0.0.46
42
+ yarl==1.15.0
43
+ aiohttp==3.10.10
44
+ referencing==0.35.1
45
+ fsspec==2024.6.1
46
+ nvidia-nccl-cu12==2.20.5
47
+ Jinja2==3.1.4
48
+ timm==0.6.13
49
+ opencv-python==4.6.0.66
50
+ tomlkit==0.12.0
51
+ pandas==2.2.3
52
+ wcwidth==0.2.13
53
+ identify==2.6.1
54
+ deepspeed==0.15.2
55
+ datasets==3.0.1
56
+ ray==2.37.0
57
+ align-anything==0.0.1.dev0
58
+ nvidia-cufft-cu12==11.0.2.54
59
+ mistral_common==1.4.4
60
+ httptools==0.6.1
61
+ scipy==1.14.1
62
+ mdurl==0.1.2
63
+ clip==0.2.0
64
+ cycler==0.12.1
65
+ pyairports==2.1.1
66
+ charset-normalizer==3.4.0
67
+ torch==2.4.0
68
+ mpmath==1.3.0
69
+ tzdata==2024.2
70
+ nest-asyncio==1.6.0
71
+ sentencepiece==0.2.0
72
+ aiohappyeyeballs==2.4.3
73
+ ffmpy==0.4.0
74
+ tiktoken==0.7.0
75
+ nvidia-curand-cu12==10.3.2.106
76
+ kiwisolver==1.4.7
77
+ audioread==3.0.1
78
+ cffi==1.17.1
79
+ clint==0.5.1
80
+ partial-json-parser==0.2.1.1.post4
81
+ dill==0.3.8
82
+ ninja==1.11.1.1
83
+ tqdm==4.66.5
84
+ gitdb==4.0.11
85
+ regex==2024.9.11
86
+ nvidia-cusparse-cu12==12.1.0.106
87
+ aiosignal==1.3.1
88
+ jsonschema-specifications==2024.10.1
89
+ yt-dlp==2024.8.6
90
+ triton==3.0.0
91
+ pydub==0.25.1
92
+ nodeenv==1.9.1
93
+ pooch==1.8.2
94
+ MarkupSafe==2.1.5
95
+ fastapi==0.115.0
96
+ setproctitle==1.3.3
97
+ pycountry==24.6.1
98
+ anyio==4.6.0
99
+ matplotlib==3.9.2
100
+ diffusers==0.30.3
101
+ librosa==0.10.2.post1
102
+ filelock==3.16.1
103
+ jiter==0.6.1
104
+ sentry-sdk==2.16.0
105
+ starlette==0.38.6
106
+ py-cpuinfo==9.0.0
107
+ typer==0.12.5
108
+ zipp==3.20.2
109
+ args==0.1.0
110
+ jsonschema==4.23.0
111
+ llvmlite==0.43.0
112
+ interegular==0.3.3
113
+ wheel==0.44.0
114
+ frechet-audio-distance==0.1.2
115
+ pytz==2024.2
116
+ pytorch-fid==0.3.0
117
+ optree==0.13.0
118
+ lark==1.2.2
119
+ msgpack==1.1.0
120
+ prometheus_client==0.21.0
121
+ typing_extensions==4.12.2
122
+ mutagen==1.47.0
123
+ pycryptodomex==3.21.0
124
+ fonttools==4.54.1
125
+ ftfy==6.3.0
126
+ orjson==3.10.7
127
+ vllm==0.6.2
128
+ watchfiles==0.24.0
129
+ iniconfig==2.0.0
130
+ idna==3.10
131
+ proglog==0.1.10
132
+ sniffio==1.3.1
133
+ pyparsing==3.1.4
134
+ h11==0.14.0
135
+ networkx==3.4.1
136
+ xformers==0.0.27.post2
137
+ hpsv2==1.2.0
138
+ pluggy==1.5.0
139
+ gguf==0.10.0
140
+ imageio==2.35.1
141
+ importlib_metadata==8.5.0
142
+ urllib3==2.2.3
143
+ nvidia-nvjitlink-cu12==12.6.77
144
+ semantic-version==2.10.0
145
+ decorator==4.4.2
146
+ attrs==24.2.0
147
+ Brotli==1.1.0
148
+ numpy==1.26.4
149
+ soxr==0.5.0.post1
150
+ requests==2.32.3
151
+ pytest-split==0.8.0
152
+ httpcore==1.0.6
153
+ webdataset==0.2.100
154
+ rpds-py==0.20.0
155
+ shellingham==1.5.4
156
+ annotated-types==0.7.0
157
+ pillow==10.4.0
158
+ nvidia-ml-py==12.560.30
159
+ packaging==24.1
160
+ peft==0.13.2
161
+ imageio-ffmpeg==0.5.1
162
+ setuptools==75.1.0
163
+ pydantic==2.9.2
164
+ Pygments==2.18.0
165
+ cloudpickle==3.1.0
166
+ torchvision==0.19.0
167
+ threadpoolctl==3.5.0
168
+ cfgv==3.4.0
169
+ tensorboard==2.18.0
170
+ moviepy==1.0.3
171
+ nvidia-cuda-cupti-cu12==12.1.105
172
+ msgspec==0.18.6
173
+ Markdown==3.7
174
+ huggingface-hub==0.25.2
175
+ scikit-learn==1.5.2
176
+ distlib==0.3.9
177
+ shortuuid==1.0.13
178
+ nvidia-cublas-cu12==12.1.3.1
179
+ pip==24.2
180
+ image-reward==1.5
181
+ click==8.1.7
182
+ lm-format-enforcer==0.10.6
183
+ joblib==1.4.2
184
+ torchaudio==2.4.0
185
+ rich==13.9.2
186
+ resampy==0.4.3
187
+ numba==0.60.0
188
+ gradio==5.0.2
189
+ transformers==4.46.0.dev0
190
+ tensorboard-data-server==0.7.2
191
+ soundfile==0.12.1
192
+ multidict==6.1.0
193
+ wandb==0.18.3
194
+ openai==1.51.2
195
+ nvidia-cudnn-cu12==9.1.0.70
196
+ python-dateutil==2.9.0.post0
197
+ psutil==6.0.0
198
+ frozenlist==1.4.1
199
+ python-multipart==0.0.12
200
+ Werkzeug==3.0.4
201
+ markdown-it-py==3.0.0
202
+ pydantic_core==2.23.4
203
+ prometheus-fastapi-instrumentator==7.0.0
204
+ sympy==1.13.3
205
+ accelerate==1.0.1
206
+ httpx==0.27.2
207
+ ruff==0.6.9
208
+ pre_commit==4.0.1
209
+ websockets==12.0
210
+ importlib_resources==6.4.0
211
+ packaging==24.1
212
+ typing_extensions==4.12.2
213
+ wheel==0.43.0
214
+ zipp==3.19.2
215
+ jaraco.text==3.12.1
216
+ inflect==7.3.1
217
+ more-itertools==10.3.0
218
+ autocommand==2.2.2
219
+ platformdirs==4.2.2
220
+ typeguard==4.3.0
221
+ jaraco.functools==4.0.1
222
+ jaraco.context==5.3.0
223
+ tomli==2.0.1
224
+ jaraco.collections==5.1.0
225
+ importlib_metadata==8.0.0
226
+ backports.tarfile==1.2.0
slice_400/wandb/run-20241111_095028-sq0dhwvm/files/wandb-metadata.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "3.11.10",
4
+ "startedAt": "2024-11-11T09:50:28.925585Z",
5
+ "args": [
6
+ "--local_rank=0",
7
+ "--model_name_or_path",
8
+ "/data/align-anything/hantao/models/0916_ti_to_ti_sft/",
9
+ "--train_datasets",
10
+ "/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs",
11
+ "--train_data_files",
12
+ "llf_ti2ti_13.5k_tokenized.pt",
13
+ "--output_dir",
14
+ "../outputs/sft_tf_cham_1111_13.5k_ti2ti",
15
+ "--train_template",
16
+ "AA_textfeedback",
17
+ "--train_split",
18
+ "train",
19
+ "--per_device_train_batch_size",
20
+ "4",
21
+ "--per_device_eval_batch_size",
22
+ "4",
23
+ "--gradient_accumulation_steps",
24
+ "2",
25
+ "--save_interval",
26
+ "400",
27
+ "--learning_rate",
28
+ "1e-6",
29
+ "--epochs",
30
+ "3",
31
+ "--lr_scheduler_type",
32
+ "cosine"
33
+ ],
34
+ "program": "-m align_anything.trainers.text_image_to_text_image.sft",
35
+ "git": {
36
+ "remote": "https://github.com/PKU-Alignment/align-anything.git",
37
+ "commit": "6fde660afc9985323f147930eedf188a5699adc7"
38
+ },
39
+ "email": "[email protected]",
40
+ "root": "../outputs/sft_tf_cham_1111_13.5k_ti2ti",
41
+ "host": "lyg0198",
42
+ "username": "align-anything",
43
+ "executable": "/data/align-anything/miniconda3/envs/hantao_cham/bin/python",
44
+ "cpu_count": 64,
45
+ "cpu_count_logical": 128,
46
+ "gpu": "[NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB]",
47
+ "gpu_count": 8,
48
+ "disk": {
49
+ "/": {
50
+ "total": "938421047296",
51
+ "used": "460055379968"
52
+ }
53
+ },
54
+ "memory": {
55
+ "total": "540633423872"
56
+ },
57
+ "cpu": {
58
+ "count": 64,
59
+ "countLogical": 128
60
+ },
61
+ "gpu_nvidia": [
62
+ {
63
+ "name": "NVIDIA A100-SXM4-80GB",
64
+ "memoryTotal": "85899345920",
65
+ "cudaCores": 6912,
66
+ "architecture": "Ampere"
67
+ },
68
+ {
69
+ "name": "NVIDIA A100-SXM4-80GB",
70
+ "memoryTotal": "85899345920",
71
+ "cudaCores": 6912,
72
+ "architecture": "Ampere"
73
+ },
74
+ {
75
+ "name": "NVIDIA A100-SXM4-80GB",
76
+ "memoryTotal": "85899345920",
77
+ "cudaCores": 6912,
78
+ "architecture": "Ampere"
79
+ },
80
+ {
81
+ "name": "NVIDIA A100-SXM4-80GB",
82
+ "memoryTotal": "85899345920",
83
+ "cudaCores": 6912,
84
+ "architecture": "Ampere"
85
+ },
86
+ {
87
+ "name": "NVIDIA A100-SXM4-80GB",
88
+ "memoryTotal": "85899345920",
89
+ "cudaCores": 6912,
90
+ "architecture": "Ampere"
91
+ },
92
+ {
93
+ "name": "NVIDIA A100-SXM4-80GB",
94
+ "memoryTotal": "85899345920",
95
+ "cudaCores": 6912,
96
+ "architecture": "Ampere"
97
+ },
98
+ {
99
+ "name": "NVIDIA A100-SXM4-80GB",
100
+ "memoryTotal": "85899345920",
101
+ "cudaCores": 6912,
102
+ "architecture": "Ampere"
103
+ },
104
+ {
105
+ "name": "NVIDIA A100-SXM4-80GB",
106
+ "memoryTotal": "85899345920",
107
+ "cudaCores": 6912,
108
+ "architecture": "Ampere"
109
+ }
110
+ ],
111
+ "cudaVersion": "12.4"
112
+ }
slice_400/wandb/run-20241111_095028-sq0dhwvm/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_timestamp":1.731325467962379e+09,"_runtime":6900.89633248,"_step":1266,"train/step":1266,"train/loss":0.6548570990562439,"train/lr":0,"train/epoch":3,"_wandb":{"runtime":6900}}
slice_400/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2024-11-11T09:50:28.93257347Z","level":"INFO","msg":"using version","core version":"0.18.3"}
2
+ {"time":"2024-11-11T09:50:28.932602745Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-core.log"}
3
+ {"time":"2024-11-11T09:50:28.93714017Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
4
+ {"time":"2024-11-11T09:50:28.968182627Z","level":"INFO","msg":"created new stream","id":"sq0dhwvm"}
5
+ {"time":"2024-11-11T09:50:28.968263994Z","level":"INFO","msg":"stream: started","id":"sq0dhwvm"}
6
+ {"time":"2024-11-11T09:50:28.968288809Z","level":"INFO","msg":"sender: started","stream_id":{"value":"sq0dhwvm"}}
7
+ {"time":"2024-11-11T09:50:28.968279236Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"sq0dhwvm"}}
8
+ {"time":"2024-11-11T09:50:28.968330962Z","level":"INFO","msg":"handler: started","stream_id":{"value":"sq0dhwvm"}}
9
+ {"time":"2024-11-11T09:50:34.632278843Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
10
+ {"time":"2024-11-11T09:50:34.636084019Z","level":"INFO","msg":"Starting system monitor"}
11
+ {"time":"2024-11-11T11:20:45.19395353Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/htlou/align-anything/sq0dhwvm/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
12
+ {"time":"2024-11-11T11:45:29.821937086Z","level":"INFO","msg":"Stopping system monitor"}
13
+ {"time":"2024-11-11T11:45:29.846984762Z","level":"INFO","msg":"Stopped system monitor"}
14
+ {"time":"2024-11-11T11:45:30.253659937Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
15
+ {"time":"2024-11-11T11:45:30.253692046Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
16
+ {"time":"2024-11-11T11:45:30.882510953Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
17
+ {"time":"2024-11-11T11:45:32.320976723Z","level":"INFO","msg":"stream: closing","id":"sq0dhwvm"}
18
+ {"time":"2024-11-11T11:45:32.321006242Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"sq0dhwvm"}}
19
+ {"time":"2024-11-11T11:45:32.321051996Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"sq0dhwvm"}}
20
+ {"time":"2024-11-11T11:45:32.321127635Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"sq0dhwvm"}}
21
+ {"time":"2024-11-11T11:45:32.323850284Z","level":"INFO","msg":"stream: closed","id":"sq0dhwvm"}
slice_400/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
2
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Configure stats pid to 3212652
3
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
4
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
5
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
6
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
7
+ 2024-11-11 09:50:28,918 WARNING MainThread:3212652 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
8
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
9
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_setup.py:_flush():79] Applying login settings: {}
10
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug.log
11
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1111_13.5k_ti2ti/wandb/run-20241111_095028-sq0dhwvm/logs/debug-internal.log
12
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():617] calling init triggers
13
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
14
+ config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_ti2ti_13.5k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1111_13.5k_ti2ti', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0916_ti_to_ti_sft/', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
15
+ 2024-11-11 09:50:28,918 INFO MainThread:3212652 [wandb_init.py:init():667] starting backend
16
+ 2024-11-11 09:50:28,919 INFO MainThread:3212652 [wandb_init.py:init():671] sending inform_init request
17
+ 2024-11-11 09:50:28,924 INFO MainThread:3212652 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-11-11 09:50:28,925 INFO MainThread:3212652 [wandb_init.py:init():684] backend started and connected
19
+ 2024-11-11 09:50:28,929 INFO MainThread:3212652 [wandb_init.py:init():779] updated telemetry
20
+ 2024-11-11 09:50:28,983 INFO MainThread:3212652 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
21
+ 2024-11-11 09:50:34,620 INFO MainThread:3212652 [wandb_init.py:init():863] starting run threads in backend
22
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_console_start():2465] atexit reg
23
+ 2024-11-11 09:50:35,177 INFO MainThread:3212652 [wandb_run.py:_redirect():2313] redirect: wrap_raw
24
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2378] Wrapping output streams.
25
+ 2024-11-11 09:50:35,178 INFO MainThread:3212652 [wandb_run.py:_redirect():2403] Redirects installed.
26
+ 2024-11-11 09:50:35,184 INFO MainThread:3212652 [wandb_init.py:init():907] run started, returning control to user process
27
+ 2024-11-11 11:45:29,778 INFO MainThread:3212652 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/sq0dhwvm
28
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
29
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2410] restore
30
+ 2024-11-11 11:45:29,779 INFO MainThread:3212652 [wandb_run.py:_restore():2416] restore done
31
+ 2024-11-11 11:45:32,305 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4049] rendering history
32
+ 2024-11-11 11:45:32,307 INFO MainThread:3212652 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
33
+ 2024-11-11 11:45:32,318 INFO MainThread:3212652 [wandb_run.py:_footer_sync_info():4008] logging synced files
slice_400/wandb/run-20241111_095028-sq0dhwvm/run-sq0dhwvm.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d94ac37632f72a3f8cd135073052c80433a6d2851df145837d3165fe8e647a8d
3
+ size 6923036
slice_800/arguments.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data_cfgs:
2
+ eval_data_files: null
3
+ eval_datasets: null
4
+ eval_optional_args: []
5
+ eval_size: null
6
+ eval_split: null
7
+ eval_subset: null
8
+ eval_template: null
9
+ train_data_files: llf_ti2ti_13.5k_tokenized.pt
10
+ train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
11
+ train_optional_args: []
12
+ train_size: null
13
+ train_split: train
14
+ train_subset: null
15
+ train_template: AA_textfeedback
16
+ logger_cfgs:
17
+ cache_dir: null
18
+ log_project: align-anything
19
+ log_run_name: sft
20
+ log_type: wandb
21
+ output_dir: ../outputs/sft_tf_cham_1111_13.5k_ti2ti
22
+ save_interval: 400.0
23
+ model_cfgs:
24
+ model_max_length: 4096
25
+ model_name_or_path: /data/align-anything/hantao/models/0916_ti_to_ti_sft/
26
+ trust_remote_code: true
27
+ special_tokens: null
28
+ train_cfgs:
29
+ adam_betas:
30
+ - 0.9
31
+ - 0.95
32
+ adam_epsilon: 1.0e-08
33
+ bf16: true
34
+ ds_cfgs: ds_z3_config.json
35
+ epochs: 3.0
36
+ eval_interval: 1000
37
+ eval_strategy: steps
38
+ fp16: false
39
+ freeze_language_model: false
40
+ gradient_accumulation_steps: 2.0
41
+ gradient_checkpointing: true
42
+ learning_rate: 1.0e-06
43
+ lr_scheduler_type: cosine
44
+ lr_warmup_ratio: 0.03
45
+ max_grad_norm: 1.0
46
+ per_device_eval_batch_size: 4.0
47
+ per_device_train_batch_size: 4.0
48
+ seed: 42
49
+ weight_decay: 0.0
slice_800/config.json ADDED
The diff for this file is too large to render. See raw diff