Commit
·
7f89522
1
Parent(s):
bd79118
reorganize
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- 24_10_19_19_15_22_epoch6.pth +0 -3
- 24_10_19_19_40_41_epoch7.pth +0 -3
- 24_10_19_20_04_15_epoch8.pth +0 -3
- 24_10_19_20_27_44_epoch9.pth +0 -3
- 24_10_19_21_14_44_epoch11.pth +0 -3
- 24_10_19_21_38_17_epoch12.pth +0 -3
- 24_10_19_22_01_49_epoch13.pth +0 -3
- 24_10_19_22_25_17_epoch14.pth +0 -3
- 24_10_19_23_12_20_epoch16.pth +0 -3
- 24_10_19_23_35_50_epoch17.pth +0 -3
- 24_10_20_18_41_30_epoch19.pth +0 -3
- 24_10_20_19_28_29_epoch21.pth +0 -3
- 24_10_20_19_51_56_epoch22.pth +0 -3
- 24_10_20_20_15_23_epoch23.pth +0 -3
- 24_10_20_20_38_52_epoch24.pth +0 -3
- 24_10_20_21_25_45_epoch26.pth +0 -3
- 24_10_20_21_49_12_epoch27.pth +0 -3
- 24_10_20_22_12_36_epoch28.pth +0 -3
- 24_10_20_22_35_58_epoch29.pth +0 -3
- 24_10_20_23_22_56_epoch31.pth +0 -3
- 24_10_20_23_46_24_epoch32.pth +0 -3
- 24_10_21_00_10_00_epoch33.pth +0 -3
- 24_10_21_00_33_29_epoch34.pth +0 -3
- 24_10_21_01_20_30_epoch36.pth +0 -3
- 24_10_21_01_43_58_epoch37.pth +0 -3
- 24_10_21_02_07_32_epoch38.pth +0 -3
- 24_10_21_02_31_04_epoch39.pth +0 -3
- 24_10_21_03_18_07_epoch41.pth +0 -3
- 24_10_21_03_41_36_epoch42.pth +0 -3
- 24_10_21_04_05_08_epoch43.pth +0 -3
- 24_10_21_04_28_33_epoch44.pth +0 -3
- 24_10_21_05_15_33_epoch46.pth +0 -3
- 24_10_21_05_39_01_epoch47.pth +0 -3
- 24_10_21_06_02_29_epoch48.pth +0 -3
- 24_10_21_06_51_48_epoch49.pth +0 -3
- 24_10_27_07_02_01_log.txt +0 -100
- 24_10_27_11_36_44_log.txt +0 -100
- 24_10_27_11_51_44_log.txt +0 -101
- 24_10_27_12_05_54_log.txt +0 -102
- 24_10_27_12_20_01_log.txt +0 -103
- 24_10_27_12_34_09_log.txt +0 -104
- 24_10_27_12_48_17_log.txt +0 -105
- 24_10_27_13_03_15_log.txt +0 -106
- 24_10_27_13_17_21_log.txt +0 -107
- 24_10_27_13_31_29_log.txt +0 -108
- 24_10_27_13_45_40_log.txt +0 -109
- 24_10_27_13_59_49_log.txt +0 -110
- 24_10_27_14_14_50_log.txt +0 -111
- 24_10_27_14_28_58_log.txt +0 -112
- 24_10_27_14_43_05_log.txt +0 -113
24_10_19_19_15_22_epoch6.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:0a6cd17775d7b12eabbe7da232fbac56a3c683e9f779f79cabb72e105ff8ea17
|
| 3 |
-
size 816441854
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_19_40_41_epoch7.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6d4701b8db89bdf82e3d15eacba1e8cbe2cd815e648e03bbfe0ca9eef17081b3
|
| 3 |
-
size 816441854
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_20_04_15_epoch8.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6506bf5ab2b66e31329c52dcea39dd8ca49f168a4233f6672ca5e3e861623b32
|
| 3 |
-
size 816441854
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_20_27_44_epoch9.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:35f61511fc9e1d980dc931b85b70bdaa99cd685133de79ae23795a4b8ed69cef
|
| 3 |
-
size 816441854
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_21_14_44_epoch11.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:448dcb80ca16f3a17e1fc11d93be401cbc9cb4f48a111f9639957ea1bf474a75
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_21_38_17_epoch12.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:cd5abd4d75c2752c6d6d8d79ad85c3ffc3d5d19609c559ce2b9aacd0afff6834
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_22_01_49_epoch13.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:fbc33b3d677969c025cbc3555970ec74f899b632cd58ad323c8717fcb3acda6e
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_22_25_17_epoch14.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:0f39fb4beb196fb8984991892875946a597f333070cf6017ab01744cb1c91858
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_23_12_20_epoch16.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:e654848f8fda8f79d8bfef0092f23c5d550ae793e80e62c155b44e9a0359c3b7
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_19_23_35_50_epoch17.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:4b2df4ab3c1b5baf77c4ca5afc20e2061869054c02c9040cb7c647f59d02bb08
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_18_41_30_epoch19.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:a7f7aa23aa3be85b9009521a38dfb117d51eaf62b28f1f8b3244e9be3fd8efd9
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_19_28_29_epoch21.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:bcf7effa693b2c4d3032a9acdb356c4f09ef10cca91fd6cb8e273ee99f528373
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_19_51_56_epoch22.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:aef8fb6b9be223a051e9e754868b3fc141fb1c6d69b4e5d782b2a91e0ec7b4f6
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_20_15_23_epoch23.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:0f9b847262c3329bf1a10c5e8d001f0de5a61cb5208708579e0a1cd90854f307
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_20_38_52_epoch24.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6d53380975b362db1ac10b1207518b1af6d28754b170f260f34c3e8e89113414
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_21_25_45_epoch26.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:25ff0e1626aa5d03caae2d8c92f57b4431e9f2daa827c61240510db4f1a5a6ea
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_21_49_12_epoch27.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:dc0d942100b32a1980f0e62b71464ece42021afa68476fe82833273b52bd8c05
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_22_12_36_epoch28.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:aa9f14999dbf09814c9738c7280ddbf9d2681550b6b32948f48447de0c51e744
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_22_35_58_epoch29.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:e28e1f18e3a8a23123ec7863c6916cddff96eb3ed59ff7d5db68d99a84e57640
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_23_22_56_epoch31.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:af604d64485ce74b728f4400ef967ada00f17bb7e01f63754797cfd6f72ce617
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_20_23_46_24_epoch32.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:b8d1b52ac5f3030f08c6792e449239f5be75734f4fc9e9a2cc1759b8c5ed19dd
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_00_10_00_epoch33.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:453b96d1dc8966b43ea30eb0f32ce5632c6d8b1ce5a6997e55a4ee87039658ed
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_00_33_29_epoch34.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:4a06fda0e8dee4b31cc03663b11cf24954ca9fc22d5eb721cbba3ab4623a434b
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_01_20_30_epoch36.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:b93f53ce00935c186275f2dd2325b10091e465944644649fe6f026976384b2d4
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_01_43_58_epoch37.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:989d20a744baa5b258ad2ce14532fa4ca5d00a8d3a531fed9b72406123d57da0
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_02_07_32_epoch38.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:48e3e412c3f9933a4463cb50e3b22e71dfe3ce30fa8b0ff569ca35f45527fe22
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_02_31_04_epoch39.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:55888009858063f40c8196b8739917de107a3fe8fe4fbcc24f00b15b9ab1318a
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_03_18_07_epoch41.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:7fb940321ef6493d6b687d3b07c659d361df5dc35ce0db95245dbdfaa66cd337
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_03_41_36_epoch42.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:db24353115991a2b8009773a66ef9d80e5a1bae0f034b41865881c6c654813fe
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_04_05_08_epoch43.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6c6d821be0a6fd359b105a7d16dbfc51b9948f05bd4d8b7cdb9d9a3912974264
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_04_28_33_epoch44.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6c17649932b0e176cc37ae02ff1be6a50da4ab823323d8229d7c06a11ed54abc
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_05_15_33_epoch46.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:87c6fdf0935e6714831b42fbb905719da7b90d6feb48b4c53412e835b4eb03fa
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_05_39_01_epoch47.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:a5a9618b7d67842ae90a9c9182e514ec666922288a789d55d8b2f14266551287
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_06_02_29_epoch48.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:32ee92b2e3a806467b97eca7c4d8cc9ece2cf138652f8a3eb135af345d55bf3b
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_21_06_51_48_epoch49.pth
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:2aa7860f1e1f4aeb6405139bf25e2ae8b73726e181f5cd077370ca4a52919a01
|
| 3 |
-
size 816441868
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_07_02_01_log.txt
DELETED
|
@@ -1,100 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 100,
|
| 10 |
-
"iters_per_epoch": 5200,
|
| 11 |
-
"batch_size_train": 25,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/MIMIC-CXR-VQA"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000029", "train_loss": "0.462359"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_11_36_44_log.txt
DELETED
|
@@ -1,100 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_11_51_44_log.txt
DELETED
|
@@ -1,101 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_12_05_54_log.txt
DELETED
|
@@ -1,102 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_12_20_01_log.txt
DELETED
|
@@ -1,103 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_12_34_09_log.txt
DELETED
|
@@ -1,104 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_12_48_17_log.txt
DELETED
|
@@ -1,105 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_13_03_15_log.txt
DELETED
|
@@ -1,106 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_13_17_21_log.txt
DELETED
|
@@ -1,107 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_13_31_29_log.txt
DELETED
|
@@ -1,108 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
| 108 |
-
{"train_lr": "0.000029", "train_loss": "0.478779"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_13_45_40_log.txt
DELETED
|
@@ -1,109 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
| 108 |
-
{"train_lr": "0.000029", "train_loss": "0.478779"}
|
| 109 |
-
{"train_lr": "0.000028", "train_loss": "0.448832"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_13_59_49_log.txt
DELETED
|
@@ -1,110 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
| 108 |
-
{"train_lr": "0.000029", "train_loss": "0.478779"}
|
| 109 |
-
{"train_lr": "0.000028", "train_loss": "0.448832"}
|
| 110 |
-
{"train_lr": "0.000028", "train_loss": "0.420626"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_14_14_50_log.txt
DELETED
|
@@ -1,111 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
| 108 |
-
{"train_lr": "0.000029", "train_loss": "0.478779"}
|
| 109 |
-
{"train_lr": "0.000028", "train_loss": "0.448832"}
|
| 110 |
-
{"train_lr": "0.000028", "train_loss": "0.420626"}
|
| 111 |
-
{"train_lr": "0.000028", "train_loss": "0.389205"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_14_28_58_log.txt
DELETED
|
@@ -1,112 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
| 108 |
-
{"train_lr": "0.000029", "train_loss": "0.478779"}
|
| 109 |
-
{"train_lr": "0.000028", "train_loss": "0.448832"}
|
| 110 |
-
{"train_lr": "0.000028", "train_loss": "0.420626"}
|
| 111 |
-
{"train_lr": "0.000028", "train_loss": "0.389205"}
|
| 112 |
-
{"train_lr": "0.000027", "train_loss": "0.360322"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24_10_27_14_43_05_log.txt
DELETED
|
@@ -1,113 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"run": {
|
| 3 |
-
"task": "image_text_pretrain",
|
| 4 |
-
"lr_sched": "linear_warmup_cosine_lr",
|
| 5 |
-
"init_lr": 3e-05,
|
| 6 |
-
"min_lr": 1e-05,
|
| 7 |
-
"warmup_lr": 1e-06,
|
| 8 |
-
"weight_decay": 0.05,
|
| 9 |
-
"max_epoch": 50,
|
| 10 |
-
"iters_per_epoch": 600,
|
| 11 |
-
"batch_size_train": 15,
|
| 12 |
-
"batch_size_eval": 1,
|
| 13 |
-
"num_workers": 4,
|
| 14 |
-
"warmup_steps": 200,
|
| 15 |
-
"seed": 42,
|
| 16 |
-
"output_dir": "output/xraygpt_vqa_finetune",
|
| 17 |
-
"amp": true,
|
| 18 |
-
"resume_ckpt_path": null,
|
| 19 |
-
"evaluate": false,
|
| 20 |
-
"train_splits": [
|
| 21 |
-
"train"
|
| 22 |
-
],
|
| 23 |
-
"device": "cuda",
|
| 24 |
-
"world_size": 2,
|
| 25 |
-
"dist_url": "env://",
|
| 26 |
-
"distributed": true,
|
| 27 |
-
"rank": 0,
|
| 28 |
-
"gpu": 0,
|
| 29 |
-
"dist_backend": "nccl"
|
| 30 |
-
},
|
| 31 |
-
"model": {
|
| 32 |
-
"arch": "mini_gpt4",
|
| 33 |
-
"image_size": 224,
|
| 34 |
-
"drop_path_rate": 0,
|
| 35 |
-
"use_grad_checkpoint": false,
|
| 36 |
-
"vit_precision": "fp16",
|
| 37 |
-
"freeze_vit": true,
|
| 38 |
-
"freeze_qformer": true,
|
| 39 |
-
"num_query_token": 32,
|
| 40 |
-
"llama_model": "Joycean0301/Llama-3.2-3B-Instruct-Medical-Conversational",
|
| 41 |
-
"prompt": "",
|
| 42 |
-
"lora_r": 16,
|
| 43 |
-
"lora_alpha": 16,
|
| 44 |
-
"lora_target_modules": [
|
| 45 |
-
"q_proj",
|
| 46 |
-
"k_proj",
|
| 47 |
-
"v_proj",
|
| 48 |
-
"o_proj",
|
| 49 |
-
"gate_proj",
|
| 50 |
-
"up_proj",
|
| 51 |
-
"down_proj"
|
| 52 |
-
],
|
| 53 |
-
"lora_dropout": 0.1,
|
| 54 |
-
"lora_bias": "none",
|
| 55 |
-
"model_type": "pretrain_vicuna",
|
| 56 |
-
"max_txt_len": 160,
|
| 57 |
-
"end_sym": "<|eot_id|>",
|
| 58 |
-
"ckpt": "24_10_26_14_03_09_pretrain100_finetune2_lora_openi.pth"
|
| 59 |
-
},
|
| 60 |
-
"preprocess": {
|
| 61 |
-
"vis_processor": {
|
| 62 |
-
"train": {
|
| 63 |
-
"name": "blip2_image_train",
|
| 64 |
-
"image_size": 224
|
| 65 |
-
},
|
| 66 |
-
"eval": {
|
| 67 |
-
"name": "blip2_image_eval",
|
| 68 |
-
"image_size": 224
|
| 69 |
-
}
|
| 70 |
-
},
|
| 71 |
-
"text_processor": {
|
| 72 |
-
"train": {
|
| 73 |
-
"name": "blip_caption"
|
| 74 |
-
},
|
| 75 |
-
"eval": {
|
| 76 |
-
"name": "blip_caption"
|
| 77 |
-
}
|
| 78 |
-
}
|
| 79 |
-
},
|
| 80 |
-
"datasets": {
|
| 81 |
-
"vqa": {
|
| 82 |
-
"data_type": "images",
|
| 83 |
-
"build_info": {
|
| 84 |
-
"storage": "dataset/llava_med_60k_instruct"
|
| 85 |
-
},
|
| 86 |
-
"vis_processor": {
|
| 87 |
-
"train": {
|
| 88 |
-
"name": "blip2_image_train",
|
| 89 |
-
"image_size": 224
|
| 90 |
-
}
|
| 91 |
-
},
|
| 92 |
-
"text_processor": {
|
| 93 |
-
"train": {
|
| 94 |
-
"name": "blip_caption"
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
}
|
| 99 |
-
}
|
| 100 |
-
{"train_lr": "0.000025", "train_loss": "1.129822"}
|
| 101 |
-
{"train_lr": "0.000030", "train_loss": "0.771452"}
|
| 102 |
-
{"train_lr": "0.000030", "train_loss": "0.699119"}
|
| 103 |
-
{"train_lr": "0.000030", "train_loss": "0.650587"}
|
| 104 |
-
{"train_lr": "0.000030", "train_loss": "0.610647"}
|
| 105 |
-
{"train_lr": "0.000029", "train_loss": "0.574458"}
|
| 106 |
-
{"train_lr": "0.000029", "train_loss": "0.542380"}
|
| 107 |
-
{"train_lr": "0.000029", "train_loss": "0.509769"}
|
| 108 |
-
{"train_lr": "0.000029", "train_loss": "0.478779"}
|
| 109 |
-
{"train_lr": "0.000028", "train_loss": "0.448832"}
|
| 110 |
-
{"train_lr": "0.000028", "train_loss": "0.420626"}
|
| 111 |
-
{"train_lr": "0.000028", "train_loss": "0.389205"}
|
| 112 |
-
{"train_lr": "0.000027", "train_loss": "0.360322"}
|
| 113 |
-
{"train_lr": "0.000027", "train_loss": "0.331934"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|