KareemElzeky commited on
Commit
904717f
·
verified ·
1 Parent(s): 5e77829

Upload folder using huggingface_hub

Browse files
config.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "deepseek-ai/deepseek-coder-1.3b-base",
3
+ "architectures": [
4
+ "DetikzifyForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 32013,
9
+ "concat_patches": 2,
10
+ "eos_token_id": 32014,
11
+ "feature_layer": 26,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2048,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 5504,
17
+ "max_position_embeddings": 16384,
18
+ "mlp_bias": false,
19
+ "mm_hidden_size": 2304,
20
+ "model_type": "detikzify",
21
+ "num_attention_heads": 16,
22
+ "num_hidden_layers": 24,
23
+ "num_key_value_heads": 16,
24
+ "num_patches": 364,
25
+ "patch_token_id": 32013,
26
+ "pretraining_tp": 1,
27
+ "rms_norm_eps": 1e-06,
28
+ "rope_scaling": {
29
+ "factor": 4.0,
30
+ "rope_type": "linear",
31
+ "type": "linear"
32
+ },
33
+ "rope_theta": 100000,
34
+ "tie_word_embeddings": false,
35
+ "torch_dtype": "float32",
36
+ "transformers_version": "4.46.2",
37
+ "use_cache": false,
38
+ "use_mm_proj": true,
39
+ "vision_config": {
40
+ "architecture": "vit_so400m_patch14_siglip_384",
41
+ "classifier": "head",
42
+ "crop_mode": "center",
43
+ "crop_pct": 0.9,
44
+ "custom_load": false,
45
+ "first_conv": "patch_embed.proj",
46
+ "fixed_input_size": true,
47
+ "hf_hub_filename": "open_clip_pytorch_model.bin",
48
+ "hf_hub_id": "timm/ViT-SO400M-14-SigLIP-384",
49
+ "input_size": [
50
+ 3,
51
+ 384,
52
+ 384
53
+ ],
54
+ "interpolation": "bicubic",
55
+ "mean": [
56
+ 0.5,
57
+ 0.5,
58
+ 0.5
59
+ ],
60
+ "num_classes": 0,
61
+ "pool_size": null,
62
+ "std": [
63
+ 0.5,
64
+ 0.5,
65
+ 0.5
66
+ ],
67
+ "tag": "webli",
68
+ "url": ""
69
+ },
70
+ "vision_tower": "vit_so400m_patch14_siglip_384.webli",
71
+ "vocab_size": 32256
72
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 32013,
4
+ "eos_token_id": 32014,
5
+ "pad_token_id": 32018,
6
+ "transformers_version": "4.46.2"
7
+ }
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cf2692594b44f2282a189265999ea42f51be94f21c4bebdc838fe2f55218716
3
+ size 4986380064
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2bf14a1981be67fbd1a5432f76aaccf8411a424307c127b40fd2434b599110b3
3
+ size 418415560
model.safetensors.index.json ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 5404770304
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "model-00002-of-00002.safetensors",
7
+ "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
8
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
9
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
10
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
11
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
12
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
13
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
14
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
15
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
16
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
17
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
18
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
19
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
20
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
21
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
22
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
23
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
24
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
25
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
26
+ "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
27
+ "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
28
+ "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
29
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
30
+ "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
31
+ "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
32
+ "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
33
+ "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
34
+ "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
35
+ "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
36
+ "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
37
+ "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
38
+ "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
39
+ "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
40
+ "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
41
+ "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
42
+ "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
43
+ "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
44
+ "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
45
+ "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
46
+ "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
47
+ "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
48
+ "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
49
+ "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
50
+ "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
51
+ "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
52
+ "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
53
+ "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
54
+ "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
55
+ "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
56
+ "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
57
+ "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
58
+ "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
59
+ "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
60
+ "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
61
+ "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
62
+ "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
63
+ "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
64
+ "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
65
+ "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
66
+ "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
67
+ "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
68
+ "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
69
+ "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
70
+ "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
71
+ "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
72
+ "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
73
+ "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
74
+ "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
75
+ "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
76
+ "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
77
+ "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
78
+ "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
79
+ "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
80
+ "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
81
+ "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
82
+ "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
83
+ "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
84
+ "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
85
+ "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
86
+ "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
87
+ "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
88
+ "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
89
+ "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
90
+ "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
91
+ "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
92
+ "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
93
+ "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
94
+ "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
95
+ "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
96
+ "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
97
+ "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
98
+ "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
99
+ "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
100
+ "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
101
+ "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
102
+ "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
103
+ "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
104
+ "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
105
+ "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
106
+ "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
107
+ "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
108
+ "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
109
+ "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
110
+ "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
111
+ "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
112
+ "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
113
+ "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
114
+ "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
115
+ "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
116
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
117
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
118
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
119
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
120
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
121
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
122
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
123
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
124
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
125
+ "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
126
+ "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
127
+ "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
128
+ "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
129
+ "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
130
+ "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
131
+ "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
132
+ "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
133
+ "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
134
+ "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
135
+ "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
136
+ "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
137
+ "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
138
+ "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
139
+ "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
140
+ "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
141
+ "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
142
+ "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
143
+ "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
144
+ "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
145
+ "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
146
+ "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
147
+ "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
148
+ "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
149
+ "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
150
+ "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
151
+ "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
152
+ "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
153
+ "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
154
+ "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
155
+ "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
156
+ "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
157
+ "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
158
+ "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
159
+ "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
160
+ "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
161
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
162
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
163
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
164
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
165
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
166
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
167
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
168
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
169
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
170
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
171
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
172
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
173
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
174
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
175
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
176
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
177
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
178
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
179
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
180
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
181
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
182
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
183
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
184
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
185
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
186
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
187
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
188
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
189
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
190
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
191
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
192
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
193
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
194
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
195
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
196
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
197
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
198
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
199
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
200
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
201
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
202
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
203
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
204
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
205
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
206
+ "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
207
+ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
208
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
209
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
210
+ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
211
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
212
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
213
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
214
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
215
+ "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
216
+ "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
217
+ "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
218
+ "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
219
+ "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
220
+ "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
221
+ "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
222
+ "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
223
+ "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
224
+ "model.mm_projector.bias": "model-00002-of-00002.safetensors",
225
+ "model.mm_projector.weight": "model-00002-of-00002.safetensors",
226
+ "model.norm.weight": "model-00002-of-00002.safetensors"
227
+ }
228
+ }
trainer_state.json ADDED
@@ -0,0 +1,763 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.995306228883699,
5
+ "eval_steps": 500,
6
+ "global_step": 1032,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.029024285163601733,
13
+ "grad_norm": 88.48429870605469,
14
+ "learning_rate": 9.677419354838708e-05,
15
+ "loss": 0.7958,
16
+ "step": 10
17
+ },
18
+ {
19
+ "epoch": 0.058048570327203466,
20
+ "grad_norm": 78.84452056884766,
21
+ "learning_rate": 0.00019354838709677416,
22
+ "loss": 0.7504,
23
+ "step": 20
24
+ },
25
+ {
26
+ "epoch": 0.08707285549080519,
27
+ "grad_norm": 213.83749389648438,
28
+ "learning_rate": 0.00029032258064516127,
29
+ "loss": 0.7391,
30
+ "step": 30
31
+ },
32
+ {
33
+ "epoch": 0.11609714065440693,
34
+ "grad_norm": 64.08855438232422,
35
+ "learning_rate": 0.00029994016586766087,
36
+ "loss": 0.7567,
37
+ "step": 40
38
+ },
39
+ {
40
+ "epoch": 0.14512142581800866,
41
+ "grad_norm": 71.85431671142578,
42
+ "learning_rate": 0.00029973339311370587,
43
+ "loss": 0.7117,
44
+ "step": 50
45
+ },
46
+ {
47
+ "epoch": 0.17414571098161039,
48
+ "grad_norm": 66.32382202148438,
49
+ "learning_rate": 0.00029937914664890375,
50
+ "loss": 0.6959,
51
+ "step": 60
52
+ },
53
+ {
54
+ "epoch": 0.20316999614521214,
55
+ "grad_norm": 52.99678039550781,
56
+ "learning_rate": 0.00029887777537365414,
57
+ "loss": 0.6835,
58
+ "step": 70
59
+ },
60
+ {
61
+ "epoch": 0.23219428130881387,
62
+ "grad_norm": 53.15193557739258,
63
+ "learning_rate": 0.0002982297730928522,
64
+ "loss": 0.6855,
65
+ "step": 80
66
+ },
67
+ {
68
+ "epoch": 0.2612185664724156,
69
+ "grad_norm": 62.969337463378906,
70
+ "learning_rate": 0.00029743577802953563,
71
+ "loss": 0.6758,
72
+ "step": 90
73
+ },
74
+ {
75
+ "epoch": 0.2902428516360173,
76
+ "grad_norm": 47.597293853759766,
77
+ "learning_rate": 0.00029649657219629316,
78
+ "loss": 0.665,
79
+ "step": 100
80
+ },
81
+ {
82
+ "epoch": 0.31926713679961904,
83
+ "grad_norm": 50.93095397949219,
84
+ "learning_rate": 0.00029541308062505385,
85
+ "loss": 0.6689,
86
+ "step": 110
87
+ },
88
+ {
89
+ "epoch": 0.34829142196322077,
90
+ "grad_norm": 44.195335388183594,
91
+ "learning_rate": 0.00029418637045601514,
92
+ "loss": 0.6553,
93
+ "step": 120
94
+ },
95
+ {
96
+ "epoch": 0.3773157071268225,
97
+ "grad_norm": 46.52369689941406,
98
+ "learning_rate": 0.00029281764988660705,
99
+ "loss": 0.6584,
100
+ "step": 130
101
+ },
102
+ {
103
+ "epoch": 0.4063399922904243,
104
+ "grad_norm": 51.798343658447266,
105
+ "learning_rate": 0.0002913082669815285,
106
+ "loss": 0.6514,
107
+ "step": 140
108
+ },
109
+ {
110
+ "epoch": 0.435364277454026,
111
+ "grad_norm": 53.8443489074707,
112
+ "learning_rate": 0.0002896597083450262,
113
+ "loss": 0.6276,
114
+ "step": 150
115
+ },
116
+ {
117
+ "epoch": 0.46438856261762773,
118
+ "grad_norm": 44.94770812988281,
119
+ "learning_rate": 0.0002878735976567259,
120
+ "loss": 0.6428,
121
+ "step": 160
122
+ },
123
+ {
124
+ "epoch": 0.49341284778122946,
125
+ "grad_norm": 38.52789306640625,
126
+ "learning_rate": 0.0002859516940724558,
127
+ "loss": 0.6415,
128
+ "step": 170
129
+ },
130
+ {
131
+ "epoch": 0.5224371329448312,
132
+ "grad_norm": 52.5710563659668,
133
+ "learning_rate": 0.0002838958904916392,
134
+ "loss": 0.6302,
135
+ "step": 180
136
+ },
137
+ {
138
+ "epoch": 0.551461418108433,
139
+ "grad_norm": 46.27107238769531,
140
+ "learning_rate": 0.00028170821169296126,
141
+ "loss": 0.6246,
142
+ "step": 190
143
+ },
144
+ {
145
+ "epoch": 0.5804857032720346,
146
+ "grad_norm": 42.310123443603516,
147
+ "learning_rate": 0.00027939081234014705,
148
+ "loss": 0.627,
149
+ "step": 200
150
+ },
151
+ {
152
+ "epoch": 0.6095099884356364,
153
+ "grad_norm": 48.09523391723633,
154
+ "learning_rate": 0.0002769459748598149,
155
+ "loss": 0.623,
156
+ "step": 210
157
+ },
158
+ {
159
+ "epoch": 0.6385342735992381,
160
+ "grad_norm": 62.250152587890625,
161
+ "learning_rate": 0.0002743761071934942,
162
+ "loss": 0.6312,
163
+ "step": 220
164
+ },
165
+ {
166
+ "epoch": 0.6675585587628399,
167
+ "grad_norm": 42.713130950927734,
168
+ "learning_rate": 0.00027168374042602366,
169
+ "loss": 0.6101,
170
+ "step": 230
171
+ },
172
+ {
173
+ "epoch": 0.6965828439264415,
174
+ "grad_norm": 49.83562469482422,
175
+ "learning_rate": 0.00026887152629266354,
176
+ "loss": 0.6,
177
+ "step": 240
178
+ },
179
+ {
180
+ "epoch": 0.7256071290900433,
181
+ "grad_norm": 39.01671600341797,
182
+ "learning_rate": 0.0002659422345673789,
183
+ "loss": 0.6038,
184
+ "step": 250
185
+ },
186
+ {
187
+ "epoch": 0.754631414253645,
188
+ "grad_norm": 35.13432693481445,
189
+ "learning_rate": 0.0002628987503348651,
190
+ "loss": 0.5956,
191
+ "step": 260
192
+ },
193
+ {
194
+ "epoch": 0.7836556994172468,
195
+ "grad_norm": 41.503684997558594,
196
+ "learning_rate": 0.00025974407114900353,
197
+ "loss": 0.6134,
198
+ "step": 270
199
+ },
200
+ {
201
+ "epoch": 0.8126799845808486,
202
+ "grad_norm": 39.328548431396484,
203
+ "learning_rate": 0.0002564813040805443,
204
+ "loss": 0.59,
205
+ "step": 280
206
+ },
207
+ {
208
+ "epoch": 0.8417042697444502,
209
+ "grad_norm": 34.63987731933594,
210
+ "learning_rate": 0.0002531136626569259,
211
+ "loss": 0.5834,
212
+ "step": 290
213
+ },
214
+ {
215
+ "epoch": 0.870728554908052,
216
+ "grad_norm": 37.82402801513672,
217
+ "learning_rate": 0.0002496444636972439,
218
+ "loss": 0.6023,
219
+ "step": 300
220
+ },
221
+ {
222
+ "epoch": 0.8997528400716537,
223
+ "grad_norm": 38.01532745361328,
224
+ "learning_rate": 0.0002460771240454877,
225
+ "loss": 0.5866,
226
+ "step": 310
227
+ },
228
+ {
229
+ "epoch": 0.9287771252352555,
230
+ "grad_norm": 37.758487701416016,
231
+ "learning_rate": 0.00024241515720526083,
232
+ "loss": 0.6001,
233
+ "step": 320
234
+ },
235
+ {
236
+ "epoch": 0.9578014103988571,
237
+ "grad_norm": 34.032989501953125,
238
+ "learning_rate": 0.0002386621698793015,
239
+ "loss": 0.5833,
240
+ "step": 330
241
+ },
242
+ {
243
+ "epoch": 0.9868256955624589,
244
+ "grad_norm": 41.784881591796875,
245
+ "learning_rate": 0.0002348218584172095,
246
+ "loss": 0.5876,
247
+ "step": 340
248
+ },
249
+ {
250
+ "epoch": 1.0158499807260606,
251
+ "grad_norm": 35.09678268432617,
252
+ "learning_rate": 0.00023089800517487986,
253
+ "loss": 0.5319,
254
+ "step": 350
255
+ },
256
+ {
257
+ "epoch": 1.0448742658896624,
258
+ "grad_norm": 32.305877685546875,
259
+ "learning_rate": 0.00022689447478922784,
260
+ "loss": 0.4666,
261
+ "step": 360
262
+ },
263
+ {
264
+ "epoch": 1.0738985510532641,
265
+ "grad_norm": 35.80933380126953,
266
+ "learning_rate": 0.0002228152103718745,
267
+ "loss": 0.4619,
268
+ "step": 370
269
+ },
270
+ {
271
+ "epoch": 1.102922836216866,
272
+ "grad_norm": 32.89548873901367,
273
+ "learning_rate": 0.00021866422962554238,
274
+ "loss": 0.4739,
275
+ "step": 380
276
+ },
277
+ {
278
+ "epoch": 1.1319471213804675,
279
+ "grad_norm": 36.34146499633789,
280
+ "learning_rate": 0.0002144456208869851,
281
+ "loss": 0.4676,
282
+ "step": 390
283
+ },
284
+ {
285
+ "epoch": 1.1609714065440693,
286
+ "grad_norm": 42.522438049316406,
287
+ "learning_rate": 0.00021016353910034938,
288
+ "loss": 0.4765,
289
+ "step": 400
290
+ },
291
+ {
292
+ "epoch": 1.189995691707671,
293
+ "grad_norm": 34.677650451660156,
294
+ "learning_rate": 0.00020582220172493467,
295
+ "loss": 0.4715,
296
+ "step": 410
297
+ },
298
+ {
299
+ "epoch": 1.2190199768712728,
300
+ "grad_norm": 33.74694061279297,
301
+ "learning_rate": 0.0002014258845813811,
302
+ "loss": 0.4655,
303
+ "step": 420
304
+ },
305
+ {
306
+ "epoch": 1.2480442620348744,
307
+ "grad_norm": 30.60100555419922,
308
+ "learning_rate": 0.00019697891764037685,
309
+ "loss": 0.461,
310
+ "step": 430
311
+ },
312
+ {
313
+ "epoch": 1.2770685471984762,
314
+ "grad_norm": 38.6037483215332,
315
+ "learning_rate": 0.00019248568075803257,
316
+ "loss": 0.4719,
317
+ "step": 440
318
+ },
319
+ {
320
+ "epoch": 1.306092832362078,
321
+ "grad_norm": 32.19020080566406,
322
+ "learning_rate": 0.00018795059936212348,
323
+ "loss": 0.4586,
324
+ "step": 450
325
+ },
326
+ {
327
+ "epoch": 1.3351171175256797,
328
+ "grad_norm": 32.962276458740234,
329
+ "learning_rate": 0.00018337814009344714,
330
+ "loss": 0.4697,
331
+ "step": 460
332
+ },
333
+ {
334
+ "epoch": 1.3641414026892815,
335
+ "grad_norm": 29.69386863708496,
336
+ "learning_rate": 0.00017877280640659068,
337
+ "loss": 0.4639,
338
+ "step": 470
339
+ },
340
+ {
341
+ "epoch": 1.393165687852883,
342
+ "grad_norm": 31.52634620666504,
343
+ "learning_rate": 0.00017413913413443915,
344
+ "loss": 0.4579,
345
+ "step": 480
346
+ },
347
+ {
348
+ "epoch": 1.4221899730164849,
349
+ "grad_norm": 35.30017852783203,
350
+ "learning_rate": 0.0001694816870207949,
351
+ "loss": 0.4684,
352
+ "step": 490
353
+ },
354
+ {
355
+ "epoch": 1.4512142581800866,
356
+ "grad_norm": 33.88492202758789,
357
+ "learning_rate": 0.00016480505222550682,
358
+ "loss": 0.4534,
359
+ "step": 500
360
+ },
361
+ {
362
+ "epoch": 1.4802385433436884,
363
+ "grad_norm": 30.00653076171875,
364
+ "learning_rate": 0.00016011383580653697,
365
+ "loss": 0.464,
366
+ "step": 510
367
+ },
368
+ {
369
+ "epoch": 1.50926282850729,
370
+ "grad_norm": 33.75349807739258,
371
+ "learning_rate": 0.00015541265818341433,
372
+ "loss": 0.4497,
373
+ "step": 520
374
+ },
375
+ {
376
+ "epoch": 1.5382871136708918,
377
+ "grad_norm": 31.689538955688477,
378
+ "learning_rate": 0.00015070614958654393,
379
+ "loss": 0.4412,
380
+ "step": 530
381
+ },
382
+ {
383
+ "epoch": 1.5673113988344936,
384
+ "grad_norm": 28.848291397094727,
385
+ "learning_rate": 0.00014599894549685273,
386
+ "loss": 0.4467,
387
+ "step": 540
388
+ },
389
+ {
390
+ "epoch": 1.5963356839980953,
391
+ "grad_norm": 27.079084396362305,
392
+ "learning_rate": 0.0001412956820802647,
393
+ "loss": 0.4428,
394
+ "step": 550
395
+ },
396
+ {
397
+ "epoch": 1.6253599691616971,
398
+ "grad_norm": 29.99922752380371,
399
+ "learning_rate": 0.0001366009916215007,
400
+ "loss": 0.4374,
401
+ "step": 560
402
+ },
403
+ {
404
+ "epoch": 1.654384254325299,
405
+ "grad_norm": 28.763559341430664,
406
+ "learning_rate": 0.00013191949796170156,
407
+ "loss": 0.4419,
408
+ "step": 570
409
+ },
410
+ {
411
+ "epoch": 1.6834085394889005,
412
+ "grad_norm": 30.430801391601562,
413
+ "learning_rate": 0.00012725581194436694,
414
+ "loss": 0.445,
415
+ "step": 580
416
+ },
417
+ {
418
+ "epoch": 1.7124328246525022,
419
+ "grad_norm": 28.43861198425293,
420
+ "learning_rate": 0.00012261452687409576,
421
+ "loss": 0.4452,
422
+ "step": 590
423
+ },
424
+ {
425
+ "epoch": 1.7414571098161038,
426
+ "grad_norm": 33.317378997802734,
427
+ "learning_rate": 0.00011800021399260094,
428
+ "loss": 0.4378,
429
+ "step": 600
430
+ },
431
+ {
432
+ "epoch": 1.7704813949797056,
433
+ "grad_norm": 27.84680938720703,
434
+ "learning_rate": 0.00011341741797645384,
435
+ "loss": 0.4375,
436
+ "step": 610
437
+ },
438
+ {
439
+ "epoch": 1.7995056801433074,
440
+ "grad_norm": 32.20744705200195,
441
+ "learning_rate": 0.0001088706524609933,
442
+ "loss": 0.4281,
443
+ "step": 620
444
+ },
445
+ {
446
+ "epoch": 1.8285299653069091,
447
+ "grad_norm": 29.68756675720215,
448
+ "learning_rate": 0.00010436439559480705,
449
+ "loss": 0.4338,
450
+ "step": 630
451
+ },
452
+ {
453
+ "epoch": 1.857554250470511,
454
+ "grad_norm": 31.973575592041016,
455
+ "learning_rate": 9.990308562916479e-05,
456
+ "loss": 0.4265,
457
+ "step": 640
458
+ },
459
+ {
460
+ "epoch": 1.8865785356341127,
461
+ "grad_norm": 26.948545455932617,
462
+ "learning_rate": 9.549111654674586e-05,
463
+ "loss": 0.4165,
464
+ "step": 650
465
+ },
466
+ {
467
+ "epoch": 1.9156028207977145,
468
+ "grad_norm": 27.91978645324707,
469
+ "learning_rate": 9.11328337339681e-05,
470
+ "loss": 0.416,
471
+ "step": 660
472
+ },
473
+ {
474
+ "epoch": 1.944627105961316,
475
+ "grad_norm": 34.58734130859375,
476
+ "learning_rate": 8.68325297011791e-05,
477
+ "loss": 0.4196,
478
+ "step": 670
479
+ },
480
+ {
481
+ "epoch": 1.9736513911249178,
482
+ "grad_norm": 24.959909439086914,
483
+ "learning_rate": 8.259443985492576e-05,
484
+ "loss": 0.4305,
485
+ "step": 680
486
+ },
487
+ {
488
+ "epoch": 2.0026756762885194,
489
+ "grad_norm": 39.029258728027344,
490
+ "learning_rate": 7.842273832646591e-05,
491
+ "loss": 0.4122,
492
+ "step": 690
493
+ },
494
+ {
495
+ "epoch": 2.031699961452121,
496
+ "grad_norm": 27.386505126953125,
497
+ "learning_rate": 7.432153386063034e-05,
498
+ "loss": 0.2751,
499
+ "step": 700
500
+ },
501
+ {
502
+ "epoch": 2.060724246615723,
503
+ "grad_norm": 30.209821701049805,
504
+ "learning_rate": 7.029486576908444e-05,
505
+ "loss": 0.2654,
506
+ "step": 710
507
+ },
508
+ {
509
+ "epoch": 2.0897485317793247,
510
+ "grad_norm": 31.79279327392578,
511
+ "learning_rate": 6.63466999519756e-05,
512
+ "loss": 0.2648,
513
+ "step": 720
514
+ },
515
+ {
516
+ "epoch": 2.1187728169429265,
517
+ "grad_norm": 31.363250732421875,
518
+ "learning_rate": 6.248092499188372e-05,
519
+ "loss": 0.2587,
520
+ "step": 730
521
+ },
522
+ {
523
+ "epoch": 2.1477971021065283,
524
+ "grad_norm": 33.62345886230469,
525
+ "learning_rate": 5.870134832392269e-05,
526
+ "loss": 0.2564,
527
+ "step": 740
528
+ },
529
+ {
530
+ "epoch": 2.17682138727013,
531
+ "grad_norm": 31.332040786743164,
532
+ "learning_rate": 5.5011692485764734e-05,
533
+ "loss": 0.253,
534
+ "step": 750
535
+ },
536
+ {
537
+ "epoch": 2.205845672433732,
538
+ "grad_norm": 30.034757614135742,
539
+ "learning_rate": 5.141559145128093e-05,
540
+ "loss": 0.26,
541
+ "step": 760
542
+ },
543
+ {
544
+ "epoch": 2.234869957597333,
545
+ "grad_norm": 30.40983772277832,
546
+ "learning_rate": 4.791658705140897e-05,
547
+ "loss": 0.2507,
548
+ "step": 770
549
+ },
550
+ {
551
+ "epoch": 2.263894242760935,
552
+ "grad_norm": 27.134634017944336,
553
+ "learning_rate": 4.451812548577333e-05,
554
+ "loss": 0.2518,
555
+ "step": 780
556
+ },
557
+ {
558
+ "epoch": 2.2929185279245368,
559
+ "grad_norm": 27.9604434967041,
560
+ "learning_rate": 4.1223553928493564e-05,
561
+ "loss": 0.2494,
562
+ "step": 790
563
+ },
564
+ {
565
+ "epoch": 2.3219428130881385,
566
+ "grad_norm": 33.73405838012695,
567
+ "learning_rate": 3.803611723152345e-05,
568
+ "loss": 0.2441,
569
+ "step": 800
570
+ },
571
+ {
572
+ "epoch": 2.3509670982517403,
573
+ "grad_norm": 31.413331985473633,
574
+ "learning_rate": 3.495895472876854e-05,
575
+ "loss": 0.2479,
576
+ "step": 810
577
+ },
578
+ {
579
+ "epoch": 2.379991383415342,
580
+ "grad_norm": 28.82455062866211,
581
+ "learning_rate": 3.199509714412901e-05,
582
+ "loss": 0.2529,
583
+ "step": 820
584
+ },
585
+ {
586
+ "epoch": 2.409015668578944,
587
+ "grad_norm": 31.402931213378906,
588
+ "learning_rate": 2.9147463606513528e-05,
589
+ "loss": 0.2499,
590
+ "step": 830
591
+ },
592
+ {
593
+ "epoch": 2.4380399537425457,
594
+ "grad_norm": 25.637739181518555,
595
+ "learning_rate": 2.6418858774763992e-05,
596
+ "loss": 0.236,
597
+ "step": 840
598
+ },
599
+ {
600
+ "epoch": 2.467064238906147,
601
+ "grad_norm": 27.47572898864746,
602
+ "learning_rate": 2.38119700753228e-05,
603
+ "loss": 0.2432,
604
+ "step": 850
605
+ },
606
+ {
607
+ "epoch": 2.496088524069749,
608
+ "grad_norm": 28.527973175048828,
609
+ "learning_rate": 2.1329365055363595e-05,
610
+ "loss": 0.2428,
611
+ "step": 860
612
+ },
613
+ {
614
+ "epoch": 2.5251128092333506,
615
+ "grad_norm": 28.3017578125,
616
+ "learning_rate": 1.89734888539916e-05,
617
+ "loss": 0.2457,
618
+ "step": 870
619
+ },
620
+ {
621
+ "epoch": 2.5541370943969524,
622
+ "grad_norm": 27.692001342773438,
623
+ "learning_rate": 1.674666179400504e-05,
624
+ "loss": 0.2409,
625
+ "step": 880
626
+ },
627
+ {
628
+ "epoch": 2.583161379560554,
629
+ "grad_norm": 30.592241287231445,
630
+ "learning_rate": 1.4651077096589486e-05,
631
+ "loss": 0.2371,
632
+ "step": 890
633
+ },
634
+ {
635
+ "epoch": 2.612185664724156,
636
+ "grad_norm": 26.051584243774414,
637
+ "learning_rate": 1.2688798721195053e-05,
638
+ "loss": 0.2389,
639
+ "step": 900
640
+ },
641
+ {
642
+ "epoch": 2.6412099498877577,
643
+ "grad_norm": 28.38836097717285,
644
+ "learning_rate": 1.086175933272514e-05,
645
+ "loss": 0.2407,
646
+ "step": 910
647
+ },
648
+ {
649
+ "epoch": 2.6702342350513595,
650
+ "grad_norm": 27.81374740600586,
651
+ "learning_rate": 9.171758398038015e-06,
652
+ "loss": 0.2389,
653
+ "step": 920
654
+ },
655
+ {
656
+ "epoch": 2.6992585202149613,
657
+ "grad_norm": 27.540956497192383,
658
+ "learning_rate": 7.620460413636342e-06,
659
+ "loss": 0.2453,
660
+ "step": 930
661
+ },
662
+ {
663
+ "epoch": 2.728282805378563,
664
+ "grad_norm": 27.374300003051758,
665
+ "learning_rate": 6.209393266290291e-06,
666
+ "loss": 0.234,
667
+ "step": 940
668
+ },
669
+ {
670
+ "epoch": 2.757307090542165,
671
+ "grad_norm": 29.071474075317383,
672
+ "learning_rate": 4.939946728208627e-06,
673
+ "loss": 0.2406,
674
+ "step": 950
675
+ },
676
+ {
677
+ "epoch": 2.786331375705766,
678
+ "grad_norm": 25.93909454345703,
679
+ "learning_rate": 3.813371088240086e-06,
680
+ "loss": 0.231,
681
+ "step": 960
682
+ },
683
+ {
684
+ "epoch": 2.815355660869368,
685
+ "grad_norm": 28.83918571472168,
686
+ "learning_rate": 2.830775920453093e-06,
687
+ "loss": 0.2303,
688
+ "step": 970
689
+ },
690
+ {
691
+ "epoch": 2.8443799460329697,
692
+ "grad_norm": 28.06920623779297,
693
+ "learning_rate": 1.9931289913066694e-06,
694
+ "loss": 0.2339,
695
+ "step": 980
696
+ },
697
+ {
698
+ "epoch": 2.8734042311965715,
699
+ "grad_norm": 28.357439041137695,
700
+ "learning_rate": 1.3012553064889631e-06,
701
+ "loss": 0.2325,
702
+ "step": 990
703
+ },
704
+ {
705
+ "epoch": 2.9024285163601733,
706
+ "grad_norm": 25.29115104675293,
707
+ "learning_rate": 7.558362983619448e-07,
708
+ "loss": 0.2374,
709
+ "step": 1000
710
+ },
711
+ {
712
+ "epoch": 2.931452801523775,
713
+ "grad_norm": 27.02465057373047,
714
+ "learning_rate": 3.57409154812871e-07,
715
+ "loss": 0.2307,
716
+ "step": 1010
717
+ },
718
+ {
719
+ "epoch": 2.960477086687377,
720
+ "grad_norm": 26.2918701171875,
721
+ "learning_rate": 1.0636629017320431e-07,
722
+ "loss": 0.232,
723
+ "step": 1020
724
+ },
725
+ {
726
+ "epoch": 2.989501371850978,
727
+ "grad_norm": 28.43804359436035,
728
+ "learning_rate": 2.9549587264754428e-09,
729
+ "loss": 0.2287,
730
+ "step": 1030
731
+ },
732
+ {
733
+ "epoch": 2.995306228883699,
734
+ "step": 1032,
735
+ "total_flos": 1.0711204212442399e+18,
736
+ "train_loss": 0.44727156865735385,
737
+ "train_runtime": 21178.1386,
738
+ "train_samples_per_second": 6.247,
739
+ "train_steps_per_second": 0.049
740
+ }
741
+ ],
742
+ "logging_steps": 10,
743
+ "max_steps": 1032,
744
+ "num_input_tokens_seen": 0,
745
+ "num_train_epochs": 3,
746
+ "save_steps": 500,
747
+ "stateful_callbacks": {
748
+ "TrainerControl": {
749
+ "args": {
750
+ "should_epoch_stop": false,
751
+ "should_evaluate": false,
752
+ "should_log": false,
753
+ "should_save": true,
754
+ "should_training_stop": true
755
+ },
756
+ "attributes": {}
757
+ }
758
+ },
759
+ "total_flos": 1.0711204212442399e+18,
760
+ "train_batch_size": 1,
761
+ "trial_name": null,
762
+ "trial_params": null
763
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:804f05ea97d2034be9157aa51cb3e47c7cee4f13f448df7f3d75429daf246732
3
+ size 5304