Training in progress, step 398

config.json

@@ -4,27 +4,27 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "eos_token_id": 0,
+  "bos_token_id": 31989,
+  "eos_token_id": 31989,
   "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 768,
-  "initializer_range": 0.02,
+  "initializer_range": 0.036084391824351615,
   "intermediate_size": 1920,
-  "max_position_embeddings": 2048,
+  "max_position_embeddings": 8192,
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 12,
-  "num_hidden_layers": 29,
+  "num_hidden_layers": 25,
   "num_key_value_heads": 4,
-  "pad_token_id": 0,
+  "pad_token_id": 31989,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
-  "rope_theta": 10000.0,
+  "rope_theta": 1000000.0,
   "tie_word_embeddings": true,
-  "torch_dtype": "float32",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.53.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f35a8dec902d7165a56c24bab7b7852ea2367d08f0a75ee0a63d9af48d7b4f6
-size 794113712
+oid sha256:187b0c014f4f4268532e53ba6211e90e1a48aa2668ca0960ee42de23f3285e83
+size 349083024

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54ac8ac530651fda895c2e75554afdc2cd7218ad99dfa39e9c9a4b7a35287208
+size 5777