pretrain-core-0

scripts/pretrain_core_model_0.yaml

@@ -20,7 +20,7 @@ model_config:
   mlp_class_name: "LLaMAMLP"
   intermediate_size: 2048 # n_embd * 4
   norm_eps: 1e-5
-  rope_base: 10000
+  rope_base: 4300 # https://arxiv.org/pdf/2405.14591
   head_size: 128 # n_embd / n_head
 
 # Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in