pretrain core 4

scripts/pretrain_core_model_4.yaml

@@ -60,7 +60,7 @@ train:
   # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
   # global_batch_size: 512
   # global_batch_size: 256
-  global_batch_size: 8
+  global_batch_size: 4
 
   # Number of samples per data-parallel rank (type: int, default: 4)
   micro_batch_size: 1
@@ -78,8 +78,7 @@ train:
   max_steps:
 
   # Limits the length of samples. Off by default (type: Optional[int], default: null)
-  # max_seq_length: 16384
-  max_seq_length:
+  max_seq_length: 16384
 
   # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
   tie_embeddings: true