pretrain core 1

README.md

@@ -197,3 +197,7 @@ Tasks                           |Version|Filter|n-shot|        Metric         |
 ```bash
 litgpt convert_pretrained_checkpoint ../out/pretrain-core-0/final ../out/pretrain-core-0/checkpoint
 ```
+
+```bash
+CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain_core_model_1.yaml
+```

scripts/pretrain_core_model_1.yaml

@@ -85,7 +85,7 @@ train:
   max_norm: 1.0
 
   #   (type: float, default: 4e-05)
-  min_lr: 1e-6
+  min_lr: 1e-5
 
 # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
 eval: