Commit
·
f9713ea
1
Parent(s):
de996a9
run.sh
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nice nohup bash -c 'echo "start $(date)" && mpirun -np 8 ./train_gpt2cu \
|
| 2 |
+
-i "dev/data/fineweb10B/fineweb_train_*.bin" \
|
| 3 |
+
-j "dev/data/fineweb10B/fineweb_val_*.bin" \
|
| 4 |
+
-o log124M \
|
| 5 |
+
-e "d12" \
|
| 6 |
+
-b 64 -t 1024 \
|
| 7 |
+
-d 524288 \
|
| 8 |
+
-r 1 \
|
| 9 |
+
-z 1 \
|
| 10 |
+
-c 0.1 \
|
| 11 |
+
-l 0.0006 \
|
| 12 |
+
-q 0.0 \
|
| 13 |
+
-u 700 \
|
| 14 |
+
-n 5000 \
|
| 15 |
+
-y 1 \
|
| 16 |
+
-v 250 -s 20000 \
|
| 17 |
+
-h 1 && echo "end $(date)"' &
|
| 18 |
+
|