Spaces:

nroggendorff
/

train-llama

Runtime error

nroggendorff commited on Jul 30, 2024

Commit

6dda86f

verified ·

1 Parent(s): f54715c

Update train.py

Files changed (1) hide show

train.py CHANGED Viewed

@@ -9,7 +9,7 @@ from tokenizers import ByteLevelBPETokenizer
 MAX_SEQ_LENGTH = 512
 BATCH_SIZE = 32
-EPOCHS = 1
 LEARNING_RATE = 2e-4
 FACTOR = 8
 VOCAB_SIZE = 32000
@@ -18,7 +18,7 @@ OUTPUT_REPO = "smallama"
 FP16 = True
 WARMUP_STEPS = 500
 DECAY = 0.01
-GRADIENT_ACCUMULATION_STEPS = 4
 CLIPPING = 1.0
 PUSH_TO_HUB = True

 MAX_SEQ_LENGTH = 512
 BATCH_SIZE = 32
+EPOCHS = 3
 LEARNING_RATE = 2e-4
 FACTOR = 8
 VOCAB_SIZE = 32000
 FP16 = True
 WARMUP_STEPS = 500
 DECAY = 0.01
+GRADIENT_ACCUMULATION_STEPS = 8
 CLIPPING = 1.0
 PUSH_TO_HUB = True