cpt core 4

scripts/cpt_core_model_4.py

@@ -85,7 +85,7 @@ def unlsoth_generator():
         break
 
 
-train_dataset = Dataset.from_generator(unlsoth_generator, max_steps=len(litgpt_streaming_dataset), streaming=True)
+train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
 
 
 from trl import SFTTrainer
@@ -101,6 +101,7 @@ trainer = UnslothTrainer(
     dataset_text_field='text',
     max_seq_length=max_seq_length,
     dataset_num_proc=32,
+    max_steps=len(litgpt_streaming_dataset),
 
     args = UnslothTrainingArguments(
         per_device_train_batch_size=8,