cpt core 4
Browse files
scripts/cpt_core_model_4.py
CHANGED
|
@@ -76,7 +76,6 @@ litgpt_streaming_dataset = StreamingDataset(
|
|
| 76 |
item_loader=TokensLoader(block_size=dataset_block_size),
|
| 77 |
)
|
| 78 |
|
| 79 |
-
|
| 80 |
def unlsoth_generator():
|
| 81 |
global litgpt_streaming_dataset
|
| 82 |
|
|
@@ -86,7 +85,7 @@ def unlsoth_generator():
|
|
| 86 |
break
|
| 87 |
|
| 88 |
|
| 89 |
-
train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
|
| 90 |
|
| 91 |
|
| 92 |
from trl import SFTTrainer
|
|
|
|
| 76 |
item_loader=TokensLoader(block_size=dataset_block_size),
|
| 77 |
)
|
| 78 |
|
|
|
|
| 79 |
def unlsoth_generator():
|
| 80 |
global litgpt_streaming_dataset
|
| 81 |
|
|
|
|
| 85 |
break
|
| 86 |
|
| 87 |
|
| 88 |
+
train_dataset = Dataset.from_generator(unlsoth_generator, max_steps=len(litgpt_streaming_dataset), streaming=True)
|
| 89 |
|
| 90 |
|
| 91 |
from trl import SFTTrainer
|