cpt core 4
Browse files
scripts/cpt_core_model_4.py
CHANGED
|
@@ -83,8 +83,7 @@ def unlsoth_generator():
|
|
| 83 |
yield {'input_ids': batch}
|
| 84 |
|
| 85 |
|
| 86 |
-
|
| 87 |
-
train_dataset = Dataset.from_generator(unlsoth_generator, batched=True)
|
| 88 |
|
| 89 |
|
| 90 |
from trl import SFTTrainer
|
|
|
|
| 83 |
yield {'input_ids': batch}
|
| 84 |
|
| 85 |
|
| 86 |
+
train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
|
|
|
|
| 87 |
|
| 88 |
|
| 89 |
from trl import SFTTrainer
|