cpt core 4

scripts/cpt_core_model_4.py

@@ -45,27 +45,6 @@ model = FastLanguageModel.get_peft_model(
 
 print(f'{model=}')
 
-'''
-from datasets import concatenate_datasets
-from cpt_base_datasets import cpt_base_datasets
-from cpt_instruct_datasets import cpt_instruct_datasets
-from unsloth_utils import load_text_dataset, load_chat_dataset
-
-core_datasets = []
-
-for dataset_config in cpt_base_datasets:
-    dataset = load_text_dataset(tokenizer, **dataset_config)
-    print(f'{dataset=}')
-    core_datasets.append(dataset)
-
-# for dataset_config in cpt_instruct_datasets:
-#     dataset = load_chat_dataset(tokenizer, **dataset_config)
-#     print(f'{dataset=}')
-#     core_datasets.append(dataset)
-
-final_dataset = concatenate_datasets(core_datasets)
-print(f'{final_dataset=}')
-'''
 
 from datasets import Dataset
 from litdata import TokensLoader, StreamingDataset
@@ -97,20 +76,23 @@ trainer = UnslothTrainer(
     model=model,
     tokenizer=tokenizer,
     train_dataset=train_dataset,
-    dataset_text_field='text',
+    # dataset_text_field='text',
     max_seq_length=max_seq_length,
     dataset_num_proc=32,
     max_steps=len(litgpt_streaming_dataset),
+    packing = False, # Can make training 5x faster for short sequences.
 
     args = UnslothTrainingArguments(
-        per_device_train_batch_size=8,
-        gradient_accumulation_steps=8,
+        per_device_train_batch_size=16,
+        gradient_accumulation_steps=64,
 
-        warmup_ratio=0.1,
+        warmup_ratio=0,
         num_train_epochs=1,
 
-        learning_rate=5e-5,
-        embedding_learning_rate=5e-6,
+        # learning_rate=5e-5,
+        # embedding_learning_rate=5e-6,
+        learning_rate = 5e-5 * 2,
+        embedding_learning_rate = 5e-5 / 2,
 
         fp16=not is_bfloat16_supported(),
         bf16=is_bfloat16_supported(),