cpt core 4

scripts/cpt_core_model_4.py

@@ -7,15 +7,14 @@ from transformers import AutoTokenizer
 os.environ['WANDB_PROJECT'] = 'tangled-alpha-0.9-core'
 run_name = 'cpt-core-4'
 
+dataset_input_dir = '../core-data-4-8193-16385-16385-1000/'
+dataset_block_size = 16385
 max_seq_length = 16385
 dtype = torch.bfloat16
 load_in_4bit = False
 model_name = '../out/pretrain-core-3/hf'
 output_dir = '../out/cpt-core-4'
 
-dataset_input_dir = '../core-data-4-8193-16385-16385-1000/'
-dataset_block_size = 16385
-
 #
 # model
 #
@@ -44,7 +43,7 @@ model = FastLanguageModel.get_peft_model(
     # lora_alpha=32,
     lora_alpha=16,
     lora_dropout=0, # Supports any, but = 0 is optimized
-    bias="none",    # Supports any, but = "none" is optimized
+    bias='none',    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
     # use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context
     use_gradient_checkpointing=False,