cpt core 4

scripts/cpt_core_model_4.py

@@ -28,14 +28,16 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 
 model = FastLanguageModel.get_peft_model(
     model,
-    r = 256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    # r = 256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
     target_modules = [
         "q_proj", "k_proj", "v_proj", "o_proj",
         "gate_proj",
         "up_proj", "down_proj",
         "embed_tokens", "lm_head",
     ],
-    lora_alpha = 32,
+    # lora_alpha = 32,
+    lora_alpha = 8,
     lora_dropout = 0, # Supports any, but = 0 is optimized
     bias = "none",    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!