cpt core 4

scripts/cpt_core_model_4.py

@@ -29,7 +29,7 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 model = FastLanguageModel.get_peft_model(
     model,
     # r = 256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
     target_modules = [
         "q_proj", "k_proj", "v_proj", "o_proj",
         "gate_proj",
@@ -37,7 +37,7 @@ model = FastLanguageModel.get_peft_model(
         "embed_tokens", "lm_head",
     ],
     # lora_alpha = 32,
-    lora_alpha = 8,
+    lora_alpha = 2,
     lora_dropout = 0, # Supports any, but = 0 is optimized
     bias = "none",    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!