Training Examples: 10-20k

adapter_config.json

@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
     "up_proj",
+    "q_proj",
     "down_proj",
+    "v_proj",
+    "k_proj",
     "o_proj",
-    "q_proj",
-    "gate_proj",
-    "v_proj"
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e5825365e8f35ad2be0fba4551dd0bb1a89c29127006b550f42dbe48103b58c
+oid sha256:7484e5ee8793b6a9d815a49cb85f1c89dceb2e005e94a3dbc5624f35beeabc5b
 size 159967880

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6572214ae96790947225891855adcce0a29f6be3999bd0eb846418c1764c449e
+oid sha256:f880ede74f46f09098edd37f094e2664bc1894201387437a09efebd43caab9ad
 size 81730644

scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c801982aae9be06d302403c1fff693e53dedf89c1d3b689ee29fedad84a96d23
+oid sha256:6c7c5a23cf686c3374465b35601286066a0bb9c35d2746ea32f4f663c7337c6a
 size 988

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5836d5feabbfa4a8a0dfb2d2daf51efcfd7a4705772f290bfa6fbba9841feacb
+oid sha256:950d7b202a8b603a9f2053cab9955a76c4fc0bb96742fce47216d10adb705966
 size 1064

trainer_state.json

@@ -3,357 +3,357 @@
   "best_model_checkpoint": null,
   "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 4849,
+  "global_step": 4850,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.00020622808826562179,
-      "grad_norm": 0.23989851772785187,
+      "epoch": 0.0002061855670103093,
+      "grad_norm": 0.5331169366836548,
       "learning_rate": 2.061855670103093e-07,
-      "loss": 0.3507,
+      "loss": 0.5976,
       "step": 1
     },
     {
-      "epoch": 0.02062280882656218,
-      "grad_norm": 0.46579787135124207,
+      "epoch": 0.020618556701030927,
+      "grad_norm": 0.42875760793685913,
       "learning_rate": 2.0618556701030927e-05,
-      "loss": 0.6141,
+      "loss": 0.5552,
       "step": 100
     },
     {
-      "epoch": 0.04124561765312436,
-      "grad_norm": 0.20079994201660156,
+      "epoch": 0.041237113402061855,
+      "grad_norm": 0.44043704867362976,
       "learning_rate": 4.1237113402061855e-05,
-      "loss": 0.6044,
+      "loss": 0.5911,
       "step": 200
     },
     {
-      "epoch": 0.06186842647968653,
-      "grad_norm": 0.33593931794166565,
+      "epoch": 0.061855670103092786,
+      "grad_norm": 0.39276885986328125,
       "learning_rate": 6.185567010309279e-05,
-      "loss": 0.5863,
+      "loss": 0.5676,
       "step": 300
     },
     {
-      "epoch": 0.08249123530624872,
-      "grad_norm": 0.5399625897407532,
+      "epoch": 0.08247422680412371,
+      "grad_norm": 0.3910420835018158,
       "learning_rate": 8.247422680412371e-05,
-      "loss": 0.5629,
+      "loss": 0.5516,
       "step": 400
     },
     {
-      "epoch": 0.10311404413281089,
-      "grad_norm": 0.5116047859191895,
+      "epoch": 0.10309278350515463,
+      "grad_norm": 0.3953075408935547,
       "learning_rate": 0.00010309278350515463,
-      "loss": 0.4852,
+      "loss": 0.5785,
       "step": 500
     },
     {
-      "epoch": 0.12373685295937306,
-      "grad_norm": 0.39057251811027527,
+      "epoch": 0.12371134020618557,
+      "grad_norm": 0.38498565554618835,
       "learning_rate": 0.00012371134020618558,
-      "loss": 0.5038,
+      "loss": 0.6043,
       "step": 600
     },
     {
-      "epoch": 0.14435966178593523,
-      "grad_norm": 0.35319826006889343,
+      "epoch": 0.14432989690721648,
+      "grad_norm": 0.36671698093414307,
       "learning_rate": 0.0001443298969072165,
-      "loss": 0.4892,
+      "loss": 0.579,
       "step": 700
     },
     {
-      "epoch": 0.16498247061249743,
-      "grad_norm": 0.2662217319011688,
+      "epoch": 0.16494845360824742,
+      "grad_norm": 0.3693369925022125,
       "learning_rate": 0.00016494845360824742,
-      "loss": 0.5089,
+      "loss": 0.5443,
       "step": 800
     },
     {
-      "epoch": 0.1856052794390596,
-      "grad_norm": 0.535358190536499,
+      "epoch": 0.18556701030927836,
+      "grad_norm": 0.3997150659561157,
       "learning_rate": 0.00018556701030927837,
-      "loss": 0.4885,
+      "loss": 0.5798,
       "step": 900
     },
     {
-      "epoch": 0.20622808826562178,
-      "grad_norm": 0.2716768682003021,
-      "learning_rate": 0.00019997048441912246,
-      "loss": 0.5124,
+      "epoch": 0.20618556701030927,
+      "grad_norm": 0.4547862708568573,
+      "learning_rate": 0.0001999704996306308,
+      "loss": 0.6046,
       "step": 1000
     },
     {
-      "epoch": 0.22685089709218395,
-      "grad_norm": 0.4777716100215912,
-      "learning_rate": 0.00019944624754044668,
-      "loss": 0.4911,
+      "epoch": 0.2268041237113402,
+      "grad_norm": 0.34885653853416443,
+      "learning_rate": 0.00019944653267951504,
+      "loss": 0.5694,
       "step": 1100
     },
     {
-      "epoch": 0.24747370591874612,
-      "grad_norm": 0.2679508626461029,
-      "learning_rate": 0.00019827006532530193,
-      "loss": 0.4793,
+      "epoch": 0.24742268041237114,
+      "grad_norm": 0.5895730257034302,
+      "learning_rate": 0.00019827095435098925,
+      "loss": 0.5858,
       "step": 1200
     },
     {
-      "epoch": 0.2680965147453083,
-      "grad_norm": 0.4718882739543915,
-      "learning_rate": 0.00019644964853733152,
-      "loss": 0.4712,
+      "epoch": 0.26804123711340205,
+      "grad_norm": 0.3080294132232666,
+      "learning_rate": 0.0001964514674798659,
+      "loss": 0.592,
       "step": 1300
     },
     {
-      "epoch": 0.28871932357187047,
-      "grad_norm": 0.2879483699798584,
-      "learning_rate": 0.00019399693138486107,
-      "loss": 0.5119,
+      "epoch": 0.28865979381443296,
+      "grad_norm": 0.2972188889980316,
+      "learning_rate": 0.00019399999403380266,
+      "loss": 0.5726,
       "step": 1400
     },
     {
-      "epoch": 0.30934213239843267,
-      "grad_norm": 0.5847165584564209,
-      "learning_rate": 0.0001909279932831403,
-      "loss": 0.4973,
+      "epoch": 0.30927835051546393,
+      "grad_norm": 0.28474223613739014,
+      "learning_rate": 0.00019093259699605125,
+      "loss": 0.5729,
       "step": 1500
     },
     {
-      "epoch": 0.32996494122499487,
-      "grad_norm": 0.3073234558105469,
-      "learning_rate": 0.0001872629534416197,
-      "loss": 0.4996,
+      "epoch": 0.32989690721649484,
+      "grad_norm": 0.39205971360206604,
+      "learning_rate": 0.00018726937511470246,
+      "loss": 0.5815,
       "step": 1600
     },
     {
-      "epoch": 0.350587750051557,
-      "grad_norm": 0.2527044713497162,
-      "learning_rate": 0.00018302583896732187,
-      "loss": 0.4805,
+      "epoch": 0.35051546391752575,
+      "grad_norm": 0.43681225180625916,
+      "learning_rate": 0.0001830343312080704,
+      "loss": 0.5555,
       "step": 1700
     },
     {
-      "epoch": 0.3712105588781192,
-      "grad_norm": 0.2731375992298126,
-      "learning_rate": 0.00017824442734898997,
-      "loss": 0.4934,
+      "epoch": 0.3711340206185567,
+      "grad_norm": 0.3381674885749817,
+      "learning_rate": 0.0001782552148891283,
+      "loss": 0.5404,
       "step": 1800
     },
     {
-      "epoch": 0.39183336770468136,
-      "grad_norm": 0.4609294533729553,
-      "learning_rate": 0.00017295006435464848,
-      "loss": 0.4947,
+      "epoch": 0.3917525773195876,
+      "grad_norm": 0.3750503361225128,
+      "learning_rate": 0.00017296334073952605,
+      "loss": 0.5795,
       "step": 1900
     },
     {
-      "epoch": 0.41245617653124356,
-      "grad_norm": 0.4430903494358063,
-      "learning_rate": 0.0001671774585363957,
-      "loss": 0.4581,
+      "epoch": 0.41237113402061853,
+      "grad_norm": 0.45220425724983215,
+      "learning_rate": 0.00016719338312458124,
+      "loss": 0.5602,
       "step": 2000
     },
     {
-      "epoch": 0.43307898535780576,
-      "grad_norm": 0.39363059401512146,
-      "learning_rate": 0.00016096445368960415,
-      "loss": 0.4923,
+      "epoch": 0.4329896907216495,
+      "grad_norm": 0.43047139048576355,
+      "learning_rate": 0.00016098314899369446,
+      "loss": 0.5571,
       "step": 2100
     },
     {
-      "epoch": 0.4537017941843679,
-      "grad_norm": 0.3113062083721161,
-      "learning_rate": 0.000154351780758231,
-      "loss": 0.5157,
+      "epoch": 0.4536082474226804,
+      "grad_norm": 0.7484344840049744,
+      "learning_rate": 0.00015437333015488587,
+      "loss": 0.5526,
       "step": 2200
     },
     {
-      "epoch": 0.4743246030109301,
-      "grad_norm": 0.5598001480102539,
-      "learning_rate": 0.00014738279081268692,
-      "loss": 0.4948,
+      "epoch": 0.4742268041237113,
+      "grad_norm": 0.39876288175582886,
+      "learning_rate": 0.00014747851305720256,
+      "loss": 0.5495,
       "step": 2300
     },
     {
-      "epoch": 0.49494741183749225,
-      "grad_norm": 0.5283440351486206,
-      "learning_rate": 0.00014010317085079503,
-      "loss": 0.4905,
+      "epoch": 0.4948453608247423,
+      "grad_norm": 0.2668340802192688,
+      "learning_rate": 0.00014020466278830452,
+      "loss": 0.5603,
       "step": 2400
     },
     {
-      "epoch": 0.5155702206640544,
-      "grad_norm": 0.6192511916160583,
-      "learning_rate": 0.00013256064428497966,
-      "loss": 0.4947,
+      "epoch": 0.5154639175257731,
+      "grad_norm": 0.2944701611995697,
+      "learning_rate": 0.0001326673763292055,
+      "loss": 0.5728,
       "step": 2500
     },
     {
-      "epoch": 0.5361930294906166,
-      "grad_norm": 0.385551393032074,
-      "learning_rate": 0.00012480465807921773,
-      "loss": 0.5283,
+      "epoch": 0.5360824742268041,
+      "grad_norm": 0.5645434856414795,
+      "learning_rate": 0.0001249160408378004,
+      "loss": 0.552,
       "step": 2600
     },
     {
-      "epoch": 0.5568158383171788,
-      "grad_norm": 0.40230363607406616,
-      "learning_rate": 0.00011688605858680692,
-      "loss": 0.5069,
+      "epoch": 0.5567010309278351,
+      "grad_norm": 0.3050793707370758,
+      "learning_rate": 0.0001170014460023793,
+      "loss": 0.5735,
       "step": 2700
     },
     {
-      "epoch": 0.5774386471437409,
-      "grad_norm": 0.4681544005870819,
-      "learning_rate": 0.00010885675821407844,
-      "loss": 0.5028,
+      "epoch": 0.5773195876288659,
+      "grad_norm": 0.4726704955101013,
+      "learning_rate": 0.00010897545124833783,
+      "loss": 0.5672,
       "step": 2800
     },
     {
-      "epoch": 0.5980614559703031,
-      "grad_norm": 0.35956326127052307,
-      "learning_rate": 0.00010076939509532679,
-      "loss": 0.5358,
+      "epoch": 0.5979381443298969,
+      "grad_norm": 0.5890568494796753,
+      "learning_rate": 0.00010089064593556474,
+      "loss": 0.5377,
       "step": 2900
     },
     {
-      "epoch": 0.6186842647968653,
-      "grad_norm": 0.4002954065799713,
-      "learning_rate": 9.267698801004341e-05,
-      "loss": 0.4941,
+      "epoch": 0.6185567010309279,
+      "grad_norm": 0.33398109674453735,
+      "learning_rate": 9.280000477302173e-05,
+      "loss": 0.5624,
       "step": 3000
     },
     {
-      "epoch": 0.6393070736234275,
-      "grad_norm": 0.5405189990997314,
-      "learning_rate": 8.463258880473373e-05,
-      "loss": 0.5329,
+      "epoch": 0.6391752577319587,
+      "grad_norm": 0.43066734075546265,
+      "learning_rate": 8.48365683413172e-05,
+      "loss": 0.5712,
       "step": 3100
     },
     {
-      "epoch": 0.6599298824499897,
-      "grad_norm": 0.3197394609451294,
-      "learning_rate": 7.668893459795486e-05,
-      "loss": 0.5125,
+      "epoch": 0.6597938144329897,
+      "grad_norm": 0.41674667596817017,
+      "learning_rate": 7.689172737117389e-05,
+      "loss": 0.5458,
       "step": 3200
     },
     {
-      "epoch": 0.6805526912765518,
-      "grad_norm": 0.3258844017982483,
-      "learning_rate": 6.889810204863274e-05,
-      "loss": 0.5353,
+      "epoch": 0.6804123711340206,
+      "grad_norm": 0.4635225832462311,
+      "learning_rate": 6.909830056250527e-05,
+      "loss": 0.5449,
       "step": 3300
     },
     {
-      "epoch": 0.701175500103114,
-      "grad_norm": 0.4742829203605652,
-      "learning_rate": 6.131116595419178e-05,
-      "loss": 0.5276,
+      "epoch": 0.7010309278350515,
+      "grad_norm": 1.0104267597198486,
+      "learning_rate": 6.15073534018669e-05,
+      "loss": 0.5475,
       "step": 3400
     },
     {
-      "epoch": 0.7217983089296762,
-      "grad_norm": 0.2836833894252777,
-      "learning_rate": 5.397786441664373e-05,
-      "loss": 0.5076,
+      "epoch": 0.7216494845360825,
+      "grad_norm": 0.24354267120361328,
+      "learning_rate": 5.416862465241033e-05,
+      "loss": 0.516,
       "step": 3500
     },
     {
-      "epoch": 0.7424211177562384,
-      "grad_norm": 0.3171875476837158,
-      "learning_rate": 4.6946272771725984e-05,
-      "loss": 0.5163,
+      "epoch": 0.7422680412371134,
+      "grad_norm": 0.500800609588623,
+      "learning_rate": 4.7130200446653475e-05,
+      "loss": 0.5138,
       "step": 3600
     },
     {
-      "epoch": 0.7630439265828006,
-      "grad_norm": 0.24590329825878143,
-      "learning_rate": 4.026248841872946e-05,
-      "loss": 0.5259,
+      "epoch": 0.7628865979381443,
+      "grad_norm": 0.26592689752578735,
+      "learning_rate": 4.043819920791322e-05,
+      "loss": 0.5543,
       "step": 3700
     },
     {
-      "epoch": 0.7836667354093627,
-      "grad_norm": 0.6508501768112183,
-      "learning_rate": 3.397032861719556e-05,
-      "loss": 0.5589,
+      "epoch": 0.7835051546391752,
+      "grad_norm": 0.549505352973938,
+      "learning_rate": 3.4136469464914575e-05,
+      "loss": 0.5576,
       "step": 3800
     },
     {
-      "epoch": 0.8042895442359249,
-      "grad_norm": 0.4867592751979828,
-      "learning_rate": 2.811104323165301e-05,
-      "loss": 0.5646,
+      "epoch": 0.8041237113402062,
+      "grad_norm": 0.38464662432670593,
+      "learning_rate": 2.8266302539609745e-05,
+      "loss": 0.5578,
       "step": 3900
     },
     {
-      "epoch": 0.8249123530624871,
-      "grad_norm": 0.312364399433136,
-      "learning_rate": 2.2723044307569775e-05,
-      "loss": 0.5721,
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.5664867758750916,
+      "learning_rate": 2.2866161990785228e-05,
+      "loss": 0.5433,
       "step": 4000
     },
     {
-      "epoch": 0.8455351618890493,
-      "grad_norm": 0.4810425341129303,
-      "learning_rate": 1.7887852500751822e-05,
-      "loss": 0.5181,
+      "epoch": 0.845360824742268,
+      "grad_norm": 0.3998001217842102,
+      "learning_rate": 1.7971431586244815e-05,
+      "loss": 0.557,
       "step": 4100
     },
     {
-      "epoch": 0.8661579707156115,
-      "grad_norm": 0.3198936879634857,
-      "learning_rate": 1.3539539439376515e-05,
-      "loss": 0.5756,
+      "epoch": 0.865979381443299,
+      "grad_norm": 0.3722541332244873,
+      "learning_rate": 1.3614183454950824e-05,
+      "loss": 0.5021,
       "step": 4200
     },
     {
-      "epoch": 0.8867807795421736,
-      "grad_norm": 0.4786675274372101,
-      "learning_rate": 9.75804006323886e-06,
-      "loss": 0.5487,
+      "epoch": 0.8865979381443299,
+      "grad_norm": 0.7536567449569702,
+      "learning_rate": 9.822967938278171e-06,
+      "loss": 0.547,
       "step": 4300
     },
     {
-      "epoch": 0.9074035883687358,
-      "grad_norm": 0.4790056347846985,
-      "learning_rate": 6.568144959657263e-06,
-      "loss": 0.5506,
+      "epoch": 0.9072164948453608,
+      "grad_norm": 0.18647919595241547,
+      "learning_rate": 6.622626517355557e-06,
+      "loss": 0.5207,
       "step": 4400
     },
     {
-      "epoch": 0.928026397195298,
-      "grad_norm": 0.587523877620697,
-      "learning_rate": 4.013449264074187e-06,
-      "loss": 0.5741,
+      "epoch": 0.9278350515463918,
+      "grad_norm": 0.4656095504760742,
+      "learning_rate": 4.034129042265066e-06,
+      "loss": 0.5498,
       "step": 4500
     },
     {
-      "epoch": 0.9486492060218602,
-      "grad_norm": 0.41147923469543457,
-      "learning_rate": 2.059119419840494e-06,
-      "loss": 0.5414,
+      "epoch": 0.9484536082474226,
+      "grad_norm": 0.544163167476654,
+      "learning_rate": 2.074436329635687e-06,
+      "loss": 0.5319,
       "step": 4600
     },
     {
-      "epoch": 0.9692720148484224,
-      "grad_norm": 0.5501502752304077,
-      "learning_rate": 7.468660935561755e-07,
-      "loss": 0.6101,
+      "epoch": 0.9690721649484536,
+      "grad_norm": 0.3850492537021637,
+      "learning_rate": 7.563890289437825e-07,
+      "loss": 0.5569,
       "step": 4700
     },
     {
-      "epoch": 0.9898948236749845,
-      "grad_norm": 0.45654186606407166,
-      "learning_rate": 8.529209787123682e-08,
-      "loss": 0.5645,
+      "epoch": 0.9896907216494846,
+      "grad_norm": 0.5212803483009338,
+      "learning_rate": 8.862348571043733e-08,
+      "loss": 0.5224,
       "step": 4800
     }
   ],
   "logging_steps": 100,
-  "max_steps": 4849,
+  "max_steps": 4850,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -369,7 +369,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.3135703548790374e+17,
+  "total_flos": 2.346791597607813e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fcdb031000815991decb49e64c1ff52b41b328f05666ceaf823e770ffa5bde97
+oid sha256:9693ae99b42c29fd67ede275c9a27879ef5bb7e8261941cc89ec0f3ccdba0aaf
 size 5624