diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 4e-08,
-      "logits/chosen": -2.6092891693115234,
-      "logits/rejected": -2.629662275314331,
-      "logps/chosen": -260.39178466796875,
-      "logps/rejected": -256.2431945800781,
+      "logits/chosen": -2.602003574371338,
+      "logits/rejected": -2.6217830181121826,
+      "logps/chosen": -260.71484375,
+      "logps/rejected": -256.2999267578125,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,1939 +25,1939 @@
     {
       "epoch": 0.01,
       "learning_rate": 4.0000000000000003e-07,
-      "logits/chosen": -2.542802095413208,
-      "logits/rejected": -2.5701138973236084,
-      "logps/chosen": -287.18267822265625,
-      "logps/rejected": -308.441162109375,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.4166666567325592,
-      "rewards/chosen": 0.0005982329603284597,
-      "rewards/margins": -0.0005221270839683712,
-      "rewards/rejected": 0.0011203601025044918,
+      "logits/chosen": -2.5351831912994385,
+      "logits/rejected": -2.56242036819458,
+      "logps/chosen": -287.53594970703125,
+      "logps/rejected": -308.8571472167969,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0010520940413698554,
+      "rewards/margins": 0.0002557992411311716,
+      "rewards/rejected": -0.0013078933116048574,
       "step": 10
     },
     {
       "epoch": 0.02,
       "learning_rate": 8.000000000000001e-07,
-      "logits/chosen": -2.520120859146118,
-      "logits/rejected": -2.5234670639038086,
-      "logps/chosen": -255.07748413085938,
-      "logps/rejected": -273.0997009277344,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.00023935179342515767,
-      "rewards/margins": 0.00027643097564578056,
-      "rewards/rejected": -0.0005157828563824296,
+      "logits/chosen": -2.5120716094970703,
+      "logits/rejected": -2.515523672103882,
+      "logps/chosen": -255.38192749023438,
+      "logps/rejected": -273.31805419921875,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0002814549079630524,
+      "rewards/margins": -0.0005342562217265368,
+      "rewards/rejected": 0.0002528014301788062,
       "step": 20
     },
     {
       "epoch": 0.02,
       "learning_rate": 1.2000000000000002e-06,
-      "logits/chosen": -2.5063071250915527,
-      "logits/rejected": -2.5213770866394043,
-      "logps/chosen": -276.6528625488281,
-      "logps/rejected": -294.9043884277344,
+      "logits/chosen": -2.498893976211548,
+      "logits/rejected": -2.5145130157470703,
+      "logps/chosen": -276.9407958984375,
+      "logps/rejected": -295.25482177734375,
       "loss": 0.693,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.0008763422374613583,
-      "rewards/margins": 0.0003111693658865988,
-      "rewards/rejected": -0.001187511719763279,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0016518374904990196,
+      "rewards/margins": 0.00022423597692977637,
+      "rewards/rejected": -0.0018760733073577285,
       "step": 30
     },
     {
       "epoch": 0.03,
       "learning_rate": 1.6000000000000001e-06,
-      "logits/chosen": -2.542746067047119,
-      "logits/rejected": -2.549612522125244,
-      "logps/chosen": -251.32003784179688,
-      "logps/rejected": -259.71685791015625,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.004003011621534824,
-      "rewards/margins": 0.0007247755420394242,
-      "rewards/rejected": -0.004727787338197231,
+      "logits/chosen": -2.5344443321228027,
+      "logits/rejected": -2.541226625442505,
+      "logps/chosen": -251.64730834960938,
+      "logps/rejected": -259.96575927734375,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.005051327869296074,
+      "rewards/margins": -0.0006530196405947208,
+      "rewards/rejected": -0.004398307763040066,
       "step": 40
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0000000000000003e-06,
-      "logits/chosen": -2.4885387420654297,
-      "logits/rejected": -2.4705405235290527,
-      "logps/chosen": -251.05563354492188,
-      "logps/rejected": -255.7452392578125,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.006660168059170246,
-      "rewards/margins": -0.00038892580778338015,
-      "rewards/rejected": -0.006271242164075375,
+      "logits/chosen": -2.479356288909912,
+      "logits/rejected": -2.461836576461792,
+      "logps/chosen": -251.314697265625,
+      "logps/rejected": -256.02020263671875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.007234510965645313,
+      "rewards/margins": 0.00020712481637019664,
+      "rewards/rejected": -0.0074416352435946465,
       "step": 50
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.4000000000000003e-06,
-      "logits/chosen": -2.5540900230407715,
-      "logits/rejected": -2.5607895851135254,
-      "logps/chosen": -262.1423034667969,
-      "logps/rejected": -280.08209228515625,
-      "loss": 0.6928,
+      "logits/chosen": -2.5456085205078125,
+      "logits/rejected": -2.5527470111846924,
+      "logps/chosen": -262.48858642578125,
+      "logps/rejected": -280.4261474609375,
+      "loss": 0.6927,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.009707225486636162,
-      "rewards/margins": 0.000684750615619123,
-      "rewards/rejected": -0.010391974821686745,
+      "rewards/chosen": -0.011781791225075722,
+      "rewards/margins": 0.0009017615811899304,
+      "rewards/rejected": -0.012683553621172905,
       "step": 60
     },
     {
       "epoch": 0.06,
       "learning_rate": 2.8000000000000003e-06,
-      "logits/chosen": -2.5549094676971436,
-      "logits/rejected": -2.5427558422088623,
-      "logps/chosen": -266.07891845703125,
-      "logps/rejected": -271.27740478515625,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.014160098508000374,
-      "rewards/margins": 0.0011990683851763606,
-      "rewards/rejected": -0.015359166078269482,
+      "logits/chosen": -2.545548915863037,
+      "logits/rejected": -2.5333781242370605,
+      "logps/chosen": -266.43798828125,
+      "logps/rejected": -271.582763671875,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.015757283195853233,
+      "rewards/margins": -0.0007508035632781684,
+      "rewards/rejected": -0.015006482601165771,
       "step": 70
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.2000000000000003e-06,
-      "logits/chosen": -2.5750041007995605,
-      "logits/rejected": -2.5791172981262207,
-      "logps/chosen": -262.36407470703125,
-      "logps/rejected": -282.44720458984375,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.015319536440074444,
-      "rewards/margins": 0.0008405897533521056,
-      "rewards/rejected": -0.01616012677550316,
+      "logits/chosen": -2.5648863315582275,
+      "logits/rejected": -2.5693516731262207,
+      "logps/chosen": -262.5594787597656,
+      "logps/rejected": -282.7106018066406,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.014923855662345886,
+      "rewards/margins": 0.0014337189495563507,
+      "rewards/rejected": -0.016357574611902237,
       "step": 80
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6000000000000003e-06,
-      "logits/chosen": -2.5781426429748535,
-      "logits/rejected": -2.5637879371643066,
-      "logps/chosen": -295.14898681640625,
-      "logps/rejected": -300.16131591796875,
-      "loss": 0.6906,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.020102154463529587,
-      "rewards/margins": 0.005331115797162056,
-      "rewards/rejected": -0.025433272123336792,
+      "logits/chosen": -2.565471649169922,
+      "logits/rejected": -2.5509932041168213,
+      "logps/chosen": -295.49615478515625,
+      "logps/rejected": -300.31103515625,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.020387709140777588,
+      "rewards/margins": 0.004231990315020084,
+      "rewards/rejected": -0.024619702249765396,
       "step": 90
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.000000000000001e-06,
-      "logits/chosen": -2.5696635246276855,
-      "logits/rejected": -2.569899797439575,
-      "logps/chosen": -263.42047119140625,
-      "logps/rejected": -276.1015319824219,
-      "loss": 0.6917,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.024685552343726158,
-      "rewards/margins": 0.0029262492898851633,
-      "rewards/rejected": -0.027611803263425827,
+      "logits/chosen": -2.5530660152435303,
+      "logits/rejected": -2.553136110305786,
+      "logps/chosen": -263.5738220214844,
+      "logps/rejected": -276.2574157714844,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.02410706877708435,
+      "rewards/margins": 0.003276412608101964,
+      "rewards/rejected": -0.027383480221033096,
       "step": 100
     },
     {
       "epoch": 0.08,
-      "eval_logits/chosen": -2.530292272567749,
-      "eval_logits/rejected": -2.5272903442382812,
-      "eval_logps/chosen": -210.00074768066406,
-      "eval_logps/rejected": -215.47608947753906,
-      "eval_loss": 0.6923908591270447,
-      "eval_rewards/accuracies": 0.5286666750907898,
-      "eval_rewards/chosen": -0.016008317470550537,
-      "eval_rewards/margins": 0.001649469486437738,
-      "eval_rewards/rejected": -0.01765778660774231,
-      "eval_runtime": 1564.6229,
-      "eval_samples_per_second": 1.914,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.508636713027954,
+      "eval_logits/rejected": -2.5057852268218994,
+      "eval_logps/chosen": -210.22964477539062,
+      "eval_logps/rejected": -215.71873474121094,
+      "eval_loss": 0.6924654245376587,
+      "eval_rewards/accuracies": 0.527999997138977,
+      "eval_rewards/chosen": -0.01618967391550541,
+      "eval_rewards/margins": 0.0015068231150507927,
+      "eval_rewards/rejected": -0.017696497961878777,
+      "eval_runtime": 1581.5421,
+      "eval_samples_per_second": 1.893,
+      "eval_steps_per_second": 0.237,
       "step": 100
     },
     {
       "epoch": 0.09,
       "learning_rate": 4.4e-06,
-      "logits/chosen": -2.573535680770874,
-      "logits/rejected": -2.5764663219451904,
-      "logps/chosen": -283.31744384765625,
-      "logps/rejected": -293.5372314453125,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.027460157871246338,
-      "rewards/margins": 0.0029141721315681934,
-      "rewards/rejected": -0.030374327674508095,
+      "logits/chosen": -2.55187726020813,
+      "logits/rejected": -2.5549445152282715,
+      "logps/chosen": -283.41143798828125,
+      "logps/rejected": -293.6915588378906,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.026097621768712997,
+      "rewards/margins": 0.004130261484533548,
+      "rewards/rejected": -0.030227884650230408,
       "step": 110
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.800000000000001e-06,
-      "logits/chosen": -2.593564033508301,
-      "logits/rejected": -2.6072158813476562,
-      "logps/chosen": -265.5439758300781,
-      "logps/rejected": -282.11444091796875,
-      "loss": 0.6911,
+      "logits/chosen": -2.561483860015869,
+      "logits/rejected": -2.575721263885498,
+      "logps/chosen": -265.60711669921875,
+      "logps/rejected": -282.2021789550781,
+      "loss": 0.691,
       "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.03183265030384064,
-      "rewards/margins": 0.004463006276637316,
-      "rewards/rejected": -0.036295659840106964,
+      "rewards/chosen": -0.030240505933761597,
+      "rewards/margins": 0.004798956215381622,
+      "rewards/rejected": -0.03503946214914322,
       "step": 120
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999755876225375e-06,
-      "logits/chosen": -2.559654712677002,
-      "logits/rejected": -2.543823480606079,
-      "logps/chosen": -279.4471435546875,
-      "logps/rejected": -297.8502502441406,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.0379730649292469,
-      "rewards/margins": 0.008286955766379833,
-      "rewards/rejected": -0.04626002162694931,
+      "logits/chosen": -2.5272843837738037,
+      "logits/rejected": -2.511995792388916,
+      "logps/chosen": -279.7336120605469,
+      "logps/rejected": -298.2309875488281,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03772541880607605,
+      "rewards/margins": 0.009567076340317726,
+      "rewards/rejected": -0.047292497009038925,
       "step": 130
     },
     {
       "epoch": 0.11,
       "learning_rate": 4.997803172081864e-06,
-      "logits/chosen": -2.5811429023742676,
-      "logits/rejected": -2.581789255142212,
-      "logps/chosen": -274.745849609375,
-      "logps/rejected": -287.67303466796875,
-      "loss": 0.6865,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.04451751708984375,
-      "rewards/margins": 0.014026105403900146,
-      "rewards/rejected": -0.058543622493743896,
+      "logits/chosen": -2.549031972885132,
+      "logits/rejected": -2.550044536590576,
+      "logps/chosen": -274.913818359375,
+      "logps/rejected": -287.93389892578125,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0441165529191494,
+      "rewards/margins": 0.01394923496991396,
+      "rewards/rejected": -0.058065783232450485,
       "step": 140
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.9938992891651825e-06,
-      "logits/chosen": -2.545161724090576,
-      "logits/rejected": -2.5361251831054688,
-      "logps/chosen": -262.94903564453125,
-      "logps/rejected": -284.0696716308594,
-      "loss": 0.6833,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.0502193346619606,
-      "rewards/margins": 0.020780760794878006,
-      "rewards/rejected": -0.07100008428096771,
+      "logits/chosen": -2.511295795440674,
+      "logits/rejected": -2.5027806758880615,
+      "logps/chosen": -263.2046203613281,
+      "logps/rejected": -284.345703125,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05038716644048691,
+      "rewards/margins": 0.021908113732933998,
+      "rewards/rejected": -0.07229527831077576,
       "step": 150
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.988047277024456e-06,
-      "logits/chosen": -2.5855746269226074,
-      "logits/rejected": -2.5990371704101562,
-      "logps/chosen": -274.5702819824219,
-      "logps/rejected": -288.3578796386719,
-      "loss": 0.6819,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.07741515338420868,
-      "rewards/margins": 0.023703744634985924,
-      "rewards/rejected": -0.10111888498067856,
+      "logits/chosen": -2.551429271697998,
+      "logits/rejected": -2.5654749870300293,
+      "logps/chosen": -274.80670166015625,
+      "logps/rejected": -288.6151428222656,
+      "loss": 0.682,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07800472527742386,
+      "rewards/margins": 0.023449204862117767,
+      "rewards/rejected": -0.10145392268896103,
       "step": 160
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.980251707005417e-06,
-      "logits/chosen": -2.5723748207092285,
-      "logits/rejected": -2.547863006591797,
-      "logps/chosen": -291.48736572265625,
-      "logps/rejected": -298.7716979980469,
-      "loss": 0.6899,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.10616284608840942,
-      "rewards/margins": 0.007837113924324512,
-      "rewards/rejected": -0.11399996280670166,
+      "logits/chosen": -2.535489320755005,
+      "logits/rejected": -2.511200428009033,
+      "logps/chosen": -291.24853515625,
+      "logps/rejected": -298.6773376464844,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10200835764408112,
+      "rewards/margins": 0.008326217532157898,
+      "rewards/rejected": -0.11033459007740021,
       "step": 170
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.970518668679459e-06,
-      "logits/chosen": -2.5809338092803955,
-      "logits/rejected": -2.566509246826172,
-      "logps/chosen": -289.52642822265625,
-      "logps/rejected": -295.8729248046875,
-      "loss": 0.6849,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.09603477269411087,
-      "rewards/margins": 0.01864427700638771,
-      "rewards/rejected": -0.11467906087636948,
+      "logits/chosen": -2.540640354156494,
+      "logits/rejected": -2.5266213417053223,
+      "logps/chosen": -289.3337707519531,
+      "logps/rejected": -295.79278564453125,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.09090762585401535,
+      "rewards/margins": 0.020125988870859146,
+      "rewards/rejected": -0.1110336035490036,
       "step": 180
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.958855765086722e-06,
-      "logits/chosen": -2.608503818511963,
-      "logits/rejected": -2.6062302589416504,
-      "logps/chosen": -269.84283447265625,
-      "logps/rejected": -280.5111083984375,
-      "loss": 0.6851,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.05638309568166733,
-      "rewards/margins": 0.017940795049071312,
-      "rewards/rejected": -0.07432388514280319,
+      "logits/chosen": -2.563791036605835,
+      "logits/rejected": -2.562145233154297,
+      "logps/chosen": -269.8388671875,
+      "logps/rejected": -280.38037109375,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.05399826169013977,
+      "rewards/margins": 0.017181050032377243,
+      "rewards/rejected": -0.07117930799722672,
       "step": 190
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.945272106796919e-06,
-      "logits/chosen": -2.605029344558716,
-      "logits/rejected": -2.6143765449523926,
-      "logps/chosen": -270.00830078125,
-      "logps/rejected": -285.1636657714844,
-      "loss": 0.6854,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.06747416406869888,
-      "rewards/margins": 0.018329909071326256,
-      "rewards/rejected": -0.08580406755208969,
+      "logits/chosen": -2.5541298389434814,
+      "logits/rejected": -2.564225673675537,
+      "logps/chosen": -269.95001220703125,
+      "logps/rejected": -285.06451416015625,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06484891474246979,
+      "rewards/margins": 0.017776602879166603,
+      "rewards/rejected": -0.08262552320957184,
       "step": 200
     },
     {
       "epoch": 0.16,
-      "eval_logits/chosen": -2.57635235786438,
-      "eval_logits/rejected": -2.573589563369751,
-      "eval_logps/chosen": -215.42254638671875,
-      "eval_logps/rejected": -222.06101989746094,
-      "eval_loss": 0.6875421404838562,
-      "eval_rewards/accuracies": 0.5563333630561829,
-      "eval_rewards/chosen": -0.07022587954998016,
-      "eval_rewards/margins": 0.013281372375786304,
-      "eval_rewards/rejected": -0.08350725471973419,
-      "eval_runtime": 1564.649,
-      "eval_samples_per_second": 1.914,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.517770528793335,
+      "eval_logits/rejected": -2.515197277069092,
+      "eval_logps/chosen": -215.1240234375,
+      "eval_logps/rejected": -221.67100524902344,
+      "eval_loss": 0.6880091428756714,
+      "eval_rewards/accuracies": 0.5613333582878113,
+      "eval_rewards/chosen": -0.06513328850269318,
+      "eval_rewards/margins": 0.012086095288395882,
+      "eval_rewards/rejected": -0.07721938192844391,
+      "eval_runtime": 1599.371,
+      "eval_samples_per_second": 1.872,
+      "eval_steps_per_second": 0.234,
       "step": 200
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.929778304792537e-06,
-      "logits/chosen": -2.594109058380127,
-      "logits/rejected": -2.5997750759124756,
-      "logps/chosen": -294.9931640625,
-      "logps/rejected": -299.47515869140625,
-      "loss": 0.6783,
+      "logits/chosen": -2.5422654151916504,
+      "logits/rejected": -2.5487327575683594,
+      "logps/chosen": -294.87188720703125,
+      "logps/rejected": -299.1329040527344,
+      "loss": 0.6792,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.08973054587841034,
-      "rewards/margins": 0.03516639396548271,
-      "rewards/rejected": -0.12489694356918335,
+      "rewards/chosen": -0.08577840030193329,
+      "rewards/margins": 0.03281060978770256,
+      "rewards/rejected": -0.11858902126550674,
       "step": 210
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.912386462179987e-06,
-      "logits/chosen": -2.6223044395446777,
-      "logits/rejected": -2.6194934844970703,
-      "logps/chosen": -282.9168701171875,
-      "logps/rejected": -308.5869140625,
-      "loss": 0.6757,
+      "logits/chosen": -2.56231951713562,
+      "logits/rejected": -2.559842348098755,
+      "logps/chosen": -283.178955078125,
+      "logps/rejected": -308.49322509765625,
+      "loss": 0.6772,
       "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.08990589529275894,
-      "rewards/margins": 0.039798181504011154,
-      "rewards/rejected": -0.1297040730714798,
+      "rewards/chosen": -0.09053866565227509,
+      "rewards/margins": 0.03568854182958603,
+      "rewards/rejected": -0.1262272149324417,
       "step": 220
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.893110164735167e-06,
-      "logits/chosen": -2.686192512512207,
-      "logits/rejected": -2.6861042976379395,
-      "logps/chosen": -288.02984619140625,
-      "logps/rejected": -301.3773193359375,
-      "loss": 0.6819,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.11503495275974274,
-      "rewards/margins": 0.02641122415661812,
-      "rewards/rejected": -0.14144618809223175,
+      "logits/chosen": -2.5928561687469482,
+      "logits/rejected": -2.5934693813323975,
+      "logps/chosen": -288.0644226074219,
+      "logps/rejected": -301.1351013183594,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.11280493438243866,
+      "rewards/margins": 0.023493468761444092,
+      "rewards/rejected": -0.13629841804504395,
       "step": 230
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.871964470290823e-06,
-      "logits/chosen": -2.660379409790039,
-      "logits/rejected": -2.6843981742858887,
-      "logps/chosen": -289.6697998046875,
-      "logps/rejected": -310.08245849609375,
-      "loss": 0.6712,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.12107042968273163,
-      "rewards/margins": 0.05008827522397041,
-      "rewards/rejected": -0.17115873098373413,
+      "logits/chosen": -2.532960891723633,
+      "logits/rejected": -2.55849027633667,
+      "logps/chosen": -289.32354736328125,
+      "logps/rejected": -309.2322692871094,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.11586654186248779,
+      "rewards/margins": 0.04449574649333954,
+      "rewards/rejected": -0.16036227345466614,
       "step": 240
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.848965896974006e-06,
-      "logits/chosen": -2.6900529861450195,
-      "logits/rejected": -2.691603183746338,
-      "logps/chosen": -281.68792724609375,
-      "logps/rejected": -301.64129638671875,
-      "loss": 0.6784,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.1229969710111618,
-      "rewards/margins": 0.03473496437072754,
-      "rewards/rejected": -0.15773193538188934,
+      "logits/chosen": -2.541764259338379,
+      "logits/rejected": -2.545576572418213,
+      "logps/chosen": -281.2950439453125,
+      "logps/rejected": -300.70416259765625,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.11654965579509735,
+      "rewards/margins": 0.02992970123887062,
+      "rewards/rejected": -0.14647935330867767,
       "step": 250
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.8241324103028055e-06,
-      "logits/chosen": -2.794003963470459,
-      "logits/rejected": -2.756747007369995,
-      "logps/chosen": -288.9064025878906,
-      "logps/rejected": -300.8970947265625,
-      "loss": 0.6737,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.1542636603116989,
-      "rewards/margins": 0.044557444751262665,
-      "rewards/rejected": -0.19882109761238098,
+      "logits/chosen": -2.5795302391052246,
+      "logits/rejected": -2.5413551330566406,
+      "logps/chosen": -287.0888977050781,
+      "logps/rejected": -298.43524169921875,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1340574324131012,
+      "rewards/margins": 0.03809996321797371,
+      "rewards/rejected": -0.1721573919057846,
       "step": 260
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.797483409152438e-06,
-      "logits/chosen": -2.8427796363830566,
-      "logits/rejected": -2.8422060012817383,
-      "logps/chosen": -280.02099609375,
-      "logps/rejected": -300.798583984375,
-      "loss": 0.667,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.17782548069953918,
-      "rewards/margins": 0.0627431869506836,
-      "rewards/rejected": -0.24056866765022278,
+      "logits/chosen": -2.565472364425659,
+      "logits/rejected": -2.569817066192627,
+      "logps/chosen": -276.9258728027344,
+      "logps/rejected": -297.29931640625,
+      "loss": 0.668,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14538586139678955,
+      "rewards/margins": 0.05918685719370842,
+      "rewards/rejected": -0.20457272231578827,
       "step": 270
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.769039710601669e-06,
-      "logits/chosen": -2.926100015640259,
-      "logits/rejected": -2.9281368255615234,
-      "logps/chosen": -280.8794250488281,
-      "logps/rejected": -298.9492492675781,
-      "loss": 0.6732,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.2086031138896942,
-      "rewards/margins": 0.047397904098033905,
-      "rewards/rejected": -0.25600099563598633,
+      "logits/chosen": -2.5913045406341553,
+      "logits/rejected": -2.592714786529541,
+      "logps/chosen": -277.0105285644531,
+      "logps/rejected": -293.9090576171875,
+      "loss": 0.678,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.16800367832183838,
+      "rewards/margins": 0.03504541888833046,
+      "rewards/rejected": -0.20304909348487854,
       "step": 280
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.738823533671383e-06,
-      "logits/chosen": -2.972533702850342,
-      "logits/rejected": -2.9721083641052246,
-      "logps/chosen": -306.55865478515625,
-      "logps/rejected": -322.1519470214844,
-      "loss": 0.6769,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.24026934802532196,
-      "rewards/margins": 0.041927777230739594,
-      "rewards/rejected": -0.28219714760780334,
+      "logits/chosen": -2.59189510345459,
+      "logits/rejected": -2.5966787338256836,
+      "logps/chosen": -300.8684387207031,
+      "logps/rejected": -316.1703186035156,
+      "loss": 0.677,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.18113958835601807,
+      "rewards/margins": 0.0392177477478981,
+      "rewards/rejected": -0.22035734355449677,
       "step": 290
     },
     {
       "epoch": 0.24,
       "learning_rate": 4.706858481968017e-06,
-      "logits/chosen": -2.922761917114258,
-      "logits/rejected": -2.9264841079711914,
-      "logps/chosen": -295.0879821777344,
-      "logps/rejected": -304.8949279785156,
-      "loss": 0.682,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.2886089086532593,
-      "rewards/margins": 0.03215286135673523,
-      "rewards/rejected": -0.3207617700099945,
+      "logits/chosen": -2.5426743030548096,
+      "logits/rejected": -2.546731472015381,
+      "logps/chosen": -288.5105895996094,
+      "logps/rejected": -297.8955993652344,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.22026526927947998,
+      "rewards/margins": 0.027653247117996216,
+      "rewards/rejected": -0.2479185312986374,
       "step": 300
     },
     {
       "epoch": 0.24,
-      "eval_logits/chosen": -2.920912981033325,
-      "eval_logits/rejected": -2.9179892539978027,
-      "eval_logps/chosen": -232.2801055908203,
-      "eval_logps/rejected": -240.21971130371094,
-      "eval_loss": 0.684112012386322,
-      "eval_rewards/accuracies": 0.5450000166893005,
-      "eval_rewards/chosen": -0.23880183696746826,
-      "eval_rewards/margins": 0.026292217895388603,
-      "eval_rewards/rejected": -0.2650940418243408,
-      "eval_runtime": 1563.5865,
-      "eval_samples_per_second": 1.915,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.519216537475586,
+      "eval_logits/rejected": -2.517465591430664,
+      "eval_logps/chosen": -227.3457489013672,
+      "eval_logps/rejected": -234.75460815429688,
+      "eval_loss": 0.6853793859481812,
+      "eval_rewards/accuracies": 0.5473333597183228,
+      "eval_rewards/chosen": -0.18735063076019287,
+      "eval_rewards/margins": 0.02070464752614498,
+      "eval_rewards/rejected": -0.2080552726984024,
+      "eval_runtime": 1584.577,
+      "eval_samples_per_second": 1.889,
+      "eval_steps_per_second": 0.237,
       "step": 300
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.673169525245416e-06,
-      "logits/chosen": -2.912137269973755,
-      "logits/rejected": -2.890232563018799,
-      "logps/chosen": -291.8182678222656,
-      "logps/rejected": -320.7854309082031,
-      "loss": 0.6652,
+      "logits/chosen": -2.534403085708618,
+      "logits/rejected": -2.514641284942627,
+      "logps/chosen": -286.3177185058594,
+      "logps/rejected": -313.84521484375,
+      "loss": 0.6698,
       "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.257565438747406,
-      "rewards/margins": 0.06874646991491318,
-      "rewards/rejected": -0.3263119161128998,
+      "rewards/chosen": -0.19986188411712646,
+      "rewards/margins": 0.05483303219079971,
+      "rewards/rejected": -0.2546949088573456,
       "step": 310
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.63778297989952e-06,
-      "logits/chosen": -3.007861375808716,
-      "logits/rejected": -2.9914684295654297,
-      "logps/chosen": -297.81695556640625,
-      "logps/rejected": -314.2324523925781,
-      "loss": 0.6678,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.28555119037628174,
-      "rewards/margins": 0.06271664053201675,
-      "rewards/rejected": -0.3482677936553955,
+      "logits/chosen": -2.5624048709869385,
+      "logits/rejected": -2.54682993888855,
+      "logps/chosen": -292.1237487792969,
+      "logps/rejected": -307.5107116699219,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2259463369846344,
+      "rewards/margins": 0.05093250423669815,
+      "rewards/rejected": -0.27687883377075195,
       "step": 320
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.60072648841109e-06,
-      "logits/chosen": -3.0622572898864746,
-      "logits/rejected": -3.0570547580718994,
-      "logps/chosen": -311.02105712890625,
-      "logps/rejected": -333.46630859375,
-      "loss": 0.6444,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.3358006477355957,
-      "rewards/margins": 0.11871840804815292,
-      "rewards/rejected": -0.454519122838974,
+      "logits/chosen": -2.540750741958618,
+      "logits/rejected": -2.536940574645996,
+      "logps/chosen": -302.9012756347656,
+      "logps/rejected": -322.0403747558594,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.25178760290145874,
+      "rewards/margins": 0.08651792258024216,
+      "rewards/rejected": -0.3383055329322815,
       "step": 330
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.562028997752574e-06,
-      "logits/chosen": -3.153225898742676,
-      "logits/rejected": -3.1641342639923096,
-      "logps/chosen": -311.9708557128906,
-      "logps/rejected": -338.3099670410156,
-      "loss": 0.6682,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.3542831838130951,
-      "rewards/margins": 0.068769671022892,
-      "rewards/rejected": -0.4230528473854065,
+      "logits/chosen": -2.5020062923431396,
+      "logits/rejected": -2.525498867034912,
+      "logps/chosen": -299.34295654296875,
+      "logps/rejected": -324.51239013671875,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.22524788975715637,
+      "rewards/margins": 0.05736035853624344,
+      "rewards/rejected": -0.2826082408428192,
       "step": 340
     },
     {
       "epoch": 0.28,
       "learning_rate": 4.521720736775947e-06,
-      "logits/chosen": -3.2916831970214844,
-      "logits/rejected": -3.3004555702209473,
-      "logps/chosen": -328.3070373535156,
-      "logps/rejected": -338.16192626953125,
-      "loss": 0.6564,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.4777124524116516,
-      "rewards/margins": 0.09214212745428085,
-      "rewards/rejected": -0.5698545575141907,
+      "logits/chosen": -2.551377534866333,
+      "logits/rejected": -2.549631118774414,
+      "logps/chosen": -312.30010986328125,
+      "logps/rejected": -318.33203125,
+      "loss": 0.6713,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.31575754284858704,
+      "rewards/margins": 0.05324278026819229,
+      "rewards/rejected": -0.36900031566619873,
       "step": 350
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.479833192599198e-06,
-      "logits/chosen": -3.4253318309783936,
-      "logits/rejected": -3.4066455364227295,
-      "logps/chosen": -329.5577697753906,
-      "logps/rejected": -350.79742431640625,
-      "loss": 0.6555,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.46095147728919983,
-      "rewards/margins": 0.09699266403913498,
-      "rewards/rejected": -0.5579441785812378,
+      "logits/chosen": -2.6122488975524902,
+      "logits/rejected": -2.597867488861084,
+      "logps/chosen": -310.14825439453125,
+      "logps/rejected": -329.4725646972656,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.26400071382522583,
+      "rewards/margins": 0.07782687991857529,
+      "rewards/rejected": -0.3418275713920593,
       "step": 360
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.436399086009928e-06,
-      "logits/chosen": -3.4043822288513184,
-      "logits/rejected": -3.3838889598846436,
-      "logps/chosen": -317.8493957519531,
-      "logps/rejected": -334.6202087402344,
-      "loss": 0.6576,
+      "logits/chosen": -2.592072010040283,
+      "logits/rejected": -2.582606554031372,
+      "logps/chosen": -298.0146789550781,
+      "logps/rejected": -312.88323974609375,
+      "loss": 0.6639,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.4469628930091858,
-      "rewards/margins": 0.08755813539028168,
-      "rewards/rejected": -0.5345209836959839,
+      "rewards/chosen": -0.24674613773822784,
+      "rewards/margins": 0.06771402060985565,
+      "rewards/rejected": -0.3144601583480835,
       "step": 370
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.391452345905239e-06,
-      "logits/chosen": -3.347275972366333,
-      "logits/rejected": -3.35723614692688,
-      "logps/chosen": -325.1240234375,
-      "logps/rejected": -339.3655090332031,
-      "loss": 0.6618,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.4934774935245514,
-      "rewards/margins": 0.09260674566030502,
-      "rewards/rejected": -0.586084246635437,
+      "logits/chosen": -2.501129150390625,
+      "logits/rejected": -2.5144436359405518,
+      "logps/chosen": -302.98785400390625,
+      "logps/rejected": -315.341552734375,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2700803875923157,
+      "rewards/margins": 0.07336559146642685,
+      "rewards/rejected": -0.3434459865093231,
       "step": 380
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.3450280827879125e-06,
-      "logits/chosen": -3.4295692443847656,
-      "logits/rejected": -3.4535210132598877,
-      "logps/chosen": -323.19415283203125,
-      "logps/rejected": -340.07989501953125,
-      "loss": 0.6639,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.4877980351448059,
-      "rewards/margins": 0.08489425480365753,
-      "rewards/rejected": -0.5726922154426575,
+      "logits/chosen": -2.5604705810546875,
+      "logits/rejected": -2.5804944038391113,
+      "logps/chosen": -301.7215881347656,
+      "logps/rejected": -316.622314453125,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.2707824110984802,
+      "rewards/margins": 0.06462465226650238,
+      "rewards/rejected": -0.3354070484638214,
       "step": 390
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.297162561339554e-06,
-      "logits/chosen": -3.366345167160034,
-      "logits/rejected": -3.3470451831817627,
-      "logps/chosen": -330.50311279296875,
-      "logps/rejected": -352.4704895019531,
-      "loss": 0.6634,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.5246509313583374,
-      "rewards/margins": 0.09757552295923233,
-      "rewards/rejected": -0.6222264766693115,
+      "logits/chosen": -2.5028789043426514,
+      "logits/rejected": -2.5059926509857178,
+      "logps/chosen": -308.93524169921875,
+      "logps/rejected": -328.21649169921875,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.30648913979530334,
+      "rewards/margins": 0.0709182545542717,
+      "rewards/rejected": -0.37740740180015564,
       "step": 400
     },
     {
       "epoch": 0.32,
-      "eval_logits/chosen": -3.460310935974121,
-      "eval_logits/rejected": -3.4548885822296143,
-      "eval_logps/chosen": -256.72369384765625,
-      "eval_logps/rejected": -266.585693359375,
-      "eval_loss": 0.681212306022644,
-      "eval_rewards/accuracies": 0.5486666560173035,
-      "eval_rewards/chosen": -0.4832378029823303,
-      "eval_rewards/margins": 0.04551612213253975,
-      "eval_rewards/rejected": -0.528753936290741,
-      "eval_runtime": 1563.8204,
-      "eval_samples_per_second": 1.915,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.536813497543335,
+      "eval_logits/rejected": -2.534651756286621,
+      "eval_logps/chosen": -237.7042236328125,
+      "eval_logps/rejected": -246.16815185546875,
+      "eval_loss": 0.68265300989151,
+      "eval_rewards/accuracies": 0.5476666688919067,
+      "eval_rewards/chosen": -0.2909356355667114,
+      "eval_rewards/margins": 0.031255174428224564,
+      "eval_rewards/rejected": -0.3221907913684845,
+      "eval_runtime": 1589.7925,
+      "eval_samples_per_second": 1.883,
+      "eval_steps_per_second": 0.236,
       "step": 400
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.247893172092157e-06,
-      "logits/chosen": -3.3930275440216064,
-      "logits/rejected": -3.3995296955108643,
-      "logps/chosen": -320.64154052734375,
-      "logps/rejected": -351.81414794921875,
-      "loss": 0.6576,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.5477547645568848,
-      "rewards/margins": 0.10663916915655136,
-      "rewards/rejected": -0.6543939709663391,
+      "logits/chosen": -2.531207799911499,
+      "logits/rejected": -2.548520565032959,
+      "logps/chosen": -297.398681640625,
+      "logps/rejected": -326.1677551269531,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.313217431306839,
+      "rewards/margins": 0.08246447145938873,
+      "rewards/rejected": -0.3956819176673889,
       "step": 410
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.197258402220187e-06,
-      "logits/chosen": -3.4664406776428223,
-      "logits/rejected": -3.4772841930389404,
-      "logps/chosen": -328.4302673339844,
-      "logps/rejected": -347.1799011230469,
-      "loss": 0.6516,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.5463472604751587,
-      "rewards/margins": 0.14043554663658142,
-      "rewards/rejected": -0.686782717704773,
+      "logits/chosen": -2.5356483459472656,
+      "logits/rejected": -2.544675350189209,
+      "logps/chosen": -305.0682373046875,
+      "logps/rejected": -318.51068115234375,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.31003984808921814,
+      "rewards/margins": 0.08686444163322449,
+      "rewards/rejected": -0.39690425992012024,
       "step": 420
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.145297805476023e-06,
-      "logits/chosen": -3.5236573219299316,
-      "logits/rejected": -3.5294857025146484,
-      "logps/chosen": -326.1194763183594,
-      "logps/rejected": -350.4049987792969,
-      "loss": 0.6477,
+      "logits/chosen": -2.5800435543060303,
+      "logits/rejected": -2.580294132232666,
+      "logps/chosen": -302.9384765625,
+      "logps/rejected": -323.58026123046875,
+      "loss": 0.6537,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.5520918965339661,
-      "rewards/margins": 0.13327525556087494,
-      "rewards/rejected": -0.685367226600647,
+      "rewards/chosen": -0.31787100434303284,
+      "rewards/margins": 0.09609754383563995,
+      "rewards/rejected": -0.413968563079834,
       "step": 430
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.092051971292228e-06,
-      "logits/chosen": -3.597280502319336,
-      "logits/rejected": -3.597381591796875,
-      "logps/chosen": -324.05584716796875,
-      "logps/rejected": -346.5114440917969,
-      "loss": 0.654,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.5706614255905151,
-      "rewards/margins": 0.11499401181936264,
-      "rewards/rejected": -0.6856554746627808,
+      "logits/chosen": -2.5852980613708496,
+      "logits/rejected": -2.595292806625366,
+      "logps/chosen": -300.8592224121094,
+      "logps/rejected": -319.3016662597656,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.3366487920284271,
+      "rewards/margins": 0.07542940974235535,
+      "rewards/rejected": -0.41207820177078247,
       "step": 440
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.037562493074792e-06,
-      "logits/chosen": -3.742100238800049,
-      "logits/rejected": -3.7566616535186768,
-      "logps/chosen": -349.8996887207031,
-      "logps/rejected": -364.50506591796875,
-      "loss": 0.6477,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.6562848091125488,
-      "rewards/margins": 0.1411934792995453,
-      "rewards/rejected": -0.7974783182144165,
+      "logits/chosen": -2.5786662101745605,
+      "logits/rejected": -2.5908896923065186,
+      "logps/chosen": -317.9658203125,
+      "logps/rejected": -327.48614501953125,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.33508211374282837,
+      "rewards/margins": 0.08991469442844391,
+      "rewards/rejected": -0.4249967634677887,
       "step": 450
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.981871935712112e-06,
-      "logits/chosen": -3.968726396560669,
-      "logits/rejected": -3.941209077835083,
-      "logps/chosen": -332.73443603515625,
-      "logps/rejected": -360.47625732421875,
-      "loss": 0.6493,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.6563684344291687,
-      "rewards/margins": 0.13294601440429688,
-      "rewards/rejected": -0.7893144488334656,
+      "logits/chosen": -2.6687827110290527,
+      "logits/rejected": -2.6695172786712646,
+      "logps/chosen": -296.9199523925781,
+      "logps/rejected": -318.85943603515625,
+      "loss": 0.666,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.2961326241493225,
+      "rewards/margins": 0.07453998178243637,
+      "rewards/rejected": -0.3706726133823395,
       "step": 460
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.925023802325094e-06,
-      "logits/chosen": -4.149798393249512,
-      "logits/rejected": -4.129707336425781,
-      "logps/chosen": -354.22802734375,
-      "logps/rejected": -380.85040283203125,
-      "loss": 0.6505,
+      "logits/chosen": -2.6613316535949707,
+      "logits/rejected": -2.6546504497528076,
+      "logps/chosen": -305.7056884765625,
+      "logps/rejected": -325.8073425292969,
+      "loss": 0.6564,
       "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.7849351763725281,
-      "rewards/margins": 0.16557727754116058,
-      "rewards/rejected": -0.9505125284194946,
+      "rewards/chosen": -0.2978198826313019,
+      "rewards/margins": 0.09947486966848373,
+      "rewards/rejected": -0.3972947299480438,
       "step": 470
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.867062500284342e-06,
-      "logits/chosen": -4.189931392669678,
-      "logits/rejected": -4.1646270751953125,
-      "logps/chosen": -344.6732482910156,
-      "logps/rejected": -376.7868957519531,
-      "loss": 0.647,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.8410800099372864,
-      "rewards/margins": 0.15282076597213745,
-      "rewards/rejected": -0.9939007759094238,
+      "logits/chosen": -2.6371190547943115,
+      "logits/rejected": -2.631121873855591,
+      "logps/chosen": -297.20257568359375,
+      "logps/rejected": -324.0657653808594,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.36376848816871643,
+      "rewards/margins": 0.09993237257003784,
+      "rewards/rejected": -0.4637008607387543,
       "step": 480
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8080333065209885e-06,
-      "logits/chosen": -4.1885223388671875,
-      "logits/rejected": -4.1956305503845215,
-      "logps/chosen": -351.62237548828125,
-      "logps/rejected": -351.5168151855469,
-      "loss": 0.6998,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.8627797961235046,
-      "rewards/margins": 0.05857674032449722,
-      "rewards/rejected": -0.9213564991950989,
+      "logits/chosen": -2.6479365825653076,
+      "logits/rejected": -2.646528720855713,
+      "logps/chosen": -306.7449035644531,
+      "logps/rejected": -303.7771301269531,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.41228657960891724,
+      "rewards/margins": 0.029567983001470566,
+      "rewards/rejected": -0.4418545663356781,
       "step": 490
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7479823321582624e-06,
-      "logits/chosen": -4.04768705368042,
-      "logits/rejected": -4.022109031677246,
-      "logps/chosen": -345.1391296386719,
-      "logps/rejected": -380.88482666015625,
-      "loss": 0.6296,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.7372187972068787,
-      "rewards/margins": 0.17785954475402832,
-      "rewards/rejected": -0.915078341960907,
+      "logits/chosen": -2.632742404937744,
+      "logits/rejected": -2.633615016937256,
+      "logps/chosen": -306.31195068359375,
+      "logps/rejected": -336.17388916015625,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.34735265374183655,
+      "rewards/margins": 0.11751687526702881,
+      "rewards/rejected": -0.46486955881118774,
       "step": 500
     },
     {
       "epoch": 0.4,
-      "eval_logits/chosen": -4.174871921539307,
-      "eval_logits/rejected": -4.1668291091918945,
-      "eval_logps/chosen": -277.3629150390625,
-      "eval_logps/rejected": -289.35430908203125,
-      "eval_loss": 0.678155779838562,
-      "eval_rewards/accuracies": 0.5600000023841858,
-      "eval_rewards/chosen": -0.6896299123764038,
-      "eval_rewards/margins": 0.06681026518344879,
-      "eval_rewards/rejected": -0.7564401626586914,
-      "eval_runtime": 1565.1126,
-      "eval_samples_per_second": 1.913,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.6363847255706787,
+      "eval_logits/rejected": -2.6328208446502686,
+      "eval_logps/chosen": -245.54345703125,
+      "eval_logps/rejected": -254.98521423339844,
+      "eval_loss": 0.6804653406143188,
+      "eval_rewards/accuracies": 0.5566666722297668,
+      "eval_rewards/chosen": -0.3693278133869171,
+      "eval_rewards/margins": 0.0410337969660759,
+      "eval_rewards/rejected": -0.41036155819892883,
+      "eval_runtime": 1591.6944,
+      "eval_samples_per_second": 1.881,
+      "eval_steps_per_second": 0.236,
       "step": 500
     },
     {
       "epoch": 0.41,
       "learning_rate": 3.686956486491419e-06,
-      "logits/chosen": -4.035499572753906,
-      "logits/rejected": -4.026083469390869,
-      "logps/chosen": -354.2256774902344,
-      "logps/rejected": -383.2754821777344,
-      "loss": 0.642,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.7423269748687744,
-      "rewards/margins": 0.21402092278003693,
-      "rewards/rejected": -0.9563478231430054,
+      "logits/chosen": -2.6169426441192627,
+      "logits/rejected": -2.6043953895568848,
+      "logps/chosen": -318.00299072265625,
+      "logps/rejected": -336.43048095703125,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.37818408012390137,
+      "rewards/margins": 0.10718680918216705,
+      "rewards/rejected": -0.485370934009552,
       "step": 510
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.625003440344166e-06,
-      "logits/chosen": -4.110401630401611,
-      "logits/rejected": -4.130579948425293,
-      "logps/chosen": -333.2506103515625,
-      "logps/rejected": -345.42529296875,
-      "loss": 0.6662,
+      "logits/chosen": -2.638087034225464,
+      "logits/rejected": -2.6651127338409424,
+      "logps/chosen": -298.8196105957031,
+      "logps/rejected": -309.6526794433594,
+      "loss": 0.6669,
       "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.6982457637786865,
-      "rewards/margins": 0.08196356892585754,
-      "rewards/rejected": -0.7802093625068665,
+      "rewards/chosen": -0.3525494933128357,
+      "rewards/margins": 0.06792327761650085,
+      "rewards/rejected": -0.42047280073165894,
       "step": 520
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.562171588830231e-06,
-      "logits/chosen": -4.092467308044434,
-      "logits/rejected": -4.074677467346191,
-      "logps/chosen": -342.26513671875,
-      "logps/rejected": -370.6748352050781,
-      "loss": 0.6627,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.6801224946975708,
-      "rewards/margins": 0.11174317449331284,
-      "rewards/rejected": -0.7918656468391418,
+      "logits/chosen": -2.682291269302368,
+      "logits/rejected": -2.690966844558716,
+      "logps/chosen": -308.031005859375,
+      "logps/rejected": -333.34600830078125,
+      "loss": 0.664,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.3355502486228943,
+      "rewards/margins": 0.08054343611001968,
+      "rewards/rejected": -0.416093647480011,
       "step": 530
     },
     {
       "epoch": 0.43,
       "learning_rate": 3.4985100135491245e-06,
-      "logits/chosen": -4.104280471801758,
-      "logits/rejected": -4.07126522064209,
-      "logps/chosen": -353.3872375488281,
-      "logps/rejected": -391.28289794921875,
-      "loss": 0.6383,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.7574983239173889,
-      "rewards/margins": 0.18485240638256073,
-      "rewards/rejected": -0.9423506855964661,
+      "logits/chosen": -2.6738035678863525,
+      "logits/rejected": -2.665532112121582,
+      "logps/chosen": -317.4346008300781,
+      "logps/rejected": -350.239990234375,
+      "loss": 0.6401,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.3944160044193268,
+      "rewards/margins": 0.13535484671592712,
+      "rewards/rejected": -0.5297707915306091,
       "step": 540
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4340684442456673e-06,
-      "logits/chosen": -4.108157634735107,
-      "logits/rejected": -4.100740432739258,
-      "logps/chosen": -353.9938049316406,
-      "logps/rejected": -375.70562744140625,
-      "loss": 0.6498,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.8029891848564148,
-      "rewards/margins": 0.12438172101974487,
-      "rewards/rejected": -0.9273708462715149,
+      "logits/chosen": -2.634321689605713,
+      "logits/rejected": -2.6402688026428223,
+      "logps/chosen": -321.502197265625,
+      "logps/rejected": -339.1592712402344,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.47532662749290466,
+      "rewards/margins": 0.08404591679573059,
+      "rewards/rejected": -0.5593725442886353,
       "step": 550
     },
     {
       "epoch": 0.45,
       "learning_rate": 3.3688972199631974e-06,
-      "logits/chosen": -4.0825605392456055,
-      "logits/rejected": -4.094653129577637,
-      "logps/chosen": -352.5472412109375,
-      "logps/rejected": -372.22125244140625,
-      "loss": 0.646,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.7980318069458008,
-      "rewards/margins": 0.1836821585893631,
-      "rewards/rejected": -0.9817139506340027,
+      "logits/chosen": -2.6393518447875977,
+      "logits/rejected": -2.6311287879943848,
+      "logps/chosen": -318.1193542480469,
+      "logps/rejected": -333.4119873046875,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.45164427161216736,
+      "rewards/margins": 0.13988950848579407,
+      "rewards/rejected": -0.5915338397026062,
       "step": 560
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.3030472497208354e-06,
-      "logits/chosen": -4.089273452758789,
-      "logits/rejected": -4.051713943481445,
-      "logps/chosen": -349.65447998046875,
-      "logps/rejected": -411.3336486816406,
-      "loss": 0.6145,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.8661645650863647,
-      "rewards/margins": 0.2521691918373108,
-      "rewards/rejected": -1.1183335781097412,
+      "logits/chosen": -2.6581737995147705,
+      "logits/rejected": -2.6506381034851074,
+      "logps/chosen": -315.7310485839844,
+      "logps/rejected": -369.3522033691406,
+      "loss": 0.6318,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5240504741668701,
+      "rewards/margins": 0.17177370190620422,
+      "rewards/rejected": -0.695824146270752,
       "step": 570
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.236569972745492e-06,
-      "logits/chosen": -4.18710994720459,
-      "logits/rejected": -4.17570161819458,
-      "logps/chosen": -347.78521728515625,
-      "logps/rejected": -368.5164489746094,
-      "loss": 0.6671,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.878218948841095,
-      "rewards/margins": 0.11807211488485336,
-      "rewards/rejected": -0.9962911605834961,
+      "logits/chosen": -2.6991753578186035,
+      "logits/rejected": -2.6973986625671387,
+      "logps/chosen": -313.09747314453125,
+      "logps/rejected": -330.6873779296875,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5288428068161011,
+      "rewards/margins": 0.08699531853199005,
+      "rewards/rejected": -0.6158381700515747,
       "step": 580
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.1695173182897126e-06,
-      "logits/chosen": -4.180907249450684,
-      "logits/rejected": -4.169510841369629,
-      "logps/chosen": -363.77081298828125,
-      "logps/rejected": -398.118408203125,
-      "loss": 0.6542,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.982130229473114,
-      "rewards/margins": 0.1635296642780304,
-      "rewards/rejected": -1.1456598043441772,
+      "logits/chosen": -2.622006893157959,
+      "logits/rejected": -2.6231589317321777,
+      "logps/chosen": -323.0411376953125,
+      "logps/rejected": -351.798583984375,
+      "loss": 0.657,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5726518034934998,
+      "rewards/margins": 0.10712896287441254,
+      "rewards/rejected": -0.6797807812690735,
       "step": 590
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.10194166506673e-06,
-      "logits/chosen": -4.227340221405029,
-      "logits/rejected": -4.191034317016602,
-      "logps/chosen": -362.04327392578125,
-      "logps/rejected": -401.2042541503906,
-      "loss": 0.6503,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.9192866086959839,
-      "rewards/margins": 0.18541966378688812,
-      "rewards/rejected": -1.1047064065933228,
+      "logits/chosen": -2.637044906616211,
+      "logits/rejected": -2.6247122287750244,
+      "logps/chosen": -323.566650390625,
+      "logps/rejected": -354.9677429199219,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5322225093841553,
+      "rewards/margins": 0.10880555957555771,
+      "rewards/rejected": -0.6410280466079712,
       "step": 600
     },
     {
       "epoch": 0.48,
-      "eval_logits/chosen": -4.443284511566162,
-      "eval_logits/rejected": -4.434456825256348,
-      "eval_logps/chosen": -304.2834167480469,
-      "eval_logps/rejected": -318.1134033203125,
-      "eval_loss": 0.6769664287567139,
-      "eval_rewards/accuracies": 0.5533333420753479,
-      "eval_rewards/chosen": -0.9588348269462585,
-      "eval_rewards/margins": 0.0851958841085434,
-      "eval_rewards/rejected": -1.0440306663513184,
-      "eval_runtime": 1565.9018,
-      "eval_samples_per_second": 1.912,
-      "eval_steps_per_second": 0.239,
+      "eval_logits/chosen": -2.6844964027404785,
+      "eval_logits/rejected": -2.6804616451263428,
+      "eval_logps/chosen": -268.708740234375,
+      "eval_logps/rejected": -279.2277526855469,
+      "eval_loss": 0.6789478063583374,
+      "eval_rewards/accuracies": 0.5559999942779541,
+      "eval_rewards/chosen": -0.6009806990623474,
+      "eval_rewards/margins": 0.05180613696575165,
+      "eval_rewards/rejected": -0.6527868509292603,
+      "eval_runtime": 1598.2559,
+      "eval_samples_per_second": 1.873,
+      "eval_steps_per_second": 0.235,
       "step": 600
     },
     {
       "epoch": 0.49,
       "learning_rate": 3.0338958003344115e-06,
-      "logits/chosen": -4.3410210609436035,
-      "logits/rejected": -4.294773101806641,
-      "logps/chosen": -360.5263977050781,
-      "logps/rejected": -391.7284240722656,
-      "loss": 0.6595,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.9942121505737305,
-      "rewards/margins": 0.21516022086143494,
-      "rewards/rejected": -1.2093722820281982,
+      "logits/chosen": -2.703444004058838,
+      "logits/rejected": -2.6941075325012207,
+      "logps/chosen": -320.3475341796875,
+      "logps/rejected": -349.988037109375,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5905410051345825,
+      "rewards/margins": 0.1995875984430313,
+      "rewards/rejected": -0.7901285886764526,
       "step": 610
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9654328786600823e-06,
-      "logits/chosen": -4.326909065246582,
-      "logits/rejected": -4.2857208251953125,
-      "logps/chosen": -359.14569091796875,
-      "logps/rejected": -399.8367004394531,
-      "loss": 0.627,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.9758028984069824,
-      "rewards/margins": 0.18271294236183167,
-      "rewards/rejected": -1.1585158109664917,
+      "logits/chosen": -2.657546281814575,
+      "logits/rejected": -2.6613922119140625,
+      "logps/chosen": -323.674560546875,
+      "logps/rejected": -357.7910461425781,
+      "loss": 0.6488,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.6175565123558044,
+      "rewards/margins": 0.11701612174510956,
+      "rewards/rejected": -0.7345726490020752,
       "step": 620
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.896606380398402e-06,
-      "logits/chosen": -4.396684169769287,
-      "logits/rejected": -4.430912971496582,
-      "logps/chosen": -377.33551025390625,
-      "logps/rejected": -398.4137878417969,
-      "loss": 0.6688,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.0871975421905518,
-      "rewards/margins": 0.13647660613059998,
-      "rewards/rejected": -1.2236744165420532,
+      "logits/chosen": -2.6770036220550537,
+      "logits/rejected": -2.697390079498291,
+      "logps/chosen": -333.11932373046875,
+      "logps/rejected": -353.01739501953125,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.6429452896118164,
+      "rewards/margins": 0.12399964034557343,
+      "rewards/rejected": -0.766944944858551,
       "step": 630
     },
     {
       "epoch": 0.51,
       "learning_rate": 2.827470069914772e-06,
-      "logits/chosen": -4.29737663269043,
-      "logits/rejected": -4.263535499572754,
-      "logps/chosen": -376.9189453125,
-      "logps/rejected": -404.60247802734375,
-      "loss": 0.6551,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.0662126541137695,
-      "rewards/margins": 0.14261041581630707,
-      "rewards/rejected": -1.2088232040405273,
+      "logits/chosen": -2.674020528793335,
+      "logits/rejected": -2.660238265991211,
+      "logps/chosen": -336.7355041503906,
+      "logps/rejected": -360.3841247558594,
+      "loss": 0.66,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.6630152463912964,
+      "rewards/margins": 0.10159214586019516,
+      "rewards/rejected": -0.7646073698997498,
       "step": 640
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7580779535868675e-06,
-      "logits/chosen": -4.269147872924805,
-      "logits/rejected": -4.276608467102051,
-      "logps/chosen": -366.36492919921875,
-      "logps/rejected": -390.6036376953125,
-      "loss": 0.6583,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.9485187530517578,
-      "rewards/margins": 0.15207722783088684,
-      "rewards/rejected": -1.1005959510803223,
+      "logits/chosen": -2.641157388687134,
+      "logits/rejected": -2.65820574760437,
+      "logps/chosen": -333.5285339355469,
+      "logps/rejected": -352.81988525390625,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.617414116859436,
+      "rewards/margins": 0.1025906577706337,
+      "rewards/rejected": -0.7200047373771667,
       "step": 650
     },
     {
       "epoch": 0.53,
       "learning_rate": 2.688484237617129e-06,
-      "logits/chosen": -4.208004474639893,
-      "logits/rejected": -4.182279586791992,
-      "logps/chosen": -362.94586181640625,
-      "logps/rejected": -393.9739990234375,
-      "loss": 0.6322,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.8923367261886597,
-      "rewards/margins": 0.20138171315193176,
-      "rewards/rejected": -1.0937182903289795,
+      "logits/chosen": -2.6740407943725586,
+      "logits/rejected": -2.6733462810516357,
+      "logps/chosen": -334.4920654296875,
+      "logps/rejected": -359.688720703125,
+      "loss": 0.6463,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6058083772659302,
+      "rewards/margins": 0.14193257689476013,
+      "rewards/rejected": -0.7477409243583679,
       "step": 660
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.6187432856891585e-06,
-      "logits/chosen": -4.1537322998046875,
-      "logits/rejected": -4.151703357696533,
-      "logps/chosen": -372.31201171875,
-      "logps/rejected": -405.5547180175781,
-      "loss": 0.6489,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.953785240650177,
-      "rewards/margins": 0.15488819777965546,
-      "rewards/rejected": -1.108673334121704,
+      "logits/chosen": -2.677578926086426,
+      "logits/rejected": -2.694331169128418,
+      "logps/chosen": -342.4553527832031,
+      "logps/rejected": -371.22515869140625,
+      "loss": 0.6558,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6532286405563354,
+      "rewards/margins": 0.11075172573328018,
+      "rewards/rejected": -0.7639803290367126,
       "step": 670
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.548909576501096e-06,
-      "logits/chosen": -4.182751655578613,
-      "logits/rejected": -4.181895732879639,
-      "logps/chosen": -372.65435791015625,
-      "logps/rejected": -398.80902099609375,
-      "loss": 0.6442,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.9745942950248718,
-      "rewards/margins": 0.160622701048851,
-      "rewards/rejected": -1.1352169513702393,
+      "logits/chosen": -2.7143263816833496,
+      "logits/rejected": -2.7066876888275146,
+      "logps/chosen": -340.72137451171875,
+      "logps/rejected": -362.9776306152344,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.6533491015434265,
+      "rewards/margins": 0.12043479830026627,
+      "rewards/rejected": -0.7737839818000793,
       "step": 680
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.4790376612091503e-06,
-      "logits/chosen": -4.2613630294799805,
-      "logits/rejected": -4.227023124694824,
-      "logps/chosen": -391.2955017089844,
-      "logps/rejected": -419.54656982421875,
-      "loss": 0.6233,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.0887335538864136,
-      "rewards/margins": 0.2128380537033081,
-      "rewards/rejected": -1.3015714883804321,
+      "logits/chosen": -2.726318359375,
+      "logits/rejected": -2.720698833465576,
+      "logps/chosen": -359.000732421875,
+      "logps/rejected": -382.2019958496094,
+      "loss": 0.6319,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.7622326612472534,
+      "rewards/margins": 0.1633436232805252,
+      "rewards/rejected": -0.9255763292312622,
       "step": 690
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.40918212081453e-06,
-      "logits/chosen": -4.308588981628418,
-      "logits/rejected": -4.264666557312012,
-      "logps/chosen": -360.5702819824219,
-      "logps/rejected": -409.14422607421875,
-      "loss": 0.5974,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.0383028984069824,
-      "rewards/margins": 0.29866236448287964,
-      "rewards/rejected": -1.3369653224945068,
+      "logits/chosen": -2.7099740505218506,
+      "logits/rejected": -2.6980385780334473,
+      "logps/chosen": -330.88818359375,
+      "logps/rejected": -372.46075439453125,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.739021897315979,
+      "rewards/margins": 0.22892406582832336,
+      "rewards/rejected": -0.9679458737373352,
       "step": 700
     },
     {
       "epoch": 0.56,
-      "eval_logits/chosen": -4.447990417480469,
-      "eval_logits/rejected": -4.436967849731445,
-      "eval_logps/chosen": -322.94854736328125,
-      "eval_logps/rejected": -338.0312194824219,
-      "eval_loss": 0.6777762174606323,
-      "eval_rewards/accuracies": 0.5653333067893982,
-      "eval_rewards/chosen": -1.1454858779907227,
-      "eval_rewards/margins": 0.09772318601608276,
-      "eval_rewards/rejected": -1.2432090044021606,
-      "eval_runtime": 1566.0394,
-      "eval_samples_per_second": 1.912,
-      "eval_steps_per_second": 0.239,
+      "eval_logits/chosen": -2.696856737136841,
+      "eval_logits/rejected": -2.691408634185791,
+      "eval_logps/chosen": -290.1963806152344,
+      "eval_logps/rejected": -301.8047180175781,
+      "eval_loss": 0.6785151362419128,
+      "eval_rewards/accuracies": 0.5550000071525574,
+      "eval_rewards/chosen": -0.8158571124076843,
+      "eval_rewards/margins": 0.06269912421703339,
+      "eval_rewards/rejected": -0.8785561323165894,
+      "eval_runtime": 1591.1417,
+      "eval_samples_per_second": 1.882,
+      "eval_steps_per_second": 0.236,
       "step": 700
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3393975235270654e-06,
-      "logits/chosen": -4.278247833251953,
-      "logits/rejected": -4.263747692108154,
-      "logps/chosen": -394.7868347167969,
-      "logps/rejected": -433.32373046875,
-      "loss": 0.6411,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.1327521800994873,
-      "rewards/margins": 0.22177498042583466,
-      "rewards/rejected": -1.354527235031128,
+      "logits/chosen": -2.684086322784424,
+      "logits/rejected": -2.68058443069458,
+      "logps/chosen": -360.47100830078125,
+      "logps/rejected": -390.83441162109375,
+      "loss": 0.6495,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.7877064943313599,
+      "rewards/margins": 0.1392781138420105,
+      "rewards/rejected": -0.9269846677780151,
       "step": 710
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.2697383821388153e-06,
-      "logits/chosen": -4.283724308013916,
-      "logits/rejected": -4.2910637855529785,
-      "logps/chosen": -383.4498596191406,
-      "logps/rejected": -406.142578125,
-      "loss": 0.6479,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.1030470132827759,
-      "rewards/margins": 0.19455528259277344,
-      "rewards/rejected": -1.2976024150848389,
+      "logits/chosen": -2.7011210918426514,
+      "logits/rejected": -2.688472270965576,
+      "logps/chosen": -351.14263916015625,
+      "logps/rejected": -368.4684143066406,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.7785929441452026,
+      "rewards/margins": 0.14113947749137878,
+      "rewards/rejected": -0.919732391834259,
       "step": 720
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.2002591114409657e-06,
-      "logits/chosen": -4.243215560913086,
-      "logits/rejected": -4.230997562408447,
-      "logps/chosen": -392.0690612792969,
-      "logps/rejected": -419.01092529296875,
-      "loss": 0.6517,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.1523319482803345,
-      "rewards/margins": 0.17104671895503998,
-      "rewards/rejected": -1.323378562927246,
+      "logits/chosen": -2.693358898162842,
+      "logits/rejected": -2.6756577491760254,
+      "logps/chosen": -362.7990417480469,
+      "logps/rejected": -383.33843994140625,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.8580299615859985,
+      "rewards/margins": 0.10724015533924103,
+      "rewards/rejected": -0.9652700424194336,
       "step": 730
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.131013985717285e-06,
-      "logits/chosen": -4.2963361740112305,
-      "logits/rejected": -4.248631477355957,
-      "logps/chosen": -402.02130126953125,
-      "logps/rejected": -451.04400634765625,
-      "loss": 0.6211,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.197863221168518,
-      "rewards/margins": 0.26271212100982666,
-      "rewards/rejected": -1.4605753421783447,
+      "logits/chosen": -2.7266604900360107,
+      "logits/rejected": -2.6952764987945557,
+      "logps/chosen": -370.53106689453125,
+      "logps/rejected": -411.46875,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.8816037178039551,
+      "rewards/margins": 0.18020646274089813,
+      "rewards/rejected": -1.0618102550506592,
       "step": 740
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.062057096347338e-06,
-      "logits/chosen": -4.2642927169799805,
-      "logits/rejected": -4.245224952697754,
-      "logps/chosen": -386.35394287109375,
-      "logps/rejected": -399.6516418457031,
-      "loss": 0.6611,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.1031478643417358,
-      "rewards/margins": 0.12747173011302948,
-      "rewards/rejected": -1.2306196689605713,
+      "logits/chosen": -2.737968921661377,
+      "logits/rejected": -2.733065605163574,
+      "logps/chosen": -361.11993408203125,
+      "logps/rejected": -372.26513671875,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.8483070135116577,
+      "rewards/margins": 0.1061801165342331,
+      "rewards/rejected": -0.954487144947052,
       "step": 750
     },
     {
       "epoch": 0.61,
       "learning_rate": 1.9934423095525733e-06,
-      "logits/chosen": -4.144771575927734,
-      "logits/rejected": -4.146181583404541,
-      "logps/chosen": -381.7077941894531,
-      "logps/rejected": -401.769287109375,
-      "loss": 0.6555,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.9921267628669739,
-      "rewards/margins": 0.19879314303398132,
-      "rewards/rejected": -1.190920114517212,
+      "logits/chosen": -2.691333055496216,
+      "logits/rejected": -2.6811554431915283,
+      "logps/chosen": -362.3067932128906,
+      "logps/rejected": -375.4438171386719,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.7950974702835083,
+      "rewards/margins": 0.12981590628623962,
+      "rewards/rejected": -0.9249132871627808,
       "step": 760
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9252232243182986e-06,
-      "logits/chosen": -4.229475498199463,
-      "logits/rejected": -4.177742958068848,
-      "logps/chosen": -345.7065734863281,
-      "logps/rejected": -388.74462890625,
-      "loss": 0.6144,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.8882685899734497,
-      "rewards/margins": 0.26380977034568787,
-      "rewards/rejected": -1.15207839012146,
+      "logits/chosen": -2.7668118476867676,
+      "logits/rejected": -2.7533440589904785,
+      "logps/chosen": -333.151611328125,
+      "logps/rejected": -369.8766784667969,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7605150938034058,
+      "rewards/margins": 0.20025065541267395,
+      "rewards/rejected": -0.9607658386230469,
       "step": 770
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8574531305244043e-06,
-      "logits/chosen": -4.0229105949401855,
-      "logits/rejected": -4.011281490325928,
-      "logps/chosen": -381.92205810546875,
-      "logps/rejected": -418.5514221191406,
-      "loss": 0.6197,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.013808250427246,
-      "rewards/margins": 0.23891642689704895,
-      "rewards/rejected": -1.2527246475219727,
+      "logits/chosen": -2.650963068008423,
+      "logits/rejected": -2.6607377529144287,
+      "logps/chosen": -363.749267578125,
+      "logps/rejected": -396.4261169433594,
+      "loss": 0.6247,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.829413115978241,
+      "rewards/margins": 0.19826601445674896,
+      "rewards/rejected": -1.0276790857315063,
       "step": 780
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.7901849673175559e-06,
-      "logits/chosen": -4.090008735656738,
-      "logits/rejected": -4.056280612945557,
-      "logps/chosen": -375.6288146972656,
-      "logps/rejected": -409.71490478515625,
-      "loss": 0.634,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.9446262121200562,
-      "rewards/margins": 0.2019636631011963,
-      "rewards/rejected": -1.1465898752212524,
+      "logits/chosen": -2.719726085662842,
+      "logits/rejected": -2.7071306705474854,
+      "logps/chosen": -357.9602355957031,
+      "logps/rejected": -386.6010437011719,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.7654613852500916,
+      "rewards/margins": 0.14790861308574677,
+      "rewards/rejected": -0.9133701324462891,
       "step": 790
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.7234712817573555e-06,
-      "logits/chosen": -4.081742763519287,
-      "logits/rejected": -4.069535255432129,
-      "logps/chosen": -408.01580810546875,
-      "logps/rejected": -428.19830322265625,
-      "loss": 0.6508,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.1142655611038208,
-      "rewards/margins": 0.18682199716567993,
-      "rewards/rejected": -1.301087498664856,
+      "logits/chosen": -2.7113006114959717,
+      "logits/rejected": -2.6781070232391357,
+      "logps/chosen": -382.4976806640625,
+      "logps/rejected": -398.0575256347656,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8575699925422668,
+      "rewards/margins": 0.13993313908576965,
+      "rewards/rejected": -0.9975031614303589,
       "step": 800
     },
     {
       "epoch": 0.64,
-      "eval_logits/chosen": -4.271145343780518,
-      "eval_logits/rejected": -4.261835098266602,
-      "eval_logps/chosen": -318.4167785644531,
-      "eval_logps/rejected": -333.9435119628906,
-      "eval_loss": 0.6748104691505432,
-      "eval_rewards/accuracies": 0.5649999976158142,
-      "eval_rewards/chosen": -1.1001687049865723,
-      "eval_rewards/margins": 0.10216327011585236,
-      "eval_rewards/rejected": -1.2023320198059082,
-      "eval_runtime": 1566.0255,
-      "eval_samples_per_second": 1.912,
-      "eval_steps_per_second": 0.239,
+      "eval_logits/chosen": -2.729461669921875,
+      "eval_logits/rejected": -2.723686933517456,
+      "eval_logps/chosen": -297.0627136230469,
+      "eval_logps/rejected": -309.38665771484375,
+      "eval_loss": 0.6770467162132263,
+      "eval_rewards/accuracies": 0.5609999895095825,
+      "eval_rewards/chosen": -0.8845203518867493,
+      "eval_rewards/margins": 0.06985554099082947,
+      "eval_rewards/rejected": -0.9543759226799011,
+      "eval_runtime": 1591.3996,
+      "eval_samples_per_second": 1.881,
+      "eval_steps_per_second": 0.236,
       "step": 800
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6573641877687936e-06,
-      "logits/chosen": -4.110816478729248,
-      "logits/rejected": -4.093571662902832,
-      "logps/chosen": -381.05877685546875,
-      "logps/rejected": -423.17364501953125,
-      "loss": 0.6202,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.0917795896530151,
-      "rewards/margins": 0.21704904735088348,
-      "rewards/rejected": -1.308828592300415,
+      "logits/chosen": -2.712907314300537,
+      "logits/rejected": -2.712637424468994,
+      "logps/chosen": -359.29473876953125,
+      "logps/rejected": -395.03173828125,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.8719032406806946,
+      "rewards/margins": 0.15319783985614777,
+      "rewards/rejected": -1.0251010656356812,
       "step": 810
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.591915325433034e-06,
-      "logits/chosen": -4.177929878234863,
-      "logits/rejected": -4.188526630401611,
-      "logps/chosen": -370.15655517578125,
-      "logps/rejected": -401.153564453125,
-      "loss": 0.6176,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.020612359046936,
-      "rewards/margins": 0.26860466599464417,
-      "rewards/rejected": -1.2892169952392578,
+      "logits/chosen": -2.771751880645752,
+      "logits/rejected": -2.764472246170044,
+      "logps/chosen": -348.4260559082031,
+      "logps/rejected": -368.5625305175781,
+      "loss": 0.6422,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8010624647140503,
+      "rewards/margins": 0.15961696207523346,
+      "rewards/rejected": -0.9606794118881226,
       "step": 820
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.5271758206483664e-06,
-      "logits/chosen": -4.208674430847168,
-      "logits/rejected": -4.197338104248047,
-      "logps/chosen": -399.302490234375,
-      "logps/rejected": -428.2151794433594,
-      "loss": 0.6565,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.2150713205337524,
-      "rewards/margins": 0.18397213518619537,
-      "rewards/rejected": -1.3990434408187866,
+      "logits/chosen": -2.740337371826172,
+      "logits/rejected": -2.732513666152954,
+      "logps/chosen": -366.95660400390625,
+      "logps/rejected": -389.9357604980469,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.8896878361701965,
+      "rewards/margins": 0.12456034123897552,
+      "rewards/rejected": -1.014248251914978,
       "step": 830
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.4631962451927966e-06,
-      "logits/chosen": -4.116250514984131,
-      "logits/rejected": -4.103257656097412,
-      "logps/chosen": -396.60467529296875,
-      "logps/rejected": -433.5016174316406,
-      "loss": 0.6155,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.1897716522216797,
-      "rewards/margins": 0.23897738754749298,
-      "rewards/rejected": -1.4287490844726562,
+      "logits/chosen": -2.6755502223968506,
+      "logits/rejected": -2.6818313598632812,
+      "logps/chosen": -362.2518005371094,
+      "logps/rejected": -391.2507019042969,
+      "loss": 0.6378,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.8435665369033813,
+      "rewards/margins": 0.16070519387722015,
+      "rewards/rejected": -1.0042717456817627,
       "step": 840
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4000265772195032e-06,
-      "logits/chosen": -4.289558410644531,
-      "logits/rejected": -4.233942985534668,
-      "logps/chosen": -391.7837219238281,
-      "logps/rejected": -433.24261474609375,
-      "loss": 0.6144,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.1339555978775024,
-      "rewards/margins": 0.25826430320739746,
-      "rewards/rejected": -1.3922197818756104,
+      "logits/chosen": -2.792430877685547,
+      "logits/rejected": -2.7686753273010254,
+      "logps/chosen": -358.8890075683594,
+      "logps/rejected": -395.0196838378906,
+      "loss": 0.6195,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8025849461555481,
+      "rewards/margins": 0.2058737725019455,
+      "rewards/rejected": -1.0084587335586548,
       "step": 850
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3377161622160077e-06,
-      "logits/chosen": -4.226014137268066,
-      "logits/rejected": -4.22386360168457,
-      "logps/chosen": -392.7188415527344,
-      "logps/rejected": -428.2996520996094,
-      "loss": 0.6116,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.1736845970153809,
-      "rewards/margins": 0.23641009628772736,
-      "rewards/rejected": -1.4100948572158813,
+      "logits/chosen": -2.709834337234497,
+      "logits/rejected": -2.7066776752471924,
+      "logps/chosen": -356.4862365722656,
+      "logps/rejected": -385.40625,
+      "loss": 0.6288,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8100967407226562,
+      "rewards/margins": 0.16853158175945282,
+      "rewards/rejected": -0.9786283373832703,
       "step": 860
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.276313674457553e-06,
-      "logits/chosen": -4.321101188659668,
-      "logits/rejected": -4.316857814788818,
-      "logps/chosen": -376.7432556152344,
-      "logps/rejected": -420.355224609375,
-      "loss": 0.6062,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.1783860921859741,
-      "rewards/margins": 0.2604502737522125,
-      "rewards/rejected": -1.4388363361358643,
+      "logits/chosen": -2.737943172454834,
+      "logits/rejected": -2.74971079826355,
+      "logps/chosen": -342.73614501953125,
+      "logps/rejected": -379.23126220703125,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8370000123977661,
+      "rewards/margins": 0.18809911608695984,
+      "rewards/rejected": -1.0250990390777588,
       "step": 870
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2158670789848095e-06,
-      "logits/chosen": -4.357269287109375,
-      "logits/rejected": -4.3427653312683105,
-      "logps/chosen": -412.49603271484375,
-      "logps/rejected": -446.0122985839844,
-      "loss": 0.6207,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.3438339233398438,
-      "rewards/margins": 0.2622073292732239,
-      "rewards/rejected": -1.6060411930084229,
+      "logits/chosen": -2.713247537612915,
+      "logits/rejected": -2.6928553581237793,
+      "logps/chosen": -373.70361328125,
+      "logps/rejected": -397.2867431640625,
+      "loss": 0.6407,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.9539483189582825,
+      "rewards/margins": 0.16210444271564484,
+      "rewards/rejected": -1.1160527467727661,
       "step": 880
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.1564235941356016e-06,
-      "logits/chosen": -4.453230381011963,
-      "logits/rejected": -4.401095390319824,
-      "logps/chosen": -398.6358337402344,
-      "logps/rejected": -450.500244140625,
-      "loss": 0.6262,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.3652451038360596,
-      "rewards/margins": 0.3177439570426941,
-      "rewards/rejected": -1.6829888820648193,
+      "logits/chosen": -2.7370991706848145,
+      "logits/rejected": -2.7364439964294434,
+      "logps/chosen": -360.29534912109375,
+      "logps/rejected": -403.6412048339844,
+      "loss": 0.6203,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.97996985912323,
+      "rewards/margins": 0.23149892687797546,
+      "rewards/rejected": -1.2114689350128174,
       "step": 890
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.0980296546599254e-06,
-      "logits/chosen": -4.331755638122559,
-      "logits/rejected": -4.3203840255737305,
-      "logps/chosen": -422.264404296875,
-      "logps/rejected": -445.89739990234375,
-      "loss": 0.6746,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.3867082595825195,
-      "rewards/margins": 0.28304582834243774,
-      "rewards/rejected": -1.6697540283203125,
+      "logits/chosen": -2.6951401233673096,
+      "logits/rejected": -2.6780078411102295,
+      "logps/chosen": -381.99420166015625,
+      "logps/rejected": -393.02215576171875,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.9827243685722351,
+      "rewards/margins": 0.15567822754383087,
+      "rewards/rejected": -1.1384025812149048,
       "step": 900
     },
     {
       "epoch": 0.72,
-      "eval_logits/chosen": -4.577242374420166,
-      "eval_logits/rejected": -4.566162586212158,
-      "eval_logps/chosen": -341.29400634765625,
-      "eval_logps/rejected": -358.15576171875,
-      "eval_loss": 0.6757350564002991,
-      "eval_rewards/accuracies": 0.5686666369438171,
-      "eval_rewards/chosen": -1.3289406299591064,
-      "eval_rewards/margins": 0.11551385372877121,
-      "eval_rewards/rejected": -1.4444544315338135,
-      "eval_runtime": 1566.2371,
-      "eval_samples_per_second": 1.912,
-      "eval_steps_per_second": 0.239,
+      "eval_logits/chosen": -2.752532482147217,
+      "eval_logits/rejected": -2.7463557720184326,
+      "eval_logps/chosen": -305.6557922363281,
+      "eval_logps/rejected": -318.45068359375,
+      "eval_loss": 0.6765736937522888,
+      "eval_rewards/accuracies": 0.5666666626930237,
+      "eval_rewards/chosen": -0.9704511761665344,
+      "eval_rewards/margins": 0.07456483691930771,
+      "eval_rewards/rejected": -1.0450160503387451,
+      "eval_runtime": 1591.8909,
+      "eval_samples_per_second": 1.881,
+      "eval_steps_per_second": 0.236,
       "step": 900
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.040730875447083e-06,
-      "logits/chosen": -4.31623649597168,
-      "logits/rejected": -4.329934597015381,
-      "logps/chosen": -407.89544677734375,
-      "logps/rejected": -431.8106384277344,
-      "loss": 0.6508,
+      "logits/chosen": -2.7219176292419434,
+      "logits/rejected": -2.7300069332122803,
+      "logps/chosen": -374.46649169921875,
+      "logps/rejected": -392.5831604003906,
+      "loss": 0.6604,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.282503604888916,
-      "rewards/margins": 0.16822777688503265,
-      "rewards/rejected": -1.4507312774658203,
+      "rewards/chosen": -0.9455657005310059,
+      "rewards/margins": 0.11010564863681793,
+      "rewards/rejected": -1.0556714534759521,
       "step": 910
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.845720158932414e-07,
-      "logits/chosen": -4.317078590393066,
-      "logits/rejected": -4.328789710998535,
-      "logps/chosen": -384.6829833984375,
-      "logps/rejected": -404.79107666015625,
-      "loss": 0.6351,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.2274483442306519,
-      "rewards/margins": 0.19847750663757324,
-      "rewards/rejected": -1.4259259700775146,
+      "logits/chosen": -2.6920173168182373,
+      "logits/rejected": -2.69575834274292,
+      "logps/chosen": -352.3219299316406,
+      "logps/rejected": -367.9543762207031,
+      "loss": 0.6388,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.9019610285758972,
+      "rewards/margins": 0.1525803804397583,
+      "rewards/rejected": -1.0545413494110107,
       "step": 920
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.295969449372796e-07,
-      "logits/chosen": -4.379042625427246,
-      "logits/rejected": -4.360438346862793,
-      "logps/chosen": -385.20245361328125,
-      "logps/rejected": -416.4458923339844,
-      "loss": 0.6352,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.1640335321426392,
-      "rewards/margins": 0.19592450559139252,
-      "rewards/rejected": -1.3599579334259033,
+      "logits/chosen": -2.768012523651123,
+      "logits/rejected": -2.773406505584717,
+      "logps/chosen": -356.3282165527344,
+      "logps/rejected": -381.96478271484375,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.873378574848175,
+      "rewards/margins": 0.13878589868545532,
+      "rewards/rejected": -1.0121644735336304,
       "step": 930
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.758486067922176e-07,
-      "logits/chosen": -4.383969306945801,
-      "logits/rejected": -4.344205379486084,
-      "logps/chosen": -384.3717346191406,
-      "logps/rejected": -427.7298889160156,
-      "loss": 0.6135,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.207880973815918,
-      "rewards/margins": 0.2520582675933838,
-      "rewards/rejected": -1.4599391222000122,
+      "logits/chosen": -2.7590746879577637,
+      "logits/rejected": -2.735861301422119,
+      "logps/chosen": -354.44818115234375,
+      "logps/rejected": -391.2872619628906,
+      "loss": 0.6291,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9065491557121277,
+      "rewards/margins": 0.18637417256832123,
+      "rewards/rejected": -1.0929234027862549,
       "step": 940
     },
     {
       "epoch": 0.76,
       "learning_rate": 8.233689873990006e-07,
-      "logits/chosen": -4.391127109527588,
-      "logits/rejected": -4.357823848724365,
-      "logps/chosen": -389.40570068359375,
-      "logps/rejected": -443.3841247558594,
-      "loss": 0.6165,
+      "logits/chosen": -2.7534782886505127,
+      "logits/rejected": -2.7382559776306152,
+      "logps/chosen": -355.0606384277344,
+      "logps/rejected": -398.70513916015625,
+      "loss": 0.6269,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.2565414905548096,
-      "rewards/margins": 0.3726215362548828,
-      "rewards/rejected": -1.6291630268096924,
+      "rewards/chosen": -0.9113260507583618,
+      "rewards/margins": 0.26781052350997925,
+      "rewards/rejected": -1.1791365146636963,
       "step": 950
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.721990816288555e-07,
-      "logits/chosen": -4.3561787605285645,
-      "logits/rejected": -4.322731971740723,
-      "logps/chosen": -363.4510803222656,
-      "logps/rejected": -400.2040710449219,
-      "loss": 0.6128,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.1432923078536987,
-      "rewards/margins": 0.24396805465221405,
-      "rewards/rejected": -1.3872603178024292,
+      "logits/chosen": -2.7826993465423584,
+      "logits/rejected": -2.763814687728882,
+      "logps/chosen": -336.1342468261719,
+      "logps/rejected": -366.10809326171875,
+      "loss": 0.63,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8676854372024536,
+      "rewards/margins": 0.1765536516904831,
+      "rewards/rejected": -1.0442390441894531,
       "step": 960
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.223788612598148e-07,
-      "logits/chosen": -4.374851703643799,
-      "logits/rejected": -4.3536810874938965,
-      "logps/chosen": -397.66680908203125,
-      "logps/rejected": -428.77227783203125,
-      "loss": 0.639,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.2013370990753174,
-      "rewards/margins": 0.15685971081256866,
-      "rewards/rejected": -1.3581968545913696,
+      "logits/chosen": -2.7675793170928955,
+      "logits/rejected": -2.776650905609131,
+      "logps/chosen": -366.38525390625,
+      "logps/rejected": -391.9957580566406,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.886553943157196,
+      "rewards/margins": 0.10182847827672958,
+      "rewards/rejected": -0.9883824586868286,
       "step": 970
     },
     {
       "epoch": 0.78,
       "learning_rate": 6.73947243752448e-07,
-      "logits/chosen": -4.431571006774902,
-      "logits/rejected": -4.440985679626465,
-      "logps/chosen": -415.3916931152344,
-      "logps/rejected": -431.7449645996094,
-      "loss": 0.656,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.3690643310546875,
-      "rewards/margins": 0.22862248122692108,
-      "rewards/rejected": -1.5976868867874146,
+      "logits/chosen": -2.7559187412261963,
+      "logits/rejected": -2.751338481903076,
+      "logps/chosen": -376.8429260253906,
+      "logps/rejected": -383.2920837402344,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9805536270141602,
+      "rewards/margins": 0.13034404814243317,
+      "rewards/rejected": -1.1108976602554321,
       "step": 980
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.269420618491759e-07,
-      "logits/chosen": -4.3968610763549805,
-      "logits/rejected": -4.365904808044434,
-      "logps/chosen": -384.7822265625,
-      "logps/rejected": -417.7525939941406,
-      "loss": 0.6459,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2827244997024536,
-      "rewards/margins": 0.17925159633159637,
-      "rewards/rejected": -1.461976170539856,
+      "logits/chosen": -2.759143352508545,
+      "logits/rejected": -2.74459171295166,
+      "logps/chosen": -351.4375305175781,
+      "logps/rejected": -376.80633544921875,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9471473693847656,
+      "rewards/margins": 0.10330625623464584,
+      "rewards/rejected": -1.0504536628723145,
       "step": 990
     },
     {
       "epoch": 0.8,
       "learning_rate": 5.814000340209267e-07,
-      "logits/chosen": -4.392641067504883,
-      "logits/rejected": -4.334468841552734,
-      "logps/chosen": -398.47662353515625,
-      "logps/rejected": -445.6675720214844,
-      "loss": 0.6151,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.2670553922653198,
-      "rewards/margins": 0.2884874939918518,
-      "rewards/rejected": -1.5555428266525269,
+      "logits/chosen": -2.746699810028076,
+      "logits/rejected": -2.7150912284851074,
+      "logps/chosen": -363.809814453125,
+      "logps/rejected": -401.50927734375,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9182988405227661,
+      "rewards/margins": 0.19320932030677795,
+      "rewards/rejected": -1.1115081310272217,
       "step": 1000
     },
     {
       "epoch": 0.8,
-      "eval_logits/chosen": -4.618439197540283,
-      "eval_logits/rejected": -4.607028484344482,
-      "eval_logps/chosen": -343.98931884765625,
-      "eval_logps/rejected": -361.1741943359375,
-      "eval_loss": 0.6755098700523376,
-      "eval_rewards/accuracies": 0.5690000057220459,
-      "eval_rewards/chosen": -1.3558939695358276,
-      "eval_rewards/margins": 0.11874517053365707,
-      "eval_rewards/rejected": -1.4746389389038086,
-      "eval_runtime": 1565.397,
-      "eval_samples_per_second": 1.913,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.7606282234191895,
+      "eval_logits/rejected": -2.7542688846588135,
+      "eval_logps/chosen": -307.0535583496094,
+      "eval_logps/rejected": -319.9799499511719,
+      "eval_loss": 0.6764061450958252,
+      "eval_rewards/accuracies": 0.5680000185966492,
+      "eval_rewards/chosen": -0.9844286441802979,
+      "eval_rewards/margins": 0.07588021457195282,
+      "eval_rewards/rejected": -1.060308814048767,
+      "eval_runtime": 1582.573,
+      "eval_samples_per_second": 1.892,
+      "eval_steps_per_second": 0.237,
       "step": 1000
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.373567357842111e-07,
-      "logits/chosen": -4.363351821899414,
-      "logits/rejected": -4.336891174316406,
-      "logps/chosen": -393.9075012207031,
-      "logps/rejected": -432.1922302246094,
-      "loss": 0.6011,
+      "logits/chosen": -2.703906536102295,
+      "logits/rejected": -2.7083990573883057,
+      "logps/chosen": -357.48388671875,
+      "logps/rejected": -387.49078369140625,
+      "loss": 0.6159,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.2202444076538086,
-      "rewards/margins": 0.2966518700122833,
-      "rewards/rejected": -1.516896367073059,
+      "rewards/chosen": -0.8543448448181152,
+      "rewards/margins": 0.21335168182849884,
+      "rewards/rejected": -1.0676965713500977,
       "step": 1010
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.948465719110226e-07,
-      "logits/chosen": -4.430235862731934,
-      "logits/rejected": -4.40949010848999,
-      "logps/chosen": -379.41375732421875,
-      "logps/rejected": -404.0538635253906,
-      "loss": 0.6481,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.1881725788116455,
-      "rewards/margins": 0.17870518565177917,
-      "rewards/rejected": -1.366877794265747,
+      "logits/chosen": -2.7659733295440674,
+      "logits/rejected": -2.7676990032196045,
+      "logps/chosen": -350.13482666015625,
+      "logps/rejected": -371.6733703613281,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8929713368415833,
+      "rewards/margins": 0.14709730446338654,
+      "rewards/rejected": -1.0400686264038086,
       "step": 1020
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.539027495532766e-07,
-      "logits/chosen": -4.403011322021484,
-      "logits/rejected": -4.427884578704834,
-      "logps/chosen": -381.40582275390625,
-      "logps/rejected": -415.64044189453125,
-      "loss": 0.6201,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.2115557193756104,
-      "rewards/margins": 0.2422032654285431,
-      "rewards/rejected": -1.4537590742111206,
+      "logits/chosen": -2.715686798095703,
+      "logits/rejected": -2.7308743000030518,
+      "logps/chosen": -346.8404235839844,
+      "logps/rejected": -375.3435363769531,
+      "loss": 0.6263,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8631433248519897,
+      "rewards/margins": 0.18629541993141174,
+      "rewards/rejected": -1.049438714981079,
       "step": 1030
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.14557252302783e-07,
-      "logits/chosen": -4.416018962860107,
-      "logits/rejected": -4.378045082092285,
-      "logps/chosen": -401.25286865234375,
-      "logps/rejected": -434.80523681640625,
-      "loss": 0.6795,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.350279450416565,
-      "rewards/margins": 0.18791106343269348,
-      "rewards/rejected": -1.538190484046936,
+      "logits/chosen": -2.7563765048980713,
+      "logits/rejected": -2.741762399673462,
+      "logps/chosen": -368.244140625,
+      "logps/rejected": -396.12298583984375,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0178955793380737,
+      "rewards/margins": 0.13189546763896942,
+      "rewards/rejected": -1.1497912406921387,
       "step": 1040
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.7684081520700884e-07,
-      "logits/chosen": -4.323966026306152,
-      "logits/rejected": -4.319721221923828,
-      "logps/chosen": -411.38812255859375,
-      "logps/rejected": -426.6982421875,
-      "loss": 0.6426,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.2739022970199585,
-      "rewards/margins": 0.20139846205711365,
-      "rewards/rejected": -1.475300908088684,
+      "logits/chosen": -2.719506025314331,
+      "logits/rejected": -2.712674140930176,
+      "logps/chosen": -373.14007568359375,
+      "logps/rejected": -384.85870361328125,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.888410210609436,
+      "rewards/margins": 0.16670987010002136,
+      "rewards/rejected": -1.0551201105117798,
       "step": 1050
     },
     {
       "epoch": 0.85,
       "learning_rate": 3.407829007601507e-07,
-      "logits/chosen": -4.3614420890808105,
-      "logits/rejected": -4.317663192749023,
-      "logps/chosen": -395.0092468261719,
-      "logps/rejected": -438.6324157714844,
-      "loss": 0.6132,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.2276242971420288,
-      "rewards/margins": 0.28230905532836914,
-      "rewards/rejected": -1.5099334716796875,
+      "logits/chosen": -2.723992109298706,
+      "logits/rejected": -2.7108616828918457,
+      "logps/chosen": -362.71112060546875,
+      "logps/rejected": -396.30413818359375,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9032829403877258,
+      "rewards/margins": 0.18060991168022156,
+      "rewards/rejected": -1.0838927030563354,
       "step": 1060
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.064116758882724e-07,
-      "logits/chosen": -4.330022811889648,
-      "logits/rejected": -4.274947643280029,
-      "logps/chosen": -406.9677429199219,
-      "logps/rejected": -459.18048095703125,
-      "loss": 0.6063,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.304404616355896,
-      "rewards/margins": 0.33513855934143066,
-      "rewards/rejected": -1.6395431756973267,
+      "logits/chosen": -2.7330386638641357,
+      "logits/rejected": -2.700990676879883,
+      "logps/chosen": -375.42425537109375,
+      "logps/rejected": -414.94635009765625,
+      "loss": 0.6324,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9860410690307617,
+      "rewards/margins": 0.20865873992443085,
+      "rewards/rejected": -1.1946996450424194,
       "step": 1070
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.737539899464908e-07,
-      "logits/chosen": -4.385619163513184,
-      "logits/rejected": -4.378711700439453,
-      "logps/chosen": -370.54156494140625,
-      "logps/rejected": -411.8634338378906,
-      "loss": 0.6197,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.1883795261383057,
-      "rewards/margins": 0.2932683825492859,
-      "rewards/rejected": -1.4816478490829468,
+      "logits/chosen": -2.7359976768493652,
+      "logits/rejected": -2.7335023880004883,
+      "logps/chosen": -345.5082092285156,
+      "logps/rejected": -376.73382568359375,
+      "loss": 0.633,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.9359216690063477,
+      "rewards/margins": 0.19209426641464233,
+      "rewards/rejected": -1.1280159950256348,
       "step": 1080
     },
     {
       "epoch": 0.87,
       "learning_rate": 2.4283535374538645e-07,
-      "logits/chosen": -4.3373942375183105,
-      "logits/rejected": -4.339685916900635,
-      "logps/chosen": -400.1960754394531,
-      "logps/rejected": -430.27557373046875,
-      "loss": 0.6378,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.2183153629302979,
-      "rewards/margins": 0.2266024649143219,
-      "rewards/rejected": -1.444917917251587,
+      "logits/chosen": -2.7470860481262207,
+      "logits/rejected": -2.759739398956299,
+      "logps/chosen": -367.7596130371094,
+      "logps/rejected": -393.4772644042969,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.8926227688789368,
+      "rewards/margins": 0.18235111236572266,
+      "rewards/rejected": -1.0749738216400146,
       "step": 1090
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.1367991962303298e-07,
-      "logits/chosen": -4.3288164138793945,
-      "logits/rejected": -4.31368350982666,
-      "logps/chosen": -382.65814208984375,
-      "logps/rejected": -405.8753967285156,
-      "loss": 0.6837,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -1.2913353443145752,
-      "rewards/margins": 0.10410255193710327,
-      "rewards/rejected": -1.3954378366470337,
+      "logits/chosen": -2.701397657394409,
+      "logits/rejected": -2.7229931354522705,
+      "logps/chosen": -348.02447509765625,
+      "logps/rejected": -368.96197509765625,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9437961578369141,
+      "rewards/margins": 0.08066528290510178,
+      "rewards/rejected": -1.0244615077972412,
       "step": 1100
     },
     {
       "epoch": 0.88,
-      "eval_logits/chosen": -4.571716785430908,
-      "eval_logits/rejected": -4.5607380867004395,
-      "eval_logps/chosen": -340.8576354980469,
-      "eval_logps/rejected": -358.08392333984375,
-      "eval_loss": 0.6748068928718567,
-      "eval_rewards/accuracies": 0.5709999799728394,
-      "eval_rewards/chosen": -1.3245766162872314,
-      "eval_rewards/margins": 0.11915931850671768,
-      "eval_rewards/rejected": -1.4437358379364014,
-      "eval_runtime": 1563.7394,
-      "eval_samples_per_second": 1.915,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.7602105140686035,
+      "eval_logits/rejected": -2.7538294792175293,
+      "eval_logps/chosen": -307.4263916015625,
+      "eval_logps/rejected": -320.4283447265625,
+      "eval_loss": 0.6762885451316833,
+      "eval_rewards/accuracies": 0.5686666369438171,
+      "eval_rewards/chosen": -0.9881570935249329,
+      "eval_rewards/margins": 0.07663587480783463,
+      "eval_rewards/rejected": -1.0647929906845093,
+      "eval_runtime": 1581.3137,
+      "eval_samples_per_second": 1.893,
+      "eval_steps_per_second": 0.237,
       "step": 1100
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.8631046257820278e-07,
-      "logits/chosen": -4.340999603271484,
-      "logits/rejected": -4.353137969970703,
-      "logps/chosen": -399.4720153808594,
-      "logps/rejected": -427.0414123535156,
-      "loss": 0.6277,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.2465407848358154,
-      "rewards/margins": 0.23380056023597717,
-      "rewards/rejected": -1.4803413152694702,
+      "logits/chosen": -2.747465133666992,
+      "logits/rejected": -2.7517151832580566,
+      "logps/chosen": -368.2339172363281,
+      "logps/rejected": -389.92803955078125,
+      "loss": 0.6368,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.9324411153793335,
+      "rewards/margins": 0.17461669445037842,
+      "rewards/rejected": -1.1070579290390015,
       "step": 1110
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.6074836247950143e-07,
-      "logits/chosen": -4.328183174133301,
-      "logits/rejected": -4.327004432678223,
-      "logps/chosen": -406.1539306640625,
-      "logps/rejected": -430.02044677734375,
-      "loss": 0.6483,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2271990776062012,
-      "rewards/margins": 0.17618249356746674,
-      "rewards/rejected": -1.4033817052841187,
+      "logits/chosen": -2.7465901374816895,
+      "logits/rejected": -2.7471508979797363,
+      "logps/chosen": -372.44866943359375,
+      "logps/rejected": -390.7837829589844,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.8873372077941895,
+      "rewards/margins": 0.12168030440807343,
+      "rewards/rejected": -1.0090174674987793,
       "step": 1120
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.370135873643097e-07,
-      "logits/chosen": -4.33071231842041,
-      "logits/rejected": -4.364874839782715,
-      "logps/chosen": -403.37628173828125,
-      "logps/rejected": -413.484375,
-      "loss": 0.6427,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.2655575275421143,
-      "rewards/margins": 0.19985264539718628,
-      "rewards/rejected": -1.4654099941253662,
+      "logits/chosen": -2.7472376823425293,
+      "logits/rejected": -2.7466180324554443,
+      "logps/chosen": -373.3634338378906,
+      "logps/rejected": -377.52569580078125,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9632595777511597,
+      "rewards/margins": 0.13923458755016327,
+      "rewards/rejected": -1.102494239807129,
       "step": 1130
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.1512467784059372e-07,
-      "logits/chosen": -4.4061384201049805,
-      "logits/rejected": -4.369538307189941,
-      "logps/chosen": -367.4423522949219,
-      "logps/rejected": -402.65911865234375,
-      "loss": 0.6186,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.1879181861877441,
-      "rewards/margins": 0.27168983221054077,
-      "rewards/rejected": -1.4596080780029297,
+      "logits/chosen": -2.7725350856781006,
+      "logits/rejected": -2.748663902282715,
+      "logps/chosen": -343.6497802734375,
+      "logps/rejected": -369.282470703125,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9474760293960571,
+      "rewards/margins": 0.1754283607006073,
+      "rewards/rejected": -1.1229044198989868,
       "step": 1140
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.509873260376251e-08,
-      "logits/chosen": -4.3491435050964355,
-      "logits/rejected": -4.2928547859191895,
-      "logps/chosen": -394.6898498535156,
-      "logps/rejected": -461.10845947265625,
-      "loss": 0.583,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.2356455326080322,
-      "rewards/margins": 0.3976030647754669,
-      "rewards/rejected": -1.6332485675811768,
+      "logits/chosen": -2.727968692779541,
+      "logits/rejected": -2.703981876373291,
+      "logps/chosen": -363.1959228515625,
+      "logps/rejected": -414.9847106933594,
+      "loss": 0.6061,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.9189531207084656,
+      "rewards/margins": 0.2506302297115326,
+      "rewards/rejected": -1.1695834398269653,
       "step": 1150
     },
     {
       "epoch": 0.93,
       "learning_rate": 7.695139507988559e-08,
-      "logits/chosen": -4.359416961669922,
-      "logits/rejected": -4.381959915161133,
-      "logps/chosen": -409.65643310546875,
-      "logps/rejected": -434.98675537109375,
-      "loss": 0.6521,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.2608633041381836,
-      "rewards/margins": 0.1726626455783844,
-      "rewards/rejected": -1.4335259199142456,
+      "logits/chosen": -2.7512032985687256,
+      "logits/rejected": -2.7607004642486572,
+      "logps/chosen": -376.4388427734375,
+      "logps/rejected": -397.3370056152344,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.925561785697937,
+      "rewards/margins": 0.1294144093990326,
+      "rewards/rejected": -1.054976224899292,
       "step": 1160
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.069684120570684e-08,
-      "logits/chosen": -4.381880283355713,
-      "logits/rejected": -4.29121208190918,
-      "logps/chosen": -395.4755554199219,
-      "logps/rejected": -438.13427734375,
-      "loss": 0.6418,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.3113107681274414,
-      "rewards/margins": 0.23519746959209442,
-      "rewards/rejected": -1.5465081930160522,
+      "logits/chosen": -2.715827703475952,
+      "logits/rejected": -2.679953098297119,
+      "logps/chosen": -362.00079345703125,
+      "logps/rejected": -398.26141357421875,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9743494987487793,
+      "rewards/margins": 0.17094501852989197,
+      "rewards/rejected": -1.145294427871704,
       "step": 1170
     },
     {
       "epoch": 0.94,
       "learning_rate": 4.634776835499871e-08,
-      "logits/chosen": -4.314273357391357,
-      "logits/rejected": -4.296536445617676,
-      "logps/chosen": -382.25958251953125,
-      "logps/rejected": -419.88897705078125,
-      "loss": 0.6354,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.2596309185028076,
-      "rewards/margins": 0.24562938511371613,
-      "rewards/rejected": -1.5052604675292969,
+      "logits/chosen": -2.679133415222168,
+      "logits/rejected": -2.6611809730529785,
+      "logps/chosen": -350.9591369628906,
+      "logps/rejected": -379.08416748046875,
+      "loss": 0.648,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.944057822227478,
+      "rewards/margins": 0.15136626362800598,
+      "rewards/rejected": -1.0954240560531616,
       "step": 1180
     },
     {
       "epoch": 0.95,
       "learning_rate": 3.3915385419908964e-08,
-      "logits/chosen": -4.292089939117432,
-      "logits/rejected": -4.3260178565979,
-      "logps/chosen": -391.3450927734375,
-      "logps/rejected": -419.9061584472656,
-      "loss": 0.6332,
+      "logits/chosen": -2.730663299560547,
+      "logits/rejected": -2.7344131469726562,
+      "logps/chosen": -362.0574951171875,
+      "logps/rejected": -382.64788818359375,
+      "loss": 0.645,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.2074798345565796,
-      "rewards/margins": 0.2409057915210724,
-      "rewards/rejected": -1.4483857154846191,
+      "rewards/chosen": -0.9124767184257507,
+      "rewards/margins": 0.15995648503303528,
+      "rewards/rejected": -1.0724332332611084,
       "step": 1190
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3409404055043938e-08,
-      "logits/chosen": -4.403561592102051,
-      "logits/rejected": -4.3759870529174805,
-      "logps/chosen": -400.16522216796875,
-      "logps/rejected": -427.40606689453125,
-      "loss": 0.6539,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.288109302520752,
-      "rewards/margins": 0.17616990208625793,
-      "rewards/rejected": -1.4642791748046875,
+      "logits/chosen": -2.7462306022644043,
+      "logits/rejected": -2.7565507888793945,
+      "logps/chosen": -369.1392517089844,
+      "logps/rejected": -390.51190185546875,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9762216806411743,
+      "rewards/margins": 0.11748246848583221,
+      "rewards/rejected": -1.093704104423523,
       "step": 1200
     },
     {
       "epoch": 0.96,
-      "eval_logits/chosen": -4.559507846832275,
-      "eval_logits/rejected": -4.548335075378418,
-      "eval_logps/chosen": -340.22308349609375,
-      "eval_logps/rejected": -357.4035949707031,
-      "eval_loss": 0.6746300458908081,
-      "eval_rewards/accuracies": 0.5730000138282776,
-      "eval_rewards/chosen": -1.318231225013733,
-      "eval_rewards/margins": 0.11870139837265015,
-      "eval_rewards/rejected": -1.4369325637817383,
-      "eval_runtime": 1563.0275,
-      "eval_samples_per_second": 1.916,
-      "eval_steps_per_second": 0.24,
+      "eval_logits/chosen": -2.7601613998413086,
+      "eval_logits/rejected": -2.7538154125213623,
+      "eval_logps/chosen": -307.4615478515625,
+      "eval_logps/rejected": -320.441162109375,
+      "eval_loss": 0.6764407753944397,
+      "eval_rewards/accuracies": 0.5663333535194397,
+      "eval_rewards/chosen": -0.9885084629058838,
+      "eval_rewards/margins": 0.07641255110502243,
+      "eval_rewards/rejected": -1.0649209022521973,
+      "eval_runtime": 1656.3369,
+      "eval_samples_per_second": 1.808,
+      "eval_steps_per_second": 0.226,
       "step": 1200
     },
     {
       "epoch": 0.97,
       "learning_rate": 1.4838031091134186e-08,
-      "logits/chosen": -4.341926574707031,
-      "logits/rejected": -4.2704925537109375,
-      "logps/chosen": -373.17681884765625,
-      "logps/rejected": -431.8099060058594,
-      "loss": 0.6065,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.2158408164978027,
-      "rewards/margins": 0.33925992250442505,
-      "rewards/rejected": -1.555100679397583,
+      "logits/chosen": -2.711413860321045,
+      "logits/rejected": -2.700925350189209,
+      "logps/chosen": -341.56524658203125,
+      "logps/rejected": -391.4617614746094,
+      "loss": 0.6092,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8985296487808228,
+      "rewards/margins": 0.250393271446228,
+      "rewards/rejected": -1.1489229202270508,
       "step": 1210
     },
     {
       "epoch": 0.98,
       "learning_rate": 8.207962124201774e-09,
-      "logits/chosen": -4.371233940124512,
-      "logits/rejected": -4.343363285064697,
-      "logps/chosen": -401.8610534667969,
-      "logps/rejected": -437.0784606933594,
-      "loss": 0.6005,
+      "logits/chosen": -2.766512393951416,
+      "logits/rejected": -2.751699924468994,
+      "logps/chosen": -370.7591857910156,
+      "logps/rejected": -398.38201904296875,
+      "loss": 0.6128,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.1877615451812744,
-      "rewards/margins": 0.29727452993392944,
-      "rewards/rejected": -1.4850361347198486,
+      "rewards/chosen": -0.873375415802002,
+      "rewards/margins": 0.22166454792022705,
+      "rewards/rejected": -1.0950400829315186,
       "step": 1220
     },
     {
       "epoch": 0.98,
       "learning_rate": 3.5243762852441023e-09,
-      "logits/chosen": -4.321781158447266,
-      "logits/rejected": -4.277449131011963,
-      "logps/chosen": -392.8436279296875,
-      "logps/rejected": -430.36279296875,
-      "loss": 0.6401,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.2518620491027832,
-      "rewards/margins": 0.2195155918598175,
-      "rewards/rejected": -1.4713777303695679,
+      "logits/chosen": -2.7122433185577393,
+      "logits/rejected": -2.7001757621765137,
+      "logps/chosen": -358.870849609375,
+      "logps/rejected": -391.4342956542969,
+      "loss": 0.6408,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.9104388952255249,
+      "rewards/margins": 0.1689816415309906,
+      "rewards/rejected": -1.0794206857681274,
       "step": 1230
     },
     {
       "epoch": 0.99,
       "learning_rate": 7.909321945129278e-10,
-      "logits/chosen": -4.296388626098633,
-      "logits/rejected": -4.257707118988037,
-      "logps/chosen": -408.13409423828125,
-      "logps/rejected": -451.42462158203125,
-      "loss": 0.6106,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2207772731781006,
-      "rewards/margins": 0.29706281423568726,
-      "rewards/rejected": -1.5178401470184326,
+      "logits/chosen": -2.7206053733825684,
+      "logits/rejected": -2.7036147117614746,
+      "logps/chosen": -379.60906982421875,
+      "logps/rejected": -411.9668884277344,
+      "loss": 0.6327,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9332448840141296,
+      "rewards/margins": 0.1873912662267685,
+      "rewards/rejected": -1.120636224746704,
       "step": 1240
     },
     {
       "epoch": 1.0,
       "step": 1249,
       "total_flos": 0.0,
-      "train_loss": 0.6508825262419409,
-      "train_runtime": 40390.1729,
-      "train_samples_per_second": 0.495,
-      "train_steps_per_second": 0.031
+      "train_loss": 0.6566078573155155,
+      "train_runtime": 38377.8314,
+      "train_samples_per_second": 0.521,
+      "train_steps_per_second": 0.033
     }
   ],
   "logging_steps": 10,