{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.161642562583493,
  "eval_steps": 500,
  "global_step": 2000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005808212812917465,
      "grad_norm": 306.3320007324219,
      "learning_rate": 4.998547356188263e-06,
      "logits/chosen": -0.7514113187789917,
      "logits/rejected": -0.6686298251152039,
      "logps/chosen": -75.72093200683594,
      "logps/rejected": -73.8106918334961,
      "loss": 13.8629,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.001161642562583493,
      "grad_norm": 301.2439270019531,
      "learning_rate": 4.997094712376526e-06,
      "logits/chosen": -0.794822096824646,
      "logits/rejected": -0.7371929287910461,
      "logps/chosen": -72.30989074707031,
      "logps/rejected": -67.51399993896484,
      "loss": 13.9577,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0054689692333340645,
      "rewards/margins": -0.009410643950104713,
      "rewards/rejected": 0.003941674716770649,
      "step": 2
    },
    {
      "epoch": 0.0017424638438752395,
      "grad_norm": 296.5701904296875,
      "learning_rate": 4.995642068564789e-06,
      "logits/chosen": -0.8363990783691406,
      "logits/rejected": -0.8187875747680664,
      "logps/chosen": -71.92262268066406,
      "logps/rejected": -72.27050018310547,
      "loss": 13.7847,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0027333260513842106,
      "rewards/margins": 0.008102846331894398,
      "rewards/rejected": -0.0053695198148489,
      "step": 3
    },
    {
      "epoch": 0.002323285125166986,
      "grad_norm": 345.2494201660156,
      "learning_rate": 4.9941894247530506e-06,
      "logits/chosen": -0.7175111174583435,
      "logits/rejected": -0.7101837396621704,
      "logps/chosen": -77.45024108886719,
      "logps/rejected": -76.15581512451172,
      "loss": 13.9709,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01066683791577816,
      "rewards/margins": -0.01057196594774723,
      "rewards/rejected": -9.487159695709124e-05,
      "step": 4
    },
    {
      "epoch": 0.0029041064064587326,
      "grad_norm": 295.31768798828125,
      "learning_rate": 4.992736780941313e-06,
      "logits/chosen": -0.7365175485610962,
      "logits/rejected": -0.7447739839553833,
      "logps/chosen": -76.33888244628906,
      "logps/rejected": -61.07477951049805,
      "loss": 13.8975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0010636046063154936,
      "rewards/margins": -0.0031540922354906797,
      "rewards/rejected": 0.004217695910483599,
      "step": 5
    },
    {
      "epoch": 0.003484927687750479,
      "grad_norm": 312.8099060058594,
      "learning_rate": 4.991284137129576e-06,
      "logits/chosen": -0.875682532787323,
      "logits/rejected": -0.8198660016059875,
      "logps/chosen": -79.96182250976562,
      "logps/rejected": -77.87804412841797,
      "loss": 13.6993,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01062581967562437,
      "rewards/margins": 0.016683798283338547,
      "rewards/rejected": -0.0060579776763916016,
      "step": 6
    },
    {
      "epoch": 0.004065748969042226,
      "grad_norm": 317.39288330078125,
      "learning_rate": 4.989831493317839e-06,
      "logits/chosen": -0.6050316095352173,
      "logits/rejected": -0.6816262602806091,
      "logps/chosen": -70.26258850097656,
      "logps/rejected": -75.84834289550781,
      "loss": 13.8254,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010981644503772259,
      "rewards/margins": 0.003875770838931203,
      "rewards/rejected": 0.007105874828994274,
      "step": 7
    },
    {
      "epoch": 0.004646570250333972,
      "grad_norm": 321.52996826171875,
      "learning_rate": 4.9883788495061015e-06,
      "logits/chosen": -0.8232254981994629,
      "logits/rejected": -0.7795180082321167,
      "logps/chosen": -72.38011169433594,
      "logps/rejected": -67.78025817871094,
      "loss": 13.6941,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.017304277047514915,
      "rewards/margins": 0.01702544279396534,
      "rewards/rejected": 0.00027883489383384585,
      "step": 8
    },
    {
      "epoch": 0.005227391531625719,
      "grad_norm": 311.1319885253906,
      "learning_rate": 4.986926205694364e-06,
      "logits/chosen": -0.7957251667976379,
      "logits/rejected": -0.748576819896698,
      "logps/chosen": -76.44227600097656,
      "logps/rejected": -72.32237243652344,
      "loss": 13.8266,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007577553391456604,
      "rewards/margins": 0.003815555479377508,
      "rewards/rejected": 0.0037619969807565212,
      "step": 9
    },
    {
      "epoch": 0.005808212812917465,
      "grad_norm": 290.04388427734375,
      "learning_rate": 4.985473561882627e-06,
      "logits/chosen": -0.851279079914093,
      "logits/rejected": -0.8175627589225769,
      "logps/chosen": -62.6363639831543,
      "logps/rejected": -66.80535125732422,
      "loss": 13.8568,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001977672567591071,
      "rewards/margins": 0.0007607266306877136,
      "rewards/rejected": 0.0012169458204880357,
      "step": 10
    },
    {
      "epoch": 0.006389034094209212,
      "grad_norm": 477.3677673339844,
      "learning_rate": 4.984020918070889e-06,
      "logits/chosen": -0.7694743871688843,
      "logits/rejected": -0.7370525598526001,
      "logps/chosen": -68.05220794677734,
      "logps/rejected": -73.65959167480469,
      "loss": 13.9053,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0020100977271795273,
      "rewards/margins": -0.004024811089038849,
      "rewards/rejected": 0.002014713129028678,
      "step": 11
    },
    {
      "epoch": 0.006969855375500958,
      "grad_norm": 339.96746826171875,
      "learning_rate": 4.982568274259152e-06,
      "logits/chosen": -0.4584922194480896,
      "logits/rejected": -0.4653104245662689,
      "logps/chosen": -68.20719146728516,
      "logps/rejected": -76.61314392089844,
      "loss": 13.789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010751953348517418,
      "rewards/margins": 0.007597408257424831,
      "rewards/rejected": 0.003154544625431299,
      "step": 12
    },
    {
      "epoch": 0.0075506766567927045,
      "grad_norm": 307.5588684082031,
      "learning_rate": 4.9811156304474144e-06,
      "logits/chosen": -0.6956412196159363,
      "logits/rejected": -0.5891402959823608,
      "logps/chosen": -66.07670593261719,
      "logps/rejected": -77.52650451660156,
      "loss": 13.8012,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0009560681646689773,
      "rewards/margins": 0.006629952695220709,
      "rewards/rejected": -0.0075860219076275826,
      "step": 13
    },
    {
      "epoch": 0.008131497938084452,
      "grad_norm": 324.5359802246094,
      "learning_rate": 4.979662986635677e-06,
      "logits/chosen": -0.5961264371871948,
      "logits/rejected": -0.6395691633224487,
      "logps/chosen": -76.06859588623047,
      "logps/rejected": -75.65780639648438,
      "loss": 13.8604,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0018664164235815406,
      "rewards/margins": 0.0006155198207125068,
      "rewards/rejected": -0.0024819376412779093,
      "step": 14
    },
    {
      "epoch": 0.008712319219376197,
      "grad_norm": 321.2744445800781,
      "learning_rate": 4.97821034282394e-06,
      "logits/chosen": -0.609241247177124,
      "logits/rejected": -0.6541947722434998,
      "logps/chosen": -72.74789428710938,
      "logps/rejected": -78.14617156982422,
      "loss": 13.6613,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.007168769836425781,
      "rewards/margins": 0.020653218030929565,
      "rewards/rejected": -0.013484450057148933,
      "step": 15
    },
    {
      "epoch": 0.009293140500667945,
      "grad_norm": 321.91412353515625,
      "learning_rate": 4.976757699012203e-06,
      "logits/chosen": -0.8816198110580444,
      "logits/rejected": -1.0502017736434937,
      "logps/chosen": -81.1220474243164,
      "logps/rejected": -78.71932220458984,
      "loss": 14.1071,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.012946806848049164,
      "rewards/margins": -0.023911792784929276,
      "rewards/rejected": 0.010964984074234962,
      "step": 16
    },
    {
      "epoch": 0.00987396178195969,
      "grad_norm": 285.6275939941406,
      "learning_rate": 4.9753050552004654e-06,
      "logits/chosen": -0.7112148404121399,
      "logits/rejected": -0.6412473917007446,
      "logps/chosen": -65.52027893066406,
      "logps/rejected": -71.49274444580078,
      "loss": 13.7638,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007086105644702911,
      "rewards/margins": 0.010165892541408539,
      "rewards/rejected": -0.00307978643104434,
      "step": 17
    },
    {
      "epoch": 0.010454783063251438,
      "grad_norm": 328.0557556152344,
      "learning_rate": 4.973852411388727e-06,
      "logits/chosen": -0.756773829460144,
      "logits/rejected": -0.853185772895813,
      "logps/chosen": -71.16859436035156,
      "logps/rejected": -70.41301727294922,
      "loss": 13.7103,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.006009445525705814,
      "rewards/margins": 0.016000624746084213,
      "rewards/rejected": -0.022010069340467453,
      "step": 18
    },
    {
      "epoch": 0.011035604344543185,
      "grad_norm": 286.8959045410156,
      "learning_rate": 4.97239976757699e-06,
      "logits/chosen": -0.7466567754745483,
      "logits/rejected": -0.8737386465072632,
      "logps/chosen": -70.04942321777344,
      "logps/rejected": -69.66856384277344,
      "loss": 13.9803,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.00499460194259882,
      "rewards/margins": -0.011114511638879776,
      "rewards/rejected": 0.006119909696280956,
      "step": 19
    },
    {
      "epoch": 0.01161642562583493,
      "grad_norm": 322.2734069824219,
      "learning_rate": 4.970947123765253e-06,
      "logits/chosen": -0.7975467443466187,
      "logits/rejected": -0.9792510271072388,
      "logps/chosen": -76.38298034667969,
      "logps/rejected": -82.6656265258789,
      "loss": 13.9642,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.015522710978984833,
      "rewards/margins": -0.009529724717140198,
      "rewards/rejected": -0.005992984864860773,
      "step": 20
    },
    {
      "epoch": 0.012197246907126678,
      "grad_norm": 316.05517578125,
      "learning_rate": 4.9694944799535164e-06,
      "logits/chosen": -0.9200956225395203,
      "logits/rejected": -0.8602321743965149,
      "logps/chosen": -69.63390350341797,
      "logps/rejected": -80.1779556274414,
      "loss": 13.8359,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017169209197163582,
      "rewards/margins": 0.004035423509776592,
      "rewards/rejected": -0.0212046317756176,
      "step": 21
    },
    {
      "epoch": 0.012778068188418423,
      "grad_norm": 335.1014404296875,
      "learning_rate": 4.968041836141778e-06,
      "logits/chosen": -0.787733256816864,
      "logits/rejected": -0.7936286926269531,
      "logps/chosen": -79.13917541503906,
      "logps/rejected": -66.25904083251953,
      "loss": 14.2123,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.030913371592760086,
      "rewards/margins": -0.03371546417474747,
      "rewards/rejected": 0.0028020956087857485,
      "step": 22
    },
    {
      "epoch": 0.01335888946971017,
      "grad_norm": 294.2160339355469,
      "learning_rate": 4.966589192330041e-06,
      "logits/chosen": -0.6786571741104126,
      "logits/rejected": -0.7551315426826477,
      "logps/chosen": -69.29032897949219,
      "logps/rejected": -69.82914733886719,
      "loss": 13.8587,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006731729954481125,
      "rewards/margins": 0.0010073954472318292,
      "rewards/rejected": -0.00773912388831377,
      "step": 23
    },
    {
      "epoch": 0.013939710751001916,
      "grad_norm": 323.32000732421875,
      "learning_rate": 4.965136548518304e-06,
      "logits/chosen": -0.9059945940971375,
      "logits/rejected": -0.7469512224197388,
      "logps/chosen": -78.02278137207031,
      "logps/rejected": -66.72486877441406,
      "loss": 13.9736,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.030265387147665024,
      "rewards/margins": -0.009611329063773155,
      "rewards/rejected": -0.02065405808389187,
      "step": 24
    },
    {
      "epoch": 0.014520532032293663,
      "grad_norm": 333.2950744628906,
      "learning_rate": 4.963683904706567e-06,
      "logits/chosen": -0.8123146891593933,
      "logits/rejected": -0.7186424136161804,
      "logps/chosen": -76.54938507080078,
      "logps/rejected": -67.63246154785156,
      "loss": 13.8885,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.013149453327059746,
      "rewards/margins": -0.0020573907531797886,
      "rewards/rejected": -0.01109206210821867,
      "step": 25
    },
    {
      "epoch": 0.015101353313585409,
      "grad_norm": 357.38775634765625,
      "learning_rate": 4.962231260894829e-06,
      "logits/chosen": -0.8334075808525085,
      "logits/rejected": -0.8764799237251282,
      "logps/chosen": -77.63631439208984,
      "logps/rejected": -77.82304382324219,
      "loss": 14.058,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.015404301695525646,
      "rewards/margins": -0.01881382428109646,
      "rewards/rejected": 0.0034095190931111574,
      "step": 26
    },
    {
      "epoch": 0.015682174594877155,
      "grad_norm": 422.6893615722656,
      "learning_rate": 4.960778617083092e-06,
      "logits/chosen": -0.9417294263839722,
      "logits/rejected": -1.0054762363433838,
      "logps/chosen": -73.45500183105469,
      "logps/rejected": -71.62086486816406,
      "loss": 13.9493,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02026461809873581,
      "rewards/margins": -0.007518300320953131,
      "rewards/rejected": -0.012746316380798817,
      "step": 27
    },
    {
      "epoch": 0.016262995876168904,
      "grad_norm": 302.3515625,
      "learning_rate": 4.959325973271355e-06,
      "logits/chosen": -0.7729172110557556,
      "logits/rejected": -0.9286600947380066,
      "logps/chosen": -74.0387191772461,
      "logps/rejected": -74.87562561035156,
      "loss": 13.7489,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.001342860283330083,
      "rewards/margins": 0.012399435043334961,
      "rewards/rejected": -0.0137422950938344,
      "step": 28
    },
    {
      "epoch": 0.01684381715746065,
      "grad_norm": 311.303466796875,
      "learning_rate": 4.957873329459617e-06,
      "logits/chosen": -0.7707468271255493,
      "logits/rejected": -0.7816058993339539,
      "logps/chosen": -80.28218078613281,
      "logps/rejected": -76.27729034423828,
      "loss": 13.7301,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.017700577154755592,
      "rewards/margins": 0.01469388883560896,
      "rewards/rejected": -0.03239446505904198,
      "step": 29
    },
    {
      "epoch": 0.017424638438752395,
      "grad_norm": 272.25091552734375,
      "learning_rate": 4.9564206856478795e-06,
      "logits/chosen": -0.9470396041870117,
      "logits/rejected": -1.0160866975784302,
      "logps/chosen": -63.82170867919922,
      "logps/rejected": -67.61204528808594,
      "loss": 13.7505,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0032949731685221195,
      "rewards/margins": 0.011799006722867489,
      "rewards/rejected": -0.008504033088684082,
      "step": 30
    },
    {
      "epoch": 0.018005459720044144,
      "grad_norm": 320.925048828125,
      "learning_rate": 4.954968041836142e-06,
      "logits/chosen": -0.8708783388137817,
      "logits/rejected": -0.8632427453994751,
      "logps/chosen": -73.67878723144531,
      "logps/rejected": -74.83086395263672,
      "loss": 13.6099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002552690450102091,
      "rewards/margins": 0.026570502668619156,
      "rewards/rejected": -0.02912319265305996,
      "step": 31
    },
    {
      "epoch": 0.01858628100133589,
      "grad_norm": 303.17852783203125,
      "learning_rate": 4.953515398024405e-06,
      "logits/chosen": -0.7567359805107117,
      "logits/rejected": -0.8445581197738647,
      "logps/chosen": -66.88877868652344,
      "logps/rejected": -71.22685241699219,
      "loss": 13.845,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01957935281097889,
      "rewards/margins": 0.0028636164497584105,
      "rewards/rejected": -0.022442970424890518,
      "step": 32
    },
    {
      "epoch": 0.019167102282627635,
      "grad_norm": 317.728515625,
      "learning_rate": 4.952062754212668e-06,
      "logits/chosen": -0.917065441608429,
      "logits/rejected": -0.8019243478775024,
      "logps/chosen": -72.54550170898438,
      "logps/rejected": -81.64886474609375,
      "loss": 13.6166,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.011156129650771618,
      "rewards/margins": 0.025838393718004227,
      "rewards/rejected": -0.03699452430009842,
      "step": 33
    },
    {
      "epoch": 0.01974792356391938,
      "grad_norm": 295.3751525878906,
      "learning_rate": 4.9506101104009305e-06,
      "logits/chosen": -0.8145904541015625,
      "logits/rejected": -0.7906870245933533,
      "logps/chosen": -64.89479064941406,
      "logps/rejected": -76.6470947265625,
      "loss": 13.7107,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01140755694359541,
      "rewards/margins": 0.017068836838006973,
      "rewards/rejected": -0.02847639098763466,
      "step": 34
    },
    {
      "epoch": 0.02032874484521113,
      "grad_norm": 305.5505676269531,
      "learning_rate": 4.949157466589193e-06,
      "logits/chosen": -0.7340711355209351,
      "logits/rejected": -0.7537750005722046,
      "logps/chosen": -75.50181579589844,
      "logps/rejected": -70.16544342041016,
      "loss": 14.1136,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.028647294268012047,
      "rewards/margins": -0.023600969463586807,
      "rewards/rejected": -0.005046320613473654,
      "step": 35
    },
    {
      "epoch": 0.020909566126502875,
      "grad_norm": 315.96038818359375,
      "learning_rate": 4.947704822777455e-06,
      "logits/chosen": -0.8396091461181641,
      "logits/rejected": -0.9497518539428711,
      "logps/chosen": -72.89092254638672,
      "logps/rejected": -83.45375061035156,
      "loss": 13.8082,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02720179595053196,
      "rewards/margins": 0.006764040794223547,
      "rewards/rejected": -0.033965837210416794,
      "step": 36
    },
    {
      "epoch": 0.02149038740779462,
      "grad_norm": 320.8994445800781,
      "learning_rate": 4.946252178965718e-06,
      "logits/chosen": -0.7043382525444031,
      "logits/rejected": -0.8497918248176575,
      "logps/chosen": -80.10662841796875,
      "logps/rejected": -80.18870544433594,
      "loss": 13.8802,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04296343773603439,
      "rewards/margins": 0.0006133742863312364,
      "rewards/rejected": -0.043576814234256744,
      "step": 37
    },
    {
      "epoch": 0.02207120868908637,
      "grad_norm": 323.31488037109375,
      "learning_rate": 4.944799535153981e-06,
      "logits/chosen": -0.9256412386894226,
      "logits/rejected": -0.9272140264511108,
      "logps/chosen": -79.0914306640625,
      "logps/rejected": -79.18199157714844,
      "loss": 13.6203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.016189271584153175,
      "rewards/margins": 0.02659023180603981,
      "rewards/rejected": -0.042779501527547836,
      "step": 38
    },
    {
      "epoch": 0.022652029970378115,
      "grad_norm": 309.1510314941406,
      "learning_rate": 4.943346891342243e-06,
      "logits/chosen": -0.7942522168159485,
      "logits/rejected": -0.9100838899612427,
      "logps/chosen": -78.5674057006836,
      "logps/rejected": -69.88294982910156,
      "loss": 13.854,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.028528030961751938,
      "rewards/margins": 0.0029734233394265175,
      "rewards/rejected": -0.03150145336985588,
      "step": 39
    },
    {
      "epoch": 0.02323285125166986,
      "grad_norm": 326.2637023925781,
      "learning_rate": 4.941894247530506e-06,
      "logits/chosen": -0.8673677444458008,
      "logits/rejected": -0.8385285139083862,
      "logps/chosen": -85.15605926513672,
      "logps/rejected": -84.39887237548828,
      "loss": 13.801,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04420323297381401,
      "rewards/margins": 0.006766452454030514,
      "rewards/rejected": -0.0509696826338768,
      "step": 40
    },
    {
      "epoch": 0.023813672532961606,
      "grad_norm": 309.16204833984375,
      "learning_rate": 4.940441603718769e-06,
      "logits/chosen": -0.8228281736373901,
      "logits/rejected": -0.8817359209060669,
      "logps/chosen": -76.8722915649414,
      "logps/rejected": -74.45531463623047,
      "loss": 13.6428,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009386795572936535,
      "rewards/margins": 0.024640636518597603,
      "rewards/rejected": -0.03402743488550186,
      "step": 41
    },
    {
      "epoch": 0.024394493814253355,
      "grad_norm": 322.296630859375,
      "learning_rate": 4.938988959907032e-06,
      "logits/chosen": -0.7055200338363647,
      "logits/rejected": -0.779880166053772,
      "logps/chosen": -75.47132873535156,
      "logps/rejected": -75.7305908203125,
      "loss": 13.6296,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.015317773446440697,
      "rewards/margins": 0.025457333773374557,
      "rewards/rejected": -0.0407751128077507,
      "step": 42
    },
    {
      "epoch": 0.0249753150955451,
      "grad_norm": 334.68182373046875,
      "learning_rate": 4.9375363160952935e-06,
      "logits/chosen": -0.7710026502609253,
      "logits/rejected": -0.82500159740448,
      "logps/chosen": -77.85375213623047,
      "logps/rejected": -74.20629119873047,
      "loss": 13.6993,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.007363433483988047,
      "rewards/margins": 0.018297644332051277,
      "rewards/rejected": -0.025661081075668335,
      "step": 43
    },
    {
      "epoch": 0.025556136376836847,
      "grad_norm": 317.21905517578125,
      "learning_rate": 4.936083672283556e-06,
      "logits/chosen": -0.988287627696991,
      "logits/rejected": -0.9961759448051453,
      "logps/chosen": -77.48571014404297,
      "logps/rejected": -74.44812774658203,
      "loss": 14.0204,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04125748947262764,
      "rewards/margins": -0.01405995525419712,
      "rewards/rejected": -0.02719753421843052,
      "step": 44
    },
    {
      "epoch": 0.026136957658128592,
      "grad_norm": 315.16363525390625,
      "learning_rate": 4.934631028471819e-06,
      "logits/chosen": -0.8442651629447937,
      "logits/rejected": -0.9505764842033386,
      "logps/chosen": -74.02120208740234,
      "logps/rejected": -72.55667114257812,
      "loss": 13.8445,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009219218045473099,
      "rewards/margins": 0.003767718095332384,
      "rewards/rejected": -0.01298693846911192,
      "step": 45
    },
    {
      "epoch": 0.02671777893942034,
      "grad_norm": 333.017578125,
      "learning_rate": 4.933178384660082e-06,
      "logits/chosen": -0.8707895278930664,
      "logits/rejected": -0.8848034739494324,
      "logps/chosen": -82.85108947753906,
      "logps/rejected": -82.15937805175781,
      "loss": 14.1048,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.05764341354370117,
      "rewards/margins": -0.019157161936163902,
      "rewards/rejected": -0.03848625347018242,
      "step": 46
    },
    {
      "epoch": 0.027298600220712087,
      "grad_norm": 328.5982971191406,
      "learning_rate": 4.9317257408483445e-06,
      "logits/chosen": -0.8642932176589966,
      "logits/rejected": -0.8195087313652039,
      "logps/chosen": -75.42310333251953,
      "logps/rejected": -78.54690551757812,
      "loss": 13.8605,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.027368813753128052,
      "rewards/margins": 0.0027305304538458586,
      "rewards/rejected": -0.03009933792054653,
      "step": 47
    },
    {
      "epoch": 0.027879421502003832,
      "grad_norm": 332.7618713378906,
      "learning_rate": 4.930273097036607e-06,
      "logits/chosen": -0.8657892346382141,
      "logits/rejected": -0.9026430249214172,
      "logps/chosen": -86.77098083496094,
      "logps/rejected": -70.10636901855469,
      "loss": 13.9202,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03694070875644684,
      "rewards/margins": -0.0033356398344039917,
      "rewards/rejected": -0.033605072647333145,
      "step": 48
    },
    {
      "epoch": 0.02846024278329558,
      "grad_norm": 326.447265625,
      "learning_rate": 4.928820453224869e-06,
      "logits/chosen": -0.931209921836853,
      "logits/rejected": -0.7586521506309509,
      "logps/chosen": -78.26404571533203,
      "logps/rejected": -75.22278594970703,
      "loss": 13.6775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009883576072752476,
      "rewards/margins": 0.02028750441968441,
      "rewards/rejected": -0.030171077698469162,
      "step": 49
    },
    {
      "epoch": 0.029041064064587327,
      "grad_norm": 323.82501220703125,
      "learning_rate": 4.927367809413132e-06,
      "logits/chosen": -0.8577003479003906,
      "logits/rejected": -0.9445897340774536,
      "logps/chosen": -80.17283630371094,
      "logps/rejected": -73.15702819824219,
      "loss": 13.8828,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04805426672101021,
      "rewards/margins": 6.009042408550158e-05,
      "rewards/rejected": -0.0481143593788147,
      "step": 50
    },
    {
      "epoch": 0.029621885345879072,
      "grad_norm": 345.8555603027344,
      "learning_rate": 4.925915165601395e-06,
      "logits/chosen": -0.8427948951721191,
      "logits/rejected": -0.6843789219856262,
      "logps/chosen": -67.90852355957031,
      "logps/rejected": -78.57261657714844,
      "loss": 13.9293,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03557172790169716,
      "rewards/margins": -0.005063153803348541,
      "rewards/rejected": -0.030508574098348618,
      "step": 51
    },
    {
      "epoch": 0.030202706627170818,
      "grad_norm": 330.11419677734375,
      "learning_rate": 4.924462521789657e-06,
      "logits/chosen": -0.8312528729438782,
      "logits/rejected": -0.8623727560043335,
      "logps/chosen": -73.96318054199219,
      "logps/rejected": -74.79844665527344,
      "loss": 14.0308,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.040969304740428925,
      "rewards/margins": -0.014493905007839203,
      "rewards/rejected": -0.02647540345788002,
      "step": 52
    },
    {
      "epoch": 0.030783527908462567,
      "grad_norm": 439.39111328125,
      "learning_rate": 4.92300987797792e-06,
      "logits/chosen": -0.8176994323730469,
      "logits/rejected": -0.7079430818557739,
      "logps/chosen": -72.11351013183594,
      "logps/rejected": -75.1050033569336,
      "loss": 13.9598,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.011684712953865528,
      "rewards/margins": -0.007551603019237518,
      "rewards/rejected": -0.004133109003305435,
      "step": 53
    },
    {
      "epoch": 0.03136434918975431,
      "grad_norm": 299.9485778808594,
      "learning_rate": 4.921557234166183e-06,
      "logits/chosen": -0.7416559457778931,
      "logits/rejected": -0.7767287492752075,
      "logps/chosen": -70.76741790771484,
      "logps/rejected": -73.53133392333984,
      "loss": 13.7678,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0056773098185658455,
      "rewards/margins": 0.010591240599751472,
      "rewards/rejected": -0.016268549486994743,
      "step": 54
    },
    {
      "epoch": 0.03194517047104606,
      "grad_norm": 294.43988037109375,
      "learning_rate": 4.920104590354446e-06,
      "logits/chosen": -0.6667272448539734,
      "logits/rejected": -0.8636151552200317,
      "logps/chosen": -63.9805793762207,
      "logps/rejected": -74.08900451660156,
      "loss": 13.9136,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.028553063049912453,
      "rewards/margins": -0.0023740821052342653,
      "rewards/rejected": -0.026178985834121704,
      "step": 55
    },
    {
      "epoch": 0.03252599175233781,
      "grad_norm": 315.2623596191406,
      "learning_rate": 4.9186519465427075e-06,
      "logits/chosen": -0.9049865007400513,
      "logits/rejected": -0.9067096710205078,
      "logps/chosen": -71.81194305419922,
      "logps/rejected": -69.87062072753906,
      "loss": 13.6316,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010484697297215462,
      "rewards/margins": 0.025315653532743454,
      "rewards/rejected": -0.03580035641789436,
      "step": 56
    },
    {
      "epoch": 0.03310681303362955,
      "grad_norm": 317.02972412109375,
      "learning_rate": 4.91719930273097e-06,
      "logits/chosen": -0.8822166323661804,
      "logits/rejected": -0.8712530136108398,
      "logps/chosen": -73.8038330078125,
      "logps/rejected": -70.23847198486328,
      "loss": 13.7197,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0062989904545247555,
      "rewards/margins": 0.016949044540524483,
      "rewards/rejected": -0.010650052689015865,
      "step": 57
    },
    {
      "epoch": 0.0336876343149213,
      "grad_norm": 379.47076416015625,
      "learning_rate": 4.915746658919233e-06,
      "logits/chosen": -0.9493061304092407,
      "logits/rejected": -0.8905242681503296,
      "logps/chosen": -77.11824798583984,
      "logps/rejected": -74.96271514892578,
      "loss": 13.7691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.024838361889123917,
      "rewards/margins": 0.011607111431658268,
      "rewards/rejected": -0.03644547611474991,
      "step": 58
    },
    {
      "epoch": 0.03426845559621305,
      "grad_norm": 306.8603210449219,
      "learning_rate": 4.914294015107496e-06,
      "logits/chosen": -0.8163889050483704,
      "logits/rejected": -0.8286741971969604,
      "logps/chosen": -65.88087463378906,
      "logps/rejected": -69.69749450683594,
      "loss": 14.0756,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.025371646508574486,
      "rewards/margins": -0.01930541917681694,
      "rewards/rejected": -0.006066230591386557,
      "step": 59
    },
    {
      "epoch": 0.03484927687750479,
      "grad_norm": 313.75762939453125,
      "learning_rate": 4.9128413712957585e-06,
      "logits/chosen": -0.8723942041397095,
      "logits/rejected": -0.852526068687439,
      "logps/chosen": -69.1073989868164,
      "logps/rejected": -78.30496978759766,
      "loss": 13.6396,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.007079926319420338,
      "rewards/margins": 0.024318810552358627,
      "rewards/rejected": -0.03139873221516609,
      "step": 60
    },
    {
      "epoch": 0.03543009815879654,
      "grad_norm": 301.9634704589844,
      "learning_rate": 4.911388727484021e-06,
      "logits/chosen": -0.9334823489189148,
      "logits/rejected": -0.8579393625259399,
      "logps/chosen": -68.89993286132812,
      "logps/rejected": -71.83196258544922,
      "loss": 13.8795,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02495257556438446,
      "rewards/margins": 0.0009910565568134189,
      "rewards/rejected": -0.025943631306290627,
      "step": 61
    },
    {
      "epoch": 0.03601091944008829,
      "grad_norm": 312.14678955078125,
      "learning_rate": 4.909936083672284e-06,
      "logits/chosen": -1.0733639001846313,
      "logits/rejected": -0.9336859583854675,
      "logps/chosen": -71.59821319580078,
      "logps/rejected": -83.17411804199219,
      "loss": 13.7566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03083074651658535,
      "rewards/margins": 0.01191837340593338,
      "rewards/rejected": -0.04274912178516388,
      "step": 62
    },
    {
      "epoch": 0.03659174072138003,
      "grad_norm": 427.48895263671875,
      "learning_rate": 4.908483439860547e-06,
      "logits/chosen": -0.9465745091438293,
      "logits/rejected": -0.9409465789794922,
      "logps/chosen": -77.52535247802734,
      "logps/rejected": -77.35426330566406,
      "loss": 13.8253,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04855041205883026,
      "rewards/margins": 0.005750913638621569,
      "rewards/rejected": -0.05430132895708084,
      "step": 63
    },
    {
      "epoch": 0.03717256200267178,
      "grad_norm": 329.34857177734375,
      "learning_rate": 4.9070307960488095e-06,
      "logits/chosen": -0.8759803771972656,
      "logits/rejected": -0.9234689474105835,
      "logps/chosen": -62.841156005859375,
      "logps/rejected": -66.76082611083984,
      "loss": 13.9698,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.018310727551579475,
      "rewards/margins": -0.008804955519735813,
      "rewards/rejected": -0.009505772963166237,
      "step": 64
    },
    {
      "epoch": 0.03775338328396353,
      "grad_norm": 290.18682861328125,
      "learning_rate": 4.905578152237072e-06,
      "logits/chosen": -0.9876300692558289,
      "logits/rejected": -0.9204443097114563,
      "logps/chosen": -70.43892669677734,
      "logps/rejected": -69.47483825683594,
      "loss": 13.7851,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.017851095646619797,
      "rewards/margins": 0.011618509888648987,
      "rewards/rejected": -0.029469609260559082,
      "step": 65
    },
    {
      "epoch": 0.03833420456525527,
      "grad_norm": 331.4013977050781,
      "learning_rate": 4.904125508425335e-06,
      "logits/chosen": -0.8038953542709351,
      "logits/rejected": -0.7992674112319946,
      "logps/chosen": -67.48692321777344,
      "logps/rejected": -85.4039077758789,
      "loss": 13.6661,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02300168201327324,
      "rewards/margins": 0.022488538175821304,
      "rewards/rejected": -0.045490216463804245,
      "step": 66
    },
    {
      "epoch": 0.03891502584654702,
      "grad_norm": 324.8753356933594,
      "learning_rate": 4.902672864613598e-06,
      "logits/chosen": -0.7837721705436707,
      "logits/rejected": -0.9071874618530273,
      "logps/chosen": -69.61351013183594,
      "logps/rejected": -68.86196899414062,
      "loss": 13.79,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02017301693558693,
      "rewards/margins": 0.008808745071291924,
      "rewards/rejected": -0.028981763869524002,
      "step": 67
    },
    {
      "epoch": 0.03949584712783876,
      "grad_norm": 317.4655456542969,
      "learning_rate": 4.90122022080186e-06,
      "logits/chosen": -0.8771296739578247,
      "logits/rejected": -0.8569726943969727,
      "logps/chosen": -73.78062438964844,
      "logps/rejected": -68.98119354248047,
      "loss": 13.5934,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009509051218628883,
      "rewards/margins": 0.02956991270184517,
      "rewards/rejected": -0.0390789620578289,
      "step": 68
    },
    {
      "epoch": 0.04007666840913051,
      "grad_norm": 349.1176452636719,
      "learning_rate": 4.8997675769901224e-06,
      "logits/chosen": -0.8722847700119019,
      "logits/rejected": -0.8638531565666199,
      "logps/chosen": -74.6583251953125,
      "logps/rejected": -73.25667572021484,
      "loss": 13.7595,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01801307685673237,
      "rewards/margins": 0.012483290396630764,
      "rewards/rejected": -0.030496370047330856,
      "step": 69
    },
    {
      "epoch": 0.04065748969042226,
      "grad_norm": 321.077880859375,
      "learning_rate": 4.898314933178385e-06,
      "logits/chosen": -0.6883363723754883,
      "logits/rejected": -0.6323266625404358,
      "logps/chosen": -73.32432556152344,
      "logps/rejected": -87.47152709960938,
      "loss": 13.6591,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0362272784113884,
      "rewards/margins": 0.024491379037499428,
      "rewards/rejected": -0.060718655586242676,
      "step": 70
    },
    {
      "epoch": 0.041238310971714,
      "grad_norm": 306.60955810546875,
      "learning_rate": 4.896862289366648e-06,
      "logits/chosen": -0.9396616816520691,
      "logits/rejected": -0.8107419013977051,
      "logps/chosen": -75.45014953613281,
      "logps/rejected": -74.94744110107422,
      "loss": 13.6692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.030824948102235794,
      "rewards/margins": 0.020616179332137108,
      "rewards/rejected": -0.051441121846437454,
      "step": 71
    },
    {
      "epoch": 0.04181913225300575,
      "grad_norm": 309.03094482421875,
      "learning_rate": 4.895409645554911e-06,
      "logits/chosen": -1.0361931324005127,
      "logits/rejected": -0.920698344707489,
      "logps/chosen": -74.54027557373047,
      "logps/rejected": -66.71863555908203,
      "loss": 14.2171,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.06477208435535431,
      "rewards/margins": -0.031630050390958786,
      "rewards/rejected": -0.03314203768968582,
      "step": 72
    },
    {
      "epoch": 0.0423999535342975,
      "grad_norm": 336.6788635253906,
      "learning_rate": 4.893957001743173e-06,
      "logits/chosen": -0.9846957921981812,
      "logits/rejected": -0.9497137069702148,
      "logps/chosen": -73.15393829345703,
      "logps/rejected": -73.5201416015625,
      "loss": 13.9208,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019983595237135887,
      "rewards/margins": -0.003768919501453638,
      "rewards/rejected": -0.016214676201343536,
      "step": 73
    },
    {
      "epoch": 0.04298077481558924,
      "grad_norm": 328.0151062011719,
      "learning_rate": 4.892504357931436e-06,
      "logits/chosen": -0.9070509672164917,
      "logits/rejected": -0.890802264213562,
      "logps/chosen": -74.13055419921875,
      "logps/rejected": -74.56624603271484,
      "loss": 14.2509,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03255216404795647,
      "rewards/margins": -0.03526480868458748,
      "rewards/rejected": 0.0027126409113407135,
      "step": 74
    },
    {
      "epoch": 0.04356159609688099,
      "grad_norm": 312.4993591308594,
      "learning_rate": 4.891051714119698e-06,
      "logits/chosen": -0.8348041772842407,
      "logits/rejected": -0.8501715660095215,
      "logps/chosen": -76.722900390625,
      "logps/rejected": -71.301025390625,
      "loss": 14.0253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024392826482653618,
      "rewards/margins": -0.01172790676355362,
      "rewards/rejected": -0.012664918787777424,
      "step": 75
    },
    {
      "epoch": 0.04414241737817274,
      "grad_norm": 308.2872314453125,
      "learning_rate": 4.889599070307961e-06,
      "logits/chosen": -0.5838706493377686,
      "logits/rejected": -0.637297511100769,
      "logps/chosen": -72.34232330322266,
      "logps/rejected": -66.2028579711914,
      "loss": 14.1823,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.019541073590517044,
      "rewards/margins": -0.02966020628809929,
      "rewards/rejected": 0.010119132697582245,
      "step": 76
    },
    {
      "epoch": 0.04472323865946448,
      "grad_norm": 305.4457702636719,
      "learning_rate": 4.8881464264962236e-06,
      "logits/chosen": -0.973824143409729,
      "logits/rejected": -0.9775202870368958,
      "logps/chosen": -72.74274444580078,
      "logps/rejected": -73.25370788574219,
      "loss": 14.0823,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.02193290740251541,
      "rewards/margins": -0.019592974334955215,
      "rewards/rejected": -0.0023399335332214832,
      "step": 77
    },
    {
      "epoch": 0.04530405994075623,
      "grad_norm": 483.3565368652344,
      "learning_rate": 4.886693782684486e-06,
      "logits/chosen": -0.8660491704940796,
      "logits/rejected": -0.8472667932510376,
      "logps/chosen": -79.67647552490234,
      "logps/rejected": -76.70512390136719,
      "loss": 14.0415,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.035908013582229614,
      "rewards/margins": -0.01337387878447771,
      "rewards/rejected": -0.02253413386642933,
      "step": 78
    },
    {
      "epoch": 0.04588488122204797,
      "grad_norm": 315.49755859375,
      "learning_rate": 4.885241138872749e-06,
      "logits/chosen": -0.6490969061851501,
      "logits/rejected": -0.7820181250572205,
      "logps/chosen": -74.25728607177734,
      "logps/rejected": -73.80535125732422,
      "loss": 13.4779,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.014750528149306774,
      "rewards/margins": 0.043530743569135666,
      "rewards/rejected": -0.058281272649765015,
      "step": 79
    },
    {
      "epoch": 0.04646570250333972,
      "grad_norm": 315.58837890625,
      "learning_rate": 4.883788495061012e-06,
      "logits/chosen": -0.8913451433181763,
      "logits/rejected": -0.8876463770866394,
      "logps/chosen": -73.64271545410156,
      "logps/rejected": -69.37626647949219,
      "loss": 13.957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01223234087228775,
      "rewards/margins": -0.005636455025523901,
      "rewards/rejected": -0.006595888640731573,
      "step": 80
    },
    {
      "epoch": 0.04704652378463147,
      "grad_norm": 313.813232421875,
      "learning_rate": 4.882335851249274e-06,
      "logits/chosen": -0.8722120523452759,
      "logits/rejected": -0.7989141345024109,
      "logps/chosen": -70.44036865234375,
      "logps/rejected": -70.73589324951172,
      "loss": 14.1477,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.026375526562333107,
      "rewards/margins": -0.026495525613427162,
      "rewards/rejected": 0.00012000077549600974,
      "step": 81
    },
    {
      "epoch": 0.04762734506592321,
      "grad_norm": 314.9659729003906,
      "learning_rate": 4.8808832074375365e-06,
      "logits/chosen": -0.7730615735054016,
      "logits/rejected": -0.8082054257392883,
      "logps/chosen": -69.96324157714844,
      "logps/rejected": -77.90928649902344,
      "loss": 13.778,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.014157066121697426,
      "rewards/margins": 0.010449771769344807,
      "rewards/rejected": -0.024606838822364807,
      "step": 82
    },
    {
      "epoch": 0.04820816634721496,
      "grad_norm": 303.14599609375,
      "learning_rate": 4.879430563625799e-06,
      "logits/chosen": -0.9874809980392456,
      "logits/rejected": -1.0569359064102173,
      "logps/chosen": -74.83180236816406,
      "logps/rejected": -76.95542907714844,
      "loss": 13.6937,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0425366535782814,
      "rewards/margins": 0.021705975756049156,
      "rewards/rejected": -0.06424263119697571,
      "step": 83
    },
    {
      "epoch": 0.04878898762850671,
      "grad_norm": 317.16961669921875,
      "learning_rate": 4.877977919814062e-06,
      "logits/chosen": -0.9083768725395203,
      "logits/rejected": -0.9116488695144653,
      "logps/chosen": -80.80293273925781,
      "logps/rejected": -72.5535888671875,
      "loss": 14.2559,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.06587149202823639,
      "rewards/margins": -0.036308903247117996,
      "rewards/rejected": -0.029562586918473244,
      "step": 84
    },
    {
      "epoch": 0.04936980890979845,
      "grad_norm": 301.5,
      "learning_rate": 4.876525276002325e-06,
      "logits/chosen": -0.9549547433853149,
      "logits/rejected": -0.9115845561027527,
      "logps/chosen": -75.24485778808594,
      "logps/rejected": -68.90257263183594,
      "loss": 14.065,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.015530148521065712,
      "rewards/margins": -0.0176745243370533,
      "rewards/rejected": 0.0021443753503262997,
      "step": 85
    },
    {
      "epoch": 0.0499506301910902,
      "grad_norm": 307.9525451660156,
      "learning_rate": 4.8750726321905875e-06,
      "logits/chosen": -0.9091174006462097,
      "logits/rejected": -0.9419649839401245,
      "logps/chosen": -76.40791320800781,
      "logps/rejected": -77.6815185546875,
      "loss": 13.7705,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015883130952715874,
      "rewards/margins": 0.01122428011149168,
      "rewards/rejected": -0.02710741199553013,
      "step": 86
    },
    {
      "epoch": 0.05053145147238195,
      "grad_norm": 321.51507568359375,
      "learning_rate": 4.87361998837885e-06,
      "logits/chosen": -0.9909089803695679,
      "logits/rejected": -0.9910022616386414,
      "logps/chosen": -77.63959503173828,
      "logps/rejected": -72.30487823486328,
      "loss": 13.8477,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006782358977943659,
      "rewards/margins": 0.003400030778720975,
      "rewards/rejected": -0.010182389989495277,
      "step": 87
    },
    {
      "epoch": 0.05111227275367369,
      "grad_norm": 321.3612060546875,
      "learning_rate": 4.872167344567112e-06,
      "logits/chosen": -0.7301020622253418,
      "logits/rejected": -0.5697265267372131,
      "logps/chosen": -70.13436126708984,
      "logps/rejected": -77.76301574707031,
      "loss": 13.7418,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.015415447764098644,
      "rewards/margins": 0.013852661475539207,
      "rewards/rejected": 0.0015627862885594368,
      "step": 88
    },
    {
      "epoch": 0.05169309403496544,
      "grad_norm": 322.7413330078125,
      "learning_rate": 4.870714700755375e-06,
      "logits/chosen": -0.8953542709350586,
      "logits/rejected": -0.8882652521133423,
      "logps/chosen": -69.642333984375,
      "logps/rejected": -66.21636962890625,
      "loss": 13.8948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015242251567542553,
      "rewards/margins": -0.00036549606011249125,
      "rewards/rejected": -0.014876757748425007,
      "step": 89
    },
    {
      "epoch": 0.052273915316257184,
      "grad_norm": 316.21905517578125,
      "learning_rate": 4.869262056943638e-06,
      "logits/chosen": -0.826396107673645,
      "logits/rejected": -0.8409526944160461,
      "logps/chosen": -84.0012435913086,
      "logps/rejected": -73.15267944335938,
      "loss": 14.2303,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03959153965115547,
      "rewards/margins": -0.03513690084218979,
      "rewards/rejected": -0.004454641602933407,
      "step": 90
    },
    {
      "epoch": 0.05285473659754893,
      "grad_norm": 314.6103515625,
      "learning_rate": 4.8678094131319e-06,
      "logits/chosen": -0.9043526649475098,
      "logits/rejected": -0.9540117979049683,
      "logps/chosen": -85.2854995727539,
      "logps/rejected": -74.1194076538086,
      "loss": 14.256,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05172392725944519,
      "rewards/margins": -0.03560470789670944,
      "rewards/rejected": -0.016119223088026047,
      "step": 91
    },
    {
      "epoch": 0.05343555787884068,
      "grad_norm": 298.3580322265625,
      "learning_rate": 4.866356769320163e-06,
      "logits/chosen": -0.83808434009552,
      "logits/rejected": -0.858515739440918,
      "logps/chosen": -67.17420959472656,
      "logps/rejected": -84.23038482666016,
      "loss": 13.4033,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0225966926664114,
      "rewards/margins": 0.05055863410234451,
      "rewards/rejected": -0.027961939573287964,
      "step": 92
    },
    {
      "epoch": 0.054016379160132424,
      "grad_norm": 293.33282470703125,
      "learning_rate": 4.864904125508426e-06,
      "logits/chosen": -0.9872828722000122,
      "logits/rejected": -1.1029959917068481,
      "logps/chosen": -72.00138854980469,
      "logps/rejected": -69.6135025024414,
      "loss": 13.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0009010225767269731,
      "rewards/margins": 0.017413010820746422,
      "rewards/rejected": -0.01831403188407421,
      "step": 93
    },
    {
      "epoch": 0.054597200441424174,
      "grad_norm": 356.63946533203125,
      "learning_rate": 4.863451481696689e-06,
      "logits/chosen": -0.8439090847969055,
      "logits/rejected": -0.8643622398376465,
      "logps/chosen": -70.82389831542969,
      "logps/rejected": -75.1737289428711,
      "loss": 13.8326,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0158695038408041,
      "rewards/margins": 0.0050217146053910255,
      "rewards/rejected": -0.0208912193775177,
      "step": 94
    },
    {
      "epoch": 0.05517802172271592,
      "grad_norm": 304.1072998046875,
      "learning_rate": 4.8619988378849505e-06,
      "logits/chosen": -1.0255842208862305,
      "logits/rejected": -1.041534662246704,
      "logps/chosen": -68.80192565917969,
      "logps/rejected": -67.79802703857422,
      "loss": 13.6247,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01212473027408123,
      "rewards/margins": 0.02532658539712429,
      "rewards/rejected": -0.01320185698568821,
      "step": 95
    },
    {
      "epoch": 0.055758843004007665,
      "grad_norm": 314.3151550292969,
      "learning_rate": 4.860546194073213e-06,
      "logits/chosen": -0.8595132827758789,
      "logits/rejected": -0.8583124279975891,
      "logps/chosen": -72.1944808959961,
      "logps/rejected": -72.77186584472656,
      "loss": 13.9049,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.031448423862457275,
      "rewards/margins": -0.0011334316805005074,
      "rewards/rejected": -0.030314987525343895,
      "step": 96
    },
    {
      "epoch": 0.056339664285299414,
      "grad_norm": 320.2240905761719,
      "learning_rate": 4.859093550261476e-06,
      "logits/chosen": -0.749364972114563,
      "logits/rejected": -0.8103491067886353,
      "logps/chosen": -79.1120376586914,
      "logps/rejected": -75.91130065917969,
      "loss": 13.9032,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03181576728820801,
      "rewards/margins": -0.0017790347337722778,
      "rewards/rejected": -0.03003673627972603,
      "step": 97
    },
    {
      "epoch": 0.05692048556659116,
      "grad_norm": 314.99896240234375,
      "learning_rate": 4.857640906449739e-06,
      "logits/chosen": -0.8008295297622681,
      "logits/rejected": -0.847716212272644,
      "logps/chosen": -79.81649017333984,
      "logps/rejected": -68.53919982910156,
      "loss": 14.0979,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.021854501217603683,
      "rewards/margins": -0.02203025482594967,
      "rewards/rejected": 0.00017575845413375646,
      "step": 98
    },
    {
      "epoch": 0.057501306847882905,
      "grad_norm": 315.1274108886719,
      "learning_rate": 4.8561882626380015e-06,
      "logits/chosen": -0.8115674257278442,
      "logits/rejected": -0.8901892900466919,
      "logps/chosen": -72.59163665771484,
      "logps/rejected": -78.07933044433594,
      "loss": 13.7458,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.000810365192592144,
      "rewards/margins": 0.014865818433463573,
      "rewards/rejected": -0.01405545137822628,
      "step": 99
    },
    {
      "epoch": 0.058082128129174654,
      "grad_norm": 318.1490173339844,
      "learning_rate": 4.854735618826264e-06,
      "logits/chosen": -0.9334144592285156,
      "logits/rejected": -0.9743694067001343,
      "logps/chosen": -74.79545593261719,
      "logps/rejected": -80.59342193603516,
      "loss": 13.5047,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.00012166835222160444,
      "rewards/margins": 0.038393907248973846,
      "rewards/rejected": -0.03827223926782608,
      "step": 100
    },
    {
      "epoch": 0.0586629494104664,
      "grad_norm": 312.1184997558594,
      "learning_rate": 4.853282975014527e-06,
      "logits/chosen": -0.8218280076980591,
      "logits/rejected": -0.848983645439148,
      "logps/chosen": -77.64916229248047,
      "logps/rejected": -77.53215026855469,
      "loss": 13.6869,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0032627955079078674,
      "rewards/margins": 0.022136736661195755,
      "rewards/rejected": -0.025399532169103622,
      "step": 101
    },
    {
      "epoch": 0.059243770691758145,
      "grad_norm": 314.2687072753906,
      "learning_rate": 4.851830331202789e-06,
      "logits/chosen": -0.8590647578239441,
      "logits/rejected": -0.8945513963699341,
      "logps/chosen": -78.69733428955078,
      "logps/rejected": -68.73847198486328,
      "loss": 13.7324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010157021693885326,
      "rewards/margins": 0.016794661059975624,
      "rewards/rejected": -0.026951681822538376,
      "step": 102
    },
    {
      "epoch": 0.059824591973049894,
      "grad_norm": 322.7899475097656,
      "learning_rate": 4.850377687391052e-06,
      "logits/chosen": -0.9431624412536621,
      "logits/rejected": -0.8844934701919556,
      "logps/chosen": -71.08782958984375,
      "logps/rejected": -76.45503234863281,
      "loss": 13.7423,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0048022083938121796,
      "rewards/margins": 0.014014339074492455,
      "rewards/rejected": -0.018816547468304634,
      "step": 103
    },
    {
      "epoch": 0.060405413254341636,
      "grad_norm": 328.67974853515625,
      "learning_rate": 4.848925043579314e-06,
      "logits/chosen": -0.7548056840896606,
      "logits/rejected": -0.7886452078819275,
      "logps/chosen": -74.668212890625,
      "logps/rejected": -74.39926147460938,
      "loss": 13.3898,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.050213318318128586,
      "rewards/margins": 0.04900939390063286,
      "rewards/rejected": 0.0012039269786328077,
      "step": 104
    },
    {
      "epoch": 0.060986234535633385,
      "grad_norm": 306.2090148925781,
      "learning_rate": 4.847472399767578e-06,
      "logits/chosen": -0.7457195520401001,
      "logits/rejected": -0.7296000123023987,
      "logps/chosen": -66.86776733398438,
      "logps/rejected": -68.6021499633789,
      "loss": 14.2794,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.01881163753569126,
      "rewards/margins": -0.039370544254779816,
      "rewards/rejected": 0.020558910444378853,
      "step": 105
    },
    {
      "epoch": 0.061567055816925134,
      "grad_norm": 429.5802917480469,
      "learning_rate": 4.84601975595584e-06,
      "logits/chosen": -0.7268589735031128,
      "logits/rejected": -0.824454665184021,
      "logps/chosen": -73.6258773803711,
      "logps/rejected": -73.26065826416016,
      "loss": 13.8393,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.019226079806685448,
      "rewards/margins": 0.005577159114181995,
      "rewards/rejected": -0.02480323798954487,
      "step": 106
    },
    {
      "epoch": 0.062147877098216876,
      "grad_norm": 310.4759521484375,
      "learning_rate": 4.844567112144103e-06,
      "logits/chosen": -0.7717695236206055,
      "logits/rejected": -0.6804165244102478,
      "logps/chosen": -73.10393524169922,
      "logps/rejected": -71.20040893554688,
      "loss": 13.5079,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.018317507579922676,
      "rewards/margins": 0.03789940103888512,
      "rewards/rejected": -0.05621690675616264,
      "step": 107
    },
    {
      "epoch": 0.06272869837950862,
      "grad_norm": 313.2925720214844,
      "learning_rate": 4.843114468332365e-06,
      "logits/chosen": -0.7994370460510254,
      "logits/rejected": -0.9286397099494934,
      "logps/chosen": -73.3687515258789,
      "logps/rejected": -73.391357421875,
      "loss": 13.7167,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0021995368879288435,
      "rewards/margins": 0.017970655113458633,
      "rewards/rejected": -0.020170193165540695,
      "step": 108
    },
    {
      "epoch": 0.06330951966080037,
      "grad_norm": 298.9688415527344,
      "learning_rate": 4.841661824520628e-06,
      "logits/chosen": -0.783902108669281,
      "logits/rejected": -0.8706483840942383,
      "logps/chosen": -72.7656021118164,
      "logps/rejected": -67.734619140625,
      "loss": 14.0378,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.010019464418292046,
      "rewards/margins": -0.016036922112107277,
      "rewards/rejected": 0.006017456762492657,
      "step": 109
    },
    {
      "epoch": 0.06389034094209212,
      "grad_norm": 292.80926513671875,
      "learning_rate": 4.840209180708891e-06,
      "logits/chosen": -0.867302417755127,
      "logits/rejected": -0.934320330619812,
      "logps/chosen": -69.18145751953125,
      "logps/rejected": -74.14984893798828,
      "loss": 13.9161,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04971027001738548,
      "rewards/margins": -0.003587187733501196,
      "rewards/rejected": -0.04612307995557785,
      "step": 110
    },
    {
      "epoch": 0.06447116222338387,
      "grad_norm": 316.4573059082031,
      "learning_rate": 4.838756536897154e-06,
      "logits/chosen": -0.7158193588256836,
      "logits/rejected": -0.703850269317627,
      "logps/chosen": -72.75650024414062,
      "logps/rejected": -74.89552307128906,
      "loss": 13.7563,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02667677029967308,
      "rewards/margins": 0.013364049606025219,
      "rewards/rejected": 0.013312721624970436,
      "step": 111
    },
    {
      "epoch": 0.06505198350467561,
      "grad_norm": 310.0760498046875,
      "learning_rate": 4.837303893085416e-06,
      "logits/chosen": -0.7871710658073425,
      "logits/rejected": -0.7631909847259521,
      "logps/chosen": -72.4153823852539,
      "logps/rejected": -69.98908233642578,
      "loss": 14.0832,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.027912933379411697,
      "rewards/margins": -0.020552167668938637,
      "rewards/rejected": -0.007360764779150486,
      "step": 112
    },
    {
      "epoch": 0.06563280478596736,
      "grad_norm": 298.72796630859375,
      "learning_rate": 4.835851249273678e-06,
      "logits/chosen": -0.8587129712104797,
      "logits/rejected": -0.828883171081543,
      "logps/chosen": -76.20726013183594,
      "logps/rejected": -70.39952087402344,
      "loss": 13.7695,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01639840006828308,
      "rewards/margins": 0.01160570327192545,
      "rewards/rejected": 0.004792699124664068,
      "step": 113
    },
    {
      "epoch": 0.0662136260672591,
      "grad_norm": 306.0551452636719,
      "learning_rate": 4.834398605461941e-06,
      "logits/chosen": -0.7858158349990845,
      "logits/rejected": -0.8138400912284851,
      "logps/chosen": -71.29947662353516,
      "logps/rejected": -74.6938247680664,
      "loss": 13.6944,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009441891685128212,
      "rewards/margins": 0.020461464300751686,
      "rewards/rejected": -0.011019574478268623,
      "step": 114
    },
    {
      "epoch": 0.06679444734855085,
      "grad_norm": 299.12298583984375,
      "learning_rate": 4.832945961650204e-06,
      "logits/chosen": -0.8294021487236023,
      "logits/rejected": -0.9068562388420105,
      "logps/chosen": -72.25274658203125,
      "logps/rejected": -74.01802062988281,
      "loss": 13.8028,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.007599563803523779,
      "rewards/margins": 0.009034421294927597,
      "rewards/rejected": -0.01663398928940296,
      "step": 115
    },
    {
      "epoch": 0.0673752686298426,
      "grad_norm": 316.1508483886719,
      "learning_rate": 4.8314933178384665e-06,
      "logits/chosen": -0.7372664213180542,
      "logits/rejected": -0.6621009707450867,
      "logps/chosen": -71.98650360107422,
      "logps/rejected": -73.5616226196289,
      "loss": 14.0602,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.040212083607912064,
      "rewards/margins": -0.01578274928033352,
      "rewards/rejected": -0.024429330602288246,
      "step": 116
    },
    {
      "epoch": 0.06795608991113435,
      "grad_norm": 311.5640869140625,
      "learning_rate": 4.830040674026729e-06,
      "logits/chosen": -0.8206332325935364,
      "logits/rejected": -0.8395715951919556,
      "logps/chosen": -71.33819580078125,
      "logps/rejected": -67.67076110839844,
      "loss": 14.0576,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04526720941066742,
      "rewards/margins": -0.017988480627536774,
      "rewards/rejected": -0.027278726920485497,
      "step": 117
    },
    {
      "epoch": 0.0685369111924261,
      "grad_norm": 319.0810852050781,
      "learning_rate": 4.828588030214992e-06,
      "logits/chosen": -0.75376957654953,
      "logits/rejected": -0.7071677446365356,
      "logps/chosen": -79.94001770019531,
      "logps/rejected": -73.17815399169922,
      "loss": 14.162,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.015543567016720772,
      "rewards/margins": -0.02663475275039673,
      "rewards/rejected": 0.011091184802353382,
      "step": 118
    },
    {
      "epoch": 0.06911773247371784,
      "grad_norm": 296.51690673828125,
      "learning_rate": 4.827135386403255e-06,
      "logits/chosen": -0.8419657945632935,
      "logits/rejected": -0.7793577313423157,
      "logps/chosen": -72.96027374267578,
      "logps/rejected": -71.80851745605469,
      "loss": 13.2528,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.016827832907438278,
      "rewards/margins": 0.06467505544424057,
      "rewards/rejected": -0.047847211360931396,
      "step": 119
    },
    {
      "epoch": 0.06969855375500958,
      "grad_norm": 305.2845458984375,
      "learning_rate": 4.825682742591517e-06,
      "logits/chosen": -0.7169400453567505,
      "logits/rejected": -0.7394998073577881,
      "logps/chosen": -69.85813903808594,
      "logps/rejected": -76.11322784423828,
      "loss": 13.4805,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0008814238244667649,
      "rewards/margins": 0.041327688843011856,
      "rewards/rejected": -0.04220911115407944,
      "step": 120
    },
    {
      "epoch": 0.07027937503630133,
      "grad_norm": 329.4168395996094,
      "learning_rate": 4.824230098779779e-06,
      "logits/chosen": -0.8370596170425415,
      "logits/rejected": -0.8346614837646484,
      "logps/chosen": -76.3817138671875,
      "logps/rejected": -69.7574691772461,
      "loss": 14.3016,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.017085228115320206,
      "rewards/margins": -0.039855439215898514,
      "rewards/rejected": 0.02277020923793316,
      "step": 121
    },
    {
      "epoch": 0.07086019631759308,
      "grad_norm": 328.85821533203125,
      "learning_rate": 4.822777454968042e-06,
      "logits/chosen": -0.8888875246047974,
      "logits/rejected": -0.8640028834342957,
      "logps/chosen": -81.20426940917969,
      "logps/rejected": -73.75362396240234,
      "loss": 14.2056,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05541349574923515,
      "rewards/margins": -0.030969763174653053,
      "rewards/rejected": -0.02444373071193695,
      "step": 122
    },
    {
      "epoch": 0.07144101759888483,
      "grad_norm": 329.655517578125,
      "learning_rate": 4.821324811156305e-06,
      "logits/chosen": -0.8150280714035034,
      "logits/rejected": -0.7874841094017029,
      "logps/chosen": -74.91133880615234,
      "logps/rejected": -76.65450286865234,
      "loss": 13.9383,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04368335008621216,
      "rewards/margins": -0.005534342024475336,
      "rewards/rejected": -0.03814900666475296,
      "step": 123
    },
    {
      "epoch": 0.07202183888017658,
      "grad_norm": 287.1902770996094,
      "learning_rate": 4.819872167344568e-06,
      "logits/chosen": -0.9967269897460938,
      "logits/rejected": -1.0370653867721558,
      "logps/chosen": -73.49361419677734,
      "logps/rejected": -76.58966827392578,
      "loss": 13.7119,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.034154172986745834,
      "rewards/margins": 0.019786100834608078,
      "rewards/rejected": -0.05394027382135391,
      "step": 124
    },
    {
      "epoch": 0.07260266016146831,
      "grad_norm": 321.77142333984375,
      "learning_rate": 4.81841952353283e-06,
      "logits/chosen": -0.8000070452690125,
      "logits/rejected": -0.8691812753677368,
      "logps/chosen": -77.01860809326172,
      "logps/rejected": -73.41801452636719,
      "loss": 13.4661,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00704178074374795,
      "rewards/margins": 0.04257983714342117,
      "rewards/rejected": -0.04962162673473358,
      "step": 125
    },
    {
      "epoch": 0.07318348144276006,
      "grad_norm": 329.365478515625,
      "learning_rate": 4.816966879721093e-06,
      "logits/chosen": -0.6257106065750122,
      "logits/rejected": -0.7529661059379578,
      "logps/chosen": -70.44479370117188,
      "logps/rejected": -71.65373229980469,
      "loss": 13.9067,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.030417144298553467,
      "rewards/margins": -0.0017956402152776718,
      "rewards/rejected": -0.028621505945920944,
      "step": 126
    },
    {
      "epoch": 0.07376430272405181,
      "grad_norm": 305.7592468261719,
      "learning_rate": 4.815514235909355e-06,
      "logits/chosen": -0.8345580101013184,
      "logits/rejected": -0.809880256652832,
      "logps/chosen": -75.27732849121094,
      "logps/rejected": -76.76701354980469,
      "loss": 13.7657,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02379775047302246,
      "rewards/margins": 0.011454248800873756,
      "rewards/rejected": -0.03525200113654137,
      "step": 127
    },
    {
      "epoch": 0.07434512400534356,
      "grad_norm": 326.51171875,
      "learning_rate": 4.814061592097618e-06,
      "logits/chosen": -0.8751303553581238,
      "logits/rejected": -0.7346702814102173,
      "logps/chosen": -71.24671936035156,
      "logps/rejected": -76.22618103027344,
      "loss": 13.7425,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.002272081095725298,
      "rewards/margins": 0.013138952665030956,
      "rewards/rejected": -0.010866871103644371,
      "step": 128
    },
    {
      "epoch": 0.0749259452866353,
      "grad_norm": 331.1041259765625,
      "learning_rate": 4.8126089482858805e-06,
      "logits/chosen": -0.840943455696106,
      "logits/rejected": -0.8373934626579285,
      "logps/chosen": -74.03942108154297,
      "logps/rejected": -69.12279510498047,
      "loss": 14.2463,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0546981580555439,
      "rewards/margins": -0.03357243537902832,
      "rewards/rejected": -0.02112571895122528,
      "step": 129
    },
    {
      "epoch": 0.07550676656792706,
      "grad_norm": 312.5914611816406,
      "learning_rate": 4.811156304474143e-06,
      "logits/chosen": -0.8838016390800476,
      "logits/rejected": -0.9450550079345703,
      "logps/chosen": -79.87310028076172,
      "logps/rejected": -80.9448013305664,
      "loss": 13.7435,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03348521143198013,
      "rewards/margins": 0.014457473531365395,
      "rewards/rejected": -0.04794268682599068,
      "step": 130
    },
    {
      "epoch": 0.07608758784921879,
      "grad_norm": 291.64898681640625,
      "learning_rate": 4.809703660662406e-06,
      "logits/chosen": -0.9270390272140503,
      "logits/rejected": -1.0079596042633057,
      "logps/chosen": -68.3660659790039,
      "logps/rejected": -75.16285705566406,
      "loss": 13.761,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01317480206489563,
      "rewards/margins": 0.013511622324585915,
      "rewards/rejected": -0.026686420664191246,
      "step": 131
    },
    {
      "epoch": 0.07666840913051054,
      "grad_norm": 325.22186279296875,
      "learning_rate": 4.808251016850669e-06,
      "logits/chosen": -0.793510913848877,
      "logits/rejected": -0.8536527752876282,
      "logps/chosen": -79.37129211425781,
      "logps/rejected": -81.64805603027344,
      "loss": 13.9288,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04622223600745201,
      "rewards/margins": -0.0038099661469459534,
      "rewards/rejected": -0.042412273585796356,
      "step": 132
    },
    {
      "epoch": 0.07724923041180229,
      "grad_norm": 332.0003662109375,
      "learning_rate": 4.8067983730389315e-06,
      "logits/chosen": -0.863021731376648,
      "logits/rejected": -0.7807949185371399,
      "logps/chosen": -70.98688507080078,
      "logps/rejected": -74.82456970214844,
      "loss": 13.6363,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03255782276391983,
      "rewards/margins": 0.027058040723204613,
      "rewards/rejected": -0.059615861624479294,
      "step": 133
    },
    {
      "epoch": 0.07783005169309404,
      "grad_norm": 419.6676025390625,
      "learning_rate": 4.8053457292271934e-06,
      "logits/chosen": -0.6968336701393127,
      "logits/rejected": -0.8155566453933716,
      "logps/chosen": -76.53175354003906,
      "logps/rejected": -80.69217681884766,
      "loss": 13.5714,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.003611106425523758,
      "rewards/margins": 0.0325997918844223,
      "rewards/rejected": -0.03621090203523636,
      "step": 134
    },
    {
      "epoch": 0.07841087297438579,
      "grad_norm": 318.14093017578125,
      "learning_rate": 4.803893085415456e-06,
      "logits/chosen": -0.8911747932434082,
      "logits/rejected": -0.8688879013061523,
      "logps/chosen": -78.88302612304688,
      "logps/rejected": -72.34574890136719,
      "loss": 13.7806,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009762173518538475,
      "rewards/margins": 0.012868879362940788,
      "rewards/rejected": -0.022631052881479263,
      "step": 135
    },
    {
      "epoch": 0.07899169425567752,
      "grad_norm": 301.05181884765625,
      "learning_rate": 4.802440441603719e-06,
      "logits/chosen": -0.9665401577949524,
      "logits/rejected": -1.0596462488174438,
      "logps/chosen": -69.1599349975586,
      "logps/rejected": -72.39261627197266,
      "loss": 13.9663,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006733216345310211,
      "rewards/margins": -0.007116011343896389,
      "rewards/rejected": 0.00038279517320916057,
      "step": 136
    },
    {
      "epoch": 0.07957251553696927,
      "grad_norm": 318.62030029296875,
      "learning_rate": 4.800987797791982e-06,
      "logits/chosen": -0.7195813059806824,
      "logits/rejected": -0.7145063281059265,
      "logps/chosen": -79.5794448852539,
      "logps/rejected": -84.13545227050781,
      "loss": 13.4993,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.002331190975382924,
      "rewards/margins": 0.03869814798235893,
      "rewards/rejected": -0.04102934151887894,
      "step": 137
    },
    {
      "epoch": 0.08015333681826102,
      "grad_norm": 425.3757019042969,
      "learning_rate": 4.7995351539802444e-06,
      "logits/chosen": -0.8645333051681519,
      "logits/rejected": -0.7608574032783508,
      "logps/chosen": -69.97611999511719,
      "logps/rejected": -85.6728515625,
      "loss": 13.1834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011998976580798626,
      "rewards/margins": 0.07343053817749023,
      "rewards/rejected": -0.06143154948949814,
      "step": 138
    },
    {
      "epoch": 0.08073415809955277,
      "grad_norm": 312.2695007324219,
      "learning_rate": 4.798082510168507e-06,
      "logits/chosen": -0.7758678197860718,
      "logits/rejected": -0.7573307752609253,
      "logps/chosen": -76.74276733398438,
      "logps/rejected": -70.28133392333984,
      "loss": 13.8316,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0051659406162798405,
      "rewards/margins": 0.006639555096626282,
      "rewards/rejected": -0.0014736183220520616,
      "step": 139
    },
    {
      "epoch": 0.08131497938084452,
      "grad_norm": 315.1054992675781,
      "learning_rate": 4.79662986635677e-06,
      "logits/chosen": -0.718429684638977,
      "logits/rejected": -0.8549942970275879,
      "logps/chosen": -70.87870788574219,
      "logps/rejected": -68.09158325195312,
      "loss": 13.8946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.027258872985839844,
      "rewards/margins": -0.0010386653011664748,
      "rewards/rejected": -0.026220208033919334,
      "step": 140
    },
    {
      "epoch": 0.08189580066213627,
      "grad_norm": 315.59136962890625,
      "learning_rate": 4.795177222545032e-06,
      "logits/chosen": -0.8170045614242554,
      "logits/rejected": -0.7605774402618408,
      "logps/chosen": -69.022705078125,
      "logps/rejected": -81.53439331054688,
      "loss": 13.625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.006630831863731146,
      "rewards/margins": 0.02589798904955387,
      "rewards/rejected": -0.019267160445451736,
      "step": 141
    },
    {
      "epoch": 0.082476621943428,
      "grad_norm": 303.656982421875,
      "learning_rate": 4.793724578733295e-06,
      "logits/chosen": -0.9443836212158203,
      "logits/rejected": -0.944778561592102,
      "logps/chosen": -77.53956604003906,
      "logps/rejected": -78.73460388183594,
      "loss": 13.8955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02790834940969944,
      "rewards/margins": -0.0015594146680086851,
      "rewards/rejected": -0.026348933577537537,
      "step": 142
    },
    {
      "epoch": 0.08305744322471975,
      "grad_norm": 316.501220703125,
      "learning_rate": 4.792271934921557e-06,
      "logits/chosen": -0.7522753477096558,
      "logits/rejected": -0.8681309819221497,
      "logps/chosen": -77.05738830566406,
      "logps/rejected": -70.8436508178711,
      "loss": 14.4085,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.031116142868995667,
      "rewards/margins": -0.05178617313504219,
      "rewards/rejected": 0.020670032128691673,
      "step": 143
    },
    {
      "epoch": 0.0836382645060115,
      "grad_norm": 673.7465209960938,
      "learning_rate": 4.79081929110982e-06,
      "logits/chosen": -0.8983888626098633,
      "logits/rejected": -0.8548039197921753,
      "logps/chosen": -78.78826141357422,
      "logps/rejected": -71.03697204589844,
      "loss": 13.8593,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004244650714099407,
      "rewards/margins": 0.004877415020018816,
      "rewards/rejected": -0.00912206619977951,
      "step": 144
    },
    {
      "epoch": 0.08421908578730325,
      "grad_norm": 308.432861328125,
      "learning_rate": 4.789366647298083e-06,
      "logits/chosen": -0.8701874613761902,
      "logits/rejected": -0.8173721432685852,
      "logps/chosen": -73.85729217529297,
      "logps/rejected": -74.20492553710938,
      "loss": 13.6175,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01854265108704567,
      "rewards/margins": 0.02803659439086914,
      "rewards/rejected": -0.04657924920320511,
      "step": 145
    },
    {
      "epoch": 0.084799907068595,
      "grad_norm": 295.6097717285156,
      "learning_rate": 4.787914003486346e-06,
      "logits/chosen": -0.8499331474304199,
      "logits/rejected": -0.8638921976089478,
      "logps/chosen": -66.98426818847656,
      "logps/rejected": -65.90644836425781,
      "loss": 13.8752,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0066559212282299995,
      "rewards/margins": 0.0012805939186364412,
      "rewards/rejected": -0.007936513982713223,
      "step": 146
    },
    {
      "epoch": 0.08538072834988673,
      "grad_norm": 331.2653503417969,
      "learning_rate": 4.786461359674608e-06,
      "logits/chosen": -0.9896368980407715,
      "logits/rejected": -1.1149613857269287,
      "logps/chosen": -77.21143341064453,
      "logps/rejected": -85.37650299072266,
      "loss": 13.5235,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.003070878330618143,
      "rewards/margins": 0.03570512682199478,
      "rewards/rejected": -0.038776006549596786,
      "step": 147
    },
    {
      "epoch": 0.08596154963117848,
      "grad_norm": 356.3915710449219,
      "learning_rate": 4.785008715862871e-06,
      "logits/chosen": -0.8506426811218262,
      "logits/rejected": -0.7640115022659302,
      "logps/chosen": -67.30644989013672,
      "logps/rejected": -71.46052551269531,
      "loss": 13.6259,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010771533474326134,
      "rewards/margins": 0.026811867952346802,
      "rewards/rejected": -0.037583399564027786,
      "step": 148
    },
    {
      "epoch": 0.08654237091247023,
      "grad_norm": 303.959716796875,
      "learning_rate": 4.783556072051134e-06,
      "logits/chosen": -0.9652125239372253,
      "logits/rejected": -1.0568349361419678,
      "logps/chosen": -78.24043273925781,
      "logps/rejected": -82.4593734741211,
      "loss": 13.4476,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0065461816266179085,
      "rewards/margins": 0.049500562250614166,
      "rewards/rejected": -0.04295438155531883,
      "step": 149
    },
    {
      "epoch": 0.08712319219376198,
      "grad_norm": 301.6387939453125,
      "learning_rate": 4.7821034282393966e-06,
      "logits/chosen": -0.9109539985656738,
      "logits/rejected": -0.8469412922859192,
      "logps/chosen": -71.34504699707031,
      "logps/rejected": -70.11753845214844,
      "loss": 14.0276,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.013672858476638794,
      "rewards/margins": -0.013126525096595287,
      "rewards/rejected": -0.0005463305860757828,
      "step": 150
    },
    {
      "epoch": 0.08770401347505373,
      "grad_norm": 385.97100830078125,
      "learning_rate": 4.780650784427659e-06,
      "logits/chosen": -0.974000096321106,
      "logits/rejected": -0.8754386901855469,
      "logps/chosen": -78.10624694824219,
      "logps/rejected": -71.39083099365234,
      "loss": 13.8936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005910863634198904,
      "rewards/margins": -0.0005791831645183265,
      "rewards/rejected": 0.006490050349384546,
      "step": 151
    },
    {
      "epoch": 0.08828483475634548,
      "grad_norm": 288.5458679199219,
      "learning_rate": 4.779198140615921e-06,
      "logits/chosen": -0.8864970207214355,
      "logits/rejected": -0.8748113512992859,
      "logps/chosen": -74.03097534179688,
      "logps/rejected": -68.74441528320312,
      "loss": 13.8866,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009058129973709583,
      "rewards/margins": 0.003120752517133951,
      "rewards/rejected": 0.005937379319220781,
      "step": 152
    },
    {
      "epoch": 0.08886565603763721,
      "grad_norm": 335.7358093261719,
      "learning_rate": 4.777745496804184e-06,
      "logits/chosen": -0.7208220362663269,
      "logits/rejected": -0.7323756217956543,
      "logps/chosen": -71.64644622802734,
      "logps/rejected": -80.21810913085938,
      "loss": 13.6955,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.004168490879237652,
      "rewards/margins": 0.020404014736413956,
      "rewards/rejected": -0.01623552106320858,
      "step": 153
    },
    {
      "epoch": 0.08944647731892896,
      "grad_norm": 302.51763916015625,
      "learning_rate": 4.776292852992447e-06,
      "logits/chosen": -0.6920525431632996,
      "logits/rejected": -0.7309268712997437,
      "logps/chosen": -69.67924499511719,
      "logps/rejected": -76.42180633544922,
      "loss": 13.5794,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.005118966568261385,
      "rewards/margins": 0.031105687841773033,
      "rewards/rejected": -0.03622465208172798,
      "step": 154
    },
    {
      "epoch": 0.09002729860022071,
      "grad_norm": 316.7055358886719,
      "learning_rate": 4.7748402091807095e-06,
      "logits/chosen": -0.7559612989425659,
      "logits/rejected": -0.6078428626060486,
      "logps/chosen": -70.20340728759766,
      "logps/rejected": -66.18345642089844,
      "loss": 13.9598,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0274501983076334,
      "rewards/margins": -0.007842998020350933,
      "rewards/rejected": -0.019607199355959892,
      "step": 155
    },
    {
      "epoch": 0.09060811988151246,
      "grad_norm": 328.9933166503906,
      "learning_rate": 4.773387565368972e-06,
      "logits/chosen": -0.7114741802215576,
      "logits/rejected": -0.8936127424240112,
      "logps/chosen": -72.7677993774414,
      "logps/rejected": -77.62763214111328,
      "loss": 13.5946,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008942861109972,
      "rewards/margins": 0.030701924115419388,
      "rewards/rejected": -0.03964478522539139,
      "step": 156
    },
    {
      "epoch": 0.09118894116280421,
      "grad_norm": 337.06060791015625,
      "learning_rate": 4.771934921557235e-06,
      "logits/chosen": -0.86445152759552,
      "logits/rejected": -0.8468233942985535,
      "logps/chosen": -84.74078369140625,
      "logps/rejected": -83.00550079345703,
      "loss": 13.9382,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03650820627808571,
      "rewards/margins": -0.002918243408203125,
      "rewards/rejected": -0.03358996659517288,
      "step": 157
    },
    {
      "epoch": 0.09176976244409595,
      "grad_norm": 318.7173767089844,
      "learning_rate": 4.770482277745498e-06,
      "logits/chosen": -0.7902041673660278,
      "logits/rejected": -0.8102203607559204,
      "logps/chosen": -72.93890380859375,
      "logps/rejected": -77.42249298095703,
      "loss": 13.745,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01063698623329401,
      "rewards/margins": 0.013827304355800152,
      "rewards/rejected": -0.02446429245173931,
      "step": 158
    },
    {
      "epoch": 0.0923505837253877,
      "grad_norm": 347.96240234375,
      "learning_rate": 4.76902963393376e-06,
      "logits/chosen": -0.8171346783638,
      "logits/rejected": -0.8012442588806152,
      "logps/chosen": -69.07205963134766,
      "logps/rejected": -80.80284118652344,
      "loss": 13.7243,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05406276509165764,
      "rewards/margins": 0.016692936420440674,
      "rewards/rejected": -0.07075570523738861,
      "step": 159
    },
    {
      "epoch": 0.09293140500667944,
      "grad_norm": 312.94622802734375,
      "learning_rate": 4.767576990122022e-06,
      "logits/chosen": -0.8780553936958313,
      "logits/rejected": -0.7555993795394897,
      "logps/chosen": -75.97615814208984,
      "logps/rejected": -73.26390838623047,
      "loss": 13.8975,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01142085064202547,
      "rewards/margins": -0.0006685241824015975,
      "rewards/rejected": -0.010752325877547264,
      "step": 160
    },
    {
      "epoch": 0.09351222628797119,
      "grad_norm": 319.6703186035156,
      "learning_rate": 4.766124346310285e-06,
      "logits/chosen": -0.9523868560791016,
      "logits/rejected": -0.9787250757217407,
      "logps/chosen": -78.85665130615234,
      "logps/rejected": -76.03120422363281,
      "loss": 13.7211,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00805568601936102,
      "rewards/margins": 0.019266730174422264,
      "rewards/rejected": -0.027322417125105858,
      "step": 161
    },
    {
      "epoch": 0.09409304756926294,
      "grad_norm": 304.69439697265625,
      "learning_rate": 4.764671702498548e-06,
      "logits/chosen": -0.7667674422264099,
      "logits/rejected": -0.8059636354446411,
      "logps/chosen": -71.68733215332031,
      "logps/rejected": -74.61915588378906,
      "loss": 13.3629,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03330715745687485,
      "rewards/margins": 0.05361800268292427,
      "rewards/rejected": -0.02031084895133972,
      "step": 162
    },
    {
      "epoch": 0.09467386885055469,
      "grad_norm": 302.20050048828125,
      "learning_rate": 4.763219058686811e-06,
      "logits/chosen": -0.7947182655334473,
      "logits/rejected": -0.8342369198799133,
      "logps/chosen": -71.64237213134766,
      "logps/rejected": -70.30358123779297,
      "loss": 13.4606,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.010867975652217865,
      "rewards/margins": 0.04301146790385246,
      "rewards/rejected": -0.053879447281360626,
      "step": 163
    },
    {
      "epoch": 0.09525469013184643,
      "grad_norm": 289.6075134277344,
      "learning_rate": 4.761766414875073e-06,
      "logits/chosen": -0.7613986134529114,
      "logits/rejected": -0.8836091160774231,
      "logps/chosen": -67.74214935302734,
      "logps/rejected": -78.71183776855469,
      "loss": 13.2522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0031068138778209686,
      "rewards/margins": 0.06571229547262192,
      "rewards/rejected": -0.06881911307573318,
      "step": 164
    },
    {
      "epoch": 0.09583551141313817,
      "grad_norm": 326.1537780761719,
      "learning_rate": 4.760313771063336e-06,
      "logits/chosen": -0.8384913206100464,
      "logits/rejected": -0.7940191030502319,
      "logps/chosen": -66.64979553222656,
      "logps/rejected": -69.57666015625,
      "loss": 13.5813,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01473635621368885,
      "rewards/margins": 0.02965986728668213,
      "rewards/rejected": -0.014923503622412682,
      "step": 165
    },
    {
      "epoch": 0.09641633269442992,
      "grad_norm": 317.4963684082031,
      "learning_rate": 4.758861127251598e-06,
      "logits/chosen": -0.7010576128959656,
      "logits/rejected": -0.7300230860710144,
      "logps/chosen": -76.84666442871094,
      "logps/rejected": -84.0490493774414,
      "loss": 13.9696,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04205438122153282,
      "rewards/margins": -0.005721705500036478,
      "rewards/rejected": -0.03633267432451248,
      "step": 166
    },
    {
      "epoch": 0.09699715397572167,
      "grad_norm": 318.7678527832031,
      "learning_rate": 4.757408483439861e-06,
      "logits/chosen": -0.9276505708694458,
      "logits/rejected": -0.9561537504196167,
      "logps/chosen": -71.64440155029297,
      "logps/rejected": -80.91627502441406,
      "loss": 13.4072,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0023384944070130587,
      "rewards/margins": 0.04981740936636925,
      "rewards/rejected": -0.05215590447187424,
      "step": 167
    },
    {
      "epoch": 0.09757797525701342,
      "grad_norm": 287.4025573730469,
      "learning_rate": 4.7559558396281235e-06,
      "logits/chosen": -0.7570281624794006,
      "logits/rejected": -0.8512780070304871,
      "logps/chosen": -68.49506378173828,
      "logps/rejected": -70.23898315429688,
      "loss": 13.8651,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.007149294018745422,
      "rewards/margins": 0.0015381794655695558,
      "rewards/rejected": -0.008687476627528667,
      "step": 168
    },
    {
      "epoch": 0.09815879653830516,
      "grad_norm": 310.3047790527344,
      "learning_rate": 4.754503195816386e-06,
      "logits/chosen": -0.8154839277267456,
      "logits/rejected": -1.0209014415740967,
      "logps/chosen": -78.32124328613281,
      "logps/rejected": -68.3904037475586,
      "loss": 13.368,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.016737347468733788,
      "rewards/margins": 0.0557125024497509,
      "rewards/rejected": -0.038975149393081665,
      "step": 169
    },
    {
      "epoch": 0.0987396178195969,
      "grad_norm": 303.63385009765625,
      "learning_rate": 4.753050552004649e-06,
      "logits/chosen": -0.7536464333534241,
      "logits/rejected": -0.7967413663864136,
      "logps/chosen": -73.55670928955078,
      "logps/rejected": -75.25071716308594,
      "loss": 13.9022,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.022630508989095688,
      "rewards/margins": -0.0017447940772399306,
      "rewards/rejected": -0.020885715261101723,
      "step": 170
    },
    {
      "epoch": 0.09932043910088866,
      "grad_norm": 328.81494140625,
      "learning_rate": 4.751597908192912e-06,
      "logits/chosen": -0.9858807325363159,
      "logits/rejected": -0.9747235178947449,
      "logps/chosen": -79.53309631347656,
      "logps/rejected": -64.26655578613281,
      "loss": 14.1977,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03506975993514061,
      "rewards/margins": -0.029723864048719406,
      "rewards/rejected": -0.005345895420759916,
      "step": 171
    },
    {
      "epoch": 0.0999012603821804,
      "grad_norm": 303.1661682128906,
      "learning_rate": 4.750145264381174e-06,
      "logits/chosen": -0.8304456472396851,
      "logits/rejected": -0.754758358001709,
      "logps/chosen": -79.57183837890625,
      "logps/rejected": -73.25392150878906,
      "loss": 13.3539,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0045074475929141045,
      "rewards/margins": 0.05535992234945297,
      "rewards/rejected": -0.059867363423109055,
      "step": 172
    },
    {
      "epoch": 0.10048208166347215,
      "grad_norm": 285.4842224121094,
      "learning_rate": 4.748692620569436e-06,
      "logits/chosen": -0.7988349199295044,
      "logits/rejected": -0.8758390545845032,
      "logps/chosen": -69.04563903808594,
      "logps/rejected": -63.191673278808594,
      "loss": 13.8072,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02564077451825142,
      "rewards/margins": 0.009737257845699787,
      "rewards/rejected": -0.03537803143262863,
      "step": 173
    },
    {
      "epoch": 0.1010629029447639,
      "grad_norm": 316.76220703125,
      "learning_rate": 4.747239976757699e-06,
      "logits/chosen": -0.8752381205558777,
      "logits/rejected": -0.9098547697067261,
      "logps/chosen": -71.7885513305664,
      "logps/rejected": -77.3386001586914,
      "loss": 13.8798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03445696085691452,
      "rewards/margins": 0.00015344536222983152,
      "rewards/rejected": -0.03461039811372757,
      "step": 174
    },
    {
      "epoch": 0.10164372422605564,
      "grad_norm": 285.7683410644531,
      "learning_rate": 4.745787332945962e-06,
      "logits/chosen": -0.8894672393798828,
      "logits/rejected": -1.066870927810669,
      "logps/chosen": -71.97786712646484,
      "logps/rejected": -70.2954330444336,
      "loss": 13.8723,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.027886096388101578,
      "rewards/margins": 0.003271915018558502,
      "rewards/rejected": -0.03115800954401493,
      "step": 175
    },
    {
      "epoch": 0.10222454550734739,
      "grad_norm": 314.2114562988281,
      "learning_rate": 4.744334689134225e-06,
      "logits/chosen": -0.9301921725273132,
      "logits/rejected": -0.9640763401985168,
      "logps/chosen": -68.97576904296875,
      "logps/rejected": -73.05828857421875,
      "loss": 13.546,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00339788431301713,
      "rewards/margins": 0.03368956595659256,
      "rewards/rejected": -0.03708745166659355,
      "step": 176
    },
    {
      "epoch": 0.10280536678863914,
      "grad_norm": 350.662353515625,
      "learning_rate": 4.742882045322487e-06,
      "logits/chosen": -0.9687705039978027,
      "logits/rejected": -0.8688680529594421,
      "logps/chosen": -76.81888580322266,
      "logps/rejected": -75.65296936035156,
      "loss": 14.2808,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07494613528251648,
      "rewards/margins": -0.03318053483963013,
      "rewards/rejected": -0.04176560416817665,
      "step": 177
    },
    {
      "epoch": 0.10338618806993088,
      "grad_norm": 335.7806396484375,
      "learning_rate": 4.74142940151075e-06,
      "logits/chosen": -0.7820795178413391,
      "logits/rejected": -0.8843638300895691,
      "logps/chosen": -83.46534729003906,
      "logps/rejected": -73.49069213867188,
      "loss": 13.5938,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.011855507269501686,
      "rewards/margins": 0.030727148056030273,
      "rewards/rejected": -0.04258265346288681,
      "step": 178
    },
    {
      "epoch": 0.10396700935122263,
      "grad_norm": 298.5807800292969,
      "learning_rate": 4.739976757699012e-06,
      "logits/chosen": -0.9966332316398621,
      "logits/rejected": -0.8384010195732117,
      "logps/chosen": -75.45916748046875,
      "logps/rejected": -79.22760009765625,
      "loss": 13.5549,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.045658472925424576,
      "rewards/margins": 0.03411347419023514,
      "rewards/rejected": -0.07977195084095001,
      "step": 179
    },
    {
      "epoch": 0.10454783063251437,
      "grad_norm": 308.5312805175781,
      "learning_rate": 4.738524113887275e-06,
      "logits/chosen": -1.1542797088623047,
      "logits/rejected": -1.1540412902832031,
      "logps/chosen": -71.09564971923828,
      "logps/rejected": -66.72710418701172,
      "loss": 13.8693,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01030341349542141,
      "rewards/margins": 0.0030408282764256,
      "rewards/rejected": -0.013344240374863148,
      "step": 180
    },
    {
      "epoch": 0.10512865191380612,
      "grad_norm": 382.357666015625,
      "learning_rate": 4.7370714700755375e-06,
      "logits/chosen": -0.8503016233444214,
      "logits/rejected": -0.8578587770462036,
      "logps/chosen": -67.1768798828125,
      "logps/rejected": -74.30778503417969,
      "loss": 13.5419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024283096194267273,
      "rewards/margins": 0.036231573671102524,
      "rewards/rejected": -0.0605146698653698,
      "step": 181
    },
    {
      "epoch": 0.10570947319509787,
      "grad_norm": 316.55859375,
      "learning_rate": 4.7356188262638e-06,
      "logits/chosen": -0.8488529324531555,
      "logits/rejected": -0.8070418238639832,
      "logps/chosen": -75.79255676269531,
      "logps/rejected": -76.4787826538086,
      "loss": 13.6775,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01909615471959114,
      "rewards/margins": 0.02385520003736019,
      "rewards/rejected": -0.04295135289430618,
      "step": 182
    },
    {
      "epoch": 0.10629029447638962,
      "grad_norm": 309.40887451171875,
      "learning_rate": 4.734166182452063e-06,
      "logits/chosen": -0.9303333163261414,
      "logits/rejected": -0.900216281414032,
      "logps/chosen": -74.81334686279297,
      "logps/rejected": -76.99653625488281,
      "loss": 14.0831,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08230753988027573,
      "rewards/margins": -0.01739531196653843,
      "rewards/rejected": -0.06491222977638245,
      "step": 183
    },
    {
      "epoch": 0.10687111575768136,
      "grad_norm": 308.8415832519531,
      "learning_rate": 4.732713538640326e-06,
      "logits/chosen": -0.9111288785934448,
      "logits/rejected": -0.8835655450820923,
      "logps/chosen": -82.20323181152344,
      "logps/rejected": -69.86083221435547,
      "loss": 13.6755,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03169438987970352,
      "rewards/margins": 0.023478079587221146,
      "rewards/rejected": -0.055172473192214966,
      "step": 184
    },
    {
      "epoch": 0.10745193703897311,
      "grad_norm": 294.7135925292969,
      "learning_rate": 4.7312608948285885e-06,
      "logits/chosen": -0.7559576630592346,
      "logits/rejected": -0.7548641562461853,
      "logps/chosen": -72.80931091308594,
      "logps/rejected": -68.47846221923828,
      "loss": 13.6632,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03249110281467438,
      "rewards/margins": 0.026273757219314575,
      "rewards/rejected": -0.05876486748456955,
      "step": 185
    },
    {
      "epoch": 0.10803275832026485,
      "grad_norm": 300.0313720703125,
      "learning_rate": 4.7298082510168504e-06,
      "logits/chosen": -0.7012637853622437,
      "logits/rejected": -0.7670835256576538,
      "logps/chosen": -69.26997375488281,
      "logps/rejected": -75.66789245605469,
      "loss": 13.8065,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05606495216488838,
      "rewards/margins": 0.008019248023629189,
      "rewards/rejected": -0.06408419460058212,
      "step": 186
    },
    {
      "epoch": 0.1086135796015566,
      "grad_norm": 305.9514465332031,
      "learning_rate": 4.728355607205113e-06,
      "logits/chosen": -0.848249614238739,
      "logits/rejected": -0.9618538022041321,
      "logps/chosen": -76.71751403808594,
      "logps/rejected": -75.86561584472656,
      "loss": 13.9142,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.058838047087192535,
      "rewards/margins": -0.0012072951067239046,
      "rewards/rejected": -0.057630755007267,
      "step": 187
    },
    {
      "epoch": 0.10919440088284835,
      "grad_norm": 299.03106689453125,
      "learning_rate": 4.726902963393376e-06,
      "logits/chosen": -0.7864362001419067,
      "logits/rejected": -0.9607146382331848,
      "logps/chosen": -73.52447509765625,
      "logps/rejected": -83.42733001708984,
      "loss": 13.0054,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.0022229477763175964,
      "rewards/margins": 0.09174972772598267,
      "rewards/rejected": -0.08952677994966507,
      "step": 188
    },
    {
      "epoch": 0.1097752221641401,
      "grad_norm": 301.38275146484375,
      "learning_rate": 4.7254503195816395e-06,
      "logits/chosen": -0.9863910675048828,
      "logits/rejected": -0.9701216816902161,
      "logps/chosen": -73.6039047241211,
      "logps/rejected": -74.17659759521484,
      "loss": 13.5522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.027939602732658386,
      "rewards/margins": 0.033790357410907745,
      "rewards/rejected": -0.06172995641827583,
      "step": 189
    },
    {
      "epoch": 0.11035604344543185,
      "grad_norm": 321.6877136230469,
      "learning_rate": 4.723997675769902e-06,
      "logits/chosen": -0.796249508857727,
      "logits/rejected": -0.7661502957344055,
      "logps/chosen": -78.24516296386719,
      "logps/rejected": -74.38370513916016,
      "loss": 13.5199,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.038561441004276276,
      "rewards/margins": 0.039519913494586945,
      "rewards/rejected": -0.07808135449886322,
      "step": 190
    },
    {
      "epoch": 0.11093686472672358,
      "grad_norm": 317.3888854980469,
      "learning_rate": 4.722545031958164e-06,
      "logits/chosen": -0.9007613062858582,
      "logits/rejected": -0.9852052927017212,
      "logps/chosen": -82.073486328125,
      "logps/rejected": -72.91219329833984,
      "loss": 13.6764,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03767850995063782,
      "rewards/margins": 0.023520758375525475,
      "rewards/rejected": -0.06119927018880844,
      "step": 191
    },
    {
      "epoch": 0.11151768600801533,
      "grad_norm": 316.7236328125,
      "learning_rate": 4.721092388146427e-06,
      "logits/chosen": -0.983284592628479,
      "logits/rejected": -0.8711267709732056,
      "logps/chosen": -76.91285705566406,
      "logps/rejected": -67.65292358398438,
      "loss": 14.1306,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03306933492422104,
      "rewards/margins": -0.022902240976691246,
      "rewards/rejected": -0.010167093947529793,
      "step": 192
    },
    {
      "epoch": 0.11209850728930708,
      "grad_norm": 313.7123718261719,
      "learning_rate": 4.71963974433469e-06,
      "logits/chosen": -0.9627790451049805,
      "logits/rejected": -0.8592319488525391,
      "logps/chosen": -76.01704406738281,
      "logps/rejected": -72.71672058105469,
      "loss": 14.0736,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.049292050302028656,
      "rewards/margins": -0.01814861036837101,
      "rewards/rejected": -0.031143436208367348,
      "step": 193
    },
    {
      "epoch": 0.11267932857059883,
      "grad_norm": 303.0953063964844,
      "learning_rate": 4.718187100522952e-06,
      "logits/chosen": -0.8993155360221863,
      "logits/rejected": -0.8849833607673645,
      "logps/chosen": -71.47299194335938,
      "logps/rejected": -75.4497299194336,
      "loss": 13.3576,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004056071862578392,
      "rewards/margins": 0.05458641052246094,
      "rewards/rejected": -0.050530336797237396,
      "step": 194
    },
    {
      "epoch": 0.11326014985189058,
      "grad_norm": 311.2488098144531,
      "learning_rate": 4.716734456711215e-06,
      "logits/chosen": -0.9742434620857239,
      "logits/rejected": -0.9584578275680542,
      "logps/chosen": -75.96842956542969,
      "logps/rejected": -71.90623474121094,
      "loss": 13.6582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01095439214259386,
      "rewards/margins": 0.023807067424058914,
      "rewards/rejected": -0.0347614586353302,
      "step": 195
    },
    {
      "epoch": 0.11384097113318233,
      "grad_norm": 300.0875549316406,
      "learning_rate": 4.715281812899478e-06,
      "logits/chosen": -0.594025194644928,
      "logits/rejected": -0.6777793765068054,
      "logps/chosen": -74.22691345214844,
      "logps/rejected": -74.91129302978516,
      "loss": 13.5482,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0018765062559396029,
      "rewards/margins": 0.03627584129571915,
      "rewards/rejected": -0.03439933806657791,
      "step": 196
    },
    {
      "epoch": 0.11442179241447406,
      "grad_norm": 299.1294860839844,
      "learning_rate": 4.713829169087741e-06,
      "logits/chosen": -0.7969453930854797,
      "logits/rejected": -0.8590051531791687,
      "logps/chosen": -74.24263000488281,
      "logps/rejected": -80.28329467773438,
      "loss": 13.9374,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029978638514876366,
      "rewards/margins": -0.0027882575523108244,
      "rewards/rejected": -0.027190377935767174,
      "step": 197
    },
    {
      "epoch": 0.11500261369576581,
      "grad_norm": 320.9808044433594,
      "learning_rate": 4.7123765252760026e-06,
      "logits/chosen": -0.9466894865036011,
      "logits/rejected": -1.0433050394058228,
      "logps/chosen": -72.66864776611328,
      "logps/rejected": -75.08808135986328,
      "loss": 14.3219,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.033920880407094955,
      "rewards/margins": -0.04234874248504639,
      "rewards/rejected": 0.008427867665886879,
      "step": 198
    },
    {
      "epoch": 0.11558343497705756,
      "grad_norm": 308.2682189941406,
      "learning_rate": 4.710923881464265e-06,
      "logits/chosen": -0.7543210387229919,
      "logits/rejected": -0.8157938718795776,
      "logps/chosen": -81.17222595214844,
      "logps/rejected": -72.63612365722656,
      "loss": 14.0541,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.022938327863812447,
      "rewards/margins": -0.015443983487784863,
      "rewards/rejected": -0.007494345307350159,
      "step": 199
    },
    {
      "epoch": 0.11616425625834931,
      "grad_norm": 312.9922180175781,
      "learning_rate": 4.709471237652528e-06,
      "logits/chosen": -0.6948032975196838,
      "logits/rejected": -0.6942026615142822,
      "logps/chosen": -73.12284851074219,
      "logps/rejected": -77.82487487792969,
      "loss": 13.9199,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029393082484602928,
      "rewards/margins": -0.0029410452116280794,
      "rewards/rejected": -0.02645203471183777,
      "step": 200
    },
    {
      "epoch": 0.11674507753964106,
      "grad_norm": 311.9326477050781,
      "learning_rate": 4.708018593840791e-06,
      "logits/chosen": -1.1453566551208496,
      "logits/rejected": -1.1129220724105835,
      "logps/chosen": -68.6366958618164,
      "logps/rejected": -73.55378723144531,
      "loss": 13.8228,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.010440112091600895,
      "rewards/margins": 0.009912310168147087,
      "rewards/rejected": -0.020352421328425407,
      "step": 201
    },
    {
      "epoch": 0.1173258988209328,
      "grad_norm": 308.12060546875,
      "learning_rate": 4.7065659500290536e-06,
      "logits/chosen": -0.8134487271308899,
      "logits/rejected": -0.7991577386856079,
      "logps/chosen": -70.990966796875,
      "logps/rejected": -78.31260681152344,
      "loss": 13.2819,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.004076024051755667,
      "rewards/margins": 0.06265188753604889,
      "rewards/rejected": -0.06672791391611099,
      "step": 202
    },
    {
      "epoch": 0.11790672010222454,
      "grad_norm": 287.57794189453125,
      "learning_rate": 4.705113306217316e-06,
      "logits/chosen": -0.9393211603164673,
      "logits/rejected": -0.9954279065132141,
      "logps/chosen": -76.2332534790039,
      "logps/rejected": -70.47115325927734,
      "loss": 13.2255,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.00021018953702878207,
      "rewards/margins": 0.06832081824541092,
      "rewards/rejected": -0.06853101402521133,
      "step": 203
    },
    {
      "epoch": 0.11848754138351629,
      "grad_norm": 305.3521423339844,
      "learning_rate": 4.703660662405578e-06,
      "logits/chosen": -0.9575725793838501,
      "logits/rejected": -1.0199588537216187,
      "logps/chosen": -80.7159423828125,
      "logps/rejected": -67.89506530761719,
      "loss": 14.0192,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06569384038448334,
      "rewards/margins": -0.01254147756844759,
      "rewards/rejected": -0.053152360022068024,
      "step": 204
    },
    {
      "epoch": 0.11906836266480804,
      "grad_norm": 278.6866760253906,
      "learning_rate": 4.702208018593841e-06,
      "logits/chosen": -0.7608988881111145,
      "logits/rejected": -0.7628680467605591,
      "logps/chosen": -70.3031005859375,
      "logps/rejected": -70.50173950195312,
      "loss": 13.39,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.021148493513464928,
      "rewards/margins": 0.052839674055576324,
      "rewards/rejected": -0.0739881694316864,
      "step": 205
    },
    {
      "epoch": 0.11964918394609979,
      "grad_norm": 304.92913818359375,
      "learning_rate": 4.700755374782104e-06,
      "logits/chosen": -0.8696325421333313,
      "logits/rejected": -0.9427449107170105,
      "logps/chosen": -74.74107360839844,
      "logps/rejected": -75.46221160888672,
      "loss": 13.7106,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.051121801137924194,
      "rewards/margins": 0.018347539007663727,
      "rewards/rejected": -0.06946934014558792,
      "step": 206
    },
    {
      "epoch": 0.12023000522739154,
      "grad_norm": 315.7526550292969,
      "learning_rate": 4.6993027309703665e-06,
      "logits/chosen": -0.8430768251419067,
      "logits/rejected": -0.7843033075332642,
      "logps/chosen": -71.7156982421875,
      "logps/rejected": -74.4084243774414,
      "loss": 13.8928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.024820514023303986,
      "rewards/margins": -0.0010401479667052627,
      "rewards/rejected": -0.023780368268489838,
      "step": 207
    },
    {
      "epoch": 0.12081082650868327,
      "grad_norm": 380.44561767578125,
      "learning_rate": 4.697850087158629e-06,
      "logits/chosen": -0.8150386810302734,
      "logits/rejected": -0.7826833724975586,
      "logps/chosen": -76.09224700927734,
      "logps/rejected": -83.32121276855469,
      "loss": 13.609,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.032852984964847565,
      "rewards/margins": 0.02871485985815525,
      "rewards/rejected": -0.061567842960357666,
      "step": 208
    },
    {
      "epoch": 0.12139164778997502,
      "grad_norm": 330.4120788574219,
      "learning_rate": 4.696397443346892e-06,
      "logits/chosen": -1.0164332389831543,
      "logits/rejected": -0.9415372610092163,
      "logps/chosen": -78.38670349121094,
      "logps/rejected": -79.13145446777344,
      "loss": 14.0359,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.061431754380464554,
      "rewards/margins": -0.01327196042984724,
      "rewards/rejected": -0.04815979301929474,
      "step": 209
    },
    {
      "epoch": 0.12197246907126677,
      "grad_norm": 336.34759521484375,
      "learning_rate": 4.694944799535155e-06,
      "logits/chosen": -0.9332435727119446,
      "logits/rejected": -0.9620237350463867,
      "logps/chosen": -78.70216369628906,
      "logps/rejected": -90.57441711425781,
      "loss": 13.5176,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00978778675198555,
      "rewards/margins": 0.0402628593146801,
      "rewards/rejected": -0.050050653517246246,
      "step": 210
    },
    {
      "epoch": 0.12255329035255852,
      "grad_norm": 303.8833923339844,
      "learning_rate": 4.693492155723417e-06,
      "logits/chosen": -0.9775910377502441,
      "logits/rejected": -0.9736326932907104,
      "logps/chosen": -78.42801666259766,
      "logps/rejected": -73.2563705444336,
      "loss": 14.1172,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.021739017218351364,
      "rewards/margins": -0.02232358045876026,
      "rewards/rejected": 0.0005845635896548629,
      "step": 211
    },
    {
      "epoch": 0.12313411163385027,
      "grad_norm": 307.4227294921875,
      "learning_rate": 4.692039511911679e-06,
      "logits/chosen": -0.8911228179931641,
      "logits/rejected": -1.0791237354278564,
      "logps/chosen": -65.61485290527344,
      "logps/rejected": -75.7813720703125,
      "loss": 13.3959,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.014038696885108948,
      "rewards/margins": 0.05425255373120308,
      "rewards/rejected": -0.04021385312080383,
      "step": 212
    },
    {
      "epoch": 0.12371493291514202,
      "grad_norm": 287.40045166015625,
      "learning_rate": 4.690586868099942e-06,
      "logits/chosen": -0.8995906710624695,
      "logits/rejected": -0.8471341133117676,
      "logps/chosen": -66.9594497680664,
      "logps/rejected": -71.04551696777344,
      "loss": 13.6394,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01811062917113304,
      "rewards/margins": 0.027448922395706177,
      "rewards/rejected": -0.04555954784154892,
      "step": 213
    },
    {
      "epoch": 0.12429575419643375,
      "grad_norm": 306.50128173828125,
      "learning_rate": 4.689134224288205e-06,
      "logits/chosen": -0.8109323382377625,
      "logits/rejected": -0.7679190635681152,
      "logps/chosen": -66.3537826538086,
      "logps/rejected": -78.93473815917969,
      "loss": 13.4859,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04216752201318741,
      "rewards/margins": 0.0444951057434082,
      "rewards/rejected": -0.08666262775659561,
      "step": 214
    },
    {
      "epoch": 0.1248765754777255,
      "grad_norm": 327.2640075683594,
      "learning_rate": 4.687681580476468e-06,
      "logits/chosen": -0.895319938659668,
      "logits/rejected": -0.9245772361755371,
      "logps/chosen": -69.65949249267578,
      "logps/rejected": -71.36875915527344,
      "loss": 14.5911,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.08888588845729828,
      "rewards/margins": -0.06758525222539902,
      "rewards/rejected": -0.021300649270415306,
      "step": 215
    },
    {
      "epoch": 0.12545739675901724,
      "grad_norm": 322.3681640625,
      "learning_rate": 4.68622893666473e-06,
      "logits/chosen": -0.8954951167106628,
      "logits/rejected": -0.81683349609375,
      "logps/chosen": -82.0992431640625,
      "logps/rejected": -71.04183197021484,
      "loss": 14.0854,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0422581322491169,
      "rewards/margins": -0.01680077239871025,
      "rewards/rejected": -0.025457357987761497,
      "step": 216
    },
    {
      "epoch": 0.12603821804030899,
      "grad_norm": 332.36920166015625,
      "learning_rate": 4.684776292852993e-06,
      "logits/chosen": -0.887475311756134,
      "logits/rejected": -0.7960731983184814,
      "logps/chosen": -74.29244995117188,
      "logps/rejected": -81.73658752441406,
      "loss": 14.184,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09383013099431992,
      "rewards/margins": -0.027784889563918114,
      "rewards/rejected": -0.06604524701833725,
      "step": 217
    },
    {
      "epoch": 0.12661903932160073,
      "grad_norm": 329.1742858886719,
      "learning_rate": 4.683323649041255e-06,
      "logits/chosen": -0.7891820073127747,
      "logits/rejected": -1.006798505783081,
      "logps/chosen": -77.01458740234375,
      "logps/rejected": -71.24549102783203,
      "loss": 14.3551,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.07353498041629791,
      "rewards/margins": -0.045538146048784256,
      "rewards/rejected": -0.02799682877957821,
      "step": 218
    },
    {
      "epoch": 0.12719986060289248,
      "grad_norm": 300.3213806152344,
      "learning_rate": 4.681871005229518e-06,
      "logits/chosen": -0.9636018872261047,
      "logits/rejected": -1.0088518857955933,
      "logps/chosen": -64.98380279541016,
      "logps/rejected": -71.98371887207031,
      "loss": 13.5981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009727184660732746,
      "rewards/margins": 0.030616506934165955,
      "rewards/rejected": -0.04034368693828583,
      "step": 219
    },
    {
      "epoch": 0.12778068188418423,
      "grad_norm": 316.0754699707031,
      "learning_rate": 4.6804183614177805e-06,
      "logits/chosen": -0.8850613832473755,
      "logits/rejected": -0.9165294766426086,
      "logps/chosen": -77.82759857177734,
      "logps/rejected": -73.77207946777344,
      "loss": 13.467,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.020776253193616867,
      "rewards/margins": 0.04324489086866379,
      "rewards/rejected": -0.022468645125627518,
      "step": 220
    },
    {
      "epoch": 0.12836150316547598,
      "grad_norm": 298.15130615234375,
      "learning_rate": 4.678965717606043e-06,
      "logits/chosen": -0.9083736538887024,
      "logits/rejected": -0.9166957139968872,
      "logps/chosen": -64.89097595214844,
      "logps/rejected": -71.65401458740234,
      "loss": 13.4666,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.004524010233581066,
      "rewards/margins": 0.045348040759563446,
      "rewards/rejected": -0.049872055649757385,
      "step": 221
    },
    {
      "epoch": 0.12894232444676773,
      "grad_norm": 328.2380065917969,
      "learning_rate": 4.677513073794306e-06,
      "logits/chosen": -0.8878594636917114,
      "logits/rejected": -0.8982254862785339,
      "logps/chosen": -74.40402221679688,
      "logps/rejected": -67.42572021484375,
      "loss": 14.4278,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.07353280484676361,
      "rewards/margins": -0.05194491147994995,
      "rewards/rejected": -0.02158789522945881,
      "step": 222
    },
    {
      "epoch": 0.12952314572805948,
      "grad_norm": 312.8938903808594,
      "learning_rate": 4.676060429982569e-06,
      "logits/chosen": -0.9499589204788208,
      "logits/rejected": -0.9511539340019226,
      "logps/chosen": -74.2853012084961,
      "logps/rejected": -74.38381958007812,
      "loss": 13.8183,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04615163058042526,
      "rewards/margins": 0.010002289898693562,
      "rewards/rejected": -0.05615391582250595,
      "step": 223
    },
    {
      "epoch": 0.13010396700935123,
      "grad_norm": 308.1676330566406,
      "learning_rate": 4.6746077861708315e-06,
      "logits/chosen": -0.9691025614738464,
      "logits/rejected": -1.0638432502746582,
      "logps/chosen": -68.75132751464844,
      "logps/rejected": -64.51249694824219,
      "loss": 14.5132,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0783403068780899,
      "rewards/margins": -0.058567456901073456,
      "rewards/rejected": -0.019772853702306747,
      "step": 224
    },
    {
      "epoch": 0.13068478829064298,
      "grad_norm": 300.42218017578125,
      "learning_rate": 4.673155142359093e-06,
      "logits/chosen": -0.9048269391059875,
      "logits/rejected": -0.9264401197433472,
      "logps/chosen": -73.9933090209961,
      "logps/rejected": -80.50711822509766,
      "loss": 13.5877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.017903203144669533,
      "rewards/margins": 0.03073006495833397,
      "rewards/rejected": -0.04863326996564865,
      "step": 225
    },
    {
      "epoch": 0.13126560957193473,
      "grad_norm": 344.2493896484375,
      "learning_rate": 4.671702498547356e-06,
      "logits/chosen": -0.8985874056816101,
      "logits/rejected": -0.843769371509552,
      "logps/chosen": -72.87957000732422,
      "logps/rejected": -78.47367095947266,
      "loss": 13.6586,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01365675963461399,
      "rewards/margins": 0.0262027345597744,
      "rewards/rejected": -0.03985949605703354,
      "step": 226
    },
    {
      "epoch": 0.13184643085322645,
      "grad_norm": 314.46856689453125,
      "learning_rate": 4.670249854735619e-06,
      "logits/chosen": -0.9261584281921387,
      "logits/rejected": -1.041046142578125,
      "logps/chosen": -76.67170715332031,
      "logps/rejected": -66.8155517578125,
      "loss": 13.9811,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03505679965019226,
      "rewards/margins": -0.007680593524128199,
      "rewards/rejected": -0.0273762084543705,
      "step": 227
    },
    {
      "epoch": 0.1324272521345182,
      "grad_norm": 341.5300598144531,
      "learning_rate": 4.668797210923882e-06,
      "logits/chosen": -0.8199512362480164,
      "logits/rejected": -0.8454850316047668,
      "logps/chosen": -63.34907150268555,
      "logps/rejected": -67.3141098022461,
      "loss": 13.2625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.009090743958950043,
      "rewards/margins": 0.06578489392995834,
      "rewards/rejected": -0.0566941499710083,
      "step": 228
    },
    {
      "epoch": 0.13300807341580995,
      "grad_norm": 301.3994445800781,
      "learning_rate": 4.667344567112144e-06,
      "logits/chosen": -1.106350302696228,
      "logits/rejected": -1.0245457887649536,
      "logps/chosen": -72.74774932861328,
      "logps/rejected": -74.6384506225586,
      "loss": 14.0679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011247700080275536,
      "rewards/margins": -0.01573079079389572,
      "rewards/rejected": 0.004483087919652462,
      "step": 229
    },
    {
      "epoch": 0.1335888946971017,
      "grad_norm": 309.1897277832031,
      "learning_rate": 4.665891923300407e-06,
      "logits/chosen": -0.9697766304016113,
      "logits/rejected": -1.008512258529663,
      "logps/chosen": -78.22880554199219,
      "logps/rejected": -62.13957977294922,
      "loss": 13.917,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.030580034479498863,
      "rewards/margins": -0.0008186303311958909,
      "rewards/rejected": -0.029761407524347305,
      "step": 230
    },
    {
      "epoch": 0.13416971597839344,
      "grad_norm": 332.3008117675781,
      "learning_rate": 4.66443927948867e-06,
      "logits/chosen": -0.9368025064468384,
      "logits/rejected": -0.8751896023750305,
      "logps/chosen": -80.72710418701172,
      "logps/rejected": -75.58308410644531,
      "loss": 13.9649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.060019601136446,
      "rewards/margins": -0.0033089532516896725,
      "rewards/rejected": -0.05671064928174019,
      "step": 231
    },
    {
      "epoch": 0.1347505372596852,
      "grad_norm": 326.7151794433594,
      "learning_rate": 4.662986635676933e-06,
      "logits/chosen": -0.9234104156494141,
      "logits/rejected": -0.9875878095626831,
      "logps/chosen": -79.9911117553711,
      "logps/rejected": -80.50080871582031,
      "loss": 13.995,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05462227389216423,
      "rewards/margins": -0.009203016757965088,
      "rewards/rejected": -0.04541926831007004,
      "step": 232
    },
    {
      "epoch": 0.13533135854097694,
      "grad_norm": 297.00897216796875,
      "learning_rate": 4.661533991865195e-06,
      "logits/chosen": -0.6637614965438843,
      "logits/rejected": -0.8283042907714844,
      "logps/chosen": -71.94387817382812,
      "logps/rejected": -65.46293640136719,
      "loss": 14.1861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0613144226372242,
      "rewards/margins": -0.02849789522588253,
      "rewards/rejected": -0.03281652554869652,
      "step": 233
    },
    {
      "epoch": 0.1359121798222687,
      "grad_norm": 308.1539306640625,
      "learning_rate": 4.660081348053458e-06,
      "logits/chosen": -1.0005970001220703,
      "logits/rejected": -1.005382776260376,
      "logps/chosen": -75.04243469238281,
      "logps/rejected": -70.30781555175781,
      "loss": 13.5591,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.005779562052339315,
      "rewards/margins": 0.03400464728474617,
      "rewards/rejected": -0.03978421539068222,
      "step": 234
    },
    {
      "epoch": 0.13649300110356044,
      "grad_norm": 314.4703674316406,
      "learning_rate": 4.658628704241721e-06,
      "logits/chosen": -0.8988968729972839,
      "logits/rejected": -0.9323342442512512,
      "logps/chosen": -70.89219665527344,
      "logps/rejected": -72.37486267089844,
      "loss": 13.8039,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03381185606122017,
      "rewards/margins": 0.010169675573706627,
      "rewards/rejected": -0.04398152977228165,
      "step": 235
    },
    {
      "epoch": 0.1370738223848522,
      "grad_norm": 318.1397705078125,
      "learning_rate": 4.657176060429983e-06,
      "logits/chosen": -0.9645525217056274,
      "logits/rejected": -0.8109456896781921,
      "logps/chosen": -74.8411865234375,
      "logps/rejected": -73.68878936767578,
      "loss": 14.1379,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.045103732496500015,
      "rewards/margins": -0.02444007806479931,
      "rewards/rejected": -0.020663652569055557,
      "step": 236
    },
    {
      "epoch": 0.13765464366614394,
      "grad_norm": 297.63604736328125,
      "learning_rate": 4.6557234166182455e-06,
      "logits/chosen": -0.9129088521003723,
      "logits/rejected": -0.9517717361450195,
      "logps/chosen": -68.26078033447266,
      "logps/rejected": -78.68707275390625,
      "loss": 13.3907,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.013581380248069763,
      "rewards/margins": 0.05378935858607292,
      "rewards/rejected": -0.06737073510885239,
      "step": 237
    },
    {
      "epoch": 0.1382354649474357,
      "grad_norm": 322.44964599609375,
      "learning_rate": 4.654270772806508e-06,
      "logits/chosen": -1.0076889991760254,
      "logits/rejected": -0.9823756217956543,
      "logps/chosen": -70.8648681640625,
      "logps/rejected": -75.48307037353516,
      "loss": 13.5039,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.021284928545355797,
      "rewards/margins": 0.039048969745635986,
      "rewards/rejected": -0.06033390760421753,
      "step": 238
    },
    {
      "epoch": 0.1388162862287274,
      "grad_norm": 320.17193603515625,
      "learning_rate": 4.652818128994771e-06,
      "logits/chosen": -0.9012085199356079,
      "logits/rejected": -1.0353925228118896,
      "logps/chosen": -77.64144134521484,
      "logps/rejected": -72.55008697509766,
      "loss": 13.9087,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0533623993396759,
      "rewards/margins": -0.0019609176088124514,
      "rewards/rejected": -0.05140148475766182,
      "step": 239
    },
    {
      "epoch": 0.13939710751001916,
      "grad_norm": 291.0409851074219,
      "learning_rate": 4.651365485183034e-06,
      "logits/chosen": -0.7694844603538513,
      "logits/rejected": -0.7582476735115051,
      "logps/chosen": -66.96981048583984,
      "logps/rejected": -69.53636169433594,
      "loss": 13.8481,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.029136648401618004,
      "rewards/margins": 0.006323431618511677,
      "rewards/rejected": -0.035460080951452255,
      "step": 240
    },
    {
      "epoch": 0.1399779287913109,
      "grad_norm": 309.0951843261719,
      "learning_rate": 4.6499128413712965e-06,
      "logits/chosen": -1.09603750705719,
      "logits/rejected": -1.1345731019973755,
      "logps/chosen": -69.8927001953125,
      "logps/rejected": -76.84407806396484,
      "loss": 13.2567,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028979092836380005,
      "rewards/margins": 0.06684218347072601,
      "rewards/rejected": -0.03786309435963631,
      "step": 241
    },
    {
      "epoch": 0.14055875007260266,
      "grad_norm": 341.45458984375,
      "learning_rate": 4.648460197559559e-06,
      "logits/chosen": -0.7778705358505249,
      "logits/rejected": -0.9100780487060547,
      "logps/chosen": -83.03035736083984,
      "logps/rejected": -76.86042785644531,
      "loss": 13.7437,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.028774719685316086,
      "rewards/margins": 0.01645464077591896,
      "rewards/rejected": -0.04522935673594475,
      "step": 242
    },
    {
      "epoch": 0.1411395713538944,
      "grad_norm": 292.6492919921875,
      "learning_rate": 4.647007553747821e-06,
      "logits/chosen": -1.0126192569732666,
      "logits/rejected": -0.8217967748641968,
      "logps/chosen": -78.85697937011719,
      "logps/rejected": -69.64402770996094,
      "loss": 13.8864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031996484845876694,
      "rewards/margins": 0.002992384135723114,
      "rewards/rejected": -0.034988872706890106,
      "step": 243
    },
    {
      "epoch": 0.14172039263518615,
      "grad_norm": 312.7855529785156,
      "learning_rate": 4.645554909936084e-06,
      "logits/chosen": -0.9456573724746704,
      "logits/rejected": -0.9571585655212402,
      "logps/chosen": -74.84808349609375,
      "logps/rejected": -71.9338150024414,
      "loss": 14.0443,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.046040650457143784,
      "rewards/margins": -0.014010493643581867,
      "rewards/rejected": -0.03203015774488449,
      "step": 244
    },
    {
      "epoch": 0.1423012139164779,
      "grad_norm": 318.9042053222656,
      "learning_rate": 4.644102266124347e-06,
      "logits/chosen": -0.7081719636917114,
      "logits/rejected": -0.8551041483879089,
      "logps/chosen": -77.06927490234375,
      "logps/rejected": -80.43268585205078,
      "loss": 13.833,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04912562295794487,
      "rewards/margins": 0.007158022373914719,
      "rewards/rejected": -0.05628364533185959,
      "step": 245
    },
    {
      "epoch": 0.14288203519776965,
      "grad_norm": 333.2130126953125,
      "learning_rate": 4.642649622312609e-06,
      "logits/chosen": -1.046338438987732,
      "logits/rejected": -1.0311440229415894,
      "logps/chosen": -76.91915893554688,
      "logps/rejected": -76.76275634765625,
      "loss": 13.7829,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005943517200648785,
      "rewards/margins": 0.011601355858147144,
      "rewards/rejected": -0.0056578353978693485,
      "step": 246
    },
    {
      "epoch": 0.1434628564790614,
      "grad_norm": 315.692626953125,
      "learning_rate": 4.641196978500872e-06,
      "logits/chosen": -0.7881887555122375,
      "logits/rejected": -0.8423610925674438,
      "logps/chosen": -72.17633056640625,
      "logps/rejected": -80.27653503417969,
      "loss": 13.5325,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.007096032612025738,
      "rewards/margins": 0.035623110830783844,
      "rewards/rejected": -0.042719148099422455,
      "step": 247
    },
    {
      "epoch": 0.14404367776035315,
      "grad_norm": 324.49945068359375,
      "learning_rate": 4.639744334689135e-06,
      "logits/chosen": -0.9588478803634644,
      "logits/rejected": -0.9462132453918457,
      "logps/chosen": -77.13792419433594,
      "logps/rejected": -72.57328796386719,
      "loss": 13.6316,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0016011804109439254,
      "rewards/margins": 0.02814745344221592,
      "rewards/rejected": -0.02974862977862358,
      "step": 248
    },
    {
      "epoch": 0.1446244990416449,
      "grad_norm": 299.9615478515625,
      "learning_rate": 4.638291690877398e-06,
      "logits/chosen": -0.857986569404602,
      "logits/rejected": -0.9681297540664673,
      "logps/chosen": -73.25657653808594,
      "logps/rejected": -80.2601547241211,
      "loss": 13.4933,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00817027036100626,
      "rewards/margins": 0.04146546870470047,
      "rewards/rejected": -0.04963573440909386,
      "step": 249
    },
    {
      "epoch": 0.14520532032293662,
      "grad_norm": 302.1356201171875,
      "learning_rate": 4.6368390470656596e-06,
      "logits/chosen": -0.9270285367965698,
      "logits/rejected": -1.053038239479065,
      "logps/chosen": -65.88447570800781,
      "logps/rejected": -72.62715148925781,
      "loss": 14.0395,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.032028716057538986,
      "rewards/margins": -0.014700132422149181,
      "rewards/rejected": -0.01732858456671238,
      "step": 250
    },
    {
      "epoch": 0.14578614160422837,
      "grad_norm": 296.97564697265625,
      "learning_rate": 4.635386403253922e-06,
      "logits/chosen": -0.8831275105476379,
      "logits/rejected": -0.8724244832992554,
      "logps/chosen": -66.05812072753906,
      "logps/rejected": -67.20362854003906,
      "loss": 13.9448,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0004748806240968406,
      "rewards/margins": -0.0028558894991874695,
      "rewards/rejected": 0.0033307753037661314,
      "step": 251
    },
    {
      "epoch": 0.14636696288552012,
      "grad_norm": 315.11810302734375,
      "learning_rate": 4.633933759442185e-06,
      "logits/chosen": -1.0385617017745972,
      "logits/rejected": -1.0641318559646606,
      "logps/chosen": -72.60139465332031,
      "logps/rejected": -70.97787475585938,
      "loss": 13.838,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.012739667668938637,
      "rewards/margins": 0.007387256715446711,
      "rewards/rejected": -0.02012692391872406,
      "step": 252
    },
    {
      "epoch": 0.14694778416681187,
      "grad_norm": 315.47259521484375,
      "learning_rate": 4.632481115630448e-06,
      "logits/chosen": -0.8671859502792358,
      "logits/rejected": -0.8790918588638306,
      "logps/chosen": -80.53587341308594,
      "logps/rejected": -78.8593978881836,
      "loss": 13.9845,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.034586913883686066,
      "rewards/margins": -0.007644425146281719,
      "rewards/rejected": -0.02694249153137207,
      "step": 253
    },
    {
      "epoch": 0.14752860544810362,
      "grad_norm": 310.9654541015625,
      "learning_rate": 4.6310284718187105e-06,
      "logits/chosen": -0.8506280183792114,
      "logits/rejected": -0.9469968676567078,
      "logps/chosen": -83.29402923583984,
      "logps/rejected": -84.33534240722656,
      "loss": 13.4946,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.028773630037903786,
      "rewards/margins": 0.0417972095310688,
      "rewards/rejected": -0.07057084143161774,
      "step": 254
    },
    {
      "epoch": 0.14810942672939537,
      "grad_norm": 304.36968994140625,
      "learning_rate": 4.629575828006973e-06,
      "logits/chosen": -0.7221881151199341,
      "logits/rejected": -0.8106364011764526,
      "logps/chosen": -73.52366638183594,
      "logps/rejected": -76.93601989746094,
      "loss": 13.6697,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.017531022429466248,
      "rewards/margins": 0.022295940667390823,
      "rewards/rejected": -0.03982696682214737,
      "step": 255
    },
    {
      "epoch": 0.14869024801068711,
      "grad_norm": 731.6671752929688,
      "learning_rate": 4.628123184195236e-06,
      "logits/chosen": -1.0281916856765747,
      "logits/rejected": -1.0932250022888184,
      "logps/chosen": -73.1572036743164,
      "logps/rejected": -76.2696304321289,
      "loss": 13.9093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009217451326549053,
      "rewards/margins": -0.0013483152724802494,
      "rewards/rejected": -0.007869137451052666,
      "step": 256
    },
    {
      "epoch": 0.14927106929197886,
      "grad_norm": 317.27630615234375,
      "learning_rate": 4.626670540383498e-06,
      "logits/chosen": -0.7391899824142456,
      "logits/rejected": -0.8331745862960815,
      "logps/chosen": -70.43421936035156,
      "logps/rejected": -76.86418151855469,
      "loss": 13.9393,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.015171433798968792,
      "rewards/margins": -0.0031314187217503786,
      "rewards/rejected": 0.018302852287888527,
      "step": 257
    },
    {
      "epoch": 0.1498518905732706,
      "grad_norm": 323.65386962890625,
      "learning_rate": 4.625217896571761e-06,
      "logits/chosen": -0.8975432515144348,
      "logits/rejected": -1.0589958429336548,
      "logps/chosen": -75.39137268066406,
      "logps/rejected": -71.28562927246094,
      "loss": 13.5174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009832754731178284,
      "rewards/margins": 0.03878428786993027,
      "rewards/rejected": -0.04861704260110855,
      "step": 258
    },
    {
      "epoch": 0.15043271185456236,
      "grad_norm": 304.92144775390625,
      "learning_rate": 4.6237652527600234e-06,
      "logits/chosen": -0.7835612297058105,
      "logits/rejected": -0.8767670392990112,
      "logps/chosen": -73.34691619873047,
      "logps/rejected": -70.6404037475586,
      "loss": 14.2398,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04404080659151077,
      "rewards/margins": -0.032345980405807495,
      "rewards/rejected": -0.011694823391735554,
      "step": 259
    },
    {
      "epoch": 0.1510135331358541,
      "grad_norm": 324.8614196777344,
      "learning_rate": 4.622312608948286e-06,
      "logits/chosen": -0.6987928152084351,
      "logits/rejected": -0.765534520149231,
      "logps/chosen": -72.92799377441406,
      "logps/rejected": -69.84391784667969,
      "loss": 13.887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.027519341558218002,
      "rewards/margins": 0.0016624340787529945,
      "rewards/rejected": -0.02918177843093872,
      "step": 260
    },
    {
      "epoch": 0.15159435441714583,
      "grad_norm": 311.1237487792969,
      "learning_rate": 4.620859965136549e-06,
      "logits/chosen": -0.7852567434310913,
      "logits/rejected": -0.9455236196517944,
      "logps/chosen": -72.65814971923828,
      "logps/rejected": -79.49076080322266,
      "loss": 13.4405,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02082516811788082,
      "rewards/margins": 0.046132300049066544,
      "rewards/rejected": -0.06695746630430222,
      "step": 261
    },
    {
      "epoch": 0.15217517569843758,
      "grad_norm": 312.7984924316406,
      "learning_rate": 4.619407321324812e-06,
      "logits/chosen": -0.9117915034294128,
      "logits/rejected": -0.9733545184135437,
      "logps/chosen": -77.04357147216797,
      "logps/rejected": -66.96760559082031,
      "loss": 14.3348,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0389360710978508,
      "rewards/margins": -0.043142665177583694,
      "rewards/rejected": 0.0042065903544425964,
      "step": 262
    },
    {
      "epoch": 0.15275599697972933,
      "grad_norm": 338.8244934082031,
      "learning_rate": 4.6179546775130744e-06,
      "logits/chosen": -0.8544296026229858,
      "logits/rejected": -0.8088324666023254,
      "logps/chosen": -76.08251190185547,
      "logps/rejected": -74.12071990966797,
      "loss": 13.896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017584245651960373,
      "rewards/margins": -0.000663819897454232,
      "rewards/rejected": -0.0169204268604517,
      "step": 263
    },
    {
      "epoch": 0.15333681826102108,
      "grad_norm": 286.11407470703125,
      "learning_rate": 4.616502033701336e-06,
      "logits/chosen": -0.8225248456001282,
      "logits/rejected": -0.8068425059318542,
      "logps/chosen": -75.96072387695312,
      "logps/rejected": -70.82938385009766,
      "loss": 13.5013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02497541345655918,
      "rewards/margins": 0.04040759429335594,
      "rewards/rejected": -0.01543218083679676,
      "step": 264
    },
    {
      "epoch": 0.15391763954231283,
      "grad_norm": 319.89794921875,
      "learning_rate": 4.615049389889599e-06,
      "logits/chosen": -0.9208014607429504,
      "logits/rejected": -0.9296444654464722,
      "logps/chosen": -75.55086517333984,
      "logps/rejected": -79.83537292480469,
      "loss": 13.6912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0359053872525692,
      "rewards/margins": 0.024030333384871483,
      "rewards/rejected": -0.05993572622537613,
      "step": 265
    },
    {
      "epoch": 0.15449846082360458,
      "grad_norm": 360.5916442871094,
      "learning_rate": 4.613596746077862e-06,
      "logits/chosen": -0.745102047920227,
      "logits/rejected": -0.7995504140853882,
      "logps/chosen": -75.1971206665039,
      "logps/rejected": -68.94092559814453,
      "loss": 13.8718,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02639504335820675,
      "rewards/margins": 0.0018470294307917356,
      "rewards/rejected": -0.028242075815796852,
      "step": 266
    },
    {
      "epoch": 0.15507928210489633,
      "grad_norm": 306.31646728515625,
      "learning_rate": 4.612144102266125e-06,
      "logits/chosen": -0.7601212859153748,
      "logits/rejected": -0.8015406727790833,
      "logps/chosen": -79.32325744628906,
      "logps/rejected": -73.96139526367188,
      "loss": 14.0699,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.05416913703083992,
      "rewards/margins": -0.017042648047208786,
      "rewards/rejected": -0.037126488983631134,
      "step": 267
    },
    {
      "epoch": 0.15566010338618808,
      "grad_norm": 371.703125,
      "learning_rate": 4.610691458454387e-06,
      "logits/chosen": -0.9163106679916382,
      "logits/rejected": -1.0520881414413452,
      "logps/chosen": -78.73876953125,
      "logps/rejected": -79.40687561035156,
      "loss": 13.5618,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009102868847548962,
      "rewards/margins": 0.032982636243104935,
      "rewards/rejected": -0.04208550602197647,
      "step": 268
    },
    {
      "epoch": 0.15624092466747982,
      "grad_norm": 309.1504211425781,
      "learning_rate": 4.60923881464265e-06,
      "logits/chosen": -0.9318111538887024,
      "logits/rejected": -0.9711005091667175,
      "logps/chosen": -74.52928924560547,
      "logps/rejected": -78.32316589355469,
      "loss": 13.5172,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03735839203000069,
      "rewards/margins": 0.03915861248970032,
      "rewards/rejected": -0.0765170007944107,
      "step": 269
    },
    {
      "epoch": 0.15682174594877157,
      "grad_norm": 297.4718017578125,
      "learning_rate": 4.607786170830912e-06,
      "logits/chosen": -0.9283093214035034,
      "logits/rejected": -0.9299993515014648,
      "logps/chosen": -73.22312927246094,
      "logps/rejected": -69.17174530029297,
      "loss": 13.9256,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0193580724298954,
      "rewards/margins": -0.0019154436886310577,
      "rewards/rejected": -0.017442626878619194,
      "step": 270
    },
    {
      "epoch": 0.15740256723006332,
      "grad_norm": 315.2553405761719,
      "learning_rate": 4.606333527019175e-06,
      "logits/chosen": -0.8597780466079712,
      "logits/rejected": -0.811455249786377,
      "logps/chosen": -67.54701232910156,
      "logps/rejected": -77.47270965576172,
      "loss": 13.5603,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0011511326301842928,
      "rewards/margins": 0.03523620590567589,
      "rewards/rejected": -0.03638733923435211,
      "step": 271
    },
    {
      "epoch": 0.15798338851135504,
      "grad_norm": 788.3867797851562,
      "learning_rate": 4.6048808832074375e-06,
      "logits/chosen": -0.8214397430419922,
      "logits/rejected": -0.791312575340271,
      "logps/chosen": -69.90260314941406,
      "logps/rejected": -70.34420776367188,
      "loss": 13.6075,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.010176298208534718,
      "rewards/margins": 0.028864461928606033,
      "rewards/rejected": -0.03904075548052788,
      "step": 272
    },
    {
      "epoch": 0.1585642097926468,
      "grad_norm": 301.57794189453125,
      "learning_rate": 4.603428239395701e-06,
      "logits/chosen": -0.8692893981933594,
      "logits/rejected": -0.8743621706962585,
      "logps/chosen": -68.12577056884766,
      "logps/rejected": -66.91731262207031,
      "loss": 13.7415,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.013723989017307758,
      "rewards/margins": 0.015195539221167564,
      "rewards/rejected": -0.028919529169797897,
      "step": 273
    },
    {
      "epoch": 0.15914503107393854,
      "grad_norm": 290.8895263671875,
      "learning_rate": 4.601975595583964e-06,
      "logits/chosen": -0.8542564511299133,
      "logits/rejected": -0.8347529172897339,
      "logps/chosen": -73.254150390625,
      "logps/rejected": -68.77384948730469,
      "loss": 13.5118,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0077955336309969425,
      "rewards/margins": 0.03801111876964569,
      "rewards/rejected": -0.030215587466955185,
      "step": 274
    },
    {
      "epoch": 0.1597258523552303,
      "grad_norm": 299.75982666015625,
      "learning_rate": 4.600522951772226e-06,
      "logits/chosen": -0.9737634658813477,
      "logits/rejected": -1.0128427743911743,
      "logps/chosen": -72.66883850097656,
      "logps/rejected": -72.12713623046875,
      "loss": 14.1505,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04277346283197403,
      "rewards/margins": -0.02521767094731331,
      "rewards/rejected": -0.01755579002201557,
      "step": 275
    },
    {
      "epoch": 0.16030667363652204,
      "grad_norm": 317.3355712890625,
      "learning_rate": 4.5990703079604885e-06,
      "logits/chosen": -0.7819968461990356,
      "logits/rejected": -0.7388423681259155,
      "logps/chosen": -68.65778350830078,
      "logps/rejected": -69.62420654296875,
      "loss": 14.0828,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.014692326076328754,
      "rewards/margins": -0.017574016004800797,
      "rewards/rejected": 0.03226633742451668,
      "step": 276
    },
    {
      "epoch": 0.1608874949178138,
      "grad_norm": 300.1111145019531,
      "learning_rate": 4.597617664148751e-06,
      "logits/chosen": -0.8221060037612915,
      "logits/rejected": -0.8107389211654663,
      "logps/chosen": -64.23518371582031,
      "logps/rejected": -74.13258361816406,
      "loss": 13.5993,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01461729221045971,
      "rewards/margins": 0.028885364532470703,
      "rewards/rejected": -0.014268075115978718,
      "step": 277
    },
    {
      "epoch": 0.16146831619910554,
      "grad_norm": 324.15899658203125,
      "learning_rate": 4.596165020337014e-06,
      "logits/chosen": -0.8910449147224426,
      "logits/rejected": -1.0317879915237427,
      "logps/chosen": -72.35569763183594,
      "logps/rejected": -86.64521789550781,
      "loss": 13.3693,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04237177222967148,
      "rewards/margins": 0.05554167553782463,
      "rewards/rejected": -0.09791344404220581,
      "step": 278
    },
    {
      "epoch": 0.1620491374803973,
      "grad_norm": 335.16619873046875,
      "learning_rate": 4.594712376525277e-06,
      "logits/chosen": -0.8945894241333008,
      "logits/rejected": -0.9022412300109863,
      "logps/chosen": -72.68575286865234,
      "logps/rejected": -77.0567626953125,
      "loss": 13.5153,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0022069262340664864,
      "rewards/margins": 0.03883010149002075,
      "rewards/rejected": -0.03662317246198654,
      "step": 279
    },
    {
      "epoch": 0.16262995876168904,
      "grad_norm": 296.8701477050781,
      "learning_rate": 4.5932597327135395e-06,
      "logits/chosen": -0.8403556942939758,
      "logits/rejected": -0.8022698163986206,
      "logps/chosen": -68.22055053710938,
      "logps/rejected": -68.78074645996094,
      "loss": 13.5981,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01986340805888176,
      "rewards/margins": 0.03163480758666992,
      "rewards/rejected": -0.05149821564555168,
      "step": 280
    },
    {
      "epoch": 0.16321078004298079,
      "grad_norm": 305.6988830566406,
      "learning_rate": 4.591807088901802e-06,
      "logits/chosen": -0.8042858839035034,
      "logits/rejected": -0.7476822137832642,
      "logps/chosen": -64.76155090332031,
      "logps/rejected": -72.77494812011719,
      "loss": 13.7642,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.013462590985000134,
      "rewards/margins": 0.014226732775568962,
      "rewards/rejected": -0.02768932655453682,
      "step": 281
    },
    {
      "epoch": 0.16379160132427253,
      "grad_norm": 317.730712890625,
      "learning_rate": 4.590354445090064e-06,
      "logits/chosen": -0.8070831298828125,
      "logits/rejected": -0.7661058306694031,
      "logps/chosen": -69.77413940429688,
      "logps/rejected": -69.04270935058594,
      "loss": 13.623,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008160887286067009,
      "rewards/margins": 0.029747169464826584,
      "rewards/rejected": -0.021586284041404724,
      "step": 282
    },
    {
      "epoch": 0.16437242260556426,
      "grad_norm": 313.9194641113281,
      "learning_rate": 4.588901801278327e-06,
      "logits/chosen": -0.8869432210922241,
      "logits/rejected": -1.0530153512954712,
      "logps/chosen": -76.52225494384766,
      "logps/rejected": -70.056884765625,
      "loss": 13.6882,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.044385336339473724,
      "rewards/margins": 0.02604733407497406,
      "rewards/rejected": -0.07043267786502838,
      "step": 283
    },
    {
      "epoch": 0.164953243886856,
      "grad_norm": 326.5664978027344,
      "learning_rate": 4.58744915746659e-06,
      "logits/chosen": -0.7581676244735718,
      "logits/rejected": -0.7644235491752625,
      "logps/chosen": -75.07527160644531,
      "logps/rejected": -67.26031494140625,
      "loss": 14.2293,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.024589911103248596,
      "rewards/margins": -0.03345941752195358,
      "rewards/rejected": 0.008869504556059837,
      "step": 284
    },
    {
      "epoch": 0.16553406516814775,
      "grad_norm": 331.2121276855469,
      "learning_rate": 4.585996513654852e-06,
      "logits/chosen": -0.6432844400405884,
      "logits/rejected": -0.6698486804962158,
      "logps/chosen": -79.6335220336914,
      "logps/rejected": -84.1219253540039,
      "loss": 13.6497,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.007236090488731861,
      "rewards/margins": 0.02394806034862995,
      "rewards/rejected": -0.03118414804339409,
      "step": 285
    },
    {
      "epoch": 0.1661148864494395,
      "grad_norm": 325.1278381347656,
      "learning_rate": 4.584543869843115e-06,
      "logits/chosen": -0.805732250213623,
      "logits/rejected": -0.810941219329834,
      "logps/chosen": -81.93310546875,
      "logps/rejected": -72.44585418701172,
      "loss": 14.3678,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.03649924322962761,
      "rewards/margins": -0.046101175248622894,
      "rewards/rejected": 0.009601928293704987,
      "step": 286
    },
    {
      "epoch": 0.16669570773073125,
      "grad_norm": 304.5572814941406,
      "learning_rate": 4.583091226031378e-06,
      "logits/chosen": -0.7995644211769104,
      "logits/rejected": -0.8172422647476196,
      "logps/chosen": -72.41849517822266,
      "logps/rejected": -67.6212387084961,
      "loss": 13.8286,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.016089126467704773,
      "rewards/margins": 0.00835293810814619,
      "rewards/rejected": 0.007736186496913433,
      "step": 287
    },
    {
      "epoch": 0.167276529012023,
      "grad_norm": 338.3421936035156,
      "learning_rate": 4.581638582219641e-06,
      "logits/chosen": -0.7788872122764587,
      "logits/rejected": -0.8800986409187317,
      "logps/chosen": -77.22239685058594,
      "logps/rejected": -84.37377166748047,
      "loss": 13.6912,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01143337320536375,
      "rewards/margins": 0.02296261489391327,
      "rewards/rejected": -0.034395989030599594,
      "step": 288
    },
    {
      "epoch": 0.16785735029331475,
      "grad_norm": 303.7924499511719,
      "learning_rate": 4.5801859384079025e-06,
      "logits/chosen": -0.8267248868942261,
      "logits/rejected": -0.8682713508605957,
      "logps/chosen": -76.82869720458984,
      "logps/rejected": -75.02543640136719,
      "loss": 13.9469,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.009327888488769531,
      "rewards/margins": -0.004273596219718456,
      "rewards/rejected": 0.013601483777165413,
      "step": 289
    },
    {
      "epoch": 0.1684381715746065,
      "grad_norm": 310.5744323730469,
      "learning_rate": 4.578733294596165e-06,
      "logits/chosen": -0.8373514413833618,
      "logits/rejected": -0.8355886340141296,
      "logps/chosen": -78.4135971069336,
      "logps/rejected": -68.44206237792969,
      "loss": 13.3873,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.01929556205868721,
      "rewards/margins": 0.05108966678380966,
      "rewards/rejected": -0.07038523256778717,
      "step": 290
    },
    {
      "epoch": 0.16901899285589825,
      "grad_norm": 294.162353515625,
      "learning_rate": 4.577280650784428e-06,
      "logits/chosen": -0.8757956624031067,
      "logits/rejected": -0.9635500907897949,
      "logps/chosen": -79.70155334472656,
      "logps/rejected": -72.04487609863281,
      "loss": 13.3423,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015520468354225159,
      "rewards/margins": 0.056552886962890625,
      "rewards/rejected": -0.041032422333955765,
      "step": 291
    },
    {
      "epoch": 0.16959981413719,
      "grad_norm": 406.17034912109375,
      "learning_rate": 4.575828006972691e-06,
      "logits/chosen": -0.712080717086792,
      "logits/rejected": -0.6416457891464233,
      "logps/chosen": -74.07041931152344,
      "logps/rejected": -75.76925659179688,
      "loss": 13.8944,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.009814701043069363,
      "rewards/margins": 0.0012444716412574053,
      "rewards/rejected": -0.0110591696575284,
      "step": 292
    },
    {
      "epoch": 0.17018063541848175,
      "grad_norm": 319.25677490234375,
      "learning_rate": 4.5743753631609535e-06,
      "logits/chosen": -0.978374183177948,
      "logits/rejected": -0.9963283538818359,
      "logps/chosen": -68.7942886352539,
      "logps/rejected": -71.56004333496094,
      "loss": 13.9775,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004177446011453867,
      "rewards/margins": -0.006889923010021448,
      "rewards/rejected": 0.002712479094043374,
      "step": 293
    },
    {
      "epoch": 0.17076145669977347,
      "grad_norm": 333.2608642578125,
      "learning_rate": 4.572922719349216e-06,
      "logits/chosen": -0.9167648553848267,
      "logits/rejected": -0.8717905282974243,
      "logps/chosen": -70.68975067138672,
      "logps/rejected": -77.90010070800781,
      "loss": 13.5974,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.002308635041117668,
      "rewards/margins": 0.027850687503814697,
      "rewards/rejected": -0.02554205060005188,
      "step": 294
    },
    {
      "epoch": 0.17134227798106522,
      "grad_norm": 312.9326477050781,
      "learning_rate": 4.571470075537478e-06,
      "logits/chosen": -0.7617141008377075,
      "logits/rejected": -0.8908483386039734,
      "logps/chosen": -70.78837585449219,
      "logps/rejected": -72.64606475830078,
      "loss": 14.292,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.008612537756562233,
      "rewards/margins": -0.03856853395700455,
      "rewards/rejected": 0.029956001788377762,
      "step": 295
    },
    {
      "epoch": 0.17192309926235697,
      "grad_norm": 308.8948669433594,
      "learning_rate": 4.570017431725741e-06,
      "logits/chosen": -0.913497269153595,
      "logits/rejected": -0.8606641888618469,
      "logps/chosen": -70.07915496826172,
      "logps/rejected": -82.69215393066406,
      "loss": 13.7394,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.011818322353065014,
      "rewards/margins": 0.017899369820952415,
      "rewards/rejected": -0.029717693105340004,
      "step": 296
    },
    {
      "epoch": 0.17250392054364871,
      "grad_norm": 291.6385803222656,
      "learning_rate": 4.568564787914004e-06,
      "logits/chosen": -0.8296724557876587,
      "logits/rejected": -0.9198773503303528,
      "logps/chosen": -67.22807312011719,
      "logps/rejected": -78.60759735107422,
      "loss": 13.3225,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0034950252156704664,
      "rewards/margins": 0.06311126798391342,
      "rewards/rejected": -0.0666062980890274,
      "step": 297
    },
    {
      "epoch": 0.17308474182494046,
      "grad_norm": 291.3044738769531,
      "learning_rate": 4.567112144102266e-06,
      "logits/chosen": -0.8050098419189453,
      "logits/rejected": -0.7676219940185547,
      "logps/chosen": -67.16458129882812,
      "logps/rejected": -78.90528869628906,
      "loss": 13.2096,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02983633242547512,
      "rewards/margins": 0.07233790308237076,
      "rewards/rejected": -0.042501576244831085,
      "step": 298
    },
    {
      "epoch": 0.1736655631062322,
      "grad_norm": 314.0892333984375,
      "learning_rate": 4.565659500290529e-06,
      "logits/chosen": -1.0540732145309448,
      "logits/rejected": -0.9160014986991882,
      "logps/chosen": -71.54045104980469,
      "logps/rejected": -75.05534362792969,
      "loss": 13.7491,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009140492416918278,
      "rewards/margins": 0.015866661444306374,
      "rewards/rejected": -0.006726170424371958,
      "step": 299
    },
    {
      "epoch": 0.17424638438752396,
      "grad_norm": 310.53985595703125,
      "learning_rate": 4.564206856478792e-06,
      "logits/chosen": -0.8692498207092285,
      "logits/rejected": -0.8690904378890991,
      "logps/chosen": -71.8248062133789,
      "logps/rejected": -82.47134399414062,
      "loss": 12.8867,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.015918642282485962,
      "rewards/margins": 0.10448668152093887,
      "rewards/rejected": -0.08856804668903351,
      "step": 300
    },
    {
      "epoch": 0.1748272056688157,
      "grad_norm": 298.11541748046875,
      "learning_rate": 4.562754212667055e-06,
      "logits/chosen": -0.8415626287460327,
      "logits/rejected": -0.8000283241271973,
      "logps/chosen": -69.10453033447266,
      "logps/rejected": -65.39994812011719,
      "loss": 13.946,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.024037543684244156,
      "rewards/margins": -0.00543618481606245,
      "rewards/rejected": -0.018601354211568832,
      "step": 301
    },
    {
      "epoch": 0.17540802695010746,
      "grad_norm": 286.9059753417969,
      "learning_rate": 4.5613015688553165e-06,
      "logits/chosen": -0.8321945071220398,
      "logits/rejected": -0.8791966438293457,
      "logps/chosen": -66.0924301147461,
      "logps/rejected": -70.165283203125,
      "loss": 13.9943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0016370766097679734,
      "rewards/margins": -0.008972769603133202,
      "rewards/rejected": 0.007335691247135401,
      "step": 302
    },
    {
      "epoch": 0.1759888482313992,
      "grad_norm": 324.43927001953125,
      "learning_rate": 4.559848925043579e-06,
      "logits/chosen": -0.8788312077522278,
      "logits/rejected": -0.8677918314933777,
      "logps/chosen": -70.78472137451172,
      "logps/rejected": -70.58416748046875,
      "loss": 13.756,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015381020493805408,
      "rewards/margins": 0.013850994408130646,
      "rewards/rejected": -0.02923201583325863,
      "step": 303
    },
    {
      "epoch": 0.17656966951269096,
      "grad_norm": 333.6717224121094,
      "learning_rate": 4.558396281231842e-06,
      "logits/chosen": -0.9329641461372375,
      "logits/rejected": -0.8686521649360657,
      "logps/chosen": -73.89896392822266,
      "logps/rejected": -78.4896011352539,
      "loss": 13.8188,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00456998823210597,
      "rewards/margins": 0.008396768942475319,
      "rewards/rejected": -0.0038267814088612795,
      "step": 304
    },
    {
      "epoch": 0.17715049079398268,
      "grad_norm": 306.7808532714844,
      "learning_rate": 4.556943637420105e-06,
      "logits/chosen": -0.8817129135131836,
      "logits/rejected": -0.7859727144241333,
      "logps/chosen": -70.53137969970703,
      "logps/rejected": -73.36188507080078,
      "loss": 13.6795,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007974056643433869,
      "rewards/margins": 0.025630056858062744,
      "rewards/rejected": -0.024832649156451225,
      "step": 305
    },
    {
      "epoch": 0.17773131207527443,
      "grad_norm": 299.79339599609375,
      "learning_rate": 4.5554909936083675e-06,
      "logits/chosen": -0.8397809863090515,
      "logits/rejected": -0.962628960609436,
      "logps/chosen": -74.5028305053711,
      "logps/rejected": -72.23627471923828,
      "loss": 14.1074,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.013882589526474476,
      "rewards/margins": -0.022180356085300446,
      "rewards/rejected": 0.03606294468045235,
      "step": 306
    },
    {
      "epoch": 0.17831213335656618,
      "grad_norm": 305.6503601074219,
      "learning_rate": 4.55403834979663e-06,
      "logits/chosen": -0.9371621012687683,
      "logits/rejected": -0.8676670789718628,
      "logps/chosen": -68.56401824951172,
      "logps/rejected": -66.05886840820312,
      "loss": 13.7674,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.023868542164564133,
      "rewards/margins": 0.013151749968528748,
      "rewards/rejected": 0.010716790333390236,
      "step": 307
    },
    {
      "epoch": 0.17889295463785793,
      "grad_norm": 316.7124328613281,
      "learning_rate": 4.552585705984893e-06,
      "logits/chosen": -0.795932412147522,
      "logits/rejected": -0.8041373491287231,
      "logps/chosen": -75.10924530029297,
      "logps/rejected": -74.57756042480469,
      "loss": 14.0814,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.026643935590982437,
      "rewards/margins": -0.018288511782884598,
      "rewards/rejected": -0.008355428464710712,
      "step": 308
    },
    {
      "epoch": 0.17947377591914968,
      "grad_norm": 316.54193115234375,
      "learning_rate": 4.551133062173155e-06,
      "logits/chosen": -0.7379294037818909,
      "logits/rejected": -0.8033340573310852,
      "logps/chosen": -70.75437927246094,
      "logps/rejected": -68.13688659667969,
      "loss": 14.334,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04628816246986389,
      "rewards/margins": -0.04288000240921974,
      "rewards/rejected": -0.0034081649500876665,
      "step": 309
    },
    {
      "epoch": 0.18005459720044142,
      "grad_norm": 320.3726501464844,
      "learning_rate": 4.549680418361418e-06,
      "logits/chosen": -0.8026307225227356,
      "logits/rejected": -0.7256749272346497,
      "logps/chosen": -77.1530532836914,
      "logps/rejected": -71.47651672363281,
      "loss": 14.3986,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.03423710912466049,
      "rewards/margins": -0.04747765138745308,
      "rewards/rejected": 0.013240538537502289,
      "step": 310
    },
    {
      "epoch": 0.18063541848173317,
      "grad_norm": 333.2999572753906,
      "learning_rate": 4.5482277745496804e-06,
      "logits/chosen": -0.9229456186294556,
      "logits/rejected": -0.9455530047416687,
      "logps/chosen": -80.14155578613281,
      "logps/rejected": -68.71078491210938,
      "loss": 14.274,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03716535493731499,
      "rewards/margins": -0.035925768315792084,
      "rewards/rejected": -0.001239586272276938,
      "step": 311
    },
    {
      "epoch": 0.18121623976302492,
      "grad_norm": 344.1839904785156,
      "learning_rate": 4.546775130737943e-06,
      "logits/chosen": -0.7100509405136108,
      "logits/rejected": -0.7682685852050781,
      "logps/chosen": -70.23936462402344,
      "logps/rejected": -70.01929473876953,
      "loss": 13.9029,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0003937946748919785,
      "rewards/margins": -7.176501821959391e-05,
      "rewards/rejected": -0.0003220273065380752,
      "step": 312
    },
    {
      "epoch": 0.18179706104431667,
      "grad_norm": 311.8206787109375,
      "learning_rate": 4.545322486926206e-06,
      "logits/chosen": -0.786939263343811,
      "logits/rejected": -0.7103012800216675,
      "logps/chosen": -73.2970962524414,
      "logps/rejected": -68.45187377929688,
      "loss": 14.034,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01686491258442402,
      "rewards/margins": -0.013414248824119568,
      "rewards/rejected": -0.0034506588708609343,
      "step": 313
    },
    {
      "epoch": 0.18237788232560842,
      "grad_norm": 290.2313537597656,
      "learning_rate": 4.543869843114469e-06,
      "logits/chosen": -0.8251843452453613,
      "logits/rejected": -0.8637291193008423,
      "logps/chosen": -76.97846984863281,
      "logps/rejected": -66.21952819824219,
      "loss": 13.9921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003126763505861163,
      "rewards/margins": -0.010613595135509968,
      "rewards/rejected": 0.007486830465495586,
      "step": 314
    },
    {
      "epoch": 0.18295870360690017,
      "grad_norm": 357.3050842285156,
      "learning_rate": 4.542417199302731e-06,
      "logits/chosen": -0.7790865898132324,
      "logits/rejected": -0.8820877075195312,
      "logps/chosen": -69.9126968383789,
      "logps/rejected": -73.85781860351562,
      "loss": 13.243,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.004135083872824907,
      "rewards/margins": 0.07195592671632767,
      "rewards/rejected": -0.06782083958387375,
      "step": 315
    },
    {
      "epoch": 0.1835395248881919,
      "grad_norm": 304.0280456542969,
      "learning_rate": 4.540964555490994e-06,
      "logits/chosen": -0.8775313496589661,
      "logits/rejected": -0.966948390007019,
      "logps/chosen": -73.7503890991211,
      "logps/rejected": -71.78192138671875,
      "loss": 13.2604,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02027755416929722,
      "rewards/margins": 0.07092587649822235,
      "rewards/rejected": -0.050648320466279984,
      "step": 316
    },
    {
      "epoch": 0.18412034616948364,
      "grad_norm": 307.97015380859375,
      "learning_rate": 4.539511911679257e-06,
      "logits/chosen": -0.8565672636032104,
      "logits/rejected": -0.9156128764152527,
      "logps/chosen": -73.54744720458984,
      "logps/rejected": -73.94361877441406,
      "loss": 13.7971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005255700554698706,
      "rewards/margins": 0.008574297651648521,
      "rewards/rejected": -0.013830000534653664,
      "step": 317
    },
    {
      "epoch": 0.1847011674507754,
      "grad_norm": 312.7808837890625,
      "learning_rate": 4.53805926786752e-06,
      "logits/chosen": -0.8373235464096069,
      "logits/rejected": -0.8614371418952942,
      "logps/chosen": -71.75985717773438,
      "logps/rejected": -80.21531677246094,
      "loss": 13.8805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002429866697639227,
      "rewards/margins": 0.001796521246433258,
      "rewards/rejected": 0.0006333448109216988,
      "step": 318
    },
    {
      "epoch": 0.18528198873206714,
      "grad_norm": 322.60736083984375,
      "learning_rate": 4.536606624055782e-06,
      "logits/chosen": -0.8135954141616821,
      "logits/rejected": -0.8624083399772644,
      "logps/chosen": -67.81532287597656,
      "logps/rejected": -81.07484436035156,
      "loss": 13.4658,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008448420092463493,
      "rewards/margins": 0.04661915823817253,
      "rewards/rejected": -0.055067580193281174,
      "step": 319
    },
    {
      "epoch": 0.1858628100133589,
      "grad_norm": 289.1242370605469,
      "learning_rate": 4.535153980244045e-06,
      "logits/chosen": -0.7294256687164307,
      "logits/rejected": -0.7239043116569519,
      "logps/chosen": -70.69343566894531,
      "logps/rejected": -76.38871765136719,
      "loss": 13.2891,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04026845842599869,
      "rewards/margins": 0.06055659055709839,
      "rewards/rejected": -0.020288124680519104,
      "step": 320
    },
    {
      "epoch": 0.18644363129465064,
      "grad_norm": 316.1600341796875,
      "learning_rate": 4.533701336432307e-06,
      "logits/chosen": -0.7641903162002563,
      "logits/rejected": -0.8571245074272156,
      "logps/chosen": -72.52635192871094,
      "logps/rejected": -72.56694030761719,
      "loss": 13.0747,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.040480513125658035,
      "rewards/margins": 0.08481685817241669,
      "rewards/rejected": -0.04433634877204895,
      "step": 321
    },
    {
      "epoch": 0.18702445257594238,
      "grad_norm": 331.61419677734375,
      "learning_rate": 4.53224869262057e-06,
      "logits/chosen": -0.7687914967536926,
      "logits/rejected": -0.862636387348175,
      "logps/chosen": -69.64253997802734,
      "logps/rejected": -72.28370666503906,
      "loss": 13.9979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.030815565958619118,
      "rewards/margins": -0.009632897563278675,
      "rewards/rejected": -0.021182667464017868,
      "step": 322
    },
    {
      "epoch": 0.18760527385723413,
      "grad_norm": 294.9023742675781,
      "learning_rate": 4.5307960488088326e-06,
      "logits/chosen": -0.7634553909301758,
      "logits/rejected": -0.8270981907844543,
      "logps/chosen": -72.29499816894531,
      "logps/rejected": -74.75524139404297,
      "loss": 13.8647,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.014828644692897797,
      "rewards/margins": 0.004178445786237717,
      "rewards/rejected": -0.019007090479135513,
      "step": 323
    },
    {
      "epoch": 0.18818609513852588,
      "grad_norm": 303.8163757324219,
      "learning_rate": 4.529343404997095e-06,
      "logits/chosen": -0.7258373498916626,
      "logits/rejected": -0.7167456746101379,
      "logps/chosen": -71.8295669555664,
      "logps/rejected": -68.2274169921875,
      "loss": 14.0144,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04376727342605591,
      "rewards/margins": -0.011802466586232185,
      "rewards/rejected": -0.03196480870246887,
      "step": 324
    },
    {
      "epoch": 0.18876691641981763,
      "grad_norm": 304.3807678222656,
      "learning_rate": 4.527890761185358e-06,
      "logits/chosen": -0.8415002822875977,
      "logits/rejected": -0.8635972738265991,
      "logps/chosen": -76.11531829833984,
      "logps/rejected": -74.70405578613281,
      "loss": 14.0264,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02958356961607933,
      "rewards/margins": -0.012393072247505188,
      "rewards/rejected": -0.017190497368574142,
      "step": 325
    },
    {
      "epoch": 0.18934773770110938,
      "grad_norm": 315.16876220703125,
      "learning_rate": 4.526438117373621e-06,
      "logits/chosen": -0.7613179683685303,
      "logits/rejected": -0.776989221572876,
      "logps/chosen": -67.979248046875,
      "logps/rejected": -69.2055435180664,
      "loss": 13.738,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007391949184238911,
      "rewards/margins": 0.015842467546463013,
      "rewards/rejected": -0.008450517430901527,
      "step": 326
    },
    {
      "epoch": 0.1899285589824011,
      "grad_norm": 310.1661682128906,
      "learning_rate": 4.524985473561883e-06,
      "logits/chosen": -0.9094734191894531,
      "logits/rejected": -0.8339530825614929,
      "logps/chosen": -76.08878326416016,
      "logps/rejected": -70.17434692382812,
      "loss": 13.821,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.020542889833450317,
      "rewards/margins": 0.006295947823673487,
      "rewards/rejected": 0.014246943406760693,
      "step": 327
    },
    {
      "epoch": 0.19050938026369285,
      "grad_norm": 313.35858154296875,
      "learning_rate": 4.5235328297501455e-06,
      "logits/chosen": -0.7620694041252136,
      "logits/rejected": -0.7337735295295715,
      "logps/chosen": -68.53215026855469,
      "logps/rejected": -75.737548828125,
      "loss": 13.5514,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0018144982168450952,
      "rewards/margins": 0.035731058567762375,
      "rewards/rejected": -0.03391656279563904,
      "step": 328
    },
    {
      "epoch": 0.1910902015449846,
      "grad_norm": 371.4503479003906,
      "learning_rate": 4.522080185938408e-06,
      "logits/chosen": -0.8343189358711243,
      "logits/rejected": -0.887412428855896,
      "logps/chosen": -69.50465393066406,
      "logps/rejected": -78.40193939208984,
      "loss": 13.6258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.025033259764313698,
      "rewards/margins": 0.02790558710694313,
      "rewards/rejected": -0.05293884128332138,
      "step": 329
    },
    {
      "epoch": 0.19167102282627635,
      "grad_norm": 368.6702575683594,
      "learning_rate": 4.520627542126671e-06,
      "logits/chosen": -0.8924549221992493,
      "logits/rejected": -0.88835608959198,
      "logps/chosen": -67.6229019165039,
      "logps/rejected": -74.09444427490234,
      "loss": 13.7015,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01569213904440403,
      "rewards/margins": 0.018520962446928024,
      "rewards/rejected": -0.0028288268949836493,
      "step": 330
    },
    {
      "epoch": 0.1922518441075681,
      "grad_norm": 324.3641052246094,
      "learning_rate": 4.519174898314934e-06,
      "logits/chosen": -0.8418426513671875,
      "logits/rejected": -0.8403279185295105,
      "logps/chosen": -72.23628234863281,
      "logps/rejected": -74.98982238769531,
      "loss": 13.8397,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.013874625787138939,
      "rewards/margins": 0.007325878832489252,
      "rewards/rejected": -0.021200504153966904,
      "step": 331
    },
    {
      "epoch": 0.19283266538885985,
      "grad_norm": 321.1864929199219,
      "learning_rate": 4.5177222545031964e-06,
      "logits/chosen": -0.8301184773445129,
      "logits/rejected": -0.748178243637085,
      "logps/chosen": -72.51194763183594,
      "logps/rejected": -72.02323913574219,
      "loss": 13.6885,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00831710733473301,
      "rewards/margins": 0.020771954208612442,
      "rewards/rejected": -0.02908906899392605,
      "step": 332
    },
    {
      "epoch": 0.1934134866701516,
      "grad_norm": 302.070556640625,
      "learning_rate": 4.516269610691459e-06,
      "logits/chosen": -0.8225947618484497,
      "logits/rejected": -0.7799954414367676,
      "logps/chosen": -73.186279296875,
      "logps/rejected": -67.39868927001953,
      "loss": 14.105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012917375192046165,
      "rewards/margins": -0.019778212532401085,
      "rewards/rejected": 0.006860838737338781,
      "step": 333
    },
    {
      "epoch": 0.19399430795144335,
      "grad_norm": 320.9998779296875,
      "learning_rate": 4.514816966879721e-06,
      "logits/chosen": -0.8660848736763,
      "logits/rejected": -0.8538210988044739,
      "logps/chosen": -77.54426574707031,
      "logps/rejected": -70.16163635253906,
      "loss": 14.0522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008558349683880806,
      "rewards/margins": -0.00849790871143341,
      "rewards/rejected": -6.044358087820001e-05,
      "step": 334
    },
    {
      "epoch": 0.1945751292327351,
      "grad_norm": 321.6021423339844,
      "learning_rate": 4.513364323067984e-06,
      "logits/chosen": -0.8346333503723145,
      "logits/rejected": -0.88763827085495,
      "logps/chosen": -74.38595581054688,
      "logps/rejected": -65.50772094726562,
      "loss": 14.0522,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.004940299782902002,
      "rewards/margins": -0.015277815982699394,
      "rewards/rejected": 0.010337515734136105,
      "step": 335
    },
    {
      "epoch": 0.19515595051402684,
      "grad_norm": 329.2897033691406,
      "learning_rate": 4.511911679256247e-06,
      "logits/chosen": -0.9041665196418762,
      "logits/rejected": -0.8650445938110352,
      "logps/chosen": -77.61279296875,
      "logps/rejected": -78.71620178222656,
      "loss": 13.5759,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0006745524588041008,
      "rewards/margins": 0.03203847259283066,
      "rewards/rejected": -0.03136391565203667,
      "step": 336
    },
    {
      "epoch": 0.1957367717953186,
      "grad_norm": 314.5374450683594,
      "learning_rate": 4.510459035444509e-06,
      "logits/chosen": -0.7889400720596313,
      "logits/rejected": -0.7860768437385559,
      "logps/chosen": -73.5044174194336,
      "logps/rejected": -74.55040740966797,
      "loss": 13.7341,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00419281842187047,
      "rewards/margins": 0.017519472166895866,
      "rewards/rejected": -0.013326652348041534,
      "step": 337
    },
    {
      "epoch": 0.19631759307661031,
      "grad_norm": 303.45733642578125,
      "learning_rate": 4.509006391632772e-06,
      "logits/chosen": -0.9617490768432617,
      "logits/rejected": -0.9624162912368774,
      "logps/chosen": -72.66578674316406,
      "logps/rejected": -73.77256774902344,
      "loss": 13.46,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024341020733118057,
      "rewards/margins": 0.044455137103796005,
      "rewards/rejected": -0.020114116370677948,
      "step": 338
    },
    {
      "epoch": 0.19689841435790206,
      "grad_norm": 297.1424560546875,
      "learning_rate": 4.507553747821035e-06,
      "logits/chosen": -0.8083046078681946,
      "logits/rejected": -0.8331934213638306,
      "logps/chosen": -71.33175659179688,
      "logps/rejected": -72.65750885009766,
      "loss": 13.8942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017249945551156998,
      "rewards/margins": 0.002586670219898224,
      "rewards/rejected": 0.014663276262581348,
      "step": 339
    },
    {
      "epoch": 0.1974792356391938,
      "grad_norm": 312.68145751953125,
      "learning_rate": 4.506101104009298e-06,
      "logits/chosen": -0.861495316028595,
      "logits/rejected": -0.8806974291801453,
      "logps/chosen": -73.86710357666016,
      "logps/rejected": -74.84933471679688,
      "loss": 13.6776,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015142458491027355,
      "rewards/margins": 0.02246389351785183,
      "rewards/rejected": -0.03760635107755661,
      "step": 340
    },
    {
      "epoch": 0.19806005692048556,
      "grad_norm": 306.0162353515625,
      "learning_rate": 4.5046484601975595e-06,
      "logits/chosen": -0.7776416540145874,
      "logits/rejected": -0.7664206624031067,
      "logps/chosen": -74.73188018798828,
      "logps/rejected": -71.7195053100586,
      "loss": 13.8777,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01987052895128727,
      "rewards/margins": 0.001284526428207755,
      "rewards/rejected": -0.02115505374968052,
      "step": 341
    },
    {
      "epoch": 0.1986408782017773,
      "grad_norm": 318.2908935546875,
      "learning_rate": 4.503195816385822e-06,
      "logits/chosen": -0.9286810159683228,
      "logits/rejected": -0.8470717668533325,
      "logps/chosen": -77.0329360961914,
      "logps/rejected": -79.6993637084961,
      "loss": 13.9516,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03609747439622879,
      "rewards/margins": -0.0019962512888014317,
      "rewards/rejected": -0.0341012142598629,
      "step": 342
    },
    {
      "epoch": 0.19922169948306906,
      "grad_norm": 343.0354309082031,
      "learning_rate": 4.501743172574085e-06,
      "logits/chosen": -0.6656073331832886,
      "logits/rejected": -0.6898230314254761,
      "logps/chosen": -75.619384765625,
      "logps/rejected": -77.71598052978516,
      "loss": 13.666,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0036884776782244444,
      "rewards/margins": 0.024708479642868042,
      "rewards/rejected": -0.028396958485245705,
      "step": 343
    },
    {
      "epoch": 0.1998025207643608,
      "grad_norm": 318.9757995605469,
      "learning_rate": 4.500290528762348e-06,
      "logits/chosen": -0.9170076251029968,
      "logits/rejected": -0.9125305414199829,
      "logps/chosen": -61.814781188964844,
      "logps/rejected": -73.81932830810547,
      "loss": 13.8999,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009178831242024899,
      "rewards/margins": 0.0020302850753068924,
      "rewards/rejected": -0.011209115386009216,
      "step": 344
    },
    {
      "epoch": 0.20038334204565256,
      "grad_norm": 301.6319580078125,
      "learning_rate": 4.4988378849506105e-06,
      "logits/chosen": -0.8031547665596008,
      "logits/rejected": -0.8383834958076477,
      "logps/chosen": -71.64036560058594,
      "logps/rejected": -69.63984680175781,
      "loss": 14.3238,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024623652920126915,
      "rewards/margins": -0.042028360068798065,
      "rewards/rejected": 0.0174047090113163,
      "step": 345
    },
    {
      "epoch": 0.2009641633269443,
      "grad_norm": 312.9844055175781,
      "learning_rate": 4.497385241138873e-06,
      "logits/chosen": -0.9153481721878052,
      "logits/rejected": -0.880916953086853,
      "logps/chosen": -75.21138000488281,
      "logps/rejected": -69.07037353515625,
      "loss": 13.5771,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0012881949078291655,
      "rewards/margins": 0.031341440975666046,
      "rewards/rejected": -0.03262963145971298,
      "step": 346
    },
    {
      "epoch": 0.20154498460823606,
      "grad_norm": 349.6585388183594,
      "learning_rate": 4.495932597327136e-06,
      "logits/chosen": -0.7509949803352356,
      "logits/rejected": -0.6910394430160522,
      "logps/chosen": -70.14628601074219,
      "logps/rejected": -71.41559600830078,
      "loss": 14.3833,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.01079236064106226,
      "rewards/margins": -0.048026543110609055,
      "rewards/rejected": 0.03723418340086937,
      "step": 347
    },
    {
      "epoch": 0.2021258058895278,
      "grad_norm": 291.7655029296875,
      "learning_rate": 4.494479953515398e-06,
      "logits/chosen": -0.8026542663574219,
      "logits/rejected": -0.803175151348114,
      "logps/chosen": -71.43840789794922,
      "logps/rejected": -71.1689224243164,
      "loss": 13.6799,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0039477264508605,
      "rewards/margins": 0.021708643063902855,
      "rewards/rejected": -0.01776091754436493,
      "step": 348
    },
    {
      "epoch": 0.20270662717081953,
      "grad_norm": 293.5408935546875,
      "learning_rate": 4.493027309703661e-06,
      "logits/chosen": -0.8886486887931824,
      "logits/rejected": -0.8125056028366089,
      "logps/chosen": -77.56742095947266,
      "logps/rejected": -66.32499694824219,
      "loss": 13.7167,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02104121446609497,
      "rewards/margins": 0.019321372732520103,
      "rewards/rejected": 0.001719839172437787,
      "step": 349
    },
    {
      "epoch": 0.20328744845211127,
      "grad_norm": 305.96697998046875,
      "learning_rate": 4.491574665891923e-06,
      "logits/chosen": -0.7537021636962891,
      "logits/rejected": -0.7224977016448975,
      "logps/chosen": -71.26932525634766,
      "logps/rejected": -69.65036010742188,
      "loss": 13.7664,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008614836260676384,
      "rewards/margins": 0.012173959985375404,
      "rewards/rejected": -0.02078879624605179,
      "step": 350
    },
    {
      "epoch": 0.20386826973340302,
      "grad_norm": 304.3200988769531,
      "learning_rate": 4.490122022080186e-06,
      "logits/chosen": -0.7840813398361206,
      "logits/rejected": -0.7836223840713501,
      "logps/chosen": -67.91246032714844,
      "logps/rejected": -72.22390747070312,
      "loss": 13.389,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.016956433653831482,
      "rewards/margins": 0.051059335470199585,
      "rewards/rejected": -0.034102894365787506,
      "step": 351
    },
    {
      "epoch": 0.20444909101469477,
      "grad_norm": 291.3819885253906,
      "learning_rate": 4.488669378268449e-06,
      "logits/chosen": -0.7469109296798706,
      "logits/rejected": -0.7488623857498169,
      "logps/chosen": -69.54528045654297,
      "logps/rejected": -72.68141174316406,
      "loss": 13.5492,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.017030106857419014,
      "rewards/margins": 0.03490540385246277,
      "rewards/rejected": -0.017875295132398605,
      "step": 352
    },
    {
      "epoch": 0.20502991229598652,
      "grad_norm": 311.4941711425781,
      "learning_rate": 4.487216734456712e-06,
      "logits/chosen": -0.7565064430236816,
      "logits/rejected": -0.814095675945282,
      "logps/chosen": -77.25660705566406,
      "logps/rejected": -70.50426483154297,
      "loss": 13.9861,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.023350324481725693,
      "rewards/margins": -0.00980983767658472,
      "rewards/rejected": -0.013540486805140972,
      "step": 353
    },
    {
      "epoch": 0.20561073357727827,
      "grad_norm": 317.1282958984375,
      "learning_rate": 4.485764090644974e-06,
      "logits/chosen": -0.6474046111106873,
      "logits/rejected": -0.5795416235923767,
      "logps/chosen": -73.16572570800781,
      "logps/rejected": -73.50746154785156,
      "loss": 13.6889,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.022182445973157883,
      "rewards/margins": 0.024161117151379585,
      "rewards/rejected": -0.0019786744378507137,
      "step": 354
    },
    {
      "epoch": 0.20619155485857002,
      "grad_norm": 353.6116638183594,
      "learning_rate": 4.484311446833236e-06,
      "logits/chosen": -0.6704899072647095,
      "logits/rejected": -0.7246066331863403,
      "logps/chosen": -75.37571716308594,
      "logps/rejected": -77.72221374511719,
      "loss": 14.1555,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.020454417914152145,
      "rewards/margins": -0.026207536458969116,
      "rewards/rejected": 0.005753117147833109,
      "step": 355
    },
    {
      "epoch": 0.20677237613986177,
      "grad_norm": 306.08367919921875,
      "learning_rate": 4.482858803021499e-06,
      "logits/chosen": -0.8278299570083618,
      "logits/rejected": -0.7702374458312988,
      "logps/chosen": -79.121337890625,
      "logps/rejected": -79.2762680053711,
      "loss": 13.3845,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.017661219462752342,
      "rewards/margins": 0.05229003354907036,
      "rewards/rejected": -0.034628815948963165,
      "step": 356
    },
    {
      "epoch": 0.20735319742115352,
      "grad_norm": 301.5957946777344,
      "learning_rate": 4.481406159209763e-06,
      "logits/chosen": -0.7846859693527222,
      "logits/rejected": -0.6674401164054871,
      "logps/chosen": -70.37995910644531,
      "logps/rejected": -72.95897674560547,
      "loss": 13.5715,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.012851124629378319,
      "rewards/margins": 0.033614885061979294,
      "rewards/rejected": -0.020763758569955826,
      "step": 357
    },
    {
      "epoch": 0.20793401870244527,
      "grad_norm": 292.0884094238281,
      "learning_rate": 4.479953515398025e-06,
      "logits/chosen": -0.7394891381263733,
      "logits/rejected": -0.7185255289077759,
      "logps/chosen": -69.58491516113281,
      "logps/rejected": -70.53492736816406,
      "loss": 13.4921,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008961153216660023,
      "rewards/margins": 0.04007042199373245,
      "rewards/rejected": -0.031109267845749855,
      "step": 358
    },
    {
      "epoch": 0.20851483998373702,
      "grad_norm": 307.0670471191406,
      "learning_rate": 4.478500871586287e-06,
      "logits/chosen": -0.7576676607131958,
      "logits/rejected": -0.8148140907287598,
      "logps/chosen": -73.82097625732422,
      "logps/rejected": -76.19036102294922,
      "loss": 13.6695,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014283361844718456,
      "rewards/margins": 0.0231223963201046,
      "rewards/rejected": -0.008839035406708717,
      "step": 359
    },
    {
      "epoch": 0.20909566126502874,
      "grad_norm": 300.3365173339844,
      "learning_rate": 4.47704822777455e-06,
      "logits/chosen": -0.8952158093452454,
      "logits/rejected": -0.7517813444137573,
      "logps/chosen": -67.8691635131836,
      "logps/rejected": -75.82969665527344,
      "loss": 14.1246,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.004317197948694229,
      "rewards/margins": -0.022498121485114098,
      "rewards/rejected": 0.01818092353641987,
      "step": 360
    },
    {
      "epoch": 0.2096764825463205,
      "grad_norm": 322.0217590332031,
      "learning_rate": 4.475595583962813e-06,
      "logits/chosen": -0.9054155349731445,
      "logits/rejected": -0.7519516348838806,
      "logps/chosen": -73.81487274169922,
      "logps/rejected": -73.7669906616211,
      "loss": 13.4128,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.025307711213827133,
      "rewards/margins": 0.04915634170174599,
      "rewards/rejected": -0.023848628625273705,
      "step": 361
    },
    {
      "epoch": 0.21025730382761224,
      "grad_norm": 305.5541687011719,
      "learning_rate": 4.4741429401510755e-06,
      "logits/chosen": -0.7097185254096985,
      "logits/rejected": -0.7574303150177002,
      "logps/chosen": -71.90687561035156,
      "logps/rejected": -68.58987426757812,
      "loss": 13.4067,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.00799614004790783,
      "rewards/margins": 0.051417239010334015,
      "rewards/rejected": -0.059413373470306396,
      "step": 362
    },
    {
      "epoch": 0.21083812510890398,
      "grad_norm": 353.3551025390625,
      "learning_rate": 4.472690296339338e-06,
      "logits/chosen": -0.925432026386261,
      "logits/rejected": -0.7938982844352722,
      "logps/chosen": -65.05897521972656,
      "logps/rejected": -73.69389343261719,
      "loss": 13.7132,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.015448915772140026,
      "rewards/margins": 0.017804250121116638,
      "rewards/rejected": -0.03325316682457924,
      "step": 363
    },
    {
      "epoch": 0.21141894639019573,
      "grad_norm": 295.042236328125,
      "learning_rate": 4.471237652527601e-06,
      "logits/chosen": -0.5853012800216675,
      "logits/rejected": -0.6785317659378052,
      "logps/chosen": -70.28089904785156,
      "logps/rejected": -81.96311950683594,
      "loss": 13.2481,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01614985801279545,
      "rewards/margins": 0.06508197635412216,
      "rewards/rejected": -0.04893212765455246,
      "step": 364
    },
    {
      "epoch": 0.21199976767148748,
      "grad_norm": 317.24664306640625,
      "learning_rate": 4.469785008715864e-06,
      "logits/chosen": -0.7670639157295227,
      "logits/rejected": -0.732401967048645,
      "logps/chosen": -73.05061340332031,
      "logps/rejected": -82.63640594482422,
      "loss": 13.4454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009124317206442356,
      "rewards/margins": 0.047481339424848557,
      "rewards/rejected": -0.05660565569996834,
      "step": 365
    },
    {
      "epoch": 0.21258058895277923,
      "grad_norm": 308.0080871582031,
      "learning_rate": 4.468332364904126e-06,
      "logits/chosen": -0.7194598913192749,
      "logits/rejected": -0.6728766560554504,
      "logps/chosen": -73.11915588378906,
      "logps/rejected": -68.853759765625,
      "loss": 13.9253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021355021744966507,
      "rewards/margins": -0.003295204136520624,
      "rewards/rejected": -0.01805981807410717,
      "step": 366
    },
    {
      "epoch": 0.21316141023407098,
      "grad_norm": 327.4511413574219,
      "learning_rate": 4.466879721092388e-06,
      "logits/chosen": -0.7333757281303406,
      "logits/rejected": -0.7388908863067627,
      "logps/chosen": -81.80522155761719,
      "logps/rejected": -69.60041046142578,
      "loss": 13.8993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011159257963299751,
      "rewards/margins": 0.005005787592381239,
      "rewards/rejected": -0.016165047883987427,
      "step": 367
    },
    {
      "epoch": 0.21374223151536273,
      "grad_norm": 298.0936279296875,
      "learning_rate": 4.465427077280651e-06,
      "logits/chosen": -0.9420258402824402,
      "logits/rejected": -0.7203485369682312,
      "logps/chosen": -78.59254455566406,
      "logps/rejected": -70.5693130493164,
      "loss": 13.4892,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02526816353201866,
      "rewards/margins": 0.04102920740842819,
      "rewards/rejected": -0.01576104201376438,
      "step": 368
    },
    {
      "epoch": 0.21432305279665448,
      "grad_norm": 297.84136962890625,
      "learning_rate": 4.463974433468914e-06,
      "logits/chosen": -0.8287162780761719,
      "logits/rejected": -0.8401540517807007,
      "logps/chosen": -70.97822570800781,
      "logps/rejected": -73.44509887695312,
      "loss": 13.4517,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03007066808640957,
      "rewards/margins": 0.047326039522886276,
      "rewards/rejected": -0.017255373299121857,
      "step": 369
    },
    {
      "epoch": 0.21490387407794623,
      "grad_norm": 322.31854248046875,
      "learning_rate": 4.462521789657177e-06,
      "logits/chosen": -0.936688244342804,
      "logits/rejected": -0.8096345663070679,
      "logps/chosen": -78.00273132324219,
      "logps/rejected": -76.44258117675781,
      "loss": 13.933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024149218574166298,
      "rewards/margins": 0.0015145957004278898,
      "rewards/rejected": -0.025663817301392555,
      "step": 370
    },
    {
      "epoch": 0.21548469535923795,
      "grad_norm": 316.048583984375,
      "learning_rate": 4.461069145845439e-06,
      "logits/chosen": -0.7057862877845764,
      "logits/rejected": -0.6469615697860718,
      "logps/chosen": -77.95980834960938,
      "logps/rejected": -83.23860931396484,
      "loss": 13.6137,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009251176379621029,
      "rewards/margins": 0.03268672153353691,
      "rewards/rejected": -0.04193788766860962,
      "step": 371
    },
    {
      "epoch": 0.2160655166405297,
      "grad_norm": 306.4630432128906,
      "learning_rate": 4.459616502033702e-06,
      "logits/chosen": -0.4769328534603119,
      "logits/rejected": -0.44898924231529236,
      "logps/chosen": -77.04486846923828,
      "logps/rejected": -78.56315612792969,
      "loss": 13.6158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02271147631108761,
      "rewards/margins": 0.02890445664525032,
      "rewards/rejected": -0.05161593109369278,
      "step": 372
    },
    {
      "epoch": 0.21664633792182145,
      "grad_norm": 304.4960021972656,
      "learning_rate": 4.458163858221964e-06,
      "logits/chosen": -0.7167657017707825,
      "logits/rejected": -0.6918506622314453,
      "logps/chosen": -71.01347351074219,
      "logps/rejected": -72.31254577636719,
      "loss": 13.732,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.000307169568259269,
      "rewards/margins": 0.01947268471121788,
      "rewards/rejected": -0.0191655196249485,
      "step": 373
    },
    {
      "epoch": 0.2172271592031132,
      "grad_norm": 296.60205078125,
      "learning_rate": 4.456711214410227e-06,
      "logits/chosen": -0.7357224225997925,
      "logits/rejected": -0.6462749242782593,
      "logps/chosen": -75.4621810913086,
      "logps/rejected": -70.76265716552734,
      "loss": 14.1072,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03969653695821762,
      "rewards/margins": -0.01930340752005577,
      "rewards/rejected": -0.02039313316345215,
      "step": 374
    },
    {
      "epoch": 0.21780798048440494,
      "grad_norm": 299.0235595703125,
      "learning_rate": 4.4552585705984895e-06,
      "logits/chosen": -0.9737260937690735,
      "logits/rejected": -0.883916974067688,
      "logps/chosen": -76.01069641113281,
      "logps/rejected": -64.29400634765625,
      "loss": 13.8576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021567821968346834,
      "rewards/margins": 0.005682178307324648,
      "rewards/rejected": -0.003525395644828677,
      "step": 375
    },
    {
      "epoch": 0.2183888017656967,
      "grad_norm": 323.23663330078125,
      "learning_rate": 4.453805926786752e-06,
      "logits/chosen": -0.7908447980880737,
      "logits/rejected": -0.7856351137161255,
      "logps/chosen": -73.55455017089844,
      "logps/rejected": -75.9771957397461,
      "loss": 14.1184,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.018270045518875122,
      "rewards/margins": -0.020638611167669296,
      "rewards/rejected": 0.0023685642518103123,
      "step": 376
    },
    {
      "epoch": 0.21896962304698844,
      "grad_norm": 322.6252746582031,
      "learning_rate": 4.452353282975015e-06,
      "logits/chosen": -0.8486648797988892,
      "logits/rejected": -0.8361449241638184,
      "logps/chosen": -71.75032043457031,
      "logps/rejected": -70.97474670410156,
      "loss": 13.4519,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0086447075009346,
      "rewards/margins": 0.04460224509239197,
      "rewards/rejected": -0.05324694514274597,
      "step": 377
    },
    {
      "epoch": 0.2195504443282802,
      "grad_norm": 533.611083984375,
      "learning_rate": 4.450900639163278e-06,
      "logits/chosen": -0.7714306116104126,
      "logits/rejected": -0.7245787382125854,
      "logps/chosen": -73.3573226928711,
      "logps/rejected": -72.71539306640625,
      "loss": 13.7809,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01556567382067442,
      "rewards/margins": 0.01385035365819931,
      "rewards/rejected": -0.029416028410196304,
      "step": 378
    },
    {
      "epoch": 0.22013126560957194,
      "grad_norm": 287.4798583984375,
      "learning_rate": 4.4494479953515405e-06,
      "logits/chosen": -0.8422917127609253,
      "logits/rejected": -0.7723952531814575,
      "logps/chosen": -67.02293395996094,
      "logps/rejected": -76.67664337158203,
      "loss": 13.2566,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.013728572055697441,
      "rewards/margins": 0.0658256933093071,
      "rewards/rejected": -0.052097123116254807,
      "step": 379
    },
    {
      "epoch": 0.2207120868908637,
      "grad_norm": 333.2846374511719,
      "learning_rate": 4.4479953515398024e-06,
      "logits/chosen": -0.7608442306518555,
      "logits/rejected": -0.7422316670417786,
      "logps/chosen": -72.54397583007812,
      "logps/rejected": -69.54707336425781,
      "loss": 14.3318,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.04744737595319748,
      "rewards/margins": -0.040470145642757416,
      "rewards/rejected": -0.00697722565382719,
      "step": 380
    },
    {
      "epoch": 0.22129290817215544,
      "grad_norm": 315.6219177246094,
      "learning_rate": 4.446542707728065e-06,
      "logits/chosen": -0.7969782948493958,
      "logits/rejected": -0.7673382759094238,
      "logps/chosen": -76.81396484375,
      "logps/rejected": -74.59119415283203,
      "loss": 14.2554,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04644475132226944,
      "rewards/margins": -0.03383153676986694,
      "rewards/rejected": -0.012613209895789623,
      "step": 381
    },
    {
      "epoch": 0.22187372945344716,
      "grad_norm": 360.37274169921875,
      "learning_rate": 4.445090063916328e-06,
      "logits/chosen": -0.7555001974105835,
      "logits/rejected": -0.8732713460922241,
      "logps/chosen": -80.61076354980469,
      "logps/rejected": -67.64311981201172,
      "loss": 14.0055,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.014537744224071503,
      "rewards/margins": -0.010739547200500965,
      "rewards/rejected": -0.0037982002831995487,
      "step": 382
    },
    {
      "epoch": 0.2224545507347389,
      "grad_norm": 305.7255554199219,
      "learning_rate": 4.443637420104591e-06,
      "logits/chosen": -0.6732084155082703,
      "logits/rejected": -0.6309961080551147,
      "logps/chosen": -69.0905532836914,
      "logps/rejected": -72.41564178466797,
      "loss": 13.2323,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.053167276084423065,
      "rewards/margins": 0.06811966747045517,
      "rewards/rejected": -0.014952393248677254,
      "step": 383
    },
    {
      "epoch": 0.22303537201603066,
      "grad_norm": 318.80755615234375,
      "learning_rate": 4.4421847762928534e-06,
      "logits/chosen": -0.8548835515975952,
      "logits/rejected": -0.7376150488853455,
      "logps/chosen": -74.19415283203125,
      "logps/rejected": -83.93922424316406,
      "loss": 14.1736,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04532115161418915,
      "rewards/margins": -0.02556682750582695,
      "rewards/rejected": -0.01975431852042675,
      "step": 384
    },
    {
      "epoch": 0.2236161932973224,
      "grad_norm": 298.5427551269531,
      "learning_rate": 4.440732132481116e-06,
      "logits/chosen": -0.7442782521247864,
      "logits/rejected": -0.7569425702095032,
      "logps/chosen": -70.04218292236328,
      "logps/rejected": -76.72520446777344,
      "loss": 13.364,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.015433462336659431,
      "rewards/margins": 0.052092112600803375,
      "rewards/rejected": -0.03665865212678909,
      "step": 385
    },
    {
      "epoch": 0.22419701457861416,
      "grad_norm": 309.5966491699219,
      "learning_rate": 4.439279488669379e-06,
      "logits/chosen": -0.6254906058311462,
      "logits/rejected": -0.6205364465713501,
      "logps/chosen": -72.59516906738281,
      "logps/rejected": -75.21646881103516,
      "loss": 13.6555,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004701803904026747,
      "rewards/margins": 0.02681594528257847,
      "rewards/rejected": -0.031517744064331055,
      "step": 386
    },
    {
      "epoch": 0.2247778358599059,
      "grad_norm": 288.473876953125,
      "learning_rate": 4.437826844857641e-06,
      "logits/chosen": -0.8325392603874207,
      "logits/rejected": -1.0077170133590698,
      "logps/chosen": -72.17142486572266,
      "logps/rejected": -77.3849105834961,
      "loss": 13.1622,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.030657533556222916,
      "rewards/margins": 0.08042553812265396,
      "rewards/rejected": -0.049768008291721344,
      "step": 387
    },
    {
      "epoch": 0.22535865714119765,
      "grad_norm": 337.0882263183594,
      "learning_rate": 4.436374201045904e-06,
      "logits/chosen": -0.6630915999412537,
      "logits/rejected": -0.7251507639884949,
      "logps/chosen": -79.76948547363281,
      "logps/rejected": -77.0470199584961,
      "loss": 14.1512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01571335829794407,
      "rewards/margins": -0.025171738117933273,
      "rewards/rejected": 0.009458379819989204,
      "step": 388
    },
    {
      "epoch": 0.2259394784224894,
      "grad_norm": 312.39605712890625,
      "learning_rate": 4.434921557234166e-06,
      "logits/chosen": -0.7051008343696594,
      "logits/rejected": -0.7591395974159241,
      "logps/chosen": -70.51537322998047,
      "logps/rejected": -74.8718032836914,
      "loss": 14.1159,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.05321550369262695,
      "rewards/margins": -0.019576644524931908,
      "rewards/rejected": -0.033638857305049896,
      "step": 389
    },
    {
      "epoch": 0.22652029970378115,
      "grad_norm": 318.12457275390625,
      "learning_rate": 4.433468913422429e-06,
      "logits/chosen": -0.7092422246932983,
      "logits/rejected": -0.7855595946311951,
      "logps/chosen": -73.6144790649414,
      "logps/rejected": -70.25662231445312,
      "loss": 13.7536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015159961767494678,
      "rewards/margins": 0.01655631884932518,
      "rewards/rejected": -0.03171628341078758,
      "step": 390
    },
    {
      "epoch": 0.2271011209850729,
      "grad_norm": 297.23565673828125,
      "learning_rate": 4.432016269610692e-06,
      "logits/chosen": -0.6911486387252808,
      "logits/rejected": -0.5650784373283386,
      "logps/chosen": -61.7591438293457,
      "logps/rejected": -79.15550231933594,
      "loss": 13.4395,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007602124474942684,
      "rewards/margins": 0.0479809045791626,
      "rewards/rejected": -0.04037877544760704,
      "step": 391
    },
    {
      "epoch": 0.22768194226636465,
      "grad_norm": 315.382080078125,
      "learning_rate": 4.4305636257989546e-06,
      "logits/chosen": -0.7994186282157898,
      "logits/rejected": -0.8166016340255737,
      "logps/chosen": -81.44297790527344,
      "logps/rejected": -74.20890808105469,
      "loss": 13.8923,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.06704780459403992,
      "rewards/margins": 0.012969402596354485,
      "rewards/rejected": -0.08001720905303955,
      "step": 392
    },
    {
      "epoch": 0.2282627635476564,
      "grad_norm": 310.18865966796875,
      "learning_rate": 4.4291109819872165e-06,
      "logits/chosen": -0.6670494079589844,
      "logits/rejected": -0.7397955656051636,
      "logps/chosen": -69.39032745361328,
      "logps/rejected": -77.76183319091797,
      "loss": 13.8815,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.033610161393880844,
      "rewards/margins": 0.0040161325596272945,
      "rewards/rejected": -0.037626296281814575,
      "step": 393
    },
    {
      "epoch": 0.22884358482894812,
      "grad_norm": 1028.6781005859375,
      "learning_rate": 4.427658338175479e-06,
      "logits/chosen": -0.6542873978614807,
      "logits/rejected": -0.6277574300765991,
      "logps/chosen": -71.77285766601562,
      "logps/rejected": -82.8670654296875,
      "loss": 13.6692,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.003381280694156885,
      "rewards/margins": 0.02418811246752739,
      "rewards/rejected": -0.02756938897073269,
      "step": 394
    },
    {
      "epoch": 0.22942440611023987,
      "grad_norm": 300.6793212890625,
      "learning_rate": 4.426205694363742e-06,
      "logits/chosen": -0.657829761505127,
      "logits/rejected": -0.6191960573196411,
      "logps/chosen": -71.67803192138672,
      "logps/rejected": -71.7804183959961,
      "loss": 13.7474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0207084771245718,
      "rewards/margins": 0.01424718089401722,
      "rewards/rejected": 0.006461297161877155,
      "step": 395
    },
    {
      "epoch": 0.23000522739153162,
      "grad_norm": 336.28759765625,
      "learning_rate": 4.424753050552005e-06,
      "logits/chosen": -0.7785569429397583,
      "logits/rejected": -0.8487392663955688,
      "logps/chosen": -82.74131774902344,
      "logps/rejected": -74.78742980957031,
      "loss": 14.0335,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03686489909887314,
      "rewards/margins": -0.003692910075187683,
      "rewards/rejected": -0.03317200019955635,
      "step": 396
    },
    {
      "epoch": 0.23058604867282337,
      "grad_norm": 300.73101806640625,
      "learning_rate": 4.4233004067402675e-06,
      "logits/chosen": -0.7787965536117554,
      "logits/rejected": -0.7510574460029602,
      "logps/chosen": -72.66011047363281,
      "logps/rejected": -75.55070495605469,
      "loss": 13.6585,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.015632599592208862,
      "rewards/margins": 0.02679685689508915,
      "rewards/rejected": -0.04242945462465286,
      "step": 397
    },
    {
      "epoch": 0.23116686995411512,
      "grad_norm": 297.7178955078125,
      "learning_rate": 4.42184776292853e-06,
      "logits/chosen": -0.6219455003738403,
      "logits/rejected": -0.619706928730011,
      "logps/chosen": -72.49166107177734,
      "logps/rejected": -64.84117126464844,
      "loss": 13.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011766968294978142,
      "rewards/margins": 0.021194420754909515,
      "rewards/rejected": -0.03296138718724251,
      "step": 398
    },
    {
      "epoch": 0.23174769123540687,
      "grad_norm": 1181.5919189453125,
      "learning_rate": 4.420395119116793e-06,
      "logits/chosen": -0.7124764323234558,
      "logits/rejected": -0.7599982023239136,
      "logps/chosen": -82.6731185913086,
      "logps/rejected": -77.68875885009766,
      "loss": 13.8524,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004232340957969427,
      "rewards/margins": 0.005291967652738094,
      "rewards/rejected": -0.001059626811183989,
      "step": 399
    },
    {
      "epoch": 0.23232851251669862,
      "grad_norm": 291.1778869628906,
      "learning_rate": 4.418942475305056e-06,
      "logits/chosen": -0.707445502281189,
      "logits/rejected": -0.7544277310371399,
      "logps/chosen": -67.69174194335938,
      "logps/rejected": -73.61653137207031,
      "loss": 13.4031,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02160044014453888,
      "rewards/margins": 0.05302933603525162,
      "rewards/rejected": -0.03142889216542244,
      "step": 400
    },
    {
      "epoch": 0.23290933379799036,
      "grad_norm": 308.5126037597656,
      "learning_rate": 4.4174898314933185e-06,
      "logits/chosen": -0.7360697984695435,
      "logits/rejected": -0.6314154863357544,
      "logps/chosen": -73.9341049194336,
      "logps/rejected": -75.5310287475586,
      "loss": 13.7204,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -7.766112685203552e-05,
      "rewards/margins": 0.02340327948331833,
      "rewards/rejected": -0.023480940610170364,
      "step": 401
    },
    {
      "epoch": 0.2334901550792821,
      "grad_norm": 297.6388244628906,
      "learning_rate": 4.416037187681581e-06,
      "logits/chosen": -0.6773185729980469,
      "logits/rejected": -0.6966699957847595,
      "logps/chosen": -71.93721008300781,
      "logps/rejected": -73.58920288085938,
      "loss": 13.8046,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014304883778095245,
      "rewards/margins": 0.012389096431434155,
      "rewards/rejected": 0.0019157860660925508,
      "step": 402
    },
    {
      "epoch": 0.23407097636057386,
      "grad_norm": 319.0065612792969,
      "learning_rate": 4.414584543869844e-06,
      "logits/chosen": -0.6499051451683044,
      "logits/rejected": -0.9121743440628052,
      "logps/chosen": -81.61251831054688,
      "logps/rejected": -78.27536010742188,
      "loss": 13.3141,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0065872580744326115,
      "rewards/margins": 0.07194839417934418,
      "rewards/rejected": -0.07853565365076065,
      "step": 403
    },
    {
      "epoch": 0.2346517976418656,
      "grad_norm": 398.7077941894531,
      "learning_rate": 4.413131900058107e-06,
      "logits/chosen": -0.6521409749984741,
      "logits/rejected": -0.7569887638092041,
      "logps/chosen": -71.24628448486328,
      "logps/rejected": -66.81242370605469,
      "loss": 13.7869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0004381166654638946,
      "rewards/margins": 0.012258688919246197,
      "rewards/rejected": -0.011820574291050434,
      "step": 404
    },
    {
      "epoch": 0.23523261892315733,
      "grad_norm": 296.407470703125,
      "learning_rate": 4.411679256246369e-06,
      "logits/chosen": -0.8323850631713867,
      "logits/rejected": -0.7685847282409668,
      "logps/chosen": -70.29356384277344,
      "logps/rejected": -73.77702331542969,
      "loss": 13.6071,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.008858191780745983,
      "rewards/margins": 0.034311629831790924,
      "rewards/rejected": -0.025453437119722366,
      "step": 405
    },
    {
      "epoch": 0.23581344020444908,
      "grad_norm": 316.6470031738281,
      "learning_rate": 4.410226612434631e-06,
      "logits/chosen": -0.8853529691696167,
      "logits/rejected": -0.8087629079818726,
      "logps/chosen": -74.18810272216797,
      "logps/rejected": -75.40741729736328,
      "loss": 13.8599,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02046876773238182,
      "rewards/margins": 0.006527154240757227,
      "rewards/rejected": 0.013941613025963306,
      "step": 406
    },
    {
      "epoch": 0.23639426148574083,
      "grad_norm": 307.0223388671875,
      "learning_rate": 4.408773968622894e-06,
      "logits/chosen": -0.5616191625595093,
      "logits/rejected": -0.5830402970314026,
      "logps/chosen": -73.06590270996094,
      "logps/rejected": -75.1976318359375,
      "loss": 13.5981,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.030349424108862877,
      "rewards/margins": 0.031421925872564316,
      "rewards/rejected": -0.0010725029278546572,
      "step": 407
    },
    {
      "epoch": 0.23697508276703258,
      "grad_norm": 306.7025146484375,
      "learning_rate": 4.407321324811157e-06,
      "logits/chosen": -0.6714197993278503,
      "logits/rejected": -0.7746154069900513,
      "logps/chosen": -76.53195190429688,
      "logps/rejected": -69.44393920898438,
      "loss": 13.6952,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0025624283589422703,
      "rewards/margins": 0.02050391212105751,
      "rewards/rejected": -0.023066340014338493,
      "step": 408
    },
    {
      "epoch": 0.23755590404832433,
      "grad_norm": 315.3130187988281,
      "learning_rate": 4.40586868099942e-06,
      "logits/chosen": -0.6545313000679016,
      "logits/rejected": -0.6212127804756165,
      "logps/chosen": -75.73979187011719,
      "logps/rejected": -75.5727310180664,
      "loss": 13.0526,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03893669322133064,
      "rewards/margins": 0.08733826875686646,
      "rewards/rejected": -0.04840157553553581,
      "step": 409
    },
    {
      "epoch": 0.23813672532961608,
      "grad_norm": 319.98065185546875,
      "learning_rate": 4.404416037187682e-06,
      "logits/chosen": -0.6923006772994995,
      "logits/rejected": -0.5738077163696289,
      "logps/chosen": -72.50543975830078,
      "logps/rejected": -73.67253112792969,
      "loss": 13.8894,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.011189279146492481,
      "rewards/margins": 0.001272419816814363,
      "rewards/rejected": -0.012461700476706028,
      "step": 410
    },
    {
      "epoch": 0.23871754661090783,
      "grad_norm": 353.4020080566406,
      "learning_rate": 4.402963393375945e-06,
      "logits/chosen": -0.6882558465003967,
      "logits/rejected": -0.6723198890686035,
      "logps/chosen": -75.6532211303711,
      "logps/rejected": -70.86685943603516,
      "loss": 13.8372,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0034420304000377655,
      "rewards/margins": 0.008326428011059761,
      "rewards/rejected": -0.004884395748376846,
      "step": 411
    },
    {
      "epoch": 0.23929836789219958,
      "grad_norm": 305.5731506347656,
      "learning_rate": 4.401510749564207e-06,
      "logits/chosen": -0.7900495529174805,
      "logits/rejected": -0.7300332188606262,
      "logps/chosen": -69.94303894042969,
      "logps/rejected": -73.22996520996094,
      "loss": 13.4269,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033657319843769073,
      "rewards/margins": 0.048354774713516235,
      "rewards/rejected": -0.014697456732392311,
      "step": 412
    },
    {
      "epoch": 0.23987918917349133,
      "grad_norm": 337.9052734375,
      "learning_rate": 4.40005810575247e-06,
      "logits/chosen": -0.7297667264938354,
      "logits/rejected": -0.8206753730773926,
      "logps/chosen": -79.34073638916016,
      "logps/rejected": -79.64215850830078,
      "loss": 13.3179,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033630721271038055,
      "rewards/margins": 0.06482113897800446,
      "rewards/rejected": -0.03119041956961155,
      "step": 413
    },
    {
      "epoch": 0.24046001045478307,
      "grad_norm": 328.78985595703125,
      "learning_rate": 4.3986054619407325e-06,
      "logits/chosen": -0.5856087803840637,
      "logits/rejected": -0.6253767013549805,
      "logps/chosen": -76.09996032714844,
      "logps/rejected": -76.34832763671875,
      "loss": 14.0925,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.030036872252821922,
      "rewards/margins": -0.014562776312232018,
      "rewards/rejected": -0.015474090352654457,
      "step": 414
    },
    {
      "epoch": 0.24104083173607482,
      "grad_norm": 297.6679382324219,
      "learning_rate": 4.397152818128995e-06,
      "logits/chosen": -0.8083240389823914,
      "logits/rejected": -0.7946100234985352,
      "logps/chosen": -72.3720703125,
      "logps/rejected": -68.49833679199219,
      "loss": 14.016,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.012605169788002968,
      "rewards/margins": -0.009788742288947105,
      "rewards/rejected": -0.0028164288960397243,
      "step": 415
    },
    {
      "epoch": 0.24162165301736654,
      "grad_norm": 310.3009338378906,
      "learning_rate": 4.395700174317258e-06,
      "logits/chosen": -0.6257360577583313,
      "logits/rejected": -0.6403561234474182,
      "logps/chosen": -76.71080017089844,
      "logps/rejected": -77.48155212402344,
      "loss": 13.5,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04251420125365257,
      "rewards/margins": 0.04183372110128403,
      "rewards/rejected": 0.0006804756703786552,
      "step": 416
    },
    {
      "epoch": 0.2422024742986583,
      "grad_norm": 325.7873840332031,
      "learning_rate": 4.394247530505521e-06,
      "logits/chosen": -0.6726334691047668,
      "logits/rejected": -0.6855700016021729,
      "logps/chosen": -75.29472351074219,
      "logps/rejected": -71.83308410644531,
      "loss": 14.0017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018920384347438812,
      "rewards/margins": -0.009463165886700153,
      "rewards/rejected": 0.02838354930281639,
      "step": 417
    },
    {
      "epoch": 0.24278329557995004,
      "grad_norm": 542.8521728515625,
      "learning_rate": 4.3927948866937835e-06,
      "logits/chosen": -0.5671173334121704,
      "logits/rejected": -0.5828499794006348,
      "logps/chosen": -80.865966796875,
      "logps/rejected": -77.36320495605469,
      "loss": 14.1966,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.010454867966473103,
      "rewards/margins": -0.0281839556992054,
      "rewards/rejected": 0.01772909425199032,
      "step": 418
    },
    {
      "epoch": 0.2433641168612418,
      "grad_norm": 313.10650634765625,
      "learning_rate": 4.391342242882045e-06,
      "logits/chosen": -0.5604667663574219,
      "logits/rejected": -0.6470273733139038,
      "logps/chosen": -72.81732177734375,
      "logps/rejected": -70.16290283203125,
      "loss": 13.2515,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017285270616412163,
      "rewards/margins": 0.07715705037117004,
      "rewards/rejected": -0.05987178534269333,
      "step": 419
    },
    {
      "epoch": 0.24394493814253354,
      "grad_norm": 322.7060241699219,
      "learning_rate": 4.389889599070308e-06,
      "logits/chosen": -0.7095221281051636,
      "logits/rejected": -0.70353764295578,
      "logps/chosen": -72.71652221679688,
      "logps/rejected": -67.59053802490234,
      "loss": 14.0589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017042094841599464,
      "rewards/margins": -0.016451817005872726,
      "rewards/rejected": -0.0005902774864807725,
      "step": 420
    },
    {
      "epoch": 0.2445257594238253,
      "grad_norm": 285.8162536621094,
      "learning_rate": 4.388436955258571e-06,
      "logits/chosen": -0.8471421003341675,
      "logits/rejected": -0.7050653696060181,
      "logps/chosen": -71.2718505859375,
      "logps/rejected": -66.46906280517578,
      "loss": 13.5971,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004547786898910999,
      "rewards/margins": 0.030440161004662514,
      "rewards/rejected": -0.02589237317442894,
      "step": 421
    },
    {
      "epoch": 0.24510658070511704,
      "grad_norm": 312.5526428222656,
      "learning_rate": 4.386984311446834e-06,
      "logits/chosen": -0.6499220728874207,
      "logits/rejected": -0.7068791389465332,
      "logps/chosen": -70.94161987304688,
      "logps/rejected": -67.37171936035156,
      "loss": 13.7637,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.014181653037667274,
      "rewards/margins": 0.019841155037283897,
      "rewards/rejected": -0.005659504793584347,
      "step": 422
    },
    {
      "epoch": 0.2456874019864088,
      "grad_norm": 306.18194580078125,
      "learning_rate": 4.385531667635096e-06,
      "logits/chosen": -0.6181563138961792,
      "logits/rejected": -0.7241290807723999,
      "logps/chosen": -63.1754035949707,
      "logps/rejected": -76.5521469116211,
      "loss": 13.3346,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06029351428151131,
      "rewards/margins": 0.05993475764989853,
      "rewards/rejected": 0.00035875439061783254,
      "step": 423
    },
    {
      "epoch": 0.24626822326770054,
      "grad_norm": 307.78765869140625,
      "learning_rate": 4.384079023823359e-06,
      "logits/chosen": -0.488178014755249,
      "logits/rejected": -0.6498016119003296,
      "logps/chosen": -70.04771423339844,
      "logps/rejected": -67.56314849853516,
      "loss": 13.8241,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.002901268657296896,
      "rewards/margins": 0.014369833283126354,
      "rewards/rejected": -0.011468565091490746,
      "step": 424
    },
    {
      "epoch": 0.24684904454899229,
      "grad_norm": 321.4908142089844,
      "learning_rate": 4.382626380011621e-06,
      "logits/chosen": -0.7453001737594604,
      "logits/rejected": -0.7693713903427124,
      "logps/chosen": -69.00504302978516,
      "logps/rejected": -69.38874816894531,
      "loss": 14.0894,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03128911182284355,
      "rewards/margins": -0.019808074459433556,
      "rewards/rejected": -0.011481037363409996,
      "step": 425
    },
    {
      "epoch": 0.24742986583028403,
      "grad_norm": 306.6564025878906,
      "learning_rate": 4.381173736199884e-06,
      "logits/chosen": -0.6882332563400269,
      "logits/rejected": -0.666528582572937,
      "logps/chosen": -71.15715026855469,
      "logps/rejected": -72.64927673339844,
      "loss": 13.5929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.045943137258291245,
      "rewards/margins": 0.03135029971599579,
      "rewards/rejected": 0.01459283847361803,
      "step": 426
    },
    {
      "epoch": 0.24801068711157576,
      "grad_norm": 319.85552978515625,
      "learning_rate": 4.3797210923881465e-06,
      "logits/chosen": -0.6911331415176392,
      "logits/rejected": -0.6865822076797485,
      "logps/chosen": -70.95181274414062,
      "logps/rejected": -69.65785217285156,
      "loss": 13.6944,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00031525566009804606,
      "rewards/margins": 0.025617409497499466,
      "rewards/rejected": -0.025302153080701828,
      "step": 427
    },
    {
      "epoch": 0.2485915083928675,
      "grad_norm": 305.7183532714844,
      "learning_rate": 4.378268448576409e-06,
      "logits/chosen": -0.5291553735733032,
      "logits/rejected": -0.5490658283233643,
      "logps/chosen": -69.1864242553711,
      "logps/rejected": -74.73960876464844,
      "loss": 13.8674,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005592182278633118,
      "rewards/margins": 0.003548829350620508,
      "rewards/rejected": 0.002043351763859391,
      "step": 428
    },
    {
      "epoch": 0.24917232967415925,
      "grad_norm": 297.0389404296875,
      "learning_rate": 4.376815804764672e-06,
      "logits/chosen": -0.6868799924850464,
      "logits/rejected": -0.71811842918396,
      "logps/chosen": -67.46659088134766,
      "logps/rejected": -66.69960021972656,
      "loss": 13.6168,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0253940187394619,
      "rewards/margins": 0.02877502143383026,
      "rewards/rejected": -0.0033809959422796965,
      "step": 429
    },
    {
      "epoch": 0.249753150955451,
      "grad_norm": 304.1397705078125,
      "learning_rate": 4.375363160952935e-06,
      "logits/chosen": -0.6851626634597778,
      "logits/rejected": -0.7611708641052246,
      "logps/chosen": -66.39443969726562,
      "logps/rejected": -74.42601013183594,
      "loss": 14.2112,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.003943081013858318,
      "rewards/margins": -0.02984614297747612,
      "rewards/rejected": 0.03378922492265701,
      "step": 430
    },
    {
      "epoch": 0.25033397223674275,
      "grad_norm": 294.4915771484375,
      "learning_rate": 4.3739105171411975e-06,
      "logits/chosen": -0.6287668943405151,
      "logits/rejected": -0.6079251170158386,
      "logps/chosen": -75.12745666503906,
      "logps/rejected": -70.1561508178711,
      "loss": 13.7044,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.015175809152424335,
      "rewards/margins": 0.02054971642792225,
      "rewards/rejected": -0.005373907275497913,
      "step": 431
    },
    {
      "epoch": 0.2509147935180345,
      "grad_norm": 331.4616394042969,
      "learning_rate": 4.3724578733294594e-06,
      "logits/chosen": -0.6616209149360657,
      "logits/rejected": -0.6820401549339294,
      "logps/chosen": -65.7193374633789,
      "logps/rejected": -67.92347717285156,
      "loss": 13.6835,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.032937806099653244,
      "rewards/margins": 0.024601612240076065,
      "rewards/rejected": 0.008336210623383522,
      "step": 432
    },
    {
      "epoch": 0.25149561479932625,
      "grad_norm": 302.4757385253906,
      "learning_rate": 4.371005229517722e-06,
      "logits/chosen": -0.5871397852897644,
      "logits/rejected": -0.6894339323043823,
      "logps/chosen": -70.20496368408203,
      "logps/rejected": -75.09275817871094,
      "loss": 13.6636,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0664600282907486,
      "rewards/margins": 0.022688765078783035,
      "rewards/rejected": 0.04377124831080437,
      "step": 433
    },
    {
      "epoch": 0.25207643608061797,
      "grad_norm": 301.0609436035156,
      "learning_rate": 4.369552585705985e-06,
      "logits/chosen": -0.72092604637146,
      "logits/rejected": -0.7814493775367737,
      "logps/chosen": -69.15807342529297,
      "logps/rejected": -69.16731262207031,
      "loss": 13.3789,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03117241896688938,
      "rewards/margins": 0.05374479293823242,
      "rewards/rejected": -0.02257237769663334,
      "step": 434
    },
    {
      "epoch": 0.25265725736190975,
      "grad_norm": 310.7433776855469,
      "learning_rate": 4.368099941894248e-06,
      "logits/chosen": -0.5796935558319092,
      "logits/rejected": -0.6400626301765442,
      "logps/chosen": -74.19486236572266,
      "logps/rejected": -78.7645263671875,
      "loss": 13.3789,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.040846049785614014,
      "rewards/margins": 0.05530615895986557,
      "rewards/rejected": -0.014460104517638683,
      "step": 435
    },
    {
      "epoch": 0.25323807864320147,
      "grad_norm": 320.4769287109375,
      "learning_rate": 4.36664729808251e-06,
      "logits/chosen": -0.7232745289802551,
      "logits/rejected": -0.850020706653595,
      "logps/chosen": -66.91292572021484,
      "logps/rejected": -63.138587951660156,
      "loss": 14.3721,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009416877292096615,
      "rewards/margins": -0.04486392065882683,
      "rewards/rejected": 0.03544704243540764,
      "step": 436
    },
    {
      "epoch": 0.25381889992449325,
      "grad_norm": 322.4789733886719,
      "learning_rate": 4.365194654270773e-06,
      "logits/chosen": -0.7586982846260071,
      "logits/rejected": -0.8501695394515991,
      "logps/chosen": -74.4521255493164,
      "logps/rejected": -72.19835662841797,
      "loss": 13.5591,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.026751860976219177,
      "rewards/margins": 0.03630850464105606,
      "rewards/rejected": -0.009556648321449757,
      "step": 437
    },
    {
      "epoch": 0.25439972120578497,
      "grad_norm": 301.9642639160156,
      "learning_rate": 4.363742010459036e-06,
      "logits/chosen": -0.7936211228370667,
      "logits/rejected": -0.8182106018066406,
      "logps/chosen": -68.75806427001953,
      "logps/rejected": -67.1334457397461,
      "loss": 13.7116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03210698813199997,
      "rewards/margins": 0.019230013713240623,
      "rewards/rejected": 0.012876978144049644,
      "step": 438
    },
    {
      "epoch": 0.25498054248707674,
      "grad_norm": 317.6552429199219,
      "learning_rate": 4.362289366647298e-06,
      "logits/chosen": -0.6283025741577148,
      "logits/rejected": -0.5947784781455994,
      "logps/chosen": -72.60492706298828,
      "logps/rejected": -81.98297882080078,
      "loss": 13.6121,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.029752427712082863,
      "rewards/margins": 0.03360595554113388,
      "rewards/rejected": -0.003853529691696167,
      "step": 439
    },
    {
      "epoch": 0.25556136376836847,
      "grad_norm": 301.3587951660156,
      "learning_rate": 4.3608367228355606e-06,
      "logits/chosen": -0.521526575088501,
      "logits/rejected": -0.6442473530769348,
      "logps/chosen": -72.14176177978516,
      "logps/rejected": -79.62040710449219,
      "loss": 13.238,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06225038319826126,
      "rewards/margins": 0.06984097510576248,
      "rewards/rejected": -0.007590600289404392,
      "step": 440
    },
    {
      "epoch": 0.25614218504966024,
      "grad_norm": 301.3314514160156,
      "learning_rate": 4.359384079023824e-06,
      "logits/chosen": -0.5870726108551025,
      "logits/rejected": -0.561357855796814,
      "logps/chosen": -69.20145416259766,
      "logps/rejected": -71.50432586669922,
      "loss": 13.8883,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.029437389224767685,
      "rewards/margins": 0.002829763339832425,
      "rewards/rejected": 0.026607628911733627,
      "step": 441
    },
    {
      "epoch": 0.25672300633095196,
      "grad_norm": 300.83056640625,
      "learning_rate": 4.357931435212087e-06,
      "logits/chosen": -0.6320289969444275,
      "logits/rejected": -0.6687763333320618,
      "logps/chosen": -69.56461334228516,
      "logps/rejected": -73.77947998046875,
      "loss": 13.5317,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04403435066342354,
      "rewards/margins": 0.036295950412750244,
      "rewards/rejected": 0.007738398853689432,
      "step": 442
    },
    {
      "epoch": 0.2573038276122437,
      "grad_norm": 299.238525390625,
      "learning_rate": 4.35647879140035e-06,
      "logits/chosen": -0.7340787053108215,
      "logits/rejected": -0.8285503387451172,
      "logps/chosen": -69.81163024902344,
      "logps/rejected": -69.9961929321289,
      "loss": 13.6267,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.020665764808654785,
      "rewards/margins": 0.035551171749830246,
      "rewards/rejected": -0.014885407872498035,
      "step": 443
    },
    {
      "epoch": 0.25788464889353546,
      "grad_norm": 332.7782897949219,
      "learning_rate": 4.3550261475886116e-06,
      "logits/chosen": -0.6751303672790527,
      "logits/rejected": -0.7156798243522644,
      "logps/chosen": -74.23109436035156,
      "logps/rejected": -77.84270477294922,
      "loss": 14.0507,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0388188436627388,
      "rewards/margins": -0.01138681173324585,
      "rewards/rejected": 0.05020565912127495,
      "step": 444
    },
    {
      "epoch": 0.2584654701748272,
      "grad_norm": 296.38922119140625,
      "learning_rate": 4.353573503776874e-06,
      "logits/chosen": -0.8190110921859741,
      "logits/rejected": -0.6782156229019165,
      "logps/chosen": -73.74525451660156,
      "logps/rejected": -72.44633483886719,
      "loss": 13.3661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07304216921329498,
      "rewards/margins": 0.05356328561902046,
      "rewards/rejected": 0.019478892907500267,
      "step": 445
    },
    {
      "epoch": 0.25904629145611896,
      "grad_norm": 310.6565246582031,
      "learning_rate": 4.352120859965137e-06,
      "logits/chosen": -0.5905870795249939,
      "logits/rejected": -0.6286332011222839,
      "logps/chosen": -73.00715637207031,
      "logps/rejected": -72.94271850585938,
      "loss": 13.9914,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0388491153717041,
      "rewards/margins": -0.007616178598254919,
      "rewards/rejected": 0.04646529257297516,
      "step": 446
    },
    {
      "epoch": 0.2596271127374107,
      "grad_norm": 352.046875,
      "learning_rate": 4.3506682161534e-06,
      "logits/chosen": -0.7802606225013733,
      "logits/rejected": -0.7054556608200073,
      "logps/chosen": -85.91800689697266,
      "logps/rejected": -72.05784606933594,
      "loss": 14.0159,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.019779253751039505,
      "rewards/margins": -0.008547335863113403,
      "rewards/rejected": 0.02832658588886261,
      "step": 447
    },
    {
      "epoch": 0.26020793401870246,
      "grad_norm": 362.30511474609375,
      "learning_rate": 4.3492155723416626e-06,
      "logits/chosen": -0.6785917282104492,
      "logits/rejected": -0.6494299173355103,
      "logps/chosen": -78.44164276123047,
      "logps/rejected": -72.85713958740234,
      "loss": 14.2555,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010164814069867134,
      "rewards/margins": -0.03138233348727226,
      "rewards/rejected": 0.04154714569449425,
      "step": 448
    },
    {
      "epoch": 0.2607887552999942,
      "grad_norm": 317.1739196777344,
      "learning_rate": 4.347762928529925e-06,
      "logits/chosen": -0.6515554189682007,
      "logits/rejected": -0.6316680908203125,
      "logps/chosen": -70.87915802001953,
      "logps/rejected": -73.50413513183594,
      "loss": 13.6688,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02080373838543892,
      "rewards/margins": 0.025081777945160866,
      "rewards/rejected": -0.0042780423536896706,
      "step": 449
    },
    {
      "epoch": 0.26136957658128596,
      "grad_norm": 315.79498291015625,
      "learning_rate": 4.346310284718187e-06,
      "logits/chosen": -0.6134510040283203,
      "logits/rejected": -0.5557273626327515,
      "logps/chosen": -74.24983215332031,
      "logps/rejected": -73.60564422607422,
      "loss": 13.8965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027813846245408058,
      "rewards/margins": 0.00493066618219018,
      "rewards/rejected": 0.022883176803588867,
      "step": 450
    },
    {
      "epoch": 0.2619503978625777,
      "grad_norm": 313.5321350097656,
      "learning_rate": 4.34485764090645e-06,
      "logits/chosen": -0.6806867718696594,
      "logits/rejected": -0.6897571086883545,
      "logps/chosen": -70.46429443359375,
      "logps/rejected": -72.20818328857422,
      "loss": 14.0127,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0054542249999940395,
      "rewards/margins": -0.009584503248333931,
      "rewards/rejected": 0.015038728713989258,
      "step": 451
    },
    {
      "epoch": 0.26253121914386945,
      "grad_norm": 314.8046875,
      "learning_rate": 4.343404997094713e-06,
      "logits/chosen": -0.5489099621772766,
      "logits/rejected": -0.6308473348617554,
      "logps/chosen": -73.40983581542969,
      "logps/rejected": -69.65528106689453,
      "loss": 13.9858,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.021905479952692986,
      "rewards/margins": -0.0018030557548627257,
      "rewards/rejected": 0.023708533495664597,
      "step": 452
    },
    {
      "epoch": 0.2631120404251612,
      "grad_norm": 567.4598388671875,
      "learning_rate": 4.3419523532829754e-06,
      "logits/chosen": -0.6974093914031982,
      "logits/rejected": -0.7631121873855591,
      "logps/chosen": -70.07582092285156,
      "logps/rejected": -81.98683166503906,
      "loss": 13.3815,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05924884229898453,
      "rewards/margins": 0.05474211648106575,
      "rewards/rejected": 0.004506723489612341,
      "step": 453
    },
    {
      "epoch": 0.2636928617064529,
      "grad_norm": 343.7535400390625,
      "learning_rate": 4.340499709471238e-06,
      "logits/chosen": -0.6991898417472839,
      "logits/rejected": -0.6066843867301941,
      "logps/chosen": -75.21280670166016,
      "logps/rejected": -74.64125061035156,
      "loss": 14.3191,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0069407084956765175,
      "rewards/margins": -0.04137198626995087,
      "rewards/rejected": 0.04831269383430481,
      "step": 454
    },
    {
      "epoch": 0.2642736829877447,
      "grad_norm": 317.2054138183594,
      "learning_rate": 4.339047065659501e-06,
      "logits/chosen": -0.7418617010116577,
      "logits/rejected": -0.7775954008102417,
      "logps/chosen": -69.52796936035156,
      "logps/rejected": -70.95177459716797,
      "loss": 13.4347,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04331202059984207,
      "rewards/margins": 0.04723655804991722,
      "rewards/rejected": -0.00392454257234931,
      "step": 455
    },
    {
      "epoch": 0.2648545042690364,
      "grad_norm": 315.6234130859375,
      "learning_rate": 4.337594421847764e-06,
      "logits/chosen": -0.7921939492225647,
      "logits/rejected": -0.9472505450248718,
      "logps/chosen": -69.9919662475586,
      "logps/rejected": -82.42518615722656,
      "loss": 13.9178,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0388253815472126,
      "rewards/margins": 0.01137163583189249,
      "rewards/rejected": 0.027453750371932983,
      "step": 456
    },
    {
      "epoch": 0.26543532555032817,
      "grad_norm": 308.3517150878906,
      "learning_rate": 4.336141778036026e-06,
      "logits/chosen": -0.6653806567192078,
      "logits/rejected": -0.5663945078849792,
      "logps/chosen": -73.91642761230469,
      "logps/rejected": -70.58480834960938,
      "loss": 13.8471,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.057994864881038666,
      "rewards/margins": 0.004966372158378363,
      "rewards/rejected": 0.05302848666906357,
      "step": 457
    },
    {
      "epoch": 0.2660161468316199,
      "grad_norm": 325.29302978515625,
      "learning_rate": 4.334689134224288e-06,
      "logits/chosen": -0.5894922018051147,
      "logits/rejected": -0.6214415431022644,
      "logps/chosen": -74.24130249023438,
      "logps/rejected": -80.94917297363281,
      "loss": 14.5074,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.005856601521372795,
      "rewards/margins": -0.0573112778365612,
      "rewards/rejected": 0.06316788494586945,
      "step": 458
    },
    {
      "epoch": 0.26659696811291167,
      "grad_norm": 283.90478515625,
      "learning_rate": 4.333236490412551e-06,
      "logits/chosen": -0.657735288143158,
      "logits/rejected": -0.6617435216903687,
      "logps/chosen": -76.9471435546875,
      "logps/rejected": -68.36034393310547,
      "loss": 13.1023,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07249955832958221,
      "rewards/margins": 0.08381040394306183,
      "rewards/rejected": -0.011310835368931293,
      "step": 459
    },
    {
      "epoch": 0.2671777893942034,
      "grad_norm": 302.0826416015625,
      "learning_rate": 4.331783846600814e-06,
      "logits/chosen": -0.7745707035064697,
      "logits/rejected": -0.6950326561927795,
      "logps/chosen": -69.80528259277344,
      "logps/rejected": -73.91972351074219,
      "loss": 13.9191,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.027396023273468018,
      "rewards/margins": -0.00236126477830112,
      "rewards/rejected": 0.029757294803857803,
      "step": 460
    },
    {
      "epoch": 0.26775861067549517,
      "grad_norm": 312.1777648925781,
      "learning_rate": 4.330331202789077e-06,
      "logits/chosen": -0.6571930050849915,
      "logits/rejected": -0.7029620409011841,
      "logps/chosen": -78.16141510009766,
      "logps/rejected": -80.50270080566406,
      "loss": 13.3549,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04237108677625656,
      "rewards/margins": 0.05985882878303528,
      "rewards/rejected": -0.01748773828148842,
      "step": 461
    },
    {
      "epoch": 0.2683394319567869,
      "grad_norm": 314.3011169433594,
      "learning_rate": 4.328878558977339e-06,
      "logits/chosen": -0.6543633937835693,
      "logits/rejected": -0.6337400674819946,
      "logps/chosen": -67.81694030761719,
      "logps/rejected": -76.94734954833984,
      "loss": 13.7697,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04975013807415962,
      "rewards/margins": 0.016220813617110252,
      "rewards/rejected": 0.03352931886911392,
      "step": 462
    },
    {
      "epoch": 0.26892025323807867,
      "grad_norm": 287.1407775878906,
      "learning_rate": 4.327425915165602e-06,
      "logits/chosen": -0.7276408076286316,
      "logits/rejected": -0.7486574649810791,
      "logps/chosen": -69.2403335571289,
      "logps/rejected": -60.5196533203125,
      "loss": 13.1441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06597266346216202,
      "rewards/margins": 0.07840771973133087,
      "rewards/rejected": -0.01243506371974945,
      "step": 463
    },
    {
      "epoch": 0.2695010745193704,
      "grad_norm": 307.4203796386719,
      "learning_rate": 4.325973271353864e-06,
      "logits/chosen": -0.7927466630935669,
      "logits/rejected": -0.8723392486572266,
      "logps/chosen": -71.81956481933594,
      "logps/rejected": -71.75994110107422,
      "loss": 13.1263,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05850011110305786,
      "rewards/margins": 0.08116643130779266,
      "rewards/rejected": -0.02266632579267025,
      "step": 464
    },
    {
      "epoch": 0.27008189580066216,
      "grad_norm": 299.2955017089844,
      "learning_rate": 4.324520627542127e-06,
      "logits/chosen": -0.6696484088897705,
      "logits/rejected": -0.6967477798461914,
      "logps/chosen": -64.9903564453125,
      "logps/rejected": -69.08036041259766,
      "loss": 13.864,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.025707131251692772,
      "rewards/margins": 0.0107659213244915,
      "rewards/rejected": 0.014941206201910973,
      "step": 465
    },
    {
      "epoch": 0.2706627170819539,
      "grad_norm": 328.59283447265625,
      "learning_rate": 4.3230679837303895e-06,
      "logits/chosen": -0.8180822134017944,
      "logits/rejected": -0.7941353917121887,
      "logps/chosen": -77.0251693725586,
      "logps/rejected": -79.32392883300781,
      "loss": 12.6981,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.08409181982278824,
      "rewards/margins": 0.12518665194511414,
      "rewards/rejected": -0.0410948283970356,
      "step": 466
    },
    {
      "epoch": 0.2712435383632456,
      "grad_norm": 302.0883483886719,
      "learning_rate": 4.321615339918652e-06,
      "logits/chosen": -0.6744131445884705,
      "logits/rejected": -0.7126671671867371,
      "logps/chosen": -82.62992858886719,
      "logps/rejected": -68.52928161621094,
      "loss": 13.8315,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04055127874016762,
      "rewards/margins": 0.00927782617509365,
      "rewards/rejected": 0.03127345070242882,
      "step": 467
    },
    {
      "epoch": 0.2718243596445374,
      "grad_norm": 296.6047668457031,
      "learning_rate": 4.320162696106915e-06,
      "logits/chosen": -0.6205381155014038,
      "logits/rejected": -0.6734114289283752,
      "logps/chosen": -72.38980865478516,
      "logps/rejected": -68.17768859863281,
      "loss": 13.3424,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.059192102402448654,
      "rewards/margins": 0.06497955322265625,
      "rewards/rejected": -0.005787448026239872,
      "step": 468
    },
    {
      "epoch": 0.2724051809258291,
      "grad_norm": 293.75018310546875,
      "learning_rate": 4.318710052295178e-06,
      "logits/chosen": -0.8345297574996948,
      "logits/rejected": -0.8630663752555847,
      "logps/chosen": -66.68868255615234,
      "logps/rejected": -69.19453430175781,
      "loss": 13.0063,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06119300797581673,
      "rewards/margins": 0.09242288768291473,
      "rewards/rejected": -0.031229889020323753,
      "step": 469
    },
    {
      "epoch": 0.2729860022071209,
      "grad_norm": 330.23846435546875,
      "learning_rate": 4.3172574084834405e-06,
      "logits/chosen": -0.6324206590652466,
      "logits/rejected": -0.6938502192497253,
      "logps/chosen": -74.11239624023438,
      "logps/rejected": -73.95758056640625,
      "loss": 14.0991,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0044637094251811504,
      "rewards/margins": -0.018323037773370743,
      "rewards/rejected": 0.01385932881385088,
      "step": 470
    },
    {
      "epoch": 0.2735668234884126,
      "grad_norm": 291.638671875,
      "learning_rate": 4.315804764671702e-06,
      "logits/chosen": -0.6697665452957153,
      "logits/rejected": -0.6289907693862915,
      "logps/chosen": -72.07844543457031,
      "logps/rejected": -68.84980773925781,
      "loss": 13.8797,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03210974857211113,
      "rewards/margins": 0.0021261770743876696,
      "rewards/rejected": 0.02998356893658638,
      "step": 471
    },
    {
      "epoch": 0.2741476447697044,
      "grad_norm": 326.5478515625,
      "learning_rate": 4.314352120859965e-06,
      "logits/chosen": -0.6647511124610901,
      "logits/rejected": -0.6528132557868958,
      "logps/chosen": -72.49028778076172,
      "logps/rejected": -72.65093231201172,
      "loss": 14.1427,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.015104440040886402,
      "rewards/margins": -0.022749561816453934,
      "rewards/rejected": 0.037853993475437164,
      "step": 472
    },
    {
      "epoch": 0.2747284660509961,
      "grad_norm": 330.836181640625,
      "learning_rate": 4.312899477048228e-06,
      "logits/chosen": -0.5921165347099304,
      "logits/rejected": -0.6261785626411438,
      "logps/chosen": -83.95507049560547,
      "logps/rejected": -69.69046020507812,
      "loss": 13.5342,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.049983587116003036,
      "rewards/margins": 0.041167039424180984,
      "rewards/rejected": 0.008816548623144627,
      "step": 473
    },
    {
      "epoch": 0.2753092873322879,
      "grad_norm": 286.28839111328125,
      "learning_rate": 4.311446833236491e-06,
      "logits/chosen": -0.5407285690307617,
      "logits/rejected": -0.5518749356269836,
      "logps/chosen": -69.38809967041016,
      "logps/rejected": -64.3993911743164,
      "loss": 13.7355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03605775907635689,
      "rewards/margins": 0.015156927518546581,
      "rewards/rejected": 0.02090083435177803,
      "step": 474
    },
    {
      "epoch": 0.2758901086135796,
      "grad_norm": 321.68450927734375,
      "learning_rate": 4.309994189424753e-06,
      "logits/chosen": -0.7276217341423035,
      "logits/rejected": -0.8036016225814819,
      "logps/chosen": -70.2737045288086,
      "logps/rejected": -66.9681625366211,
      "loss": 13.9062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0001468472182750702,
      "rewards/margins": -0.00017342269711662084,
      "rewards/rejected": 0.00032026879489421844,
      "step": 475
    },
    {
      "epoch": 0.2764709298948714,
      "grad_norm": 306.3502502441406,
      "learning_rate": 4.308541545613016e-06,
      "logits/chosen": -0.7900197505950928,
      "logits/rejected": -0.6438810229301453,
      "logps/chosen": -69.17347717285156,
      "logps/rejected": -62.60564422607422,
      "loss": 13.6038,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04666885733604431,
      "rewards/margins": 0.03157535940408707,
      "rewards/rejected": 0.015093490481376648,
      "step": 476
    },
    {
      "epoch": 0.2770517511761631,
      "grad_norm": 382.4345397949219,
      "learning_rate": 4.307088901801279e-06,
      "logits/chosen": -0.8302785158157349,
      "logits/rejected": -0.830335795879364,
      "logps/chosen": -83.38411712646484,
      "logps/rejected": -72.17436981201172,
      "loss": 13.7722,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02929898537695408,
      "rewards/margins": 0.015593672171235085,
      "rewards/rejected": 0.013705313205718994,
      "step": 477
    },
    {
      "epoch": 0.2776325724574548,
      "grad_norm": 292.42779541015625,
      "learning_rate": 4.305636257989541e-06,
      "logits/chosen": -0.6505283713340759,
      "logits/rejected": -0.5827508568763733,
      "logps/chosen": -64.67672729492188,
      "logps/rejected": -68.16065979003906,
      "loss": 13.2237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05520979315042496,
      "rewards/margins": 0.0683043897151947,
      "rewards/rejected": -0.013094606809318066,
      "step": 478
    },
    {
      "epoch": 0.2782133937387466,
      "grad_norm": 321.1835632324219,
      "learning_rate": 4.3041836141778035e-06,
      "logits/chosen": -0.4862455725669861,
      "logits/rejected": -0.4693034589290619,
      "logps/chosen": -70.4814224243164,
      "logps/rejected": -84.19317626953125,
      "loss": 13.6491,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.016184937208890915,
      "rewards/margins": 0.03135867789387703,
      "rewards/rejected": -0.015173738822340965,
      "step": 479
    },
    {
      "epoch": 0.2787942150200383,
      "grad_norm": 311.97564697265625,
      "learning_rate": 4.302730970366066e-06,
      "logits/chosen": -0.554233193397522,
      "logits/rejected": -0.5739291310310364,
      "logps/chosen": -71.45389556884766,
      "logps/rejected": -76.47578430175781,
      "loss": 13.6573,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.028831113129854202,
      "rewards/margins": 0.03090267814695835,
      "rewards/rejected": -0.0020715640857815742,
      "step": 480
    },
    {
      "epoch": 0.2793750363013301,
      "grad_norm": 309.7761535644531,
      "learning_rate": 4.301278326554329e-06,
      "logits/chosen": -0.6836757659912109,
      "logits/rejected": -0.7563328742980957,
      "logps/chosen": -65.9247817993164,
      "logps/rejected": -76.11883544921875,
      "loss": 14.0186,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06773559749126434,
      "rewards/margins": -0.002912606345489621,
      "rewards/rejected": 0.0706482082605362,
      "step": 481
    },
    {
      "epoch": 0.2799558575826218,
      "grad_norm": 301.5060729980469,
      "learning_rate": 4.299825682742592e-06,
      "logits/chosen": -0.7972155809402466,
      "logits/rejected": -0.7350047826766968,
      "logps/chosen": -67.74440002441406,
      "logps/rejected": -71.73310089111328,
      "loss": 13.6848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.025806616991758347,
      "rewards/margins": 0.022161057218909264,
      "rewards/rejected": 0.0036455602385103703,
      "step": 482
    },
    {
      "epoch": 0.2805366788639136,
      "grad_norm": 301.0857849121094,
      "learning_rate": 4.2983730389308545e-06,
      "logits/chosen": -0.6307353973388672,
      "logits/rejected": -0.5791727900505066,
      "logps/chosen": -70.24183654785156,
      "logps/rejected": -69.36378479003906,
      "loss": 13.4857,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06295748054981232,
      "rewards/margins": 0.043685734272003174,
      "rewards/rejected": 0.019271746277809143,
      "step": 483
    },
    {
      "epoch": 0.2811175001452053,
      "grad_norm": 685.1707763671875,
      "learning_rate": 4.296920395119117e-06,
      "logits/chosen": -0.7495703101158142,
      "logits/rejected": -0.678167998790741,
      "logps/chosen": -77.27181243896484,
      "logps/rejected": -72.3003158569336,
      "loss": 13.8755,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.010486298240721226,
      "rewards/margins": 0.006247940473258495,
      "rewards/rejected": 0.004238357301801443,
      "step": 484
    },
    {
      "epoch": 0.2816983214264971,
      "grad_norm": 294.2120056152344,
      "learning_rate": 4.29546775130738e-06,
      "logits/chosen": -0.5442952513694763,
      "logits/rejected": -0.6088670492172241,
      "logps/chosen": -73.0514907836914,
      "logps/rejected": -73.78404235839844,
      "loss": 13.6341,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0040179165080189705,
      "rewards/margins": 0.027353759855031967,
      "rewards/rejected": -0.03137167543172836,
      "step": 485
    },
    {
      "epoch": 0.2822791427077888,
      "grad_norm": 306.35723876953125,
      "learning_rate": 4.294015107495643e-06,
      "logits/chosen": -0.6691688299179077,
      "logits/rejected": -0.7110737562179565,
      "logps/chosen": -68.50287628173828,
      "logps/rejected": -65.523193359375,
      "loss": 13.5978,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014910449273884296,
      "rewards/margins": 0.03183292597532272,
      "rewards/rejected": -0.016922477632761,
      "step": 486
    },
    {
      "epoch": 0.2828599639890806,
      "grad_norm": 314.25897216796875,
      "learning_rate": 4.2925624636839055e-06,
      "logits/chosen": -0.6867062449455261,
      "logits/rejected": -0.6559956073760986,
      "logps/chosen": -86.4985122680664,
      "logps/rejected": -67.22306060791016,
      "loss": 13.7589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01645682193338871,
      "rewards/margins": 0.017809275537729263,
      "rewards/rejected": -0.001352452440187335,
      "step": 487
    },
    {
      "epoch": 0.2834407852703723,
      "grad_norm": 288.4145812988281,
      "learning_rate": 4.291109819872168e-06,
      "logits/chosen": -0.6363990902900696,
      "logits/rejected": -0.6217866539955139,
      "logps/chosen": -72.58358764648438,
      "logps/rejected": -71.9157485961914,
      "loss": 13.0157,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0661824643611908,
      "rewards/margins": 0.09238552302122116,
      "rewards/rejected": -0.026203066110610962,
      "step": 488
    },
    {
      "epoch": 0.28402160655166403,
      "grad_norm": 304.28765869140625,
      "learning_rate": 4.28965717606043e-06,
      "logits/chosen": -0.7047585248947144,
      "logits/rejected": -0.6908853054046631,
      "logps/chosen": -69.95263671875,
      "logps/rejected": -71.68666076660156,
      "loss": 13.3886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04382898285984993,
      "rewards/margins": 0.059452660381793976,
      "rewards/rejected": -0.015623673796653748,
      "step": 489
    },
    {
      "epoch": 0.2846024278329558,
      "grad_norm": 314.20196533203125,
      "learning_rate": 4.288204532248693e-06,
      "logits/chosen": -0.7270024418830872,
      "logits/rejected": -0.7925176620483398,
      "logps/chosen": -73.37562561035156,
      "logps/rejected": -85.60382843017578,
      "loss": 13.1235,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03009481355547905,
      "rewards/margins": 0.0827663242816925,
      "rewards/rejected": -0.052671510726213455,
      "step": 490
    },
    {
      "epoch": 0.28518324911424753,
      "grad_norm": 296.90582275390625,
      "learning_rate": 4.286751888436956e-06,
      "logits/chosen": -0.6854633092880249,
      "logits/rejected": -0.7035878896713257,
      "logps/chosen": -68.50250244140625,
      "logps/rejected": -67.49859619140625,
      "loss": 13.7267,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.030942970886826515,
      "rewards/margins": 0.019369639456272125,
      "rewards/rejected": 0.011573335155844688,
      "step": 491
    },
    {
      "epoch": 0.2857640703955393,
      "grad_norm": 292.898681640625,
      "learning_rate": 4.285299244625218e-06,
      "logits/chosen": -0.5533262491226196,
      "logits/rejected": -0.6104485392570496,
      "logps/chosen": -72.7857666015625,
      "logps/rejected": -67.43394470214844,
      "loss": 13.9817,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.005316922441124916,
      "rewards/margins": -0.0064797671511769295,
      "rewards/rejected": 0.0011628434294834733,
      "step": 492
    },
    {
      "epoch": 0.286344891676831,
      "grad_norm": 323.1135559082031,
      "learning_rate": 4.283846600813481e-06,
      "logits/chosen": -0.5882741808891296,
      "logits/rejected": -0.6077271699905396,
      "logps/chosen": -78.02590942382812,
      "logps/rejected": -83.53060150146484,
      "loss": 13.4642,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007745922543108463,
      "rewards/margins": 0.050111640244722366,
      "rewards/rejected": -0.04236571118235588,
      "step": 493
    },
    {
      "epoch": 0.2869257129581228,
      "grad_norm": 377.0531921386719,
      "learning_rate": 4.282393957001744e-06,
      "logits/chosen": -0.659203827381134,
      "logits/rejected": -0.6870671510696411,
      "logps/chosen": -68.08976745605469,
      "logps/rejected": -78.85200500488281,
      "loss": 13.4473,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.050796620547771454,
      "rewards/margins": 0.04922497272491455,
      "rewards/rejected": 0.0015716440975666046,
      "step": 494
    },
    {
      "epoch": 0.2875065342394145,
      "grad_norm": 575.7816162109375,
      "learning_rate": 4.280941313190007e-06,
      "logits/chosen": -0.6494874954223633,
      "logits/rejected": -0.6947387456893921,
      "logps/chosen": -73.44721984863281,
      "logps/rejected": -77.6927719116211,
      "loss": 14.4423,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00039952099905349314,
      "rewards/margins": -0.050260018557310104,
      "rewards/rejected": 0.05065953731536865,
      "step": 495
    },
    {
      "epoch": 0.2880873555207063,
      "grad_norm": 292.5953369140625,
      "learning_rate": 4.2794886693782685e-06,
      "logits/chosen": -0.745873749256134,
      "logits/rejected": -0.6551159024238586,
      "logps/chosen": -68.93890380859375,
      "logps/rejected": -70.55396270751953,
      "loss": 13.5985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01787867583334446,
      "rewards/margins": 0.032074641436338425,
      "rewards/rejected": -0.01419596653431654,
      "step": 496
    },
    {
      "epoch": 0.288668176801998,
      "grad_norm": 317.0806884765625,
      "learning_rate": 4.278036025566531e-06,
      "logits/chosen": -0.6900259256362915,
      "logits/rejected": -0.6669089198112488,
      "logps/chosen": -72.48905944824219,
      "logps/rejected": -75.67109680175781,
      "loss": 13.8434,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.016828546300530434,
      "rewards/margins": 0.006076467223465443,
      "rewards/rejected": 0.010752077214419842,
      "step": 497
    },
    {
      "epoch": 0.2892489980832898,
      "grad_norm": 450.179931640625,
      "learning_rate": 4.276583381754794e-06,
      "logits/chosen": -0.7485173344612122,
      "logits/rejected": -0.8365989923477173,
      "logps/chosen": -72.33958435058594,
      "logps/rejected": -69.51062774658203,
      "loss": 13.3385,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0609331838786602,
      "rewards/margins": 0.060974687337875366,
      "rewards/rejected": -4.150420500081964e-05,
      "step": 498
    },
    {
      "epoch": 0.2898298193645815,
      "grad_norm": 319.4369201660156,
      "learning_rate": 4.275130737943057e-06,
      "logits/chosen": -0.659496545791626,
      "logits/rejected": -0.6515854001045227,
      "logps/chosen": -70.12030029296875,
      "logps/rejected": -75.98362731933594,
      "loss": 13.3334,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.026825930923223495,
      "rewards/margins": 0.0615021288394928,
      "rewards/rejected": -0.03467618674039841,
      "step": 499
    },
    {
      "epoch": 0.29041064064587324,
      "grad_norm": 317.2661437988281,
      "learning_rate": 4.2736780941313195e-06,
      "logits/chosen": -0.6833754777908325,
      "logits/rejected": -0.6367133855819702,
      "logps/chosen": -81.86927795410156,
      "logps/rejected": -78.07447814941406,
      "loss": 13.1256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07352960854768753,
      "rewards/margins": 0.08241195976734161,
      "rewards/rejected": -0.008882349357008934,
      "step": 500
    },
    {
      "epoch": 0.290991461927165,
      "grad_norm": 318.4466247558594,
      "learning_rate": 4.272225450319582e-06,
      "logits/chosen": -1.007275104522705,
      "logits/rejected": -0.8824909329414368,
      "logps/chosen": -70.7936782836914,
      "logps/rejected": -71.09184265136719,
      "loss": 13.608,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.039610691368579865,
      "rewards/margins": 0.032731592655181885,
      "rewards/rejected": 0.006879097782075405,
      "step": 501
    },
    {
      "epoch": 0.29157228320845674,
      "grad_norm": 293.3976745605469,
      "learning_rate": 4.270772806507845e-06,
      "logits/chosen": -0.6601558923721313,
      "logits/rejected": -0.6545599699020386,
      "logps/chosen": -71.0650405883789,
      "logps/rejected": -72.53407287597656,
      "loss": 13.7676,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00019608512229751796,
      "rewards/margins": 0.01757933758199215,
      "rewards/rejected": -0.01777542382478714,
      "step": 502
    },
    {
      "epoch": 0.2921531044897485,
      "grad_norm": 310.49322509765625,
      "learning_rate": 4.269320162696107e-06,
      "logits/chosen": -0.5019787549972534,
      "logits/rejected": -0.5980736017227173,
      "logps/chosen": -74.40044403076172,
      "logps/rejected": -67.54830169677734,
      "loss": 13.3456,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03700960800051689,
      "rewards/margins": 0.0595683753490448,
      "rewards/rejected": -0.02255876362323761,
      "step": 503
    },
    {
      "epoch": 0.29273392577104024,
      "grad_norm": 292.01019287109375,
      "learning_rate": 4.26786751888437e-06,
      "logits/chosen": -0.5457882881164551,
      "logits/rejected": -0.7330330610275269,
      "logps/chosen": -66.13023376464844,
      "logps/rejected": -67.7120132446289,
      "loss": 13.7292,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0030907634645700455,
      "rewards/margins": 0.03074917569756508,
      "rewards/rejected": -0.033839933574199677,
      "step": 504
    },
    {
      "epoch": 0.293314747052332,
      "grad_norm": 314.1589050292969,
      "learning_rate": 4.2664148750726324e-06,
      "logits/chosen": -0.6293397545814514,
      "logits/rejected": -0.7415448427200317,
      "logps/chosen": -75.83900451660156,
      "logps/rejected": -84.2178726196289,
      "loss": 13.2021,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.055055178701877594,
      "rewards/margins": 0.0833793580532074,
      "rewards/rejected": -0.0283241868019104,
      "step": 505
    },
    {
      "epoch": 0.29389556833362374,
      "grad_norm": 288.65093994140625,
      "learning_rate": 4.264962231260895e-06,
      "logits/chosen": -0.5864494442939758,
      "logits/rejected": -0.7309472560882568,
      "logps/chosen": -67.95295715332031,
      "logps/rejected": -80.39786529541016,
      "loss": 13.0597,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03946956992149353,
      "rewards/margins": 0.09237784147262573,
      "rewards/rejected": -0.0529082827270031,
      "step": 506
    },
    {
      "epoch": 0.2944763896149155,
      "grad_norm": 323.42449951171875,
      "learning_rate": 4.263509587449158e-06,
      "logits/chosen": -0.49582844972610474,
      "logits/rejected": -0.6143544912338257,
      "logps/chosen": -74.20238494873047,
      "logps/rejected": -66.28921508789062,
      "loss": 13.5348,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014446879737079144,
      "rewards/margins": 0.040076714009046555,
      "rewards/rejected": -0.025629838928580284,
      "step": 507
    },
    {
      "epoch": 0.29505721089620723,
      "grad_norm": 304.49407958984375,
      "learning_rate": 4.262056943637421e-06,
      "logits/chosen": -0.550538957118988,
      "logits/rejected": -0.5864927172660828,
      "logps/chosen": -68.0365982055664,
      "logps/rejected": -74.60637664794922,
      "loss": 13.7157,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.017872009426355362,
      "rewards/margins": 0.020814960822463036,
      "rewards/rejected": -0.002942953957244754,
      "step": 508
    },
    {
      "epoch": 0.295638032177499,
      "grad_norm": 396.60736083984375,
      "learning_rate": 4.2606042998256834e-06,
      "logits/chosen": -0.7205631732940674,
      "logits/rejected": -0.6172083616256714,
      "logps/chosen": -69.93831634521484,
      "logps/rejected": -71.62530517578125,
      "loss": 13.0703,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.06218590587377548,
      "rewards/margins": 0.08707333356142044,
      "rewards/rejected": -0.02488742023706436,
      "step": 509
    },
    {
      "epoch": 0.29621885345879073,
      "grad_norm": 340.6492919921875,
      "learning_rate": 4.259151656013945e-06,
      "logits/chosen": -0.7098259925842285,
      "logits/rejected": -0.8209171295166016,
      "logps/chosen": -79.39151000976562,
      "logps/rejected": -73.8201904296875,
      "loss": 13.3678,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02004718780517578,
      "rewards/margins": 0.06106545776128769,
      "rewards/rejected": -0.04101826995611191,
      "step": 510
    },
    {
      "epoch": 0.29679967474008245,
      "grad_norm": 304.18780517578125,
      "learning_rate": 4.257699012202208e-06,
      "logits/chosen": -0.6429446339607239,
      "logits/rejected": -0.8323208093643188,
      "logps/chosen": -70.1782455444336,
      "logps/rejected": -70.72254943847656,
      "loss": 13.6496,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.007325345184653997,
      "rewards/margins": 0.03689438849687576,
      "rewards/rejected": -0.02956903912127018,
      "step": 511
    },
    {
      "epoch": 0.29738049602137423,
      "grad_norm": 292.5777587890625,
      "learning_rate": 4.256246368390471e-06,
      "logits/chosen": -0.6322038173675537,
      "logits/rejected": -0.7191158533096313,
      "logps/chosen": -71.34642028808594,
      "logps/rejected": -73.82039642333984,
      "loss": 13.4242,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007254153490066528,
      "rewards/margins": 0.05077975243330002,
      "rewards/rejected": -0.04352560266852379,
      "step": 512
    },
    {
      "epoch": 0.29796131730266595,
      "grad_norm": 288.7735595703125,
      "learning_rate": 4.2547937245787336e-06,
      "logits/chosen": -0.7022128701210022,
      "logits/rejected": -0.7011411786079407,
      "logps/chosen": -65.72151184082031,
      "logps/rejected": -66.15309143066406,
      "loss": 13.7914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0430249348282814,
      "rewards/margins": 0.011196841485798359,
      "rewards/rejected": 0.03182809054851532,
      "step": 513
    },
    {
      "epoch": 0.29854213858395773,
      "grad_norm": 330.7345275878906,
      "learning_rate": 4.253341080766996e-06,
      "logits/chosen": -0.6906915307044983,
      "logits/rejected": -0.7374723553657532,
      "logps/chosen": -78.03739929199219,
      "logps/rejected": -93.80352783203125,
      "loss": 13.7291,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022170495241880417,
      "rewards/margins": 0.025943556800484657,
      "rewards/rejected": -0.048114050179719925,
      "step": 514
    },
    {
      "epoch": 0.29912295986524945,
      "grad_norm": 322.0339050292969,
      "learning_rate": 4.251888436955259e-06,
      "logits/chosen": -0.6764459609985352,
      "logits/rejected": -0.5635499954223633,
      "logps/chosen": -76.67605590820312,
      "logps/rejected": -72.45964813232422,
      "loss": 13.7177,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0028892538975924253,
      "rewards/margins": 0.023371810093522072,
      "rewards/rejected": -0.020482560619711876,
      "step": 515
    },
    {
      "epoch": 0.2997037811465412,
      "grad_norm": 321.15399169921875,
      "learning_rate": 4.250435793143521e-06,
      "logits/chosen": -0.6874633431434631,
      "logits/rejected": -0.7325695157051086,
      "logps/chosen": -72.60140228271484,
      "logps/rejected": -78.3973159790039,
      "loss": 14.8322,
      "rewards/accuracies": 0.15000000596046448,
      "rewards/chosen": -0.08024777472019196,
      "rewards/margins": -0.08848480880260468,
      "rewards/rejected": 0.008237037807703018,
      "step": 516
    },
    {
      "epoch": 0.30028460242783295,
      "grad_norm": 309.9378967285156,
      "learning_rate": 4.248983149331784e-06,
      "logits/chosen": -0.6688744425773621,
      "logits/rejected": -0.6749259233474731,
      "logps/chosen": -77.09115600585938,
      "logps/rejected": -67.28768157958984,
      "loss": 13.9949,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02630709670484066,
      "rewards/margins": -0.007807762827724218,
      "rewards/rejected": 0.034114859998226166,
      "step": 517
    },
    {
      "epoch": 0.3008654237091247,
      "grad_norm": 319.2854919433594,
      "learning_rate": 4.2475305055200465e-06,
      "logits/chosen": -0.6039861440658569,
      "logits/rejected": -0.6350539922714233,
      "logps/chosen": -80.51014709472656,
      "logps/rejected": -76.9443130493164,
      "loss": 13.3171,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.047122977674007416,
      "rewards/margins": 0.07121749967336655,
      "rewards/rejected": -0.024094533175230026,
      "step": 518
    },
    {
      "epoch": 0.30144624499041645,
      "grad_norm": 296.62738037109375,
      "learning_rate": 4.246077861708309e-06,
      "logits/chosen": -0.6995037794113159,
      "logits/rejected": -0.6872465014457703,
      "logps/chosen": -77.89250946044922,
      "logps/rejected": -75.6382827758789,
      "loss": 13.3481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011319695971906185,
      "rewards/margins": 0.056264109909534454,
      "rewards/rejected": -0.044944409281015396,
      "step": 519
    },
    {
      "epoch": 0.3020270662717082,
      "grad_norm": 344.6690979003906,
      "learning_rate": 4.244625217896572e-06,
      "logits/chosen": -0.7577833533287048,
      "logits/rejected": -0.7368292808532715,
      "logps/chosen": -80.45695495605469,
      "logps/rejected": -76.34242248535156,
      "loss": 14.2532,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03447725996375084,
      "rewards/margins": -0.027280423790216446,
      "rewards/rejected": 0.061757683753967285,
      "step": 520
    },
    {
      "epoch": 0.30260788755299994,
      "grad_norm": 307.37774658203125,
      "learning_rate": 4.243172574084835e-06,
      "logits/chosen": -0.7661414742469788,
      "logits/rejected": -0.7345612645149231,
      "logps/chosen": -71.1924057006836,
      "logps/rejected": -78.25225830078125,
      "loss": 12.5541,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.10229741036891937,
      "rewards/margins": 0.14318832755088806,
      "rewards/rejected": -0.04089091718196869,
      "step": 521
    },
    {
      "epoch": 0.30318870883429166,
      "grad_norm": 336.8166809082031,
      "learning_rate": 4.2417199302730975e-06,
      "logits/chosen": -0.9011874198913574,
      "logits/rejected": -0.9594923257827759,
      "logps/chosen": -84.79351806640625,
      "logps/rejected": -76.28190612792969,
      "loss": 13.2933,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008011607453227043,
      "rewards/margins": 0.06244700402021408,
      "rewards/rejected": -0.05443539097905159,
      "step": 522
    },
    {
      "epoch": 0.30376953011558344,
      "grad_norm": 305.69476318359375,
      "learning_rate": 4.240267286461359e-06,
      "logits/chosen": -0.41301050782203674,
      "logits/rejected": -0.5842508673667908,
      "logps/chosen": -73.02799987792969,
      "logps/rejected": -72.72782897949219,
      "loss": 13.5815,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01740368828177452,
      "rewards/margins": 0.06376995146274567,
      "rewards/rejected": -0.046366266906261444,
      "step": 523
    },
    {
      "epoch": 0.30435035139687516,
      "grad_norm": 359.3056640625,
      "learning_rate": 4.238814642649622e-06,
      "logits/chosen": -0.8724691271781921,
      "logits/rejected": -0.8090575933456421,
      "logps/chosen": -80.69252014160156,
      "logps/rejected": -70.90336608886719,
      "loss": 13.8932,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0029185772873461246,
      "rewards/margins": 0.007794302888214588,
      "rewards/rejected": -0.0048757269978523254,
      "step": 524
    },
    {
      "epoch": 0.30493117267816694,
      "grad_norm": 307.8243408203125,
      "learning_rate": 4.237361998837886e-06,
      "logits/chosen": -0.6465967893600464,
      "logits/rejected": -0.5403602123260498,
      "logps/chosen": -72.42932891845703,
      "logps/rejected": -76.02727508544922,
      "loss": 13.8421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.019543122500181198,
      "rewards/margins": 0.0053735459223389626,
      "rewards/rejected": 0.014169578440487385,
      "step": 525
    },
    {
      "epoch": 0.30551199395945866,
      "grad_norm": 317.7290344238281,
      "learning_rate": 4.2359093550261485e-06,
      "logits/chosen": -0.7441942691802979,
      "logits/rejected": -0.739268958568573,
      "logps/chosen": -70.67054748535156,
      "logps/rejected": -73.28771209716797,
      "loss": 13.9572,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.008803701028227806,
      "rewards/margins": -0.006234263069927692,
      "rewards/rejected": 0.015037964098155499,
      "step": 526
    },
    {
      "epoch": 0.30609281524075044,
      "grad_norm": 321.7218017578125,
      "learning_rate": 4.234456711214411e-06,
      "logits/chosen": -0.5514706373214722,
      "logits/rejected": -0.6030277609825134,
      "logps/chosen": -80.82585144042969,
      "logps/rejected": -68.28355407714844,
      "loss": 13.5752,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.025033999234437943,
      "rewards/margins": 0.037477243691682816,
      "rewards/rejected": -0.012443247251212597,
      "step": 527
    },
    {
      "epoch": 0.30667363652204216,
      "grad_norm": 317.5115661621094,
      "learning_rate": 4.233004067402673e-06,
      "logits/chosen": -0.6700873374938965,
      "logits/rejected": -0.6243543028831482,
      "logps/chosen": -71.29347229003906,
      "logps/rejected": -69.8293228149414,
      "loss": 13.4781,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.019291725009679794,
      "rewards/margins": 0.047863397747278214,
      "rewards/rejected": -0.02857167460024357,
      "step": 528
    },
    {
      "epoch": 0.30725445780333394,
      "grad_norm": 304.30230712890625,
      "learning_rate": 4.231551423590936e-06,
      "logits/chosen": -0.6290581226348877,
      "logits/rejected": -0.7105584144592285,
      "logps/chosen": -76.74227142333984,
      "logps/rejected": -70.03807067871094,
      "loss": 13.8268,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.030986541882157326,
      "rewards/margins": 0.015768680721521378,
      "rewards/rejected": -0.046755217015743256,
      "step": 529
    },
    {
      "epoch": 0.30783527908462566,
      "grad_norm": 308.83099365234375,
      "learning_rate": 4.230098779779199e-06,
      "logits/chosen": -0.7084294557571411,
      "logits/rejected": -0.6327206492424011,
      "logps/chosen": -79.52645874023438,
      "logps/rejected": -72.7778549194336,
      "loss": 13.9131,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0055747805163264275,
      "rewards/margins": 0.0061087412759661674,
      "rewards/rejected": -0.0005339615163393319,
      "step": 530
    },
    {
      "epoch": 0.30841610036591743,
      "grad_norm": 309.6053466796875,
      "learning_rate": 4.228646135967461e-06,
      "logits/chosen": -0.6223837733268738,
      "logits/rejected": -0.6086059808731079,
      "logps/chosen": -72.55455017089844,
      "logps/rejected": -67.36306762695312,
      "loss": 13.8436,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02795691415667534,
      "rewards/margins": 0.01112289633601904,
      "rewards/rejected": 0.016834020614624023,
      "step": 531
    },
    {
      "epoch": 0.30899692164720916,
      "grad_norm": 313.63336181640625,
      "learning_rate": 4.227193492155724e-06,
      "logits/chosen": -0.7594996690750122,
      "logits/rejected": -0.7488009333610535,
      "logps/chosen": -72.03538513183594,
      "logps/rejected": -79.82178497314453,
      "loss": 13.0208,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.010113591328263283,
      "rewards/margins": 0.09955569356679916,
      "rewards/rejected": -0.08944210410118103,
      "step": 532
    },
    {
      "epoch": 0.3095777429285009,
      "grad_norm": 306.1441650390625,
      "learning_rate": 4.225740848343987e-06,
      "logits/chosen": -0.7741316556930542,
      "logits/rejected": -0.8882448077201843,
      "logps/chosen": -73.84909057617188,
      "logps/rejected": -75.96207427978516,
      "loss": 13.5817,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.004449290223419666,
      "rewards/margins": 0.04892003908753395,
      "rewards/rejected": -0.04447074979543686,
      "step": 533
    },
    {
      "epoch": 0.31015856420979265,
      "grad_norm": 289.4524230957031,
      "learning_rate": 4.22428820453225e-06,
      "logits/chosen": -0.5990484356880188,
      "logits/rejected": -0.7283953428268433,
      "logps/chosen": -68.75221252441406,
      "logps/rejected": -70.73362731933594,
      "loss": 13.4645,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03096533939242363,
      "rewards/margins": 0.04442495107650757,
      "rewards/rejected": -0.013459615409374237,
      "step": 534
    },
    {
      "epoch": 0.3107393854910844,
      "grad_norm": 306.3465576171875,
      "learning_rate": 4.2228355607205115e-06,
      "logits/chosen": -0.7504904866218567,
      "logits/rejected": -0.740320086479187,
      "logps/chosen": -69.62543487548828,
      "logps/rejected": -78.7964859008789,
      "loss": 13.2895,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.06250744313001633,
      "rewards/margins": 0.06913135945796967,
      "rewards/rejected": -0.006623915396630764,
      "step": 535
    },
    {
      "epoch": 0.31132020677237615,
      "grad_norm": 313.4544982910156,
      "learning_rate": 4.221382916908774e-06,
      "logits/chosen": -0.6303409337997437,
      "logits/rejected": -0.6835408210754395,
      "logps/chosen": -78.17256164550781,
      "logps/rejected": -71.18514251708984,
      "loss": 13.3348,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02659953013062477,
      "rewards/margins": 0.06356380879878998,
      "rewards/rejected": -0.03696427494287491,
      "step": 536
    },
    {
      "epoch": 0.3119010280536679,
      "grad_norm": 337.2955627441406,
      "learning_rate": 4.219930273097037e-06,
      "logits/chosen": -0.8487906455993652,
      "logits/rejected": -0.872395396232605,
      "logps/chosen": -77.28082275390625,
      "logps/rejected": -82.19414520263672,
      "loss": 13.4067,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012853232212364674,
      "rewards/margins": 0.053110599517822266,
      "rewards/rejected": -0.04025736823678017,
      "step": 537
    },
    {
      "epoch": 0.31248184933495965,
      "grad_norm": 286.7469482421875,
      "learning_rate": 4.2184776292853e-06,
      "logits/chosen": -0.6404751539230347,
      "logits/rejected": -0.6587169766426086,
      "logps/chosen": -74.94679260253906,
      "logps/rejected": -73.58045959472656,
      "loss": 12.8176,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.037719763815402985,
      "rewards/margins": 0.11207199096679688,
      "rewards/rejected": -0.07435222715139389,
      "step": 538
    },
    {
      "epoch": 0.31306267061625137,
      "grad_norm": 303.54852294921875,
      "learning_rate": 4.2170249854735625e-06,
      "logits/chosen": -0.5835073590278625,
      "logits/rejected": -0.5741956830024719,
      "logps/chosen": -71.42955017089844,
      "logps/rejected": -68.7015151977539,
      "loss": 13.2086,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.06677176058292389,
      "rewards/margins": 0.07689666002988815,
      "rewards/rejected": -0.010124899446964264,
      "step": 539
    },
    {
      "epoch": 0.31364349189754315,
      "grad_norm": 299.73089599609375,
      "learning_rate": 4.215572341661825e-06,
      "logits/chosen": -0.8799319267272949,
      "logits/rejected": -0.8263736963272095,
      "logps/chosen": -71.5801010131836,
      "logps/rejected": -74.73466491699219,
      "loss": 13.3193,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.056644029915332794,
      "rewards/margins": 0.0625743716955185,
      "rewards/rejected": -0.005930337123572826,
      "step": 540
    },
    {
      "epoch": 0.31422431317883487,
      "grad_norm": 348.8278503417969,
      "learning_rate": 4.214119697850088e-06,
      "logits/chosen": -0.6225399971008301,
      "logits/rejected": -0.6066278219223022,
      "logps/chosen": -67.95753479003906,
      "logps/rejected": -70.62969207763672,
      "loss": 14.2989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.025478297844529152,
      "rewards/margins": -0.03527476638555527,
      "rewards/rejected": 0.009796475991606712,
      "step": 541
    },
    {
      "epoch": 0.31480513446012665,
      "grad_norm": 320.58349609375,
      "learning_rate": 4.21266705403835e-06,
      "logits/chosen": -0.6626735925674438,
      "logits/rejected": -0.7045550346374512,
      "logps/chosen": -68.6723403930664,
      "logps/rejected": -76.953125,
      "loss": 13.6882,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05210607126355171,
      "rewards/margins": 0.027962008491158485,
      "rewards/rejected": 0.02414405718445778,
      "step": 542
    },
    {
      "epoch": 0.31538595574141837,
      "grad_norm": 294.38104248046875,
      "learning_rate": 4.211214410226613e-06,
      "logits/chosen": -0.6852847337722778,
      "logits/rejected": -0.7603325843811035,
      "logps/chosen": -71.50347137451172,
      "logps/rejected": -73.55186462402344,
      "loss": 13.2244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03986804932355881,
      "rewards/margins": 0.07143954932689667,
      "rewards/rejected": -0.03157149627804756,
      "step": 543
    },
    {
      "epoch": 0.3159667770227101,
      "grad_norm": 292.24896240234375,
      "learning_rate": 4.209761766414875e-06,
      "logits/chosen": -0.6846610307693481,
      "logits/rejected": -0.7384731769561768,
      "logps/chosen": -69.51171112060547,
      "logps/rejected": -70.13493347167969,
      "loss": 13.8235,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0011608407367020845,
      "rewards/margins": 0.007756291422992945,
      "rewards/rejected": -0.008917133323848248,
      "step": 544
    },
    {
      "epoch": 0.31654759830400186,
      "grad_norm": 303.1371765136719,
      "learning_rate": 4.208309122603138e-06,
      "logits/chosen": -0.6676656007766724,
      "logits/rejected": -0.6589547395706177,
      "logps/chosen": -68.53245544433594,
      "logps/rejected": -65.3570556640625,
      "loss": 13.4721,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04229654744267464,
      "rewards/margins": 0.047294534742832184,
      "rewards/rejected": -0.0049979896284639835,
      "step": 545
    },
    {
      "epoch": 0.3171284195852936,
      "grad_norm": 304.0082092285156,
      "learning_rate": 4.206856478791401e-06,
      "logits/chosen": -0.7934265732765198,
      "logits/rejected": -0.7060130834579468,
      "logps/chosen": -65.05821990966797,
      "logps/rejected": -70.21916198730469,
      "loss": 14.4302,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.013832311145961285,
      "rewards/margins": -0.04456400126218796,
      "rewards/rejected": 0.058396317064762115,
      "step": 546
    },
    {
      "epoch": 0.31770924086658536,
      "grad_norm": 318.7176513671875,
      "learning_rate": 4.205403834979664e-06,
      "logits/chosen": -0.6346458196640015,
      "logits/rejected": -0.7181236147880554,
      "logps/chosen": -77.43191528320312,
      "logps/rejected": -72.55345153808594,
      "loss": 13.3574,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.015245789662003517,
      "rewards/margins": 0.06777335703372955,
      "rewards/rejected": -0.08301915228366852,
      "step": 547
    },
    {
      "epoch": 0.3182900621478771,
      "grad_norm": 307.634521484375,
      "learning_rate": 4.2039511911679255e-06,
      "logits/chosen": -0.79632169008255,
      "logits/rejected": -0.7473276853561401,
      "logps/chosen": -77.22640991210938,
      "logps/rejected": -75.24634552001953,
      "loss": 13.5692,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03856954723596573,
      "rewards/margins": 0.03735596686601639,
      "rewards/rejected": 0.0012135781580582261,
      "step": 548
    },
    {
      "epoch": 0.31887088342916886,
      "grad_norm": 299.0860595703125,
      "learning_rate": 4.202498547356188e-06,
      "logits/chosen": -0.5579553842544556,
      "logits/rejected": -0.5251676440238953,
      "logps/chosen": -66.94325256347656,
      "logps/rejected": -70.40986633300781,
      "loss": 13.9272,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008126763626933098,
      "rewards/margins": 0.0029366514645516872,
      "rewards/rejected": -0.011063413694500923,
      "step": 549
    },
    {
      "epoch": 0.3194517047104606,
      "grad_norm": 294.1581726074219,
      "learning_rate": 4.201045903544451e-06,
      "logits/chosen": -0.6971367001533508,
      "logits/rejected": -0.6503037214279175,
      "logps/chosen": -72.15474700927734,
      "logps/rejected": -72.1407470703125,
      "loss": 13.4412,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04408375173807144,
      "rewards/margins": 0.05293964222073555,
      "rewards/rejected": -0.00885589700192213,
      "step": 550
    },
    {
      "epoch": 0.32003252599175236,
      "grad_norm": 1313.975830078125,
      "learning_rate": 4.199593259732714e-06,
      "logits/chosen": -0.8329079747200012,
      "logits/rejected": -0.7704097032546997,
      "logps/chosen": -72.53883361816406,
      "logps/rejected": -70.23878479003906,
      "loss": 13.435,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.07676436007022858,
      "rewards/margins": 0.04872946813702583,
      "rewards/rejected": 0.028034895658493042,
      "step": 551
    },
    {
      "epoch": 0.3206133472730441,
      "grad_norm": 331.4753112792969,
      "learning_rate": 4.1981406159209765e-06,
      "logits/chosen": -0.7476204633712769,
      "logits/rejected": -0.7286363840103149,
      "logps/chosen": -69.5918960571289,
      "logps/rejected": -63.991371154785156,
      "loss": 14.1313,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013785046525299549,
      "rewards/margins": -0.01810169778764248,
      "rewards/rejected": 0.004316650331020355,
      "step": 552
    },
    {
      "epoch": 0.32119416855433586,
      "grad_norm": 332.4710388183594,
      "learning_rate": 4.196687972109239e-06,
      "logits/chosen": -0.6895079612731934,
      "logits/rejected": -0.7061454653739929,
      "logps/chosen": -77.14973449707031,
      "logps/rejected": -76.41380310058594,
      "loss": 13.5007,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.059791963547468185,
      "rewards/margins": 0.047647859901189804,
      "rewards/rejected": 0.012144106440246105,
      "step": 553
    },
    {
      "epoch": 0.3217749898356276,
      "grad_norm": 306.3582763671875,
      "learning_rate": 4.195235328297502e-06,
      "logits/chosen": -0.7227594256401062,
      "logits/rejected": -0.6854357123374939,
      "logps/chosen": -75.45829010009766,
      "logps/rejected": -71.11771392822266,
      "loss": 13.5619,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03397790342569351,
      "rewards/margins": 0.04335654899477959,
      "rewards/rejected": -0.009378653019666672,
      "step": 554
    },
    {
      "epoch": 0.3223558111169193,
      "grad_norm": 313.3603210449219,
      "learning_rate": 4.193782684485764e-06,
      "logits/chosen": -0.788299024105072,
      "logits/rejected": -0.7475656270980835,
      "logps/chosen": -68.71570587158203,
      "logps/rejected": -70.02986145019531,
      "loss": 14.0656,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.042551252990961075,
      "rewards/margins": -0.008566422387957573,
      "rewards/rejected": 0.0511176697909832,
      "step": 555
    },
    {
      "epoch": 0.3229366323982111,
      "grad_norm": 335.6896667480469,
      "learning_rate": 4.192330040674027e-06,
      "logits/chosen": -0.7440964579582214,
      "logits/rejected": -0.7291234731674194,
      "logps/chosen": -82.71476745605469,
      "logps/rejected": -77.9942398071289,
      "loss": 13.8855,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.008972082287073135,
      "rewards/margins": 0.009817652404308319,
      "rewards/rejected": -0.000845567905344069,
      "step": 556
    },
    {
      "epoch": 0.3235174536795028,
      "grad_norm": 351.78411865234375,
      "learning_rate": 4.190877396862289e-06,
      "logits/chosen": -0.7939974069595337,
      "logits/rejected": -0.7254185080528259,
      "logps/chosen": -91.87419128417969,
      "logps/rejected": -90.27113342285156,
      "loss": 12.7765,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.044230926781892776,
      "rewards/margins": 0.12266170978546143,
      "rewards/rejected": -0.07843078672885895,
      "step": 557
    },
    {
      "epoch": 0.3240982749607946,
      "grad_norm": 295.10577392578125,
      "learning_rate": 4.189424753050552e-06,
      "logits/chosen": -0.532805323600769,
      "logits/rejected": -0.6035286784172058,
      "logps/chosen": -69.48627471923828,
      "logps/rejected": -70.66758728027344,
      "loss": 13.5119,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.034174613654613495,
      "rewards/margins": 0.04042964056134224,
      "rewards/rejected": -0.006255028303712606,
      "step": 558
    },
    {
      "epoch": 0.3246790962420863,
      "grad_norm": 288.3070373535156,
      "learning_rate": 4.187972109238815e-06,
      "logits/chosen": -0.6821750998497009,
      "logits/rejected": -0.7404053807258606,
      "logps/chosen": -70.35401916503906,
      "logps/rejected": -72.6601791381836,
      "loss": 13.8817,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.017289195209741592,
      "rewards/margins": 0.004057192709296942,
      "rewards/rejected": -0.02134638838469982,
      "step": 559
    },
    {
      "epoch": 0.3252599175233781,
      "grad_norm": 320.47100830078125,
      "learning_rate": 4.186519465427078e-06,
      "logits/chosen": -0.6497399210929871,
      "logits/rejected": -0.607397735118866,
      "logps/chosen": -74.75005340576172,
      "logps/rejected": -78.32804870605469,
      "loss": 13.5264,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04527430608868599,
      "rewards/margins": 0.04255044087767601,
      "rewards/rejected": 0.00272386590950191,
      "step": 560
    },
    {
      "epoch": 0.3258407388046698,
      "grad_norm": 301.3592834472656,
      "learning_rate": 4.18506682161534e-06,
      "logits/chosen": -0.6746230721473694,
      "logits/rejected": -0.6951724886894226,
      "logps/chosen": -71.5419921875,
      "logps/rejected": -73.84922790527344,
      "loss": 13.1544,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03311220183968544,
      "rewards/margins": 0.08440899848937988,
      "rewards/rejected": -0.05129680037498474,
      "step": 561
    },
    {
      "epoch": 0.32642156008596157,
      "grad_norm": 313.3168029785156,
      "learning_rate": 4.183614177803602e-06,
      "logits/chosen": -0.6119886636734009,
      "logits/rejected": -0.6854387521743774,
      "logps/chosen": -70.71475982666016,
      "logps/rejected": -73.73226165771484,
      "loss": 13.9581,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0057665156200528145,
      "rewards/margins": -0.0022575571201741695,
      "rewards/rejected": 0.008024071343243122,
      "step": 562
    },
    {
      "epoch": 0.3270023813672533,
      "grad_norm": 295.86163330078125,
      "learning_rate": 4.182161533991865e-06,
      "logits/chosen": -0.5245779752731323,
      "logits/rejected": -0.589116096496582,
      "logps/chosen": -70.53639221191406,
      "logps/rejected": -73.27334594726562,
      "loss": 13.6625,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033234186470508575,
      "rewards/margins": 0.02967524155974388,
      "rewards/rejected": 0.0035589407198131084,
      "step": 563
    },
    {
      "epoch": 0.32758320264854507,
      "grad_norm": 292.6914978027344,
      "learning_rate": 4.180708890180128e-06,
      "logits/chosen": -0.7187098264694214,
      "logits/rejected": -0.7310807108879089,
      "logps/chosen": -70.57524871826172,
      "logps/rejected": -71.39189147949219,
      "loss": 13.7081,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.041135989129543304,
      "rewards/margins": 0.019587086513638496,
      "rewards/rejected": 0.021548902615904808,
      "step": 564
    },
    {
      "epoch": 0.3281640239298368,
      "grad_norm": 298.4253234863281,
      "learning_rate": 4.1792562463683906e-06,
      "logits/chosen": -0.7534239292144775,
      "logits/rejected": -0.7867181897163391,
      "logps/chosen": -72.37812805175781,
      "logps/rejected": -69.36402130126953,
      "loss": 13.7307,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06334944069385529,
      "rewards/margins": 0.022134965285658836,
      "rewards/rejected": 0.0412144735455513,
      "step": 565
    },
    {
      "epoch": 0.3287448452111285,
      "grad_norm": 305.56884765625,
      "learning_rate": 4.177803602556653e-06,
      "logits/chosen": -0.8273922204971313,
      "logits/rejected": -0.6768549084663391,
      "logps/chosen": -72.92654418945312,
      "logps/rejected": -68.28416442871094,
      "loss": 13.5351,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06688196212053299,
      "rewards/margins": 0.040909625589847565,
      "rewards/rejected": 0.025972336530685425,
      "step": 566
    },
    {
      "epoch": 0.3293256664924203,
      "grad_norm": 317.9597473144531,
      "learning_rate": 4.176350958744916e-06,
      "logits/chosen": -0.6657764315605164,
      "logits/rejected": -0.632070779800415,
      "logps/chosen": -74.51126861572266,
      "logps/rejected": -81.23558807373047,
      "loss": 14.0378,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016053617000579834,
      "rewards/margins": -0.010346947237849236,
      "rewards/rejected": 0.02640056610107422,
      "step": 567
    },
    {
      "epoch": 0.329906487773712,
      "grad_norm": 296.336669921875,
      "learning_rate": 4.174898314933179e-06,
      "logits/chosen": -0.44092756509780884,
      "logits/rejected": -0.4833409786224365,
      "logps/chosen": -79.0365982055664,
      "logps/rejected": -65.05804443359375,
      "loss": 13.3725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02982897125184536,
      "rewards/margins": 0.061524223536252975,
      "rewards/rejected": -0.031695254147052765,
      "step": 568
    },
    {
      "epoch": 0.3304873090550038,
      "grad_norm": 301.1853942871094,
      "learning_rate": 4.1734456711214416e-06,
      "logits/chosen": -0.7429706454277039,
      "logits/rejected": -0.7991895079612732,
      "logps/chosen": -72.23892974853516,
      "logps/rejected": -74.65027618408203,
      "loss": 13.5499,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04059157520532608,
      "rewards/margins": 0.03457489609718323,
      "rewards/rejected": 0.00601667445152998,
      "step": 569
    },
    {
      "epoch": 0.3310681303362955,
      "grad_norm": 318.2872009277344,
      "learning_rate": 4.171993027309704e-06,
      "logits/chosen": -0.8133655786514282,
      "logits/rejected": -0.7936081886291504,
      "logps/chosen": -75.30345153808594,
      "logps/rejected": -74.62310791015625,
      "loss": 13.7668,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02610892988741398,
      "rewards/margins": 0.020346611738204956,
      "rewards/rejected": 0.0057623242028057575,
      "step": 570
    },
    {
      "epoch": 0.3316489516175873,
      "grad_norm": 339.5540466308594,
      "learning_rate": 4.170540383497967e-06,
      "logits/chosen": -0.6674878001213074,
      "logits/rejected": -0.7041738629341125,
      "logps/chosen": -81.82828521728516,
      "logps/rejected": -85.7889175415039,
      "loss": 14.0594,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014918116852641106,
      "rewards/margins": -0.00931711494922638,
      "rewards/rejected": 0.024235233664512634,
      "step": 571
    },
    {
      "epoch": 0.332229772898879,
      "grad_norm": 291.53070068359375,
      "learning_rate": 4.16908773968623e-06,
      "logits/chosen": -0.5750855803489685,
      "logits/rejected": -0.6670510172843933,
      "logps/chosen": -70.49334716796875,
      "logps/rejected": -79.74695587158203,
      "loss": 13.5056,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.000834398262668401,
      "rewards/margins": 0.04568032547831535,
      "rewards/rejected": -0.04484592750668526,
      "step": 572
    },
    {
      "epoch": 0.3328105941801708,
      "grad_norm": 338.2127990722656,
      "learning_rate": 4.167635095874492e-06,
      "logits/chosen": -0.6680286526679993,
      "logits/rejected": -0.6583356857299805,
      "logps/chosen": -74.26518249511719,
      "logps/rejected": -76.2101821899414,
      "loss": 13.9376,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06941697746515274,
      "rewards/margins": -0.0016284823650494218,
      "rewards/rejected": 0.07104545831680298,
      "step": 573
    },
    {
      "epoch": 0.3333914154614625,
      "grad_norm": 324.9139709472656,
      "learning_rate": 4.1661824520627544e-06,
      "logits/chosen": -0.6829845309257507,
      "logits/rejected": -0.7503547668457031,
      "logps/chosen": -79.72300720214844,
      "logps/rejected": -77.9569320678711,
      "loss": 13.4892,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05273396894335747,
      "rewards/margins": 0.041454561054706573,
      "rewards/rejected": 0.011279400438070297,
      "step": 574
    },
    {
      "epoch": 0.3339722367427543,
      "grad_norm": 294.2154235839844,
      "learning_rate": 4.164729808251017e-06,
      "logits/chosen": -0.6569613814353943,
      "logits/rejected": -0.6229342222213745,
      "logps/chosen": -72.42926025390625,
      "logps/rejected": -68.15298461914062,
      "loss": 14.7445,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.039252303540706635,
      "rewards/margins": -0.08114682883024216,
      "rewards/rejected": 0.04189452528953552,
      "step": 575
    },
    {
      "epoch": 0.334553058024046,
      "grad_norm": 335.7951965332031,
      "learning_rate": 4.16327716443928e-06,
      "logits/chosen": -0.6173557639122009,
      "logits/rejected": -0.7220408916473389,
      "logps/chosen": -69.47132873535156,
      "logps/rejected": -70.61470031738281,
      "loss": 12.8536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.047998152673244476,
      "rewards/margins": 0.11041458696126938,
      "rewards/rejected": -0.062416426837444305,
      "step": 576
    },
    {
      "epoch": 0.3351338793053377,
      "grad_norm": 305.68243408203125,
      "learning_rate": 4.161824520627543e-06,
      "logits/chosen": -0.6545889377593994,
      "logits/rejected": -0.6952469348907471,
      "logps/chosen": -82.79303741455078,
      "logps/rejected": -67.61451721191406,
      "loss": 14.1696,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.011021709069609642,
      "rewards/margins": -0.020772431045770645,
      "rewards/rejected": 0.009750718250870705,
      "step": 577
    },
    {
      "epoch": 0.3357147005866295,
      "grad_norm": 306.0280456542969,
      "learning_rate": 4.1603718768158054e-06,
      "logits/chosen": -0.6899808645248413,
      "logits/rejected": -0.6691157221794128,
      "logps/chosen": -75.88886260986328,
      "logps/rejected": -70.24473571777344,
      "loss": 13.5661,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05026114732027054,
      "rewards/margins": 0.04536793380975723,
      "rewards/rejected": 0.004893226083368063,
      "step": 578
    },
    {
      "epoch": 0.3362955218679212,
      "grad_norm": 314.8397521972656,
      "learning_rate": 4.158919233004068e-06,
      "logits/chosen": -0.5284186601638794,
      "logits/rejected": -0.5781315565109253,
      "logps/chosen": -71.56379699707031,
      "logps/rejected": -67.72520446777344,
      "loss": 14.1152,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.008697940036654472,
      "rewards/margins": -0.018017660826444626,
      "rewards/rejected": 0.009319724515080452,
      "step": 579
    },
    {
      "epoch": 0.336876343149213,
      "grad_norm": 312.38427734375,
      "learning_rate": 4.15746658919233e-06,
      "logits/chosen": -0.7509424686431885,
      "logits/rejected": -0.8194720149040222,
      "logps/chosen": -68.1953125,
      "logps/rejected": -79.12274169921875,
      "loss": 14.2305,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.021652111783623695,
      "rewards/margins": -0.0276623722165823,
      "rewards/rejected": 0.0493144765496254,
      "step": 580
    },
    {
      "epoch": 0.3374571644305047,
      "grad_norm": 296.4921875,
      "learning_rate": 4.156013945380593e-06,
      "logits/chosen": -0.5414460897445679,
      "logits/rejected": -0.579011082649231,
      "logps/chosen": -71.58470153808594,
      "logps/rejected": -77.13636016845703,
      "loss": 13.9073,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.022373903542757034,
      "rewards/margins": 0.008005992509424686,
      "rewards/rejected": 0.014367911033332348,
      "step": 581
    },
    {
      "epoch": 0.3380379857117965,
      "grad_norm": 295.32965087890625,
      "learning_rate": 4.154561301568856e-06,
      "logits/chosen": -0.8184449076652527,
      "logits/rejected": -0.946854293346405,
      "logps/chosen": -65.59197235107422,
      "logps/rejected": -69.1659927368164,
      "loss": 13.6964,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02355037070810795,
      "rewards/margins": 0.024608146399259567,
      "rewards/rejected": -0.0010577782522886992,
      "step": 582
    },
    {
      "epoch": 0.3386188069930882,
      "grad_norm": 316.63592529296875,
      "learning_rate": 4.153108657757118e-06,
      "logits/chosen": -0.6438121199607849,
      "logits/rejected": -0.7025144696235657,
      "logps/chosen": -77.89268493652344,
      "logps/rejected": -75.57435607910156,
      "loss": 13.9757,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03375036641955376,
      "rewards/margins": 0.004329390823841095,
      "rewards/rejected": 0.02942098118364811,
      "step": 583
    },
    {
      "epoch": 0.33919962827438,
      "grad_norm": 303.33251953125,
      "learning_rate": 4.151656013945381e-06,
      "logits/chosen": -0.6150007247924805,
      "logits/rejected": -0.6166488528251648,
      "logps/chosen": -73.3652572631836,
      "logps/rejected": -78.53809356689453,
      "loss": 13.1565,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04595949128270149,
      "rewards/margins": 0.08009113371372223,
      "rewards/rejected": -0.03413163870573044,
      "step": 584
    },
    {
      "epoch": 0.3397804495556717,
      "grad_norm": 301.8458557128906,
      "learning_rate": 4.150203370133644e-06,
      "logits/chosen": -0.6662709712982178,
      "logits/rejected": -0.8405240774154663,
      "logps/chosen": -68.98908996582031,
      "logps/rejected": -80.71705627441406,
      "loss": 13.6482,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0004700004938058555,
      "rewards/margins": 0.030774127691984177,
      "rewards/rejected": -0.03030412830412388,
      "step": 585
    },
    {
      "epoch": 0.3403612708369635,
      "grad_norm": 315.66986083984375,
      "learning_rate": 4.148750726321907e-06,
      "logits/chosen": -0.6200467348098755,
      "logits/rejected": -0.6849344968795776,
      "logps/chosen": -72.97185516357422,
      "logps/rejected": -66.83480834960938,
      "loss": 13.8744,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03314518183469772,
      "rewards/margins": 0.006186559796333313,
      "rewards/rejected": 0.02695862017571926,
      "step": 586
    },
    {
      "epoch": 0.3409420921182552,
      "grad_norm": 316.5364074707031,
      "learning_rate": 4.1472980825101685e-06,
      "logits/chosen": -0.6957502365112305,
      "logits/rejected": -0.77708899974823,
      "logps/chosen": -73.742919921875,
      "logps/rejected": -72.82991027832031,
      "loss": 14.2651,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.02233811281621456,
      "rewards/margins": -0.03094848431646824,
      "rewards/rejected": 0.008610370568931103,
      "step": 587
    },
    {
      "epoch": 0.34152291339954693,
      "grad_norm": 300.6039733886719,
      "learning_rate": 4.145845438698431e-06,
      "logits/chosen": -0.5576103925704956,
      "logits/rejected": -0.4413486421108246,
      "logps/chosen": -76.01347351074219,
      "logps/rejected": -71.07667541503906,
      "loss": 13.5076,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.014086894690990448,
      "rewards/margins": 0.0441095307469368,
      "rewards/rejected": -0.03002263978123665,
      "step": 588
    },
    {
      "epoch": 0.3421037346808387,
      "grad_norm": 312.4639587402344,
      "learning_rate": 4.144392794886694e-06,
      "logits/chosen": -0.7106123566627502,
      "logits/rejected": -0.6808815002441406,
      "logps/chosen": -72.44802856445312,
      "logps/rejected": -76.73309326171875,
      "loss": 13.6203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.022707967087626457,
      "rewards/margins": 0.03484155982732773,
      "rewards/rejected": -0.01213359646499157,
      "step": 589
    },
    {
      "epoch": 0.34268455596213043,
      "grad_norm": 302.58642578125,
      "learning_rate": 4.142940151074957e-06,
      "logits/chosen": -0.6447229981422424,
      "logits/rejected": -0.75730299949646,
      "logps/chosen": -73.82032775878906,
      "logps/rejected": -72.29612731933594,
      "loss": 13.4369,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04192671552300453,
      "rewards/margins": 0.04805769771337509,
      "rewards/rejected": -0.006130979862064123,
      "step": 590
    },
    {
      "epoch": 0.3432653772434222,
      "grad_norm": 427.1463928222656,
      "learning_rate": 4.1414875072632195e-06,
      "logits/chosen": -0.6302953958511353,
      "logits/rejected": -0.7209846377372742,
      "logps/chosen": -71.4793930053711,
      "logps/rejected": -66.0015640258789,
      "loss": 13.3978,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03149376064538956,
      "rewards/margins": 0.05494686961174011,
      "rewards/rejected": -0.02345309965312481,
      "step": 591
    },
    {
      "epoch": 0.34384619852471393,
      "grad_norm": 305.7786560058594,
      "learning_rate": 4.140034863451482e-06,
      "logits/chosen": -0.6327844858169556,
      "logits/rejected": -0.7896640300750732,
      "logps/chosen": -73.62690734863281,
      "logps/rejected": -71.11927795410156,
      "loss": 14.0315,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.018691398203372955,
      "rewards/margins": -0.008565169759094715,
      "rewards/rejected": 0.027256567031145096,
      "step": 592
    },
    {
      "epoch": 0.3444270198060057,
      "grad_norm": 291.48944091796875,
      "learning_rate": 4.138582219639745e-06,
      "logits/chosen": -0.6120963096618652,
      "logits/rejected": -0.6668158769607544,
      "logps/chosen": -71.43421936035156,
      "logps/rejected": -73.65585327148438,
      "loss": 13.6873,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.009499874897301197,
      "rewards/margins": 0.033064182847738266,
      "rewards/rejected": -0.04256405681371689,
      "step": 593
    },
    {
      "epoch": 0.34500784108729743,
      "grad_norm": 283.23406982421875,
      "learning_rate": 4.137129575828007e-06,
      "logits/chosen": -0.69483482837677,
      "logits/rejected": -0.8557512164115906,
      "logps/chosen": -71.92134094238281,
      "logps/rejected": -74.01726531982422,
      "loss": 12.8864,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01784166321158409,
      "rewards/margins": 0.10965867340564728,
      "rewards/rejected": -0.09181700646877289,
      "step": 594
    },
    {
      "epoch": 0.3455886623685892,
      "grad_norm": 281.31951904296875,
      "learning_rate": 4.13567693201627e-06,
      "logits/chosen": -0.702051043510437,
      "logits/rejected": -0.7037491202354431,
      "logps/chosen": -69.55567932128906,
      "logps/rejected": -65.29097747802734,
      "loss": 13.4975,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.025551121681928635,
      "rewards/margins": 0.045175038278102875,
      "rewards/rejected": -0.07072616368532181,
      "step": 595
    },
    {
      "epoch": 0.3461694836498809,
      "grad_norm": 310.319091796875,
      "learning_rate": 4.134224288204532e-06,
      "logits/chosen": -0.604898989200592,
      "logits/rejected": -0.5756534337997437,
      "logps/chosen": -72.46310424804688,
      "logps/rejected": -75.34378051757812,
      "loss": 13.812,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.018788345158100128,
      "rewards/margins": 0.01757490262389183,
      "rewards/rejected": 0.0012134440476074815,
      "step": 596
    },
    {
      "epoch": 0.3467503049311727,
      "grad_norm": 329.1076354980469,
      "learning_rate": 4.132771644392795e-06,
      "logits/chosen": -0.6654219031333923,
      "logits/rejected": -0.5940228700637817,
      "logps/chosen": -77.3206558227539,
      "logps/rejected": -70.39866638183594,
      "loss": 14.0139,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.008175027556717396,
      "rewards/margins": -0.004936867859214544,
      "rewards/rejected": 0.013111898675560951,
      "step": 597
    },
    {
      "epoch": 0.3473311262124644,
      "grad_norm": 356.73992919921875,
      "learning_rate": 4.131319000581058e-06,
      "logits/chosen": -0.6939708590507507,
      "logits/rejected": -0.7016677856445312,
      "logps/chosen": -78.16722869873047,
      "logps/rejected": -69.61579895019531,
      "loss": 13.4649,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014713537879288197,
      "rewards/margins": 0.04950027912855148,
      "rewards/rejected": -0.03478673845529556,
      "step": 598
    },
    {
      "epoch": 0.34791194749375615,
      "grad_norm": 335.206298828125,
      "learning_rate": 4.129866356769321e-06,
      "logits/chosen": -0.5904199481010437,
      "logits/rejected": -0.603577196598053,
      "logps/chosen": -79.11907196044922,
      "logps/rejected": -83.3130874633789,
      "loss": 13.9855,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0011968526523560286,
      "rewards/margins": 0.002481907606124878,
      "rewards/rejected": -0.0012850582133978605,
      "step": 599
    },
    {
      "epoch": 0.3484927687750479,
      "grad_norm": 318.6190490722656,
      "learning_rate": 4.128413712957583e-06,
      "logits/chosen": -0.572067379951477,
      "logits/rejected": -0.647987961769104,
      "logps/chosen": -85.52361297607422,
      "logps/rejected": -79.0700454711914,
      "loss": 13.2384,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.010197488591074944,
      "rewards/margins": 0.08274447917938232,
      "rewards/rejected": -0.09294196218252182,
      "step": 600
    },
    {
      "epoch": 0.34907359005633964,
      "grad_norm": 317.1435241699219,
      "learning_rate": 4.126961069145845e-06,
      "logits/chosen": -0.7126006484031677,
      "logits/rejected": -0.7353073358535767,
      "logps/chosen": -68.36890411376953,
      "logps/rejected": -76.00941467285156,
      "loss": 14.2786,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.012436933815479279,
      "rewards/margins": -0.03413182869553566,
      "rewards/rejected": 0.02169489674270153,
      "step": 601
    },
    {
      "epoch": 0.3496544113376314,
      "grad_norm": 303.97283935546875,
      "learning_rate": 4.125508425334108e-06,
      "logits/chosen": -0.5907556414604187,
      "logits/rejected": -0.6256478428840637,
      "logps/chosen": -76.39266204833984,
      "logps/rejected": -74.95109558105469,
      "loss": 12.4707,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04735926538705826,
      "rewards/margins": 0.1522711217403412,
      "rewards/rejected": -0.10491186380386353,
      "step": 602
    },
    {
      "epoch": 0.35023523261892314,
      "grad_norm": 313.0004577636719,
      "learning_rate": 4.124055781522371e-06,
      "logits/chosen": -0.6598242521286011,
      "logits/rejected": -0.6428278684616089,
      "logps/chosen": -64.64137268066406,
      "logps/rejected": -74.44950866699219,
      "loss": 13.724,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.016485584899783134,
      "rewards/margins": 0.02066924050450325,
      "rewards/rejected": -0.004183652810752392,
      "step": 603
    },
    {
      "epoch": 0.3508160539002149,
      "grad_norm": 367.84808349609375,
      "learning_rate": 4.1226031377106335e-06,
      "logits/chosen": -0.5852268934249878,
      "logits/rejected": -0.6519114971160889,
      "logps/chosen": -69.59758758544922,
      "logps/rejected": -69.01988983154297,
      "loss": 13.7946,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.04319590702652931,
      "rewards/margins": 0.01627770997583866,
      "rewards/rejected": 0.0269182026386261,
      "step": 604
    },
    {
      "epoch": 0.35139687518150664,
      "grad_norm": 316.7720947265625,
      "learning_rate": 4.121150493898896e-06,
      "logits/chosen": -0.6948081851005554,
      "logits/rejected": -0.7896615266799927,
      "logps/chosen": -79.47821807861328,
      "logps/rejected": -79.4210433959961,
      "loss": 12.9175,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.08956082910299301,
      "rewards/margins": 0.10826803743839264,
      "rewards/rejected": -0.018707215785980225,
      "step": 605
    },
    {
      "epoch": 0.3519776964627984,
      "grad_norm": 320.3518371582031,
      "learning_rate": 4.119697850087159e-06,
      "logits/chosen": -0.6471167802810669,
      "logits/rejected": -0.5845264196395874,
      "logps/chosen": -72.15476989746094,
      "logps/rejected": -66.76841735839844,
      "loss": 14.2575,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.012289335019886494,
      "rewards/margins": -0.03404436260461807,
      "rewards/rejected": 0.021755026653409004,
      "step": 606
    },
    {
      "epoch": 0.35255851774409014,
      "grad_norm": 307.3863525390625,
      "learning_rate": 4.118245206275422e-06,
      "logits/chosen": -0.5887011289596558,
      "logits/rejected": -0.6126518249511719,
      "logps/chosen": -81.18836975097656,
      "logps/rejected": -77.85643005371094,
      "loss": 13.6217,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024307237938046455,
      "rewards/margins": 0.029000231996178627,
      "rewards/rejected": -0.004692991729825735,
      "step": 607
    },
    {
      "epoch": 0.3531393390253819,
      "grad_norm": 281.9200744628906,
      "learning_rate": 4.116792562463684e-06,
      "logits/chosen": -0.8018544912338257,
      "logits/rejected": -0.7951962351799011,
      "logps/chosen": -73.1496810913086,
      "logps/rejected": -74.61567687988281,
      "loss": 13.2176,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009971092455089092,
      "rewards/margins": 0.08315368741750717,
      "rewards/rejected": -0.0731825903058052,
      "step": 608
    },
    {
      "epoch": 0.35372016030667364,
      "grad_norm": 303.743896484375,
      "learning_rate": 4.115339918651947e-06,
      "logits/chosen": -0.7372728586196899,
      "logits/rejected": -0.8942030668258667,
      "logps/chosen": -74.2212905883789,
      "logps/rejected": -72.51829528808594,
      "loss": 12.8897,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.10641467571258545,
      "rewards/margins": 0.13827785849571228,
      "rewards/rejected": -0.03186319023370743,
      "step": 609
    },
    {
      "epoch": 0.35430098158796536,
      "grad_norm": 293.53631591796875,
      "learning_rate": 4.11388727484021e-06,
      "logits/chosen": -0.8246177434921265,
      "logits/rejected": -0.7705774903297424,
      "logps/chosen": -70.34178161621094,
      "logps/rejected": -73.58016204833984,
      "loss": 13.598,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.030447527766227722,
      "rewards/margins": 0.03866151347756386,
      "rewards/rejected": -0.008213978260755539,
      "step": 610
    },
    {
      "epoch": 0.35488180286925713,
      "grad_norm": 307.77294921875,
      "learning_rate": 4.112434631028473e-06,
      "logits/chosen": -0.7274325489997864,
      "logits/rejected": -0.7559345364570618,
      "logps/chosen": -76.30690002441406,
      "logps/rejected": -75.15937042236328,
      "loss": 13.8293,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01979799196124077,
      "rewards/margins": 0.014523372054100037,
      "rewards/rejected": -0.0343213714659214,
      "step": 611
    },
    {
      "epoch": 0.35546262415054886,
      "grad_norm": 314.6964416503906,
      "learning_rate": 4.110981987216735e-06,
      "logits/chosen": -0.7580679655075073,
      "logits/rejected": -0.6873196363449097,
      "logps/chosen": -72.52796173095703,
      "logps/rejected": -71.6462631225586,
      "loss": 13.9075,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024890460073947906,
      "rewards/margins": 0.01054720301181078,
      "rewards/rejected": 0.01434325985610485,
      "step": 612
    },
    {
      "epoch": 0.35604344543184063,
      "grad_norm": 315.9555358886719,
      "learning_rate": 4.109529343404997e-06,
      "logits/chosen": -0.779710590839386,
      "logits/rejected": -0.726919412612915,
      "logps/chosen": -77.7830810546875,
      "logps/rejected": -71.8597183227539,
      "loss": 14.1527,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.022592689841985703,
      "rewards/margins": -0.015459999442100525,
      "rewards/rejected": -0.0071326917968690395,
      "step": 613
    },
    {
      "epoch": 0.35662426671313235,
      "grad_norm": 301.09161376953125,
      "learning_rate": 4.10807669959326e-06,
      "logits/chosen": -0.7886669635772705,
      "logits/rejected": -0.8101975321769714,
      "logps/chosen": -73.55926513671875,
      "logps/rejected": -79.67219543457031,
      "loss": 13.0973,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06979019194841385,
      "rewards/margins": 0.08688534796237946,
      "rewards/rejected": -0.017095154151320457,
      "step": 614
    },
    {
      "epoch": 0.35720508799442413,
      "grad_norm": 290.6911926269531,
      "learning_rate": 4.106624055781523e-06,
      "logits/chosen": -0.721502423286438,
      "logits/rejected": -0.7246500253677368,
      "logps/chosen": -69.41618347167969,
      "logps/rejected": -73.62068176269531,
      "loss": 13.3402,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04512856528162956,
      "rewards/margins": 0.0591517873108387,
      "rewards/rejected": -0.014023219235241413,
      "step": 615
    },
    {
      "epoch": 0.35778590927571585,
      "grad_norm": 284.9295959472656,
      "learning_rate": 4.105171411969786e-06,
      "logits/chosen": -0.6129464507102966,
      "logits/rejected": -0.6507026553153992,
      "logps/chosen": -75.41661071777344,
      "logps/rejected": -68.95967102050781,
      "loss": 12.9987,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.029339304193854332,
      "rewards/margins": 0.09721994400024414,
      "rewards/rejected": -0.06788064539432526,
      "step": 616
    },
    {
      "epoch": 0.35836673055700763,
      "grad_norm": 318.4294738769531,
      "learning_rate": 4.103718768158048e-06,
      "logits/chosen": -0.594603419303894,
      "logits/rejected": -0.542455792427063,
      "logps/chosen": -76.71803283691406,
      "logps/rejected": -76.74525451660156,
      "loss": 13.5157,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04475555941462517,
      "rewards/margins": 0.04647917300462723,
      "rewards/rejected": -0.00172361359000206,
      "step": 617
    },
    {
      "epoch": 0.35894755183829935,
      "grad_norm": 307.6109924316406,
      "learning_rate": 4.102266124346311e-06,
      "logits/chosen": -0.6024777889251709,
      "logits/rejected": -0.6433155536651611,
      "logps/chosen": -70.24485778808594,
      "logps/rejected": -69.94918823242188,
      "loss": 13.3803,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00755837420001626,
      "rewards/margins": 0.064852774143219,
      "rewards/rejected": -0.05729439854621887,
      "step": 618
    },
    {
      "epoch": 0.3595283731195911,
      "grad_norm": 329.19183349609375,
      "learning_rate": 4.100813480534573e-06,
      "logits/chosen": -0.521633505821228,
      "logits/rejected": -0.5646509528160095,
      "logps/chosen": -78.86627197265625,
      "logps/rejected": -71.89064025878906,
      "loss": 14.1275,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.00912876520305872,
      "rewards/margins": -0.02030937746167183,
      "rewards/rejected": 0.011180608533322811,
      "step": 619
    },
    {
      "epoch": 0.36010919440088285,
      "grad_norm": 306.20867919921875,
      "learning_rate": 4.099360836722836e-06,
      "logits/chosen": -0.6866437792778015,
      "logits/rejected": -0.7564648389816284,
      "logps/chosen": -74.31228637695312,
      "logps/rejected": -65.19970703125,
      "loss": 13.9814,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00010943338565994054,
      "rewards/margins": -0.00688022980466485,
      "rewards/rejected": 0.00677079102024436,
      "step": 620
    },
    {
      "epoch": 0.36069001568217457,
      "grad_norm": 307.6351318359375,
      "learning_rate": 4.0979081929110985e-06,
      "logits/chosen": -0.6264073252677917,
      "logits/rejected": -0.6101059913635254,
      "logps/chosen": -79.4601821899414,
      "logps/rejected": -68.94159698486328,
      "loss": 13.2178,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00908391922712326,
      "rewards/margins": 0.07702421396970749,
      "rewards/rejected": -0.06794029474258423,
      "step": 621
    },
    {
      "epoch": 0.36127083696346635,
      "grad_norm": 309.8197326660156,
      "learning_rate": 4.096455549099361e-06,
      "logits/chosen": -0.6580110788345337,
      "logits/rejected": -0.8408079147338867,
      "logps/chosen": -73.57799530029297,
      "logps/rejected": -81.4070816040039,
      "loss": 14.01,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03712352365255356,
      "rewards/margins": -0.004934538155794144,
      "rewards/rejected": -0.03218898922204971,
      "step": 622
    },
    {
      "epoch": 0.36185165824475807,
      "grad_norm": 293.77130126953125,
      "learning_rate": 4.095002905287624e-06,
      "logits/chosen": -0.6701300740242004,
      "logits/rejected": -0.7274219393730164,
      "logps/chosen": -68.0548095703125,
      "logps/rejected": -68.81280517578125,
      "loss": 13.7275,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0037030756939202547,
      "rewards/margins": 0.027376368641853333,
      "rewards/rejected": -0.031079450622200966,
      "step": 623
    },
    {
      "epoch": 0.36243247952604984,
      "grad_norm": 288.8678894042969,
      "learning_rate": 4.093550261475887e-06,
      "logits/chosen": -0.7441659569740295,
      "logits/rejected": -0.7965889573097229,
      "logps/chosen": -70.99964904785156,
      "logps/rejected": -71.14028930664062,
      "loss": 13.362,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02267099916934967,
      "rewards/margins": 0.05958705022931099,
      "rewards/rejected": -0.03691605478525162,
      "step": 624
    },
    {
      "epoch": 0.36301330080734157,
      "grad_norm": 373.985595703125,
      "learning_rate": 4.0920976176641495e-06,
      "logits/chosen": -0.783255398273468,
      "logits/rejected": -0.7588286995887756,
      "logps/chosen": -74.08125305175781,
      "logps/rejected": -80.33110809326172,
      "loss": 13.2311,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.038171131163835526,
      "rewards/margins": 0.07666865736246109,
      "rewards/rejected": -0.03849751502275467,
      "step": 625
    },
    {
      "epoch": 0.36359412208863334,
      "grad_norm": 332.46234130859375,
      "learning_rate": 4.0906449738524114e-06,
      "logits/chosen": -0.6622897982597351,
      "logits/rejected": -0.5771545767784119,
      "logps/chosen": -78.99581909179688,
      "logps/rejected": -71.4215316772461,
      "loss": 14.3235,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07639651745557785,
      "rewards/margins": -0.0335925929248333,
      "rewards/rejected": -0.04280392453074455,
      "step": 626
    },
    {
      "epoch": 0.36417494336992506,
      "grad_norm": 339.90875244140625,
      "learning_rate": 4.089192330040674e-06,
      "logits/chosen": -0.7199238538742065,
      "logits/rejected": -0.7270562052726746,
      "logps/chosen": -76.53150939941406,
      "logps/rejected": -88.87334442138672,
      "loss": 13.3714,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010254562832415104,
      "rewards/margins": 0.06578972935676575,
      "rewards/rejected": -0.05553516745567322,
      "step": 627
    },
    {
      "epoch": 0.36475576465121684,
      "grad_norm": 318.9749755859375,
      "learning_rate": 4.087739686228937e-06,
      "logits/chosen": -0.6694357395172119,
      "logits/rejected": -0.7174801826477051,
      "logps/chosen": -69.3748779296875,
      "logps/rejected": -77.28684997558594,
      "loss": 14.3935,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.024402162060141563,
      "rewards/margins": -0.044836897403001785,
      "rewards/rejected": 0.020434733480215073,
      "step": 628
    },
    {
      "epoch": 0.36533658593250856,
      "grad_norm": 308.3586120605469,
      "learning_rate": 4.0862870424172e-06,
      "logits/chosen": -0.4848438799381256,
      "logits/rejected": -0.47951608896255493,
      "logps/chosen": -66.2980728149414,
      "logps/rejected": -80.6948471069336,
      "loss": 13.5319,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0488138273358345,
      "rewards/margins": 0.04704555124044418,
      "rewards/rejected": -0.09585938602685928,
      "step": 629
    },
    {
      "epoch": 0.36591740721380034,
      "grad_norm": 329.3526306152344,
      "learning_rate": 4.0848343986054624e-06,
      "logits/chosen": -0.5832070112228394,
      "logits/rejected": -0.6127845644950867,
      "logps/chosen": -77.78378295898438,
      "logps/rejected": -67.96507263183594,
      "loss": 14.4786,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00027630740078166127,
      "rewards/margins": -0.04388640448451042,
      "rewards/rejected": 0.04416271299123764,
      "step": 630
    },
    {
      "epoch": 0.36649822849509206,
      "grad_norm": 282.7289733886719,
      "learning_rate": 4.083381754793725e-06,
      "logits/chosen": -0.6684743165969849,
      "logits/rejected": -0.7016184329986572,
      "logps/chosen": -69.01554870605469,
      "logps/rejected": -68.71202087402344,
      "loss": 13.2268,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04856640845537186,
      "rewards/margins": 0.07856104522943497,
      "rewards/rejected": -0.029994633048772812,
      "step": 631
    },
    {
      "epoch": 0.3670790497763838,
      "grad_norm": 1237.3873291015625,
      "learning_rate": 4.081929110981988e-06,
      "logits/chosen": -0.5690406560897827,
      "logits/rejected": -0.6339157819747925,
      "logps/chosen": -79.32388305664062,
      "logps/rejected": -73.99163818359375,
      "loss": 13.3389,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02145705372095108,
      "rewards/margins": 0.06668312102556229,
      "rewards/rejected": -0.045226071029901505,
      "step": 632
    },
    {
      "epoch": 0.36765987105767556,
      "grad_norm": 309.4449157714844,
      "learning_rate": 4.08047646717025e-06,
      "logits/chosen": -0.7146767377853394,
      "logits/rejected": -0.7952632904052734,
      "logps/chosen": -71.60173797607422,
      "logps/rejected": -70.31050872802734,
      "loss": 13.4012,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01931237056851387,
      "rewards/margins": 0.07422088086605072,
      "rewards/rejected": -0.09353326261043549,
      "step": 633
    },
    {
      "epoch": 0.3682406923389673,
      "grad_norm": 321.6402282714844,
      "learning_rate": 4.0790238233585126e-06,
      "logits/chosen": -0.4627179503440857,
      "logits/rejected": -0.49417465925216675,
      "logps/chosen": -68.27827453613281,
      "logps/rejected": -68.9576187133789,
      "loss": 14.036,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0748579353094101,
      "rewards/margins": -0.010526349768042564,
      "rewards/rejected": -0.06433158367872238,
      "step": 634
    },
    {
      "epoch": 0.36882151362025906,
      "grad_norm": 315.15155029296875,
      "learning_rate": 4.077571179546775e-06,
      "logits/chosen": -0.4579242765903473,
      "logits/rejected": -0.4775335192680359,
      "logps/chosen": -66.7532958984375,
      "logps/rejected": -72.78587341308594,
      "loss": 13.4604,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.024356910958886147,
      "rewards/margins": 0.052337825298309326,
      "rewards/rejected": -0.07669473439455032,
      "step": 635
    },
    {
      "epoch": 0.3694023349015508,
      "grad_norm": 314.5369567871094,
      "learning_rate": 4.076118535735038e-06,
      "logits/chosen": -0.7437750101089478,
      "logits/rejected": -0.7035426497459412,
      "logps/chosen": -74.56098937988281,
      "logps/rejected": -82.19285583496094,
      "loss": 13.6072,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.027040233835577965,
      "rewards/margins": 0.03697334975004196,
      "rewards/rejected": -0.00993311870843172,
      "step": 636
    },
    {
      "epoch": 0.36998315618284255,
      "grad_norm": 316.7352294921875,
      "learning_rate": 4.074665891923301e-06,
      "logits/chosen": -0.7988698482513428,
      "logits/rejected": -0.6878986358642578,
      "logps/chosen": -68.0035629272461,
      "logps/rejected": -67.69490051269531,
      "loss": 13.5846,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0013492100406438112,
      "rewards/margins": 0.03704356402158737,
      "rewards/rejected": -0.038392774760723114,
      "step": 637
    },
    {
      "epoch": 0.3705639774641343,
      "grad_norm": 332.4465637207031,
      "learning_rate": 4.0732132481115636e-06,
      "logits/chosen": -0.432908833026886,
      "logits/rejected": -0.49958959221839905,
      "logps/chosen": -74.47289276123047,
      "logps/rejected": -73.33644104003906,
      "loss": 13.719,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03480667248368263,
      "rewards/margins": 0.02545056864619255,
      "rewards/rejected": 0.009356101974844933,
      "step": 638
    },
    {
      "epoch": 0.37114479874542605,
      "grad_norm": 306.9449768066406,
      "learning_rate": 4.0717606042998255e-06,
      "logits/chosen": -0.6738191843032837,
      "logits/rejected": -0.6642774343490601,
      "logps/chosen": -71.95086669921875,
      "logps/rejected": -81.12242126464844,
      "loss": 13.009,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.056968025863170624,
      "rewards/margins": 0.0969221368432045,
      "rewards/rejected": -0.039954110980033875,
      "step": 639
    },
    {
      "epoch": 0.3717256200267178,
      "grad_norm": 278.2969970703125,
      "learning_rate": 4.070307960488088e-06,
      "logits/chosen": -0.680871844291687,
      "logits/rejected": -0.6395547389984131,
      "logps/chosen": -64.48146057128906,
      "logps/rejected": -70.27690124511719,
      "loss": 12.5765,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04453757777810097,
      "rewards/margins": 0.14197415113449097,
      "rewards/rejected": -0.0974365621805191,
      "step": 640
    },
    {
      "epoch": 0.37230644130800955,
      "grad_norm": 297.1010437011719,
      "learning_rate": 4.068855316676351e-06,
      "logits/chosen": -0.6302939653396606,
      "logits/rejected": -0.7064687609672546,
      "logps/chosen": -71.5230941772461,
      "logps/rejected": -68.34951782226562,
      "loss": 12.9383,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04373275116086006,
      "rewards/margins": 0.10687211900949478,
      "rewards/rejected": -0.06313937157392502,
      "step": 641
    },
    {
      "epoch": 0.37288726258930127,
      "grad_norm": 310.2055969238281,
      "learning_rate": 4.067402672864614e-06,
      "logits/chosen": -0.8405004739761353,
      "logits/rejected": -0.8176850080490112,
      "logps/chosen": -71.03034210205078,
      "logps/rejected": -77.72269439697266,
      "loss": 13.2716,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.011298018507659435,
      "rewards/margins": 0.076210156083107,
      "rewards/rejected": -0.06491214036941528,
      "step": 642
    },
    {
      "epoch": 0.373468083870593,
      "grad_norm": 288.0509033203125,
      "learning_rate": 4.0659500290528765e-06,
      "logits/chosen": -0.807684063911438,
      "logits/rejected": -0.8896517753601074,
      "logps/chosen": -71.74691009521484,
      "logps/rejected": -67.34007263183594,
      "loss": 13.4595,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03239784389734268,
      "rewards/margins": 0.04948094114661217,
      "rewards/rejected": -0.017083102837204933,
      "step": 643
    },
    {
      "epoch": 0.37404890515188477,
      "grad_norm": 295.96917724609375,
      "learning_rate": 4.064497385241139e-06,
      "logits/chosen": -0.6759235858917236,
      "logits/rejected": -0.5782681703567505,
      "logps/chosen": -70.57847595214844,
      "logps/rejected": -74.41899108886719,
      "loss": 13.0928,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05720372125506401,
      "rewards/margins": 0.09038721024990082,
      "rewards/rejected": -0.03318347781896591,
      "step": 644
    },
    {
      "epoch": 0.3746297264331765,
      "grad_norm": 301.96588134765625,
      "learning_rate": 4.063044741429402e-06,
      "logits/chosen": -0.7978429198265076,
      "logits/rejected": -0.8465709686279297,
      "logps/chosen": -70.97563171386719,
      "logps/rejected": -74.35396575927734,
      "loss": 13.3645,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.030164528638124466,
      "rewards/margins": 0.06472036987543106,
      "rewards/rejected": -0.03455584496259689,
      "step": 645
    },
    {
      "epoch": 0.37521054771446827,
      "grad_norm": 307.4619140625,
      "learning_rate": 4.061592097617664e-06,
      "logits/chosen": -0.6767681837081909,
      "logits/rejected": -0.7364662885665894,
      "logps/chosen": -75.87703704833984,
      "logps/rejected": -73.02762603759766,
      "loss": 13.3096,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02578345499932766,
      "rewards/margins": 0.06501609086990356,
      "rewards/rejected": -0.039232634007930756,
      "step": 646
    },
    {
      "epoch": 0.37579136899576,
      "grad_norm": 302.466552734375,
      "learning_rate": 4.060139453805927e-06,
      "logits/chosen": -0.7271603941917419,
      "logits/rejected": -0.8793846368789673,
      "logps/chosen": -72.50651550292969,
      "logps/rejected": -75.49837493896484,
      "loss": 13.0485,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03700340539216995,
      "rewards/margins": 0.09631631523370743,
      "rewards/rejected": -0.05931291729211807,
      "step": 647
    },
    {
      "epoch": 0.37637219027705177,
      "grad_norm": 305.48046875,
      "learning_rate": 4.058686809994189e-06,
      "logits/chosen": -0.803033709526062,
      "logits/rejected": -0.7026882171630859,
      "logps/chosen": -72.45674133300781,
      "logps/rejected": -76.67012786865234,
      "loss": 12.9947,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.007967616431415081,
      "rewards/margins": 0.09880019724369049,
      "rewards/rejected": -0.09083259105682373,
      "step": 648
    },
    {
      "epoch": 0.3769530115583435,
      "grad_norm": 332.3240051269531,
      "learning_rate": 4.057234166182452e-06,
      "logits/chosen": -0.6038522720336914,
      "logits/rejected": -0.555856466293335,
      "logps/chosen": -76.23857116699219,
      "logps/rejected": -73.07332611083984,
      "loss": 13.5949,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.037921372801065445,
      "rewards/margins": 0.046288467943668365,
      "rewards/rejected": -0.008367091417312622,
      "step": 649
    },
    {
      "epoch": 0.37753383283963526,
      "grad_norm": 415.86383056640625,
      "learning_rate": 4.055781522370715e-06,
      "logits/chosen": -0.7161723375320435,
      "logits/rejected": -0.7207925915718079,
      "logps/chosen": -68.08644104003906,
      "logps/rejected": -69.88211059570312,
      "loss": 13.6038,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.022032026201486588,
      "rewards/margins": 0.038105495274066925,
      "rewards/rejected": -0.016073474660515785,
      "step": 650
    },
    {
      "epoch": 0.378114654120927,
      "grad_norm": 314.11492919921875,
      "learning_rate": 4.054328878558978e-06,
      "logits/chosen": -0.7593899369239807,
      "logits/rejected": -0.7028077840805054,
      "logps/chosen": -72.53091430664062,
      "logps/rejected": -77.84877014160156,
      "loss": 13.655,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016945505514740944,
      "rewards/margins": 0.02826940454542637,
      "rewards/rejected": -0.011323900893330574,
      "step": 651
    },
    {
      "epoch": 0.37869547540221876,
      "grad_norm": 311.8132019042969,
      "learning_rate": 4.05287623474724e-06,
      "logits/chosen": -0.7351340651512146,
      "logits/rejected": -0.7461525797843933,
      "logps/chosen": -79.3344497680664,
      "logps/rejected": -68.52467346191406,
      "loss": 13.3454,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.013737152330577374,
      "rewards/margins": 0.07491800934076309,
      "rewards/rejected": -0.08865516632795334,
      "step": 652
    },
    {
      "epoch": 0.3792762966835105,
      "grad_norm": 321.1096496582031,
      "learning_rate": 4.051423590935503e-06,
      "logits/chosen": -0.6296363472938538,
      "logits/rejected": -0.6592835783958435,
      "logps/chosen": -74.88213348388672,
      "logps/rejected": -78.33106231689453,
      "loss": 13.6806,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.038465339690446854,
      "rewards/margins": 0.029468858614563942,
      "rewards/rejected": -0.06793420016765594,
      "step": 653
    },
    {
      "epoch": 0.3798571179648022,
      "grad_norm": 299.4139404296875,
      "learning_rate": 4.049970947123766e-06,
      "logits/chosen": -0.7071855664253235,
      "logits/rejected": -0.749565839767456,
      "logps/chosen": -75.56929016113281,
      "logps/rejected": -70.52806854248047,
      "loss": 13.3743,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03643953427672386,
      "rewards/margins": 0.06584478914737701,
      "rewards/rejected": -0.10228432714939117,
      "step": 654
    },
    {
      "epoch": 0.380437939246094,
      "grad_norm": 334.07879638671875,
      "learning_rate": 4.048518303312029e-06,
      "logits/chosen": -0.68670654296875,
      "logits/rejected": -0.893659234046936,
      "logps/chosen": -72.4461898803711,
      "logps/rejected": -86.50956726074219,
      "loss": 13.315,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.010635538958013058,
      "rewards/margins": 0.09464681148529053,
      "rewards/rejected": -0.08401128649711609,
      "step": 655
    },
    {
      "epoch": 0.3810187605273857,
      "grad_norm": 315.373779296875,
      "learning_rate": 4.047065659500291e-06,
      "logits/chosen": -0.6514602899551392,
      "logits/rejected": -0.7396805286407471,
      "logps/chosen": -77.65937805175781,
      "logps/rejected": -70.10638427734375,
      "loss": 14.1567,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06592129915952682,
      "rewards/margins": -0.020820606499910355,
      "rewards/rejected": -0.04510069265961647,
      "step": 656
    },
    {
      "epoch": 0.3815995818086775,
      "grad_norm": 321.6916198730469,
      "learning_rate": 4.045613015688554e-06,
      "logits/chosen": -0.7136542201042175,
      "logits/rejected": -0.7592549920082092,
      "logps/chosen": -76.73454284667969,
      "logps/rejected": -73.61782836914062,
      "loss": 13.8061,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009097671136260033,
      "rewards/margins": 0.025288304314017296,
      "rewards/rejected": -0.03438597545027733,
      "step": 657
    },
    {
      "epoch": 0.3821804030899692,
      "grad_norm": 296.6112365722656,
      "learning_rate": 4.044160371876816e-06,
      "logits/chosen": -0.6855738759040833,
      "logits/rejected": -0.5937049984931946,
      "logps/chosen": -68.712158203125,
      "logps/rejected": -74.79548645019531,
      "loss": 13.4851,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006267908029258251,
      "rewards/margins": 0.046390462666749954,
      "rewards/rejected": -0.05265836790204048,
      "step": 658
    },
    {
      "epoch": 0.382761224371261,
      "grad_norm": 322.6234130859375,
      "learning_rate": 4.042707728065079e-06,
      "logits/chosen": -0.8190171122550964,
      "logits/rejected": -0.8573258519172668,
      "logps/chosen": -72.13541412353516,
      "logps/rejected": -65.77295684814453,
      "loss": 14.5607,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.004783238284289837,
      "rewards/margins": -0.06286787241697311,
      "rewards/rejected": 0.06765110790729523,
      "step": 659
    },
    {
      "epoch": 0.3833420456525527,
      "grad_norm": 326.6965026855469,
      "learning_rate": 4.0412550842533415e-06,
      "logits/chosen": -0.6807196736335754,
      "logits/rejected": -0.7124016284942627,
      "logps/chosen": -80.24552917480469,
      "logps/rejected": -75.31925964355469,
      "loss": 14.4393,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.10739662498235703,
      "rewards/margins": -0.041134439408779144,
      "rewards/rejected": -0.06626218557357788,
      "step": 660
    },
    {
      "epoch": 0.3839228669338445,
      "grad_norm": 310.8951721191406,
      "learning_rate": 4.039802440441604e-06,
      "logits/chosen": -0.7667916417121887,
      "logits/rejected": -0.7749382257461548,
      "logps/chosen": -78.43385314941406,
      "logps/rejected": -76.484130859375,
      "loss": 14.1133,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.033638693392276764,
      "rewards/margins": -0.00871949177235365,
      "rewards/rejected": -0.024919208139181137,
      "step": 661
    },
    {
      "epoch": 0.3845036882151362,
      "grad_norm": 332.6522216796875,
      "learning_rate": 4.038349796629867e-06,
      "logits/chosen": -0.6729914546012878,
      "logits/rejected": -0.8632933497428894,
      "logps/chosen": -80.35395812988281,
      "logps/rejected": -80.90164947509766,
      "loss": 13.6627,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.049274541437625885,
      "rewards/margins": 0.03929256647825241,
      "rewards/rejected": -0.0885671079158783,
      "step": 662
    },
    {
      "epoch": 0.385084509496428,
      "grad_norm": 323.0083312988281,
      "learning_rate": 4.03689715281813e-06,
      "logits/chosen": -0.9102508425712585,
      "logits/rejected": -0.8042302131652832,
      "logps/chosen": -72.66389465332031,
      "logps/rejected": -71.84483337402344,
      "loss": 14.1066,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.07181964069604874,
      "rewards/margins": -0.017825614660978317,
      "rewards/rejected": -0.05399402976036072,
      "step": 663
    },
    {
      "epoch": 0.3856653307777197,
      "grad_norm": 340.7964782714844,
      "learning_rate": 4.0354445090063925e-06,
      "logits/chosen": -0.7347263097763062,
      "logits/rejected": -0.774405300617218,
      "logps/chosen": -86.1633529663086,
      "logps/rejected": -79.38214874267578,
      "loss": 14.2786,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.020557690411806107,
      "rewards/margins": -0.019774410873651505,
      "rewards/rejected": -0.0007832825067453086,
      "step": 664
    },
    {
      "epoch": 0.3862461520590114,
      "grad_norm": 345.67315673828125,
      "learning_rate": 4.033991865194654e-06,
      "logits/chosen": -0.6604214906692505,
      "logits/rejected": -0.7178537249565125,
      "logps/chosen": -72.80216979980469,
      "logps/rejected": -72.74732971191406,
      "loss": 14.3951,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07244611531496048,
      "rewards/margins": -0.021248834207654,
      "rewards/rejected": -0.05119727924466133,
      "step": 665
    },
    {
      "epoch": 0.3868269733403032,
      "grad_norm": 332.7640686035156,
      "learning_rate": 4.032539221382917e-06,
      "logits/chosen": -0.7640831470489502,
      "logits/rejected": -0.7370232343673706,
      "logps/chosen": -70.40046691894531,
      "logps/rejected": -69.4461898803711,
      "loss": 13.6271,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.01473932433873415,
      "rewards/margins": 0.044324107468128204,
      "rewards/rejected": -0.05906342715024948,
      "step": 666
    },
    {
      "epoch": 0.3874077946215949,
      "grad_norm": 334.3003845214844,
      "learning_rate": 4.03108657757118e-06,
      "logits/chosen": -0.7360302209854126,
      "logits/rejected": -0.7869777679443359,
      "logps/chosen": -85.70533752441406,
      "logps/rejected": -72.21440887451172,
      "loss": 13.4059,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04054093360900879,
      "rewards/margins": 0.05683417245745659,
      "rewards/rejected": -0.016293242573738098,
      "step": 667
    },
    {
      "epoch": 0.3879886159028867,
      "grad_norm": 288.31170654296875,
      "learning_rate": 4.029633933759443e-06,
      "logits/chosen": -0.7544078826904297,
      "logits/rejected": -0.7067466974258423,
      "logps/chosen": -72.47239685058594,
      "logps/rejected": -64.64818572998047,
      "loss": 12.8853,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06467044353485107,
      "rewards/margins": 0.11924330145120621,
      "rewards/rejected": -0.05457286164164543,
      "step": 668
    },
    {
      "epoch": 0.3885694371841784,
      "grad_norm": 319.8020935058594,
      "learning_rate": 4.028181289947705e-06,
      "logits/chosen": -0.51985102891922,
      "logits/rejected": -0.6410090327262878,
      "logps/chosen": -72.35859680175781,
      "logps/rejected": -74.97998809814453,
      "loss": 13.5131,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.022069012746214867,
      "rewards/margins": 0.04354357719421387,
      "rewards/rejected": -0.021474560722708702,
      "step": 669
    },
    {
      "epoch": 0.3891502584654702,
      "grad_norm": 306.6014709472656,
      "learning_rate": 4.026728646135968e-06,
      "logits/chosen": -0.6216963529586792,
      "logits/rejected": -0.6753697395324707,
      "logps/chosen": -77.22586822509766,
      "logps/rejected": -70.61882019042969,
      "loss": 14.0923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01738358661532402,
      "rewards/margins": -0.0009111147373914719,
      "rewards/rejected": -0.01647247187793255,
      "step": 670
    },
    {
      "epoch": 0.3897310797467619,
      "grad_norm": 307.93670654296875,
      "learning_rate": 4.02527600232423e-06,
      "logits/chosen": -0.7877734303474426,
      "logits/rejected": -0.8353005647659302,
      "logps/chosen": -79.74364471435547,
      "logps/rejected": -78.38971710205078,
      "loss": 13.4594,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.013755464926362038,
      "rewards/margins": 0.05986330658197403,
      "rewards/rejected": -0.07361876964569092,
      "step": 671
    },
    {
      "epoch": 0.3903119010280537,
      "grad_norm": 317.7315368652344,
      "learning_rate": 4.023823358512493e-06,
      "logits/chosen": -0.6735803484916687,
      "logits/rejected": -0.732440173625946,
      "logps/chosen": -76.98451232910156,
      "logps/rejected": -80.71636962890625,
      "loss": 13.8575,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008590728044509888,
      "rewards/margins": 0.008975962176918983,
      "rewards/rejected": -0.01756669022142887,
      "step": 672
    },
    {
      "epoch": 0.3908927223093454,
      "grad_norm": 311.2159729003906,
      "learning_rate": 4.0223707147007555e-06,
      "logits/chosen": -0.7430940866470337,
      "logits/rejected": -0.7209886312484741,
      "logps/chosen": -72.24745178222656,
      "logps/rejected": -72.40113830566406,
      "loss": 14.7028,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05548017472028732,
      "rewards/margins": -0.061144404113292694,
      "rewards/rejected": 0.005664219614118338,
      "step": 673
    },
    {
      "epoch": 0.3914735435906372,
      "grad_norm": 308.31805419921875,
      "learning_rate": 4.020918070889018e-06,
      "logits/chosen": -0.5726041793823242,
      "logits/rejected": -0.6990719437599182,
      "logps/chosen": -68.8240737915039,
      "logps/rejected": -69.23278045654297,
      "loss": 14.2499,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0369163416326046,
      "rewards/margins": -0.02730955183506012,
      "rewards/rejected": -0.009606788866221905,
      "step": 674
    },
    {
      "epoch": 0.3920543648719289,
      "grad_norm": 313.16650390625,
      "learning_rate": 4.019465427077281e-06,
      "logits/chosen": -0.6481889486312866,
      "logits/rejected": -0.6497616171836853,
      "logps/chosen": -73.71761322021484,
      "logps/rejected": -70.89939880371094,
      "loss": 13.6169,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017801934853196144,
      "rewards/margins": 0.03877174109220505,
      "rewards/rejected": -0.02096981182694435,
      "step": 675
    },
    {
      "epoch": 0.39263518615322063,
      "grad_norm": 313.9520263671875,
      "learning_rate": 4.018012783265544e-06,
      "logits/chosen": -0.7253094911575317,
      "logits/rejected": -0.6935799717903137,
      "logps/chosen": -74.28784942626953,
      "logps/rejected": -76.39752197265625,
      "loss": 14.0332,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01669791154563427,
      "rewards/margins": -0.0043237642385065556,
      "rewards/rejected": -0.012374145910143852,
      "step": 676
    },
    {
      "epoch": 0.3932160074345124,
      "grad_norm": 327.840576171875,
      "learning_rate": 4.0165601394538065e-06,
      "logits/chosen": -0.5964494943618774,
      "logits/rejected": -0.8218010663986206,
      "logps/chosen": -76.28643035888672,
      "logps/rejected": -73.67375183105469,
      "loss": 13.48,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00023033171601127833,
      "rewards/margins": 0.07958341389894485,
      "rewards/rejected": -0.07935307919979095,
      "step": 677
    },
    {
      "epoch": 0.3937968287158041,
      "grad_norm": 469.1325378417969,
      "learning_rate": 4.015107495642068e-06,
      "logits/chosen": -0.7487168312072754,
      "logits/rejected": -0.7056422829627991,
      "logps/chosen": -71.22139739990234,
      "logps/rejected": -77.16386413574219,
      "loss": 13.3125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.013796500861644745,
      "rewards/margins": 0.06354296207427979,
      "rewards/rejected": -0.04974645376205444,
      "step": 678
    },
    {
      "epoch": 0.3943776499970959,
      "grad_norm": 298.0015869140625,
      "learning_rate": 4.013654851830331e-06,
      "logits/chosen": -0.7079442739486694,
      "logits/rejected": -0.9590281248092651,
      "logps/chosen": -74.9395980834961,
      "logps/rejected": -75.6585922241211,
      "loss": 12.8306,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03988940268754959,
      "rewards/margins": 0.14326909184455872,
      "rewards/rejected": -0.10337970405817032,
      "step": 679
    },
    {
      "epoch": 0.3949584712783876,
      "grad_norm": 318.6708984375,
      "learning_rate": 4.012202208018594e-06,
      "logits/chosen": -0.5922810435295105,
      "logits/rejected": -0.5826822519302368,
      "logps/chosen": -72.672119140625,
      "logps/rejected": -74.83512115478516,
      "loss": 13.6615,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.07369053363800049,
      "rewards/margins": 0.028689509257674217,
      "rewards/rejected": 0.045001011341810226,
      "step": 680
    },
    {
      "epoch": 0.3955392925596794,
      "grad_norm": 296.9277648925781,
      "learning_rate": 4.010749564206857e-06,
      "logits/chosen": -0.6584534049034119,
      "logits/rejected": -0.6963584423065186,
      "logps/chosen": -72.95672607421875,
      "logps/rejected": -71.49995422363281,
      "loss": 13.4348,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.031108522787690163,
      "rewards/margins": 0.05346295237541199,
      "rewards/rejected": -0.022354427725076675,
      "step": 681
    },
    {
      "epoch": 0.3961201138409711,
      "grad_norm": 296.8775329589844,
      "learning_rate": 4.009296920395119e-06,
      "logits/chosen": -0.7727608680725098,
      "logits/rejected": -0.753118634223938,
      "logps/chosen": -75.11225128173828,
      "logps/rejected": -68.10901641845703,
      "loss": 13.6226,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.010256567969918251,
      "rewards/margins": 0.03369814157485962,
      "rewards/rejected": -0.023441573604941368,
      "step": 682
    },
    {
      "epoch": 0.3967009351222629,
      "grad_norm": 304.0804748535156,
      "learning_rate": 4.007844276583382e-06,
      "logits/chosen": -0.6444306373596191,
      "logits/rejected": -0.7014695405960083,
      "logps/chosen": -67.61308288574219,
      "logps/rejected": -67.55976104736328,
      "loss": 14.3106,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.028560440987348557,
      "rewards/margins": -0.03599892184138298,
      "rewards/rejected": 0.007438488304615021,
      "step": 683
    },
    {
      "epoch": 0.3972817564035546,
      "grad_norm": 345.8822021484375,
      "learning_rate": 4.006391632771645e-06,
      "logits/chosen": -0.5845493078231812,
      "logits/rejected": -0.6856909990310669,
      "logps/chosen": -68.45945739746094,
      "logps/rejected": -67.73657989501953,
      "loss": 13.9901,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.026378994807600975,
      "rewards/margins": 0.004468211438506842,
      "rewards/rejected": -0.03084721229970455,
      "step": 684
    },
    {
      "epoch": 0.3978625776848464,
      "grad_norm": 308.62628173828125,
      "learning_rate": 4.004938988959907e-06,
      "logits/chosen": -0.6908701062202454,
      "logits/rejected": -0.7285133600234985,
      "logps/chosen": -68.93060302734375,
      "logps/rejected": -76.362548828125,
      "loss": 13.9869,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.020935364067554474,
      "rewards/margins": 0.005568015389144421,
      "rewards/rejected": -0.02650338038802147,
      "step": 685
    },
    {
      "epoch": 0.3984433989661381,
      "grad_norm": 307.853271484375,
      "learning_rate": 4.0034863451481696e-06,
      "logits/chosen": -0.6336562633514404,
      "logits/rejected": -0.601762592792511,
      "logps/chosen": -73.2461929321289,
      "logps/rejected": -64.52156066894531,
      "loss": 13.7892,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.021032018586993217,
      "rewards/margins": 0.01943250373005867,
      "rewards/rejected": 0.0015995114808902144,
      "step": 686
    },
    {
      "epoch": 0.39902422024742984,
      "grad_norm": 304.0710754394531,
      "learning_rate": 4.002033701336432e-06,
      "logits/chosen": -0.8108295202255249,
      "logits/rejected": -0.8165252804756165,
      "logps/chosen": -68.44810485839844,
      "logps/rejected": -71.20729064941406,
      "loss": 13.428,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06437697261571884,
      "rewards/margins": 0.05376725271344185,
      "rewards/rejected": 0.010609723627567291,
      "step": 687
    },
    {
      "epoch": 0.3996050415287216,
      "grad_norm": 302.7625732421875,
      "learning_rate": 4.000581057524695e-06,
      "logits/chosen": -0.7618246078491211,
      "logits/rejected": -0.7047253251075745,
      "logps/chosen": -69.78257751464844,
      "logps/rejected": -78.85441589355469,
      "loss": 13.4723,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.013262261636555195,
      "rewards/margins": 0.05348924919962883,
      "rewards/rejected": -0.06675150990486145,
      "step": 688
    },
    {
      "epoch": 0.40018586281001334,
      "grad_norm": 326.1325988769531,
      "learning_rate": 3.999128413712958e-06,
      "logits/chosen": -0.7000328302383423,
      "logits/rejected": -0.7599672675132751,
      "logps/chosen": -79.16525268554688,
      "logps/rejected": -72.787109375,
      "loss": 14.4071,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010734537616372108,
      "rewards/margins": -0.04592505842447281,
      "rewards/rejected": 0.03519051522016525,
      "step": 689
    },
    {
      "epoch": 0.4007666840913051,
      "grad_norm": 398.7939758300781,
      "learning_rate": 3.9976757699012206e-06,
      "logits/chosen": -0.6336180567741394,
      "logits/rejected": -0.5584805607795715,
      "logps/chosen": -79.39990234375,
      "logps/rejected": -70.52845764160156,
      "loss": 13.6773,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02765648439526558,
      "rewards/margins": 0.03300664573907852,
      "rewards/rejected": -0.005350158549845219,
      "step": 690
    },
    {
      "epoch": 0.40134750537259684,
      "grad_norm": 316.0892028808594,
      "learning_rate": 3.996223126089483e-06,
      "logits/chosen": -0.7683295011520386,
      "logits/rejected": -0.5641528964042664,
      "logps/chosen": -81.93292236328125,
      "logps/rejected": -74.35237121582031,
      "loss": 14.2073,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.032339874655008316,
      "rewards/margins": -0.020309090614318848,
      "rewards/rejected": -0.01203078217804432,
      "step": 691
    },
    {
      "epoch": 0.4019283266538886,
      "grad_norm": 322.30126953125,
      "learning_rate": 3.994770482277745e-06,
      "logits/chosen": -0.7082656025886536,
      "logits/rejected": -0.7538729906082153,
      "logps/chosen": -74.92417907714844,
      "logps/rejected": -71.00489807128906,
      "loss": 14.2258,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.02695903740823269,
      "rewards/margins": -0.026692455634474754,
      "rewards/rejected": -0.0002665802894625813,
      "step": 692
    },
    {
      "epoch": 0.40250914793518033,
      "grad_norm": 325.52874755859375,
      "learning_rate": 3.993317838466009e-06,
      "logits/chosen": -0.7899920344352722,
      "logits/rejected": -0.846684455871582,
      "logps/chosen": -75.40380859375,
      "logps/rejected": -71.71354675292969,
      "loss": 13.1936,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.058164648711681366,
      "rewards/margins": 0.08477498590946198,
      "rewards/rejected": -0.026610326021909714,
      "step": 693
    },
    {
      "epoch": 0.4030899692164721,
      "grad_norm": 336.93310546875,
      "learning_rate": 3.9918651946542715e-06,
      "logits/chosen": -0.6738449335098267,
      "logits/rejected": -0.6702437400817871,
      "logps/chosen": -68.76301574707031,
      "logps/rejected": -81.96266174316406,
      "loss": 12.6865,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04724300652742386,
      "rewards/margins": 0.13397014141082764,
      "rewards/rejected": -0.08672711998224258,
      "step": 694
    },
    {
      "epoch": 0.40367079049776383,
      "grad_norm": 276.9514465332031,
      "learning_rate": 3.990412550842534e-06,
      "logits/chosen": -0.7575210332870483,
      "logits/rejected": -0.6729222536087036,
      "logps/chosen": -70.20577239990234,
      "logps/rejected": -74.03892517089844,
      "loss": 13.151,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.061166275292634964,
      "rewards/margins": 0.08962966501712799,
      "rewards/rejected": -0.02846338413655758,
      "step": 695
    },
    {
      "epoch": 0.4042516117790556,
      "grad_norm": 319.8224792480469,
      "learning_rate": 3.988959907030796e-06,
      "logits/chosen": -0.575911819934845,
      "logits/rejected": -0.5608241558074951,
      "logps/chosen": -70.46253967285156,
      "logps/rejected": -82.12687683105469,
      "loss": 13.5207,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.048672787845134735,
      "rewards/margins": 0.048407696187496185,
      "rewards/rejected": 0.0002650946262292564,
      "step": 696
    },
    {
      "epoch": 0.40483243306034733,
      "grad_norm": 325.2471618652344,
      "learning_rate": 3.987507263219059e-06,
      "logits/chosen": -0.6701699495315552,
      "logits/rejected": -0.6926618218421936,
      "logps/chosen": -79.00935363769531,
      "logps/rejected": -78.64802551269531,
      "loss": 13.9549,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.009344990365207195,
      "rewards/margins": -0.002072087023407221,
      "rewards/rejected": 0.01141707319766283,
      "step": 697
    },
    {
      "epoch": 0.40541325434163905,
      "grad_norm": 319.7401428222656,
      "learning_rate": 3.986054619407322e-06,
      "logits/chosen": -0.606377124786377,
      "logits/rejected": -0.6375631093978882,
      "logps/chosen": -74.11344909667969,
      "logps/rejected": -65.52316284179688,
      "loss": 13.5328,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.07139308750629425,
      "rewards/margins": 0.043192584067583084,
      "rewards/rejected": 0.02820049785077572,
      "step": 698
    },
    {
      "epoch": 0.40599407562293083,
      "grad_norm": 290.7586669921875,
      "learning_rate": 3.9846019755955844e-06,
      "logits/chosen": -0.6408273577690125,
      "logits/rejected": -0.6153540015220642,
      "logps/chosen": -74.5358657836914,
      "logps/rejected": -74.61564636230469,
      "loss": 13.6657,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011930713430047035,
      "rewards/margins": 0.03009539470076561,
      "rewards/rejected": -0.018164681270718575,
      "step": 699
    },
    {
      "epoch": 0.40657489690422255,
      "grad_norm": 324.70916748046875,
      "learning_rate": 3.983149331783847e-06,
      "logits/chosen": -0.6709033250808716,
      "logits/rejected": -0.7083495259284973,
      "logps/chosen": -77.2063980102539,
      "logps/rejected": -75.69718170166016,
      "loss": 13.9698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004046584479510784,
      "rewards/margins": 0.004778122063726187,
      "rewards/rejected": -0.0007315344992093742,
      "step": 700
    },
    {
      "epoch": 0.4071557181855143,
      "grad_norm": 308.9340515136719,
      "learning_rate": 3.98169668797211e-06,
      "logits/chosen": -0.603878378868103,
      "logits/rejected": -0.657692015171051,
      "logps/chosen": -74.08457946777344,
      "logps/rejected": -67.98051452636719,
      "loss": 13.4656,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.017448043450713158,
      "rewards/margins": 0.04989555850625038,
      "rewards/rejected": -0.03244751691818237,
      "step": 701
    },
    {
      "epoch": 0.40773653946680605,
      "grad_norm": 304.8455505371094,
      "learning_rate": 3.980244044160373e-06,
      "logits/chosen": -0.6844228506088257,
      "logits/rejected": -0.7053964138031006,
      "logps/chosen": -69.97819519042969,
      "logps/rejected": -72.03315734863281,
      "loss": 13.7659,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02907332219183445,
      "rewards/margins": 0.025167405605316162,
      "rewards/rejected": 0.003905917750671506,
      "step": 702
    },
    {
      "epoch": 0.4083173607480978,
      "grad_norm": 308.1776428222656,
      "learning_rate": 3.978791400348635e-06,
      "logits/chosen": -0.5791782140731812,
      "logits/rejected": -0.646981954574585,
      "logps/chosen": -82.76728820800781,
      "logps/rejected": -72.67459869384766,
      "loss": 13.6071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.035702355206012726,
      "rewards/margins": 0.03648295998573303,
      "rewards/rejected": -0.0007806119392625988,
      "step": 703
    },
    {
      "epoch": 0.40889818202938955,
      "grad_norm": 313.1890869140625,
      "learning_rate": 3.977338756536897e-06,
      "logits/chosen": -0.7870761156082153,
      "logits/rejected": -0.7041381001472473,
      "logps/chosen": -74.79627227783203,
      "logps/rejected": -77.46690368652344,
      "loss": 13.0616,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.07957975566387177,
      "rewards/margins": 0.09422776103019714,
      "rewards/rejected": -0.014648010022938251,
      "step": 704
    },
    {
      "epoch": 0.4094790033106813,
      "grad_norm": 324.8765869140625,
      "learning_rate": 3.97588611272516e-06,
      "logits/chosen": -0.8196079134941101,
      "logits/rejected": -0.7677024602890015,
      "logps/chosen": -68.56859588623047,
      "logps/rejected": -76.06591033935547,
      "loss": 13.521,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03140668943524361,
      "rewards/margins": 0.04369328171014786,
      "rewards/rejected": -0.01228659600019455,
      "step": 705
    },
    {
      "epoch": 0.41005982459197304,
      "grad_norm": 334.3929138183594,
      "learning_rate": 3.974433468913423e-06,
      "logits/chosen": -0.6489372849464417,
      "logits/rejected": -0.6309608221054077,
      "logps/chosen": -76.56379699707031,
      "logps/rejected": -82.5246353149414,
      "loss": 14.4875,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03186134248971939,
      "rewards/margins": -0.04680603742599487,
      "rewards/rejected": 0.014944696798920631,
      "step": 706
    },
    {
      "epoch": 0.4106406458732648,
      "grad_norm": 277.21441650390625,
      "learning_rate": 3.972980825101686e-06,
      "logits/chosen": -0.6851536631584167,
      "logits/rejected": -0.6442720890045166,
      "logps/chosen": -71.58158874511719,
      "logps/rejected": -65.97663116455078,
      "loss": 12.4476,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.09448430687189102,
      "rewards/margins": 0.1556243896484375,
      "rewards/rejected": -0.06114007905125618,
      "step": 707
    },
    {
      "epoch": 0.41122146715455654,
      "grad_norm": 304.2071533203125,
      "learning_rate": 3.971528181289948e-06,
      "logits/chosen": -0.6629344820976257,
      "logits/rejected": -0.6967271566390991,
      "logps/chosen": -73.93403625488281,
      "logps/rejected": -69.1995849609375,
      "loss": 13.107,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.025395860895514488,
      "rewards/margins": 0.08553169667720795,
      "rewards/rejected": -0.06013583019375801,
      "step": 708
    },
    {
      "epoch": 0.41180228843584826,
      "grad_norm": 300.8368835449219,
      "learning_rate": 3.970075537478211e-06,
      "logits/chosen": -0.5319725871086121,
      "logits/rejected": -0.6029818654060364,
      "logps/chosen": -74.1766128540039,
      "logps/rejected": -71.83555603027344,
      "loss": 13.7521,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.042827047407627106,
      "rewards/margins": 0.020685147494077682,
      "rewards/rejected": 0.022141892462968826,
      "step": 709
    },
    {
      "epoch": 0.41238310971714004,
      "grad_norm": 386.9114685058594,
      "learning_rate": 3.968622893666473e-06,
      "logits/chosen": -0.7132548093795776,
      "logits/rejected": -0.5851150751113892,
      "logps/chosen": -79.25279235839844,
      "logps/rejected": -73.51028442382812,
      "loss": 14.7161,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.05419192835688591,
      "rewards/margins": -0.07584583759307861,
      "rewards/rejected": 0.021653901785612106,
      "step": 710
    },
    {
      "epoch": 0.41296393099843176,
      "grad_norm": 322.0220947265625,
      "learning_rate": 3.967170249854736e-06,
      "logits/chosen": -0.6925621628761292,
      "logits/rejected": -0.7238684892654419,
      "logps/chosen": -75.48567199707031,
      "logps/rejected": -68.52146911621094,
      "loss": 13.8128,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.054952871054410934,
      "rewards/margins": 0.009693034924566746,
      "rewards/rejected": 0.045259833335876465,
      "step": 711
    },
    {
      "epoch": 0.41354475227972354,
      "grad_norm": 332.9398498535156,
      "learning_rate": 3.9657176060429985e-06,
      "logits/chosen": -0.6430686712265015,
      "logits/rejected": -0.7087141275405884,
      "logps/chosen": -68.45174407958984,
      "logps/rejected": -67.28535461425781,
      "loss": 14.5686,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.015753645449876785,
      "rewards/margins": -0.05839390307664871,
      "rewards/rejected": 0.0741475448012352,
      "step": 712
    },
    {
      "epoch": 0.41412557356101526,
      "grad_norm": 313.2064514160156,
      "learning_rate": 3.964264962231261e-06,
      "logits/chosen": -0.6678661108016968,
      "logits/rejected": -0.6293448209762573,
      "logps/chosen": -81.44425201416016,
      "logps/rejected": -72.38615417480469,
      "loss": 13.5039,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.019158076494932175,
      "rewards/margins": 0.047387607395648956,
      "rewards/rejected": -0.028229529038071632,
      "step": 713
    },
    {
      "epoch": 0.41470639484230704,
      "grad_norm": 301.8228454589844,
      "learning_rate": 3.962812318419524e-06,
      "logits/chosen": -0.6319082975387573,
      "logits/rejected": -0.6024832129478455,
      "logps/chosen": -62.35749053955078,
      "logps/rejected": -71.47017669677734,
      "loss": 13.4994,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0663997158408165,
      "rewards/margins": 0.04610716551542282,
      "rewards/rejected": 0.02029253914952278,
      "step": 714
    },
    {
      "epoch": 0.41528721612359876,
      "grad_norm": 345.0201110839844,
      "learning_rate": 3.961359674607787e-06,
      "logits/chosen": -0.6447056531906128,
      "logits/rejected": -0.6276105642318726,
      "logps/chosen": -79.46293640136719,
      "logps/rejected": -75.56305694580078,
      "loss": 14.4818,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03468679264187813,
      "rewards/margins": -0.044044964015483856,
      "rewards/rejected": 0.009358169510960579,
      "step": 715
    },
    {
      "epoch": 0.41586803740489053,
      "grad_norm": 341.76220703125,
      "learning_rate": 3.9599070307960495e-06,
      "logits/chosen": -0.5557613372802734,
      "logits/rejected": -0.6789140701293945,
      "logps/chosen": -79.2572021484375,
      "logps/rejected": -89.22966003417969,
      "loss": 14.2231,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06551501154899597,
      "rewards/margins": -0.027714461088180542,
      "rewards/rejected": -0.03780054301023483,
      "step": 716
    },
    {
      "epoch": 0.41644885868618225,
      "grad_norm": 282.11700439453125,
      "learning_rate": 3.958454386984311e-06,
      "logits/chosen": -0.6314088106155396,
      "logits/rejected": -0.7064960598945618,
      "logps/chosen": -65.93815612792969,
      "logps/rejected": -75.64500427246094,
      "loss": 13.2297,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01477456372231245,
      "rewards/margins": 0.07601601630449295,
      "rewards/rejected": -0.06124146655201912,
      "step": 717
    },
    {
      "epoch": 0.41702967996747403,
      "grad_norm": 273.31231689453125,
      "learning_rate": 3.957001743172574e-06,
      "logits/chosen": -0.659504771232605,
      "logits/rejected": -0.8665167689323425,
      "logps/chosen": -73.95689392089844,
      "logps/rejected": -73.23133087158203,
      "loss": 12.612,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.08090134710073471,
      "rewards/margins": 0.15707240998744965,
      "rewards/rejected": -0.07617107778787613,
      "step": 718
    },
    {
      "epoch": 0.41761050124876575,
      "grad_norm": 314.1131591796875,
      "learning_rate": 3.955549099360837e-06,
      "logits/chosen": -0.6384779214859009,
      "logits/rejected": -0.6633267998695374,
      "logps/chosen": -71.73622131347656,
      "logps/rejected": -75.96559143066406,
      "loss": 13.3543,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05409403517842293,
      "rewards/margins": 0.060982413589954376,
      "rewards/rejected": -0.0068883770145475864,
      "step": 719
    },
    {
      "epoch": 0.4181913225300575,
      "grad_norm": 316.5398864746094,
      "learning_rate": 3.9540964555491e-06,
      "logits/chosen": -0.7581623196601868,
      "logits/rejected": -0.6908892393112183,
      "logps/chosen": -78.83635711669922,
      "logps/rejected": -73.35903930664062,
      "loss": 13.6177,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0010162651306018233,
      "rewards/margins": 0.038523249328136444,
      "rewards/rejected": -0.03750698268413544,
      "step": 720
    },
    {
      "epoch": 0.41877214381134925,
      "grad_norm": 289.4864807128906,
      "learning_rate": 3.952643811737362e-06,
      "logits/chosen": -0.6493207216262817,
      "logits/rejected": -0.6764460802078247,
      "logps/chosen": -75.92554473876953,
      "logps/rejected": -68.46549224853516,
      "loss": 14.2656,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07545448839664459,
      "rewards/margins": -0.029577041044831276,
      "rewards/rejected": -0.045877449214458466,
      "step": 721
    },
    {
      "epoch": 0.419352965092641,
      "grad_norm": 307.9372253417969,
      "learning_rate": 3.951191167925625e-06,
      "logits/chosen": -0.6669474244117737,
      "logits/rejected": -0.8115142583847046,
      "logps/chosen": -67.41337585449219,
      "logps/rejected": -64.86238098144531,
      "loss": 13.7911,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005310344509780407,
      "rewards/margins": 0.02083405666053295,
      "rewards/rejected": -0.015523704700171947,
      "step": 722
    },
    {
      "epoch": 0.41993378637393275,
      "grad_norm": 283.71624755859375,
      "learning_rate": 3.949738524113888e-06,
      "logits/chosen": -0.7168434858322144,
      "logits/rejected": -0.7291404008865356,
      "logps/chosen": -73.59349060058594,
      "logps/rejected": -68.3053207397461,
      "loss": 13.7698,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.002716714981943369,
      "rewards/margins": 0.01622510887682438,
      "rewards/rejected": -0.013508396223187447,
      "step": 723
    },
    {
      "epoch": 0.42051460765522447,
      "grad_norm": 349.4669494628906,
      "learning_rate": 3.94828588030215e-06,
      "logits/chosen": -0.7875105738639832,
      "logits/rejected": -0.7584863901138306,
      "logps/chosen": -72.06592559814453,
      "logps/rejected": -77.07530212402344,
      "loss": 14.4762,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.032033637166023254,
      "rewards/margins": -0.050473153591156006,
      "rewards/rejected": 0.0184395220130682,
      "step": 724
    },
    {
      "epoch": 0.42109542893651625,
      "grad_norm": 359.53485107421875,
      "learning_rate": 3.9468332364904125e-06,
      "logits/chosen": -0.6558941602706909,
      "logits/rejected": -0.5819476842880249,
      "logps/chosen": -68.26545715332031,
      "logps/rejected": -73.47137451171875,
      "loss": 14.0153,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.005019567906856537,
      "rewards/margins": 0.0016324628377333283,
      "rewards/rejected": 0.0033871070481836796,
      "step": 725
    },
    {
      "epoch": 0.42167625021780797,
      "grad_norm": 323.96868896484375,
      "learning_rate": 3.945380592678675e-06,
      "logits/chosen": -0.6153490543365479,
      "logits/rejected": -0.6308630704879761,
      "logps/chosen": -72.20024108886719,
      "logps/rejected": -67.79884338378906,
      "loss": 13.4744,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.002849989803507924,
      "rewards/margins": 0.07088102400302887,
      "rewards/rejected": -0.06803102791309357,
      "step": 726
    },
    {
      "epoch": 0.42225707149909975,
      "grad_norm": 448.19921875,
      "learning_rate": 3.943927948866938e-06,
      "logits/chosen": -0.7376095056533813,
      "logits/rejected": -0.6571551561355591,
      "logps/chosen": -76.57099151611328,
      "logps/rejected": -74.70809173583984,
      "loss": 14.4323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01626337133347988,
      "rewards/margins": -0.03361039236187935,
      "rewards/rejected": 0.01734701171517372,
      "step": 727
    },
    {
      "epoch": 0.42283789278039147,
      "grad_norm": 301.0350646972656,
      "learning_rate": 3.942475305055201e-06,
      "logits/chosen": -0.7029098272323608,
      "logits/rejected": -0.7672086358070374,
      "logps/chosen": -74.73590850830078,
      "logps/rejected": -72.44758605957031,
      "loss": 13.2734,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.021909218281507492,
      "rewards/margins": 0.06785444915294647,
      "rewards/rejected": -0.04594522714614868,
      "step": 728
    },
    {
      "epoch": 0.42341871406168324,
      "grad_norm": 323.43463134765625,
      "learning_rate": 3.9410226612434635e-06,
      "logits/chosen": -0.70557701587677,
      "logits/rejected": -0.7261152267456055,
      "logps/chosen": -77.63336181640625,
      "logps/rejected": -71.49602508544922,
      "loss": 14.8529,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05248032882809639,
      "rewards/margins": -0.09009293466806412,
      "rewards/rejected": 0.037612609565258026,
      "step": 729
    },
    {
      "epoch": 0.42399953534297496,
      "grad_norm": 330.1050720214844,
      "learning_rate": 3.939570017431726e-06,
      "logits/chosen": -0.5593830347061157,
      "logits/rejected": -0.6348336935043335,
      "logps/chosen": -84.4022445678711,
      "logps/rejected": -81.31836700439453,
      "loss": 13.0719,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0025830883532762527,
      "rewards/margins": 0.09452484548091888,
      "rewards/rejected": -0.09710792452096939,
      "step": 730
    },
    {
      "epoch": 0.4245803566242667,
      "grad_norm": 317.7475280761719,
      "learning_rate": 3.938117373619988e-06,
      "logits/chosen": -0.4515294134616852,
      "logits/rejected": -0.38896283507347107,
      "logps/chosen": -69.86441040039062,
      "logps/rejected": -78.91719055175781,
      "loss": 14.0736,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04327527433633804,
      "rewards/margins": -0.009526817128062248,
      "rewards/rejected": -0.033748455345630646,
      "step": 731
    },
    {
      "epoch": 0.42516117790555846,
      "grad_norm": 319.0084533691406,
      "learning_rate": 3.936664729808251e-06,
      "logits/chosen": -0.6351519823074341,
      "logits/rejected": -0.6803760528564453,
      "logps/chosen": -69.61072540283203,
      "logps/rejected": -71.7401351928711,
      "loss": 14.1246,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05607505515217781,
      "rewards/margins": -0.01812058314681053,
      "rewards/rejected": -0.03795447573065758,
      "step": 732
    },
    {
      "epoch": 0.4257419991868502,
      "grad_norm": 335.4173889160156,
      "learning_rate": 3.935212085996514e-06,
      "logits/chosen": -0.7924081087112427,
      "logits/rejected": -0.7546756267547607,
      "logps/chosen": -70.29083251953125,
      "logps/rejected": -76.52457427978516,
      "loss": 14.3631,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.028643876314163208,
      "rewards/margins": -0.033245109021663666,
      "rewards/rejected": 0.004601232707500458,
      "step": 733
    },
    {
      "epoch": 0.42632282046814196,
      "grad_norm": 302.1365051269531,
      "learning_rate": 3.933759442184776e-06,
      "logits/chosen": -0.7318800687789917,
      "logits/rejected": -0.7138211727142334,
      "logps/chosen": -74.42980194091797,
      "logps/rejected": -76.93816375732422,
      "loss": 14.0023,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.018741881474852562,
      "rewards/margins": -0.004794149659574032,
      "rewards/rejected": -0.013947735540568829,
      "step": 734
    },
    {
      "epoch": 0.4269036417494337,
      "grad_norm": 304.879638671875,
      "learning_rate": 3.932306798373039e-06,
      "logits/chosen": -0.6316365003585815,
      "logits/rejected": -0.6677497029304504,
      "logps/chosen": -71.40536499023438,
      "logps/rejected": -72.45552825927734,
      "loss": 13.5122,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06338571012020111,
      "rewards/margins": 0.04115435481071472,
      "rewards/rejected": 0.02223135158419609,
      "step": 735
    },
    {
      "epoch": 0.42748446303072546,
      "grad_norm": 298.0213317871094,
      "learning_rate": 3.930854154561302e-06,
      "logits/chosen": -0.7024403214454651,
      "logits/rejected": -0.7682080864906311,
      "logps/chosen": -71.77532196044922,
      "logps/rejected": -69.44412994384766,
      "loss": 13.3706,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03764911741018295,
      "rewards/margins": 0.06105116754770279,
      "rewards/rejected": -0.023402050137519836,
      "step": 736
    },
    {
      "epoch": 0.4280652843120172,
      "grad_norm": 280.1487121582031,
      "learning_rate": 3.929401510749565e-06,
      "logits/chosen": -0.9538941383361816,
      "logits/rejected": -0.8766900300979614,
      "logps/chosen": -66.9570541381836,
      "logps/rejected": -72.75699615478516,
      "loss": 12.845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06486214697360992,
      "rewards/margins": 0.11776605993509293,
      "rewards/rejected": -0.05290389060974121,
      "step": 737
    },
    {
      "epoch": 0.42864610559330896,
      "grad_norm": 325.70440673828125,
      "learning_rate": 3.927948866937827e-06,
      "logits/chosen": -0.6374528408050537,
      "logits/rejected": -0.5448911190032959,
      "logps/chosen": -74.01681518554688,
      "logps/rejected": -62.33977127075195,
      "loss": 13.9094,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0036867789458483458,
      "rewards/margins": 0.0037749551702290773,
      "rewards/rejected": -8.817538764560595e-05,
      "step": 738
    },
    {
      "epoch": 0.4292269268746007,
      "grad_norm": 324.1231689453125,
      "learning_rate": 3.92649622312609e-06,
      "logits/chosen": -0.7090293169021606,
      "logits/rejected": -0.7085317373275757,
      "logps/chosen": -80.50122833251953,
      "logps/rejected": -66.37286376953125,
      "loss": 14.212,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005439291708171368,
      "rewards/margins": -0.020895807072520256,
      "rewards/rejected": 0.015456515364348888,
      "step": 739
    },
    {
      "epoch": 0.42980774815589246,
      "grad_norm": 310.73968505859375,
      "learning_rate": 3.925043579314353e-06,
      "logits/chosen": -0.6998378038406372,
      "logits/rejected": -0.649873673915863,
      "logps/chosen": -75.84722137451172,
      "logps/rejected": -73.048095703125,
      "loss": 13.2256,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.017816346138715744,
      "rewards/margins": 0.0821092426776886,
      "rewards/rejected": -0.06429289281368256,
      "step": 740
    },
    {
      "epoch": 0.4303885694371842,
      "grad_norm": 297.516845703125,
      "learning_rate": 3.923590935502616e-06,
      "logits/chosen": -0.6497145891189575,
      "logits/rejected": -0.6620525121688843,
      "logps/chosen": -80.6782455444336,
      "logps/rejected": -72.28392028808594,
      "loss": 13.2971,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0035281956661492586,
      "rewards/margins": 0.07035094499588013,
      "rewards/rejected": -0.07387915253639221,
      "step": 741
    },
    {
      "epoch": 0.4309693907184759,
      "grad_norm": 322.9284973144531,
      "learning_rate": 3.9221382916908775e-06,
      "logits/chosen": -0.5986420512199402,
      "logits/rejected": -0.5791078209877014,
      "logps/chosen": -68.31107330322266,
      "logps/rejected": -71.18339538574219,
      "loss": 13.9564,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.011520462110638618,
      "rewards/margins": 0.00581027427688241,
      "rewards/rejected": -0.017330732196569443,
      "step": 742
    },
    {
      "epoch": 0.4315502119997677,
      "grad_norm": 309.1309814453125,
      "learning_rate": 3.92068564787914e-06,
      "logits/chosen": -0.7334194183349609,
      "logits/rejected": -0.5711901783943176,
      "logps/chosen": -75.96092224121094,
      "logps/rejected": -72.91703796386719,
      "loss": 13.3731,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.012258807197213173,
      "rewards/margins": 0.057714782655239105,
      "rewards/rejected": -0.06997359544038773,
      "step": 743
    },
    {
      "epoch": 0.4321310332810594,
      "grad_norm": 342.9898376464844,
      "learning_rate": 3.919233004067403e-06,
      "logits/chosen": -0.7844117283821106,
      "logits/rejected": -0.7111651301383972,
      "logps/chosen": -79.83903503417969,
      "logps/rejected": -80.42810821533203,
      "loss": 14.6354,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.030741948634386063,
      "rewards/margins": -0.07022452354431152,
      "rewards/rejected": 0.03948257490992546,
      "step": 744
    },
    {
      "epoch": 0.4327118545623512,
      "grad_norm": 298.7255859375,
      "learning_rate": 3.917780360255666e-06,
      "logits/chosen": -0.7623270750045776,
      "logits/rejected": -0.8348603248596191,
      "logps/chosen": -74.46296691894531,
      "logps/rejected": -72.27961730957031,
      "loss": 13.4668,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03371516615152359,
      "rewards/margins": 0.04851061850786209,
      "rewards/rejected": -0.08222578465938568,
      "step": 745
    },
    {
      "epoch": 0.4332926758436429,
      "grad_norm": 294.17254638671875,
      "learning_rate": 3.9163277164439285e-06,
      "logits/chosen": -0.7279725670814514,
      "logits/rejected": -0.739454984664917,
      "logps/chosen": -70.19053649902344,
      "logps/rejected": -70.75625610351562,
      "loss": 13.4564,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03082049824297428,
      "rewards/margins": 0.059337783604860306,
      "rewards/rejected": -0.09015828371047974,
      "step": 746
    },
    {
      "epoch": 0.43387349712493467,
      "grad_norm": 307.3867492675781,
      "learning_rate": 3.914875072632191e-06,
      "logits/chosen": -0.6892760396003723,
      "logits/rejected": -0.7267829179763794,
      "logps/chosen": -69.66451263427734,
      "logps/rejected": -73.37333679199219,
      "loss": 13.2647,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07057398557662964,
      "rewards/margins": 0.07388408482074738,
      "rewards/rejected": -0.0033100892324000597,
      "step": 747
    },
    {
      "epoch": 0.4344543184062264,
      "grad_norm": 334.6365661621094,
      "learning_rate": 3.913422428820454e-06,
      "logits/chosen": -0.5453966856002808,
      "logits/rejected": -0.6484876871109009,
      "logps/chosen": -75.01347351074219,
      "logps/rejected": -84.60029602050781,
      "loss": 13.4385,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02981463447213173,
      "rewards/margins": 0.05149499326944351,
      "rewards/rejected": -0.021680355072021484,
      "step": 748
    },
    {
      "epoch": 0.43503513968751817,
      "grad_norm": 293.3527526855469,
      "learning_rate": 3.911969785008716e-06,
      "logits/chosen": -0.6956106424331665,
      "logits/rejected": -0.7478946447372437,
      "logps/chosen": -69.3331069946289,
      "logps/rejected": -75.3602523803711,
      "loss": 13.4493,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.025318946689367294,
      "rewards/margins": 0.05369370058178902,
      "rewards/rejected": -0.028374750167131424,
      "step": 749
    },
    {
      "epoch": 0.4356159609688099,
      "grad_norm": 312.72625732421875,
      "learning_rate": 3.910517141196979e-06,
      "logits/chosen": -0.6957664489746094,
      "logits/rejected": -0.6999244689941406,
      "logps/chosen": -76.08050537109375,
      "logps/rejected": -71.7776870727539,
      "loss": 13.3181,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.011510677635669708,
      "rewards/margins": 0.061960190534591675,
      "rewards/rejected": -0.07347087562084198,
      "step": 750
    },
    {
      "epoch": 0.43619678225010167,
      "grad_norm": 322.7213134765625,
      "learning_rate": 3.9090644973852414e-06,
      "logits/chosen": -0.7363187074661255,
      "logits/rejected": -0.7484365105628967,
      "logps/chosen": -77.98309326171875,
      "logps/rejected": -74.41910552978516,
      "loss": 13.3354,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03591137006878853,
      "rewards/margins": 0.07070378959178925,
      "rewards/rejected": -0.034792426973581314,
      "step": 751
    },
    {
      "epoch": 0.4367776035313934,
      "grad_norm": 308.99310302734375,
      "learning_rate": 3.907611853573504e-06,
      "logits/chosen": -0.6841967701911926,
      "logits/rejected": -0.7418473362922668,
      "logps/chosen": -74.3568115234375,
      "logps/rejected": -72.77391052246094,
      "loss": 12.8602,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.026743004098534584,
      "rewards/margins": 0.11576484143733978,
      "rewards/rejected": -0.08902183920145035,
      "step": 752
    },
    {
      "epoch": 0.4373584248126851,
      "grad_norm": 327.3843994140625,
      "learning_rate": 3.906159209761767e-06,
      "logits/chosen": -0.7423384189605713,
      "logits/rejected": -0.7147141098976135,
      "logps/chosen": -70.23924255371094,
      "logps/rejected": -68.62517547607422,
      "loss": 13.8643,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.012057294137775898,
      "rewards/margins": 0.011633175425231457,
      "rewards/rejected": 0.00042412133188918233,
      "step": 753
    },
    {
      "epoch": 0.4379392460939769,
      "grad_norm": 318.1657409667969,
      "learning_rate": 3.90470656595003e-06,
      "logits/chosen": -0.823145866394043,
      "logits/rejected": -0.8383482694625854,
      "logps/chosen": -73.15534973144531,
      "logps/rejected": -74.34989929199219,
      "loss": 14.5024,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.057351987808942795,
      "rewards/margins": -0.05695341154932976,
      "rewards/rejected": -0.0003985777439083904,
      "step": 754
    },
    {
      "epoch": 0.4385200673752686,
      "grad_norm": 308.12493896484375,
      "learning_rate": 3.903253922138292e-06,
      "logits/chosen": -0.7838844060897827,
      "logits/rejected": -0.7964550256729126,
      "logps/chosen": -69.17980194091797,
      "logps/rejected": -77.6092758178711,
      "loss": 13.97,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05046354979276657,
      "rewards/margins": 0.004970133304595947,
      "rewards/rejected": -0.05543368309736252,
      "step": 755
    },
    {
      "epoch": 0.4391008886565604,
      "grad_norm": 328.02618408203125,
      "learning_rate": 3.901801278326554e-06,
      "logits/chosen": -0.6274627447128296,
      "logits/rejected": -0.7087150812149048,
      "logps/chosen": -67.25186920166016,
      "logps/rejected": -69.48518371582031,
      "loss": 14.5885,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.02703443542122841,
      "rewards/margins": -0.061674535274505615,
      "rewards/rejected": 0.03464009612798691,
      "step": 756
    },
    {
      "epoch": 0.4396817099378521,
      "grad_norm": 345.1214904785156,
      "learning_rate": 3.900348634514817e-06,
      "logits/chosen": -0.6962085962295532,
      "logits/rejected": -0.6879252195358276,
      "logps/chosen": -74.38581848144531,
      "logps/rejected": -74.284423828125,
      "loss": 14.0462,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004917553160339594,
      "rewards/margins": -0.009653748944401741,
      "rewards/rejected": 0.00473619531840086,
      "step": 757
    },
    {
      "epoch": 0.4402625312191439,
      "grad_norm": 360.69549560546875,
      "learning_rate": 3.89889599070308e-06,
      "logits/chosen": -0.770926833152771,
      "logits/rejected": -0.7564770579338074,
      "logps/chosen": -71.93128204345703,
      "logps/rejected": -76.0265121459961,
      "loss": 14.132,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.017853304743766785,
      "rewards/margins": -0.016330275684595108,
      "rewards/rejected": -0.0015230312710627913,
      "step": 758
    },
    {
      "epoch": 0.4408433525004356,
      "grad_norm": 325.82342529296875,
      "learning_rate": 3.8974433468913426e-06,
      "logits/chosen": -0.7091246843338013,
      "logits/rejected": -0.7488458752632141,
      "logps/chosen": -67.05516815185547,
      "logps/rejected": -79.63505554199219,
      "loss": 13.8617,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04035795480012894,
      "rewards/margins": 0.01930631510913372,
      "rewards/rejected": -0.0596642792224884,
      "step": 759
    },
    {
      "epoch": 0.4414241737817274,
      "grad_norm": 324.606201171875,
      "learning_rate": 3.895990703079605e-06,
      "logits/chosen": -0.8655685186386108,
      "logits/rejected": -0.8269084095954895,
      "logps/chosen": -71.62133026123047,
      "logps/rejected": -81.47102355957031,
      "loss": 13.8222,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03722354397177696,
      "rewards/margins": 0.020576555281877518,
      "rewards/rejected": -0.05780010297894478,
      "step": 760
    },
    {
      "epoch": 0.4420049950630191,
      "grad_norm": 311.0895690917969,
      "learning_rate": 3.894538059267868e-06,
      "logits/chosen": -0.5948249697685242,
      "logits/rejected": -0.6009396314620972,
      "logps/chosen": -71.13340759277344,
      "logps/rejected": -74.0931625366211,
      "loss": 13.6398,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004656590521335602,
      "rewards/margins": 0.03076646290719509,
      "rewards/rejected": -0.035423047840595245,
      "step": 761
    },
    {
      "epoch": 0.4425858163443109,
      "grad_norm": 348.3786315917969,
      "learning_rate": 3.89308541545613e-06,
      "logits/chosen": -0.6613047122955322,
      "logits/rejected": -0.7789413332939148,
      "logps/chosen": -80.16218566894531,
      "logps/rejected": -78.0029296875,
      "loss": 13.6857,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03409738838672638,
      "rewards/margins": 0.029163142666220665,
      "rewards/rejected": 0.004934241063892841,
      "step": 762
    },
    {
      "epoch": 0.4431666376256026,
      "grad_norm": 273.7310485839844,
      "learning_rate": 3.891632771644393e-06,
      "logits/chosen": -0.8774948120117188,
      "logits/rejected": -0.8197600245475769,
      "logps/chosen": -67.24043273925781,
      "logps/rejected": -73.55924987792969,
      "loss": 12.5638,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.040607668459415436,
      "rewards/margins": 0.15121008455753326,
      "rewards/rejected": -0.11060241609811783,
      "step": 763
    },
    {
      "epoch": 0.4437474589068943,
      "grad_norm": 289.9358825683594,
      "learning_rate": 3.8901801278326555e-06,
      "logits/chosen": -0.7543174028396606,
      "logits/rejected": -0.7527318596839905,
      "logps/chosen": -70.50340270996094,
      "logps/rejected": -71.85820007324219,
      "loss": 12.648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0302159134298563,
      "rewards/margins": 0.13685820996761322,
      "rewards/rejected": -0.10664232075214386,
      "step": 764
    },
    {
      "epoch": 0.4443282801881861,
      "grad_norm": 310.6090393066406,
      "learning_rate": 3.888727484020918e-06,
      "logits/chosen": -0.7520283460617065,
      "logits/rejected": -0.8482815027236938,
      "logps/chosen": -80.78767395019531,
      "logps/rejected": -71.63502502441406,
      "loss": 13.935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02459750324487686,
      "rewards/margins": 7.411837577819824e-05,
      "rewards/rejected": -0.02467162348330021,
      "step": 765
    },
    {
      "epoch": 0.4449091014694778,
      "grad_norm": 320.955322265625,
      "learning_rate": 3.887274840209181e-06,
      "logits/chosen": -0.6493512988090515,
      "logits/rejected": -0.7719279527664185,
      "logps/chosen": -71.09584045410156,
      "logps/rejected": -79.13584899902344,
      "loss": 13.7825,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004698459059000015,
      "rewards/margins": 0.015006857924163342,
      "rewards/rejected": -0.01970531791448593,
      "step": 766
    },
    {
      "epoch": 0.4454899227507696,
      "grad_norm": 420.12744140625,
      "learning_rate": 3.885822196397444e-06,
      "logits/chosen": -0.5989329218864441,
      "logits/rejected": -0.5799925923347473,
      "logps/chosen": -67.3458251953125,
      "logps/rejected": -70.0871810913086,
      "loss": 14.2179,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.024614132940769196,
      "rewards/margins": -0.01728874072432518,
      "rewards/rejected": -0.00732539314776659,
      "step": 767
    },
    {
      "epoch": 0.4460707440320613,
      "grad_norm": 310.5950927734375,
      "learning_rate": 3.8843695525857065e-06,
      "logits/chosen": -0.6708532571792603,
      "logits/rejected": -0.6406176686286926,
      "logps/chosen": -67.67707824707031,
      "logps/rejected": -73.09920501708984,
      "loss": 13.3177,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.022770235314965248,
      "rewards/margins": 0.07169272005558014,
      "rewards/rejected": -0.04892248287796974,
      "step": 768
    },
    {
      "epoch": 0.4466515653133531,
      "grad_norm": 346.4231872558594,
      "learning_rate": 3.882916908773968e-06,
      "logits/chosen": -0.6678956151008606,
      "logits/rejected": -0.7639406323432922,
      "logps/chosen": -76.6402359008789,
      "logps/rejected": -70.21337127685547,
      "loss": 13.5121,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009954368695616722,
      "rewards/margins": 0.04720446467399597,
      "rewards/rejected": -0.057158827781677246,
      "step": 769
    },
    {
      "epoch": 0.4472323865946448,
      "grad_norm": 315.72998046875,
      "learning_rate": 3.881464264962231e-06,
      "logits/chosen": -0.6422103643417358,
      "logits/rejected": -0.6924742460250854,
      "logps/chosen": -77.5414047241211,
      "logps/rejected": -85.16349792480469,
      "loss": 13.3908,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02027234062552452,
      "rewards/margins": 0.06246469169855118,
      "rewards/rejected": -0.04219234734773636,
      "step": 770
    },
    {
      "epoch": 0.4478132078759366,
      "grad_norm": 325.33782958984375,
      "learning_rate": 3.880011621150494e-06,
      "logits/chosen": -0.8680871725082397,
      "logits/rejected": -0.8178070187568665,
      "logps/chosen": -77.91289520263672,
      "logps/rejected": -76.54703521728516,
      "loss": 13.9257,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004537890199571848,
      "rewards/margins": 0.007611017674207687,
      "rewards/rejected": -0.012148907408118248,
      "step": 771
    },
    {
      "epoch": 0.4483940291572283,
      "grad_norm": 332.87200927734375,
      "learning_rate": 3.878558977338757e-06,
      "logits/chosen": -0.7658424377441406,
      "logits/rejected": -0.7849222421646118,
      "logps/chosen": -72.97148132324219,
      "logps/rejected": -69.93640899658203,
      "loss": 14.4375,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02565593644976616,
      "rewards/margins": -0.046471305191516876,
      "rewards/rejected": 0.020815376192331314,
      "step": 772
    },
    {
      "epoch": 0.4489748504385201,
      "grad_norm": 298.5015563964844,
      "learning_rate": 3.877106333527019e-06,
      "logits/chosen": -0.715753436088562,
      "logits/rejected": -0.7763512134552002,
      "logps/chosen": -69.95814514160156,
      "logps/rejected": -67.6918716430664,
      "loss": 13.4113,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04581587761640549,
      "rewards/margins": 0.05313152074813843,
      "rewards/rejected": -0.007315640337765217,
      "step": 773
    },
    {
      "epoch": 0.4495556717198118,
      "grad_norm": 325.48419189453125,
      "learning_rate": 3.875653689715282e-06,
      "logits/chosen": -0.6810447573661804,
      "logits/rejected": -0.6709048748016357,
      "logps/chosen": -88.08033752441406,
      "logps/rejected": -82.59185791015625,
      "loss": 14.0073,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03442404791712761,
      "rewards/margins": -0.004874535836279392,
      "rewards/rejected": -0.02954951487481594,
      "step": 774
    },
    {
      "epoch": 0.4501364930011036,
      "grad_norm": 316.40673828125,
      "learning_rate": 3.874201045903545e-06,
      "logits/chosen": -0.648461639881134,
      "logits/rejected": -0.6868349313735962,
      "logps/chosen": -74.5299301147461,
      "logps/rejected": -67.671875,
      "loss": 13.9253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010094478726387024,
      "rewards/margins": 0.0021387473680078983,
      "rewards/rejected": 0.007955733686685562,
      "step": 775
    },
    {
      "epoch": 0.4507173142823953,
      "grad_norm": 306.03436279296875,
      "learning_rate": 3.872748402091807e-06,
      "logits/chosen": -0.6466223001480103,
      "logits/rejected": -0.6228175759315491,
      "logps/chosen": -72.484619140625,
      "logps/rejected": -65.22151947021484,
      "loss": 13.3478,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02175520732998848,
      "rewards/margins": 0.06977047026157379,
      "rewards/rejected": -0.04801527410745621,
      "step": 776
    },
    {
      "epoch": 0.45129813556368703,
      "grad_norm": 303.6878662109375,
      "learning_rate": 3.87129575828007e-06,
      "logits/chosen": -0.7520937919616699,
      "logits/rejected": -0.6366826295852661,
      "logps/chosen": -72.18943786621094,
      "logps/rejected": -68.44010162353516,
      "loss": 14.4002,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.10646836459636688,
      "rewards/margins": -0.042999267578125,
      "rewards/rejected": -0.06346909701824188,
      "step": 777
    },
    {
      "epoch": 0.4518789568449788,
      "grad_norm": 319.6017761230469,
      "learning_rate": 3.869843114468333e-06,
      "logits/chosen": -0.6259938478469849,
      "logits/rejected": -0.6519014835357666,
      "logps/chosen": -68.73445129394531,
      "logps/rejected": -68.32264709472656,
      "loss": 13.8294,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.008890246972441673,
      "rewards/margins": 0.014760665595531464,
      "rewards/rejected": -0.005870418157428503,
      "step": 778
    },
    {
      "epoch": 0.45245977812627053,
      "grad_norm": 299.91241455078125,
      "learning_rate": 3.868390470656596e-06,
      "logits/chosen": -0.6745251417160034,
      "logits/rejected": -0.7223367691040039,
      "logps/chosen": -77.0238037109375,
      "logps/rejected": -68.16448211669922,
      "loss": 12.6462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06470943987369537,
      "rewards/margins": 0.1429821252822876,
      "rewards/rejected": -0.07827268540859222,
      "step": 779
    },
    {
      "epoch": 0.4530405994075623,
      "grad_norm": 299.5854797363281,
      "learning_rate": 3.866937826844859e-06,
      "logits/chosen": -0.7268974781036377,
      "logits/rejected": -0.7254965901374817,
      "logps/chosen": -72.63690185546875,
      "logps/rejected": -73.64038848876953,
      "loss": 13.6416,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.021657172590494156,
      "rewards/margins": 0.045961178839206696,
      "rewards/rejected": -0.06761835515499115,
      "step": 780
    },
    {
      "epoch": 0.453621420688854,
      "grad_norm": 315.7631530761719,
      "learning_rate": 3.8654851830331205e-06,
      "logits/chosen": -0.7705460786819458,
      "logits/rejected": -0.7526998519897461,
      "logps/chosen": -71.8909912109375,
      "logps/rejected": -75.11150360107422,
      "loss": 14.061,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.007896973751485348,
      "rewards/margins": -0.006453951355069876,
      "rewards/rejected": -0.0014430228620767593,
      "step": 781
    },
    {
      "epoch": 0.4542022419701458,
      "grad_norm": 306.499267578125,
      "learning_rate": 3.864032539221383e-06,
      "logits/chosen": -0.7486709356307983,
      "logits/rejected": -0.7750841379165649,
      "logps/chosen": -73.98805236816406,
      "logps/rejected": -71.86274719238281,
      "loss": 13.4212,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008844633586704731,
      "rewards/margins": 0.058701179921627045,
      "rewards/rejected": -0.04985655099153519,
      "step": 782
    },
    {
      "epoch": 0.4547830632514375,
      "grad_norm": 289.93267822265625,
      "learning_rate": 3.862579895409646e-06,
      "logits/chosen": -0.6648403406143188,
      "logits/rejected": -0.5542594790458679,
      "logps/chosen": -72.4288558959961,
      "logps/rejected": -62.92414474487305,
      "loss": 13.8115,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.025623226538300514,
      "rewards/margins": 0.012874387204647064,
      "rewards/rejected": 0.012748842127621174,
      "step": 783
    },
    {
      "epoch": 0.4553638845327293,
      "grad_norm": 320.468505859375,
      "learning_rate": 3.861127251597909e-06,
      "logits/chosen": -0.6467171907424927,
      "logits/rejected": -0.6780973672866821,
      "logps/chosen": -70.74858856201172,
      "logps/rejected": -75.70240020751953,
      "loss": 13.4407,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.019734477624297142,
      "rewards/margins": 0.056104935705661774,
      "rewards/rejected": -0.07583941519260406,
      "step": 784
    },
    {
      "epoch": 0.455944705814021,
      "grad_norm": 314.7945556640625,
      "learning_rate": 3.8596746077861715e-06,
      "logits/chosen": -0.7252448797225952,
      "logits/rejected": -0.6654443144798279,
      "logps/chosen": -69.35020446777344,
      "logps/rejected": -74.48258209228516,
      "loss": 13.722,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004625787492841482,
      "rewards/margins": 0.02148066833615303,
      "rewards/rejected": -0.01685487851500511,
      "step": 785
    },
    {
      "epoch": 0.4565255270953128,
      "grad_norm": 336.58001708984375,
      "learning_rate": 3.858221963974434e-06,
      "logits/chosen": -0.9182072877883911,
      "logits/rejected": -0.8336294889450073,
      "logps/chosen": -70.61786651611328,
      "logps/rejected": -96.56700134277344,
      "loss": 13.3177,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.005032673943787813,
      "rewards/margins": 0.06978379189968109,
      "rewards/rejected": -0.07481645792722702,
      "step": 786
    },
    {
      "epoch": 0.4571063483766045,
      "grad_norm": 340.71490478515625,
      "learning_rate": 3.856769320162697e-06,
      "logits/chosen": -0.8411046266555786,
      "logits/rejected": -0.8996657133102417,
      "logps/chosen": -74.63349914550781,
      "logps/rejected": -84.13905334472656,
      "loss": 13.212,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04600357636809349,
      "rewards/margins": 0.07600485533475876,
      "rewards/rejected": -0.030001282691955566,
      "step": 787
    },
    {
      "epoch": 0.45768716965789624,
      "grad_norm": 333.482666015625,
      "learning_rate": 3.855316676350959e-06,
      "logits/chosen": -0.6594542264938354,
      "logits/rejected": -0.6667267084121704,
      "logps/chosen": -70.84710693359375,
      "logps/rejected": -72.32903289794922,
      "loss": 13.3626,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029414311051368713,
      "rewards/margins": 0.060748837888240814,
      "rewards/rejected": -0.0313345268368721,
      "step": 788
    },
    {
      "epoch": 0.458267990939188,
      "grad_norm": 312.21856689453125,
      "learning_rate": 3.853864032539222e-06,
      "logits/chosen": -0.8260235786437988,
      "logits/rejected": -0.853685200214386,
      "logps/chosen": -62.875640869140625,
      "logps/rejected": -75.08868408203125,
      "loss": 13.8724,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05585920810699463,
      "rewards/margins": 0.006377121899276972,
      "rewards/rejected": -0.062236327677965164,
      "step": 789
    },
    {
      "epoch": 0.45884881222047974,
      "grad_norm": 312.63446044921875,
      "learning_rate": 3.852411388727484e-06,
      "logits/chosen": -0.651515781879425,
      "logits/rejected": -0.6716042160987854,
      "logps/chosen": -76.17277526855469,
      "logps/rejected": -69.4102554321289,
      "loss": 13.3182,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02180982567369938,
      "rewards/margins": 0.06877848505973816,
      "rewards/rejected": -0.04696866124868393,
      "step": 790
    },
    {
      "epoch": 0.4594296335017715,
      "grad_norm": 294.8563537597656,
      "learning_rate": 3.850958744915747e-06,
      "logits/chosen": -0.9102421998977661,
      "logits/rejected": -0.8507450819015503,
      "logps/chosen": -70.69921112060547,
      "logps/rejected": -72.28413391113281,
      "loss": 14.0738,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.016772408038377762,
      "rewards/margins": -0.005055961664766073,
      "rewards/rejected": -0.011716444976627827,
      "step": 791
    },
    {
      "epoch": 0.46001045478306324,
      "grad_norm": 303.2449035644531,
      "learning_rate": 3.84950610110401e-06,
      "logits/chosen": -0.8211100697517395,
      "logits/rejected": -0.7845950126647949,
      "logps/chosen": -67.99166870117188,
      "logps/rejected": -68.65469360351562,
      "loss": 12.9882,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008265440352261066,
      "rewards/margins": 0.10116876661777496,
      "rewards/rejected": -0.09290332347154617,
      "step": 792
    },
    {
      "epoch": 0.460591276064355,
      "grad_norm": 313.7750549316406,
      "learning_rate": 3.848053457292273e-06,
      "logits/chosen": -0.6798004508018494,
      "logits/rejected": -0.6693185567855835,
      "logps/chosen": -82.59542083740234,
      "logps/rejected": -71.71035766601562,
      "loss": 13.5624,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.00769240316003561,
      "rewards/margins": 0.05136420577764511,
      "rewards/rejected": -0.0590566024184227,
      "step": 793
    },
    {
      "epoch": 0.46117209734564674,
      "grad_norm": 287.33343505859375,
      "learning_rate": 3.8466008134805345e-06,
      "logits/chosen": -0.8326593637466431,
      "logits/rejected": -0.8718840479850769,
      "logps/chosen": -67.961669921875,
      "logps/rejected": -73.32498168945312,
      "loss": 13.2243,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022751104086637497,
      "rewards/margins": 0.08069998770952225,
      "rewards/rejected": -0.05794887617230415,
      "step": 794
    },
    {
      "epoch": 0.4617529186269385,
      "grad_norm": 330.590087890625,
      "learning_rate": 3.845148169668797e-06,
      "logits/chosen": -0.8093253374099731,
      "logits/rejected": -0.8372796177864075,
      "logps/chosen": -73.56087493896484,
      "logps/rejected": -84.00132751464844,
      "loss": 14.7452,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.07513497769832611,
      "rewards/margins": -0.07671995460987091,
      "rewards/rejected": 0.0015849672490730882,
      "step": 795
    },
    {
      "epoch": 0.46233373990823023,
      "grad_norm": 295.850830078125,
      "learning_rate": 3.84369552585706e-06,
      "logits/chosen": -0.8418253064155579,
      "logits/rejected": -0.8993352055549622,
      "logps/chosen": -69.29906463623047,
      "logps/rejected": -72.6314697265625,
      "loss": 13.7799,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00324619235470891,
      "rewards/margins": 0.02390274405479431,
      "rewards/rejected": -0.027148932218551636,
      "step": 796
    },
    {
      "epoch": 0.462914561189522,
      "grad_norm": 335.1182556152344,
      "learning_rate": 3.842242882045323e-06,
      "logits/chosen": -0.7779780626296997,
      "logits/rejected": -0.6619454026222229,
      "logps/chosen": -64.12891387939453,
      "logps/rejected": -74.85227966308594,
      "loss": 14.454,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.031012335792183876,
      "rewards/margins": -0.047410137951374054,
      "rewards/rejected": 0.01639780029654503,
      "step": 797
    },
    {
      "epoch": 0.46349538247081373,
      "grad_norm": 310.0404052734375,
      "learning_rate": 3.8407902382335855e-06,
      "logits/chosen": -0.8367874026298523,
      "logits/rejected": -0.7925940155982971,
      "logps/chosen": -75.65746307373047,
      "logps/rejected": -71.82279205322266,
      "loss": 13.9021,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008593537844717503,
      "rewards/margins": 0.010417133569717407,
      "rewards/rejected": -0.001823595492169261,
      "step": 798
    },
    {
      "epoch": 0.46407620375210545,
      "grad_norm": 305.9939270019531,
      "learning_rate": 3.839337594421848e-06,
      "logits/chosen": -0.7069296836853027,
      "logits/rejected": -0.7354210615158081,
      "logps/chosen": -81.55580139160156,
      "logps/rejected": -76.04948425292969,
      "loss": 13.6306,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.01436761487275362,
      "rewards/margins": 0.03523886203765869,
      "rewards/rejected": -0.020871248096227646,
      "step": 799
    },
    {
      "epoch": 0.46465702503339723,
      "grad_norm": 319.43377685546875,
      "learning_rate": 3.837884950610111e-06,
      "logits/chosen": -0.775700032711029,
      "logits/rejected": -0.6986560225486755,
      "logps/chosen": -74.52439880371094,
      "logps/rejected": -77.12367248535156,
      "loss": 13.7436,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015211820602416992,
      "rewards/margins": 0.019106799736618996,
      "rewards/rejected": -0.0038949784357100725,
      "step": 800
    },
    {
      "epoch": 0.46523784631468895,
      "grad_norm": 306.27374267578125,
      "learning_rate": 3.836432306798373e-06,
      "logits/chosen": -0.8053959012031555,
      "logits/rejected": -0.8362959623336792,
      "logps/chosen": -70.24461364746094,
      "logps/rejected": -73.2827377319336,
      "loss": 13.0576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0017929266905412078,
      "rewards/margins": 0.09252931922674179,
      "rewards/rejected": -0.09073638916015625,
      "step": 801
    },
    {
      "epoch": 0.46581866759598073,
      "grad_norm": 310.7746887207031,
      "learning_rate": 3.834979662986636e-06,
      "logits/chosen": -1.0116592645645142,
      "logits/rejected": -0.8014705777168274,
      "logps/chosen": -71.8570785522461,
      "logps/rejected": -82.65800476074219,
      "loss": 13.0408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03449561074376106,
      "rewards/margins": 0.09185705333948135,
      "rewards/rejected": -0.05736144259572029,
      "step": 802
    },
    {
      "epoch": 0.46639948887727245,
      "grad_norm": 341.84814453125,
      "learning_rate": 3.833527019174898e-06,
      "logits/chosen": -0.7799655199050903,
      "logits/rejected": -0.7931917309761047,
      "logps/chosen": -76.88513946533203,
      "logps/rejected": -71.80252075195312,
      "loss": 14.5748,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08196653425693512,
      "rewards/margins": -0.050925351679325104,
      "rewards/rejected": -0.031041180714964867,
      "step": 803
    },
    {
      "epoch": 0.4669803101585642,
      "grad_norm": 317.75946044921875,
      "learning_rate": 3.832074375363161e-06,
      "logits/chosen": -0.830287754535675,
      "logits/rejected": -0.8624798655509949,
      "logps/chosen": -71.75201416015625,
      "logps/rejected": -73.34031677246094,
      "loss": 13.3196,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0004104435502085835,
      "rewards/margins": 0.0665319561958313,
      "rewards/rejected": -0.06694237887859344,
      "step": 804
    },
    {
      "epoch": 0.46756113143985595,
      "grad_norm": 321.6256103515625,
      "learning_rate": 3.830621731551424e-06,
      "logits/chosen": -0.7428877353668213,
      "logits/rejected": -0.7949348092079163,
      "logps/chosen": -72.76325988769531,
      "logps/rejected": -71.17098236083984,
      "loss": 14.5415,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.015930796042084694,
      "rewards/margins": -0.05372260883450508,
      "rewards/rejected": 0.06965340673923492,
      "step": 805
    },
    {
      "epoch": 0.4681419527211477,
      "grad_norm": 323.96905517578125,
      "learning_rate": 3.829169087739687e-06,
      "logits/chosen": -0.8501350283622742,
      "logits/rejected": -0.8383558392524719,
      "logps/chosen": -71.15087127685547,
      "logps/rejected": -70.85906982421875,
      "loss": 14.7471,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.09234993904829025,
      "rewards/margins": -0.0763244479894638,
      "rewards/rejected": -0.016025494784116745,
      "step": 806
    },
    {
      "epoch": 0.46872277400243945,
      "grad_norm": 336.2712097167969,
      "learning_rate": 3.827716443927949e-06,
      "logits/chosen": -0.8244204521179199,
      "logits/rejected": -0.8803207278251648,
      "logps/chosen": -86.31137084960938,
      "logps/rejected": -71.70460510253906,
      "loss": 14.0914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.029995271936058998,
      "rewards/margins": -0.010159234516322613,
      "rewards/rejected": -0.01983603462576866,
      "step": 807
    },
    {
      "epoch": 0.4693035952837312,
      "grad_norm": 286.52447509765625,
      "learning_rate": 3.826263800116211e-06,
      "logits/chosen": -0.8492335081100464,
      "logits/rejected": -0.9490715265274048,
      "logps/chosen": -67.36164855957031,
      "logps/rejected": -68.58895111083984,
      "loss": 13.456,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0037143989466130733,
      "rewards/margins": 0.05153452232480049,
      "rewards/rejected": -0.05524892359972,
      "step": 808
    },
    {
      "epoch": 0.46988441656502294,
      "grad_norm": 335.1893005371094,
      "learning_rate": 3.824811156304474e-06,
      "logits/chosen": -0.8309356570243835,
      "logits/rejected": -0.8245996236801147,
      "logps/chosen": -72.35955047607422,
      "logps/rejected": -76.44219970703125,
      "loss": 14.0092,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006397458724677563,
      "rewards/margins": -0.003385643707588315,
      "rewards/rejected": -0.003011818276718259,
      "step": 809
    },
    {
      "epoch": 0.47046523784631467,
      "grad_norm": 326.7082824707031,
      "learning_rate": 3.823358512492737e-06,
      "logits/chosen": -0.6885795593261719,
      "logits/rejected": -0.7032235860824585,
      "logps/chosen": -74.98950958251953,
      "logps/rejected": -68.4511947631836,
      "loss": 13.7991,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.007976588793098927,
      "rewards/margins": 0.016375992447137833,
      "rewards/rejected": -0.024352580308914185,
      "step": 810
    },
    {
      "epoch": 0.47104605912760644,
      "grad_norm": 336.2662048339844,
      "learning_rate": 3.8219058686809996e-06,
      "logits/chosen": -0.7334426641464233,
      "logits/rejected": -0.8731748461723328,
      "logps/chosen": -78.27384948730469,
      "logps/rejected": -82.20344543457031,
      "loss": 14.1666,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01889634132385254,
      "rewards/margins": -0.0168316587805748,
      "rewards/rejected": -0.0020646885968744755,
      "step": 811
    },
    {
      "epoch": 0.47162688040889816,
      "grad_norm": 317.0302734375,
      "learning_rate": 3.820453224869262e-06,
      "logits/chosen": -0.7193567752838135,
      "logits/rejected": -0.7776497006416321,
      "logps/chosen": -81.74263000488281,
      "logps/rejected": -81.36003875732422,
      "loss": 13.3265,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.033215902745723724,
      "rewards/margins": 0.10018181800842285,
      "rewards/rejected": -0.06696590781211853,
      "step": 812
    },
    {
      "epoch": 0.47220770169018994,
      "grad_norm": 308.6961975097656,
      "learning_rate": 3.819000581057525e-06,
      "logits/chosen": -0.7122281193733215,
      "logits/rejected": -0.7115441560745239,
      "logps/chosen": -75.91162872314453,
      "logps/rejected": -70.84449768066406,
      "loss": 13.6834,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.016342181712388992,
      "rewards/margins": 0.0316503569483757,
      "rewards/rejected": -0.015308169648051262,
      "step": 813
    },
    {
      "epoch": 0.47278852297148166,
      "grad_norm": 307.2214050292969,
      "learning_rate": 3.817547937245788e-06,
      "logits/chosen": -0.8540679812431335,
      "logits/rejected": -0.9650880694389343,
      "logps/chosen": -69.38043975830078,
      "logps/rejected": -89.74822235107422,
      "loss": 13.2044,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05181754380464554,
      "rewards/margins": 0.07774057984352112,
      "rewards/rejected": -0.02592303231358528,
      "step": 814
    },
    {
      "epoch": 0.47336934425277344,
      "grad_norm": 310.0686340332031,
      "learning_rate": 3.81609529343405e-06,
      "logits/chosen": -0.8766164779663086,
      "logits/rejected": -0.7675495147705078,
      "logps/chosen": -75.18499755859375,
      "logps/rejected": -70.85122680664062,
      "loss": 13.5494,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.037073832005262375,
      "rewards/margins": 0.04066385701298714,
      "rewards/rejected": -0.0035900219809263945,
      "step": 815
    },
    {
      "epoch": 0.47395016553406516,
      "grad_norm": 319.04913330078125,
      "learning_rate": 3.814642649622313e-06,
      "logits/chosen": -0.773638904094696,
      "logits/rejected": -0.688806414604187,
      "logps/chosen": -69.4584732055664,
      "logps/rejected": -78.58277130126953,
      "loss": 13.5887,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.04278705641627312,
      "rewards/margins": 0.03746388480067253,
      "rewards/rejected": 0.005323170684278011,
      "step": 816
    },
    {
      "epoch": 0.47453098681535694,
      "grad_norm": 314.92266845703125,
      "learning_rate": 3.813190005810575e-06,
      "logits/chosen": -0.8834367990493774,
      "logits/rejected": -0.892521858215332,
      "logps/chosen": -77.84300231933594,
      "logps/rejected": -79.00919342041016,
      "loss": 13.2775,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.016293682157993317,
      "rewards/margins": 0.06721033900976181,
      "rewards/rejected": -0.050916653126478195,
      "step": 817
    },
    {
      "epoch": 0.47511180809664866,
      "grad_norm": 293.46875,
      "learning_rate": 3.811737361998838e-06,
      "logits/chosen": -0.9297618865966797,
      "logits/rejected": -0.944604218006134,
      "logps/chosen": -79.2077407836914,
      "logps/rejected": -73.26544952392578,
      "loss": 13.6234,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0014585237950086594,
      "rewards/margins": 0.03561336174607277,
      "rewards/rejected": -0.037071891129016876,
      "step": 818
    },
    {
      "epoch": 0.47569262937794043,
      "grad_norm": 377.02056884765625,
      "learning_rate": 3.810284718187101e-06,
      "logits/chosen": -0.7812774777412415,
      "logits/rejected": -0.8613381385803223,
      "logps/chosen": -72.1882553100586,
      "logps/rejected": -74.88095092773438,
      "loss": 13.8795,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0032719247974455357,
      "rewards/margins": 0.010365369729697704,
      "rewards/rejected": -0.013637298718094826,
      "step": 819
    },
    {
      "epoch": 0.47627345065923216,
      "grad_norm": 316.15924072265625,
      "learning_rate": 3.808832074375364e-06,
      "logits/chosen": -0.8011975288391113,
      "logits/rejected": -0.9280617833137512,
      "logps/chosen": -73.37976837158203,
      "logps/rejected": -81.17606353759766,
      "loss": 13.2769,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005650301463901997,
      "rewards/margins": 0.08938013017177582,
      "rewards/rejected": -0.0837298333644867,
      "step": 820
    },
    {
      "epoch": 0.4768542719405239,
      "grad_norm": 337.64898681640625,
      "learning_rate": 3.807379430563626e-06,
      "logits/chosen": -0.6541992425918579,
      "logits/rejected": -0.6755486130714417,
      "logps/chosen": -68.98448181152344,
      "logps/rejected": -71.53800201416016,
      "loss": 14.094,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.021977489814162254,
      "rewards/margins": -0.01157850306481123,
      "rewards/rejected": -0.010398988611996174,
      "step": 821
    },
    {
      "epoch": 0.47743509322181565,
      "grad_norm": 395.7552795410156,
      "learning_rate": 3.805926786751889e-06,
      "logits/chosen": -0.8055717349052429,
      "logits/rejected": -0.7719739079475403,
      "logps/chosen": -78.54650115966797,
      "logps/rejected": -69.30370330810547,
      "loss": 14.2552,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04533190652728081,
      "rewards/margins": -0.03052378073334694,
      "rewards/rejected": -0.014808130450546741,
      "step": 822
    },
    {
      "epoch": 0.4780159145031074,
      "grad_norm": 329.39019775390625,
      "learning_rate": 3.8044741429401517e-06,
      "logits/chosen": -0.9106775522232056,
      "logits/rejected": -0.9728819131851196,
      "logps/chosen": -75.94145202636719,
      "logps/rejected": -72.80001831054688,
      "loss": 14.326,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04924512654542923,
      "rewards/margins": -0.034947142004966736,
      "rewards/rejected": -0.014297977089881897,
      "step": 823
    },
    {
      "epoch": 0.47859673578439915,
      "grad_norm": 311.20428466796875,
      "learning_rate": 3.8030214991284144e-06,
      "logits/chosen": -0.7255929708480835,
      "logits/rejected": -0.8986749649047852,
      "logps/chosen": -72.01505279541016,
      "logps/rejected": -78.78022766113281,
      "loss": 13.9084,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.033934831619262695,
      "rewards/margins": 0.015138444490730762,
      "rewards/rejected": -0.04907327890396118,
      "step": 824
    },
    {
      "epoch": 0.4791775570656909,
      "grad_norm": 350.76275634765625,
      "learning_rate": 3.8015688553166768e-06,
      "logits/chosen": -0.8962694406509399,
      "logits/rejected": -0.9466017484664917,
      "logps/chosen": -77.05352783203125,
      "logps/rejected": -73.25821685791016,
      "loss": 14.4856,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09311369061470032,
      "rewards/margins": -0.03986470773816109,
      "rewards/rejected": -0.05324899032711983,
      "step": 825
    },
    {
      "epoch": 0.47975837834698265,
      "grad_norm": 331.6654968261719,
      "learning_rate": 3.8001162115049395e-06,
      "logits/chosen": -0.8453457951545715,
      "logits/rejected": -0.8091884851455688,
      "logps/chosen": -75.66307067871094,
      "logps/rejected": -78.55387115478516,
      "loss": 13.6351,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.022285010665655136,
      "rewards/margins": 0.04095644876360893,
      "rewards/rejected": -0.018671434372663498,
      "step": 826
    },
    {
      "epoch": 0.48033919962827437,
      "grad_norm": 319.9637451171875,
      "learning_rate": 3.7986635676932023e-06,
      "logits/chosen": -0.7850121259689331,
      "logits/rejected": -0.887397289276123,
      "logps/chosen": -63.03386688232422,
      "logps/rejected": -72.07405090332031,
      "loss": 13.5181,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0016908079851418734,
      "rewards/margins": 0.0456506684422493,
      "rewards/rejected": -0.043959856033325195,
      "step": 827
    },
    {
      "epoch": 0.48092002090956615,
      "grad_norm": 335.4803771972656,
      "learning_rate": 3.7972109238814646e-06,
      "logits/chosen": -0.8599593043327332,
      "logits/rejected": -0.8137737512588501,
      "logps/chosen": -83.40064239501953,
      "logps/rejected": -74.57695007324219,
      "loss": 14.3082,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014900142326951027,
      "rewards/margins": -0.03146820515394211,
      "rewards/rejected": 0.04636834189295769,
      "step": 828
    },
    {
      "epoch": 0.48150084219085787,
      "grad_norm": 341.3481750488281,
      "learning_rate": 3.7957582800697273e-06,
      "logits/chosen": -0.8422654867172241,
      "logits/rejected": -0.9839199185371399,
      "logps/chosen": -82.99832916259766,
      "logps/rejected": -72.62747192382812,
      "loss": 13.2486,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0003313060733489692,
      "rewards/margins": 0.07307750731706619,
      "rewards/rejected": -0.07274620234966278,
      "step": 829
    },
    {
      "epoch": 0.48208166347214965,
      "grad_norm": 324.2658386230469,
      "learning_rate": 3.79430563625799e-06,
      "logits/chosen": -0.9340323209762573,
      "logits/rejected": -0.8132963180541992,
      "logps/chosen": -76.088623046875,
      "logps/rejected": -78.1231689453125,
      "loss": 13.8514,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.044568758457899094,
      "rewards/margins": 0.016439538449048996,
      "rewards/rejected": -0.061008304357528687,
      "step": 830
    },
    {
      "epoch": 0.48266248475344137,
      "grad_norm": 599.396240234375,
      "learning_rate": 3.7928529924462524e-06,
      "logits/chosen": -0.9430292248725891,
      "logits/rejected": -1.02345871925354,
      "logps/chosen": -72.04682922363281,
      "logps/rejected": -71.67384338378906,
      "loss": 13.8161,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021543148905038834,
      "rewards/margins": 0.017365697771310806,
      "rewards/rejected": 0.004177444148808718,
      "step": 831
    },
    {
      "epoch": 0.4832433060347331,
      "grad_norm": 385.9155578613281,
      "learning_rate": 3.791400348634515e-06,
      "logits/chosen": -0.8752473592758179,
      "logits/rejected": -0.8917462229728699,
      "logps/chosen": -74.92131042480469,
      "logps/rejected": -85.57743835449219,
      "loss": 13.2866,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.053486812859773636,
      "rewards/margins": 0.06886574625968933,
      "rewards/rejected": -0.01537893433123827,
      "step": 832
    },
    {
      "epoch": 0.48382412731602487,
      "grad_norm": 292.564697265625,
      "learning_rate": 3.789947704822778e-06,
      "logits/chosen": -0.8002208471298218,
      "logits/rejected": -0.7895336151123047,
      "logps/chosen": -71.14689636230469,
      "logps/rejected": -73.53022766113281,
      "loss": 13.5167,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02734098769724369,
      "rewards/margins": 0.043625812977552414,
      "rewards/rejected": -0.016284827142953873,
      "step": 833
    },
    {
      "epoch": 0.4844049485973166,
      "grad_norm": 319.7361755371094,
      "learning_rate": 3.7884950610110407e-06,
      "logits/chosen": -0.7712616920471191,
      "logits/rejected": -0.727273166179657,
      "logps/chosen": -88.34223937988281,
      "logps/rejected": -80.57120513916016,
      "loss": 13.1972,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04582667723298073,
      "rewards/margins": 0.07808013260364532,
      "rewards/rejected": -0.03225346654653549,
      "step": 834
    },
    {
      "epoch": 0.48498576987860836,
      "grad_norm": 325.38385009765625,
      "learning_rate": 3.787042417199303e-06,
      "logits/chosen": -0.9470648765563965,
      "logits/rejected": -0.9024521708488464,
      "logps/chosen": -72.9203872680664,
      "logps/rejected": -80.79722595214844,
      "loss": 13.8307,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.028173645958304405,
      "rewards/margins": 0.016614312306046486,
      "rewards/rejected": -0.04478795826435089,
      "step": 835
    },
    {
      "epoch": 0.4855665911599001,
      "grad_norm": 346.70098876953125,
      "learning_rate": 3.7855897733875657e-06,
      "logits/chosen": -0.9459084272384644,
      "logits/rejected": -0.9673193693161011,
      "logps/chosen": -70.69756317138672,
      "logps/rejected": -76.44866943359375,
      "loss": 13.1924,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01108443271368742,
      "rewards/margins": 0.0785098746418953,
      "rewards/rejected": -0.0674254447221756,
      "step": 836
    },
    {
      "epoch": 0.48614741244119186,
      "grad_norm": 346.8296813964844,
      "learning_rate": 3.7841371295758285e-06,
      "logits/chosen": -0.9128061532974243,
      "logits/rejected": -0.8887417912483215,
      "logps/chosen": -87.18028259277344,
      "logps/rejected": -85.31761169433594,
      "loss": 14.1875,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.070538230240345,
      "rewards/margins": -0.020030764862895012,
      "rewards/rejected": -0.050507474690675735,
      "step": 837
    },
    {
      "epoch": 0.4867282337224836,
      "grad_norm": 308.1683349609375,
      "learning_rate": 3.782684485764091e-06,
      "logits/chosen": -0.8262773752212524,
      "logits/rejected": -0.7639673948287964,
      "logps/chosen": -76.27080535888672,
      "logps/rejected": -80.69708251953125,
      "loss": 13.3472,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01220276765525341,
      "rewards/margins": 0.06231879070401192,
      "rewards/rejected": -0.050116024911403656,
      "step": 838
    },
    {
      "epoch": 0.48730905500377536,
      "grad_norm": 327.6427001953125,
      "learning_rate": 3.7812318419523535e-06,
      "logits/chosen": -0.7329220175743103,
      "logits/rejected": -0.8371773958206177,
      "logps/chosen": -80.13806915283203,
      "logps/rejected": -85.14008331298828,
      "loss": 12.5923,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.040527790784835815,
      "rewards/margins": 0.15911123156547546,
      "rewards/rejected": -0.11858340352773666,
      "step": 839
    },
    {
      "epoch": 0.4878898762850671,
      "grad_norm": 347.03125,
      "learning_rate": 3.7797791981406163e-06,
      "logits/chosen": -0.842019259929657,
      "logits/rejected": -0.8081402778625488,
      "logps/chosen": -85.04810333251953,
      "logps/rejected": -70.10143280029297,
      "loss": 13.535,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05649472028017044,
      "rewards/margins": 0.04177948087453842,
      "rewards/rejected": 0.014715233817696571,
      "step": 840
    },
    {
      "epoch": 0.48847069756635886,
      "grad_norm": 343.4723205566406,
      "learning_rate": 3.778326554328879e-06,
      "logits/chosen": -0.9070305824279785,
      "logits/rejected": -0.8659006953239441,
      "logps/chosen": -75.98943328857422,
      "logps/rejected": -72.68668365478516,
      "loss": 14.1313,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.046057023108005524,
      "rewards/margins": -0.016837935894727707,
      "rewards/rejected": -0.029219087213277817,
      "step": 841
    },
    {
      "epoch": 0.4890515188476506,
      "grad_norm": 297.91790771484375,
      "learning_rate": 3.7768739105171414e-06,
      "logits/chosen": -0.8112656474113464,
      "logits/rejected": -0.7686861753463745,
      "logps/chosen": -71.14762115478516,
      "logps/rejected": -66.70329284667969,
      "loss": 13.8801,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013187101110816002,
      "rewards/margins": 0.013149671256542206,
      "rewards/rejected": 3.743171691894531e-05,
      "step": 842
    },
    {
      "epoch": 0.4896323401289423,
      "grad_norm": 324.6368408203125,
      "learning_rate": 3.775421266705404e-06,
      "logits/chosen": -0.7056946754455566,
      "logits/rejected": -0.7951919436454773,
      "logps/chosen": -78.37447357177734,
      "logps/rejected": -74.64752960205078,
      "loss": 13.5777,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004278040025383234,
      "rewards/margins": 0.03796951100230217,
      "rewards/rejected": -0.042247556149959564,
      "step": 843
    },
    {
      "epoch": 0.4902131614102341,
      "grad_norm": 300.9444885253906,
      "learning_rate": 3.773968622893667e-06,
      "logits/chosen": -1.035330891609192,
      "logits/rejected": -1.0094799995422363,
      "logps/chosen": -72.4145278930664,
      "logps/rejected": -69.17391204833984,
      "loss": 13.6103,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0112453643232584,
      "rewards/margins": 0.031555455178022385,
      "rewards/rejected": -0.042800821363925934,
      "step": 844
    },
    {
      "epoch": 0.4907939826915258,
      "grad_norm": 314.42364501953125,
      "learning_rate": 3.772515979081929e-06,
      "logits/chosen": -0.8987109065055847,
      "logits/rejected": -0.771704912185669,
      "logps/chosen": -76.92586517333984,
      "logps/rejected": -70.66104888916016,
      "loss": 14.2906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0432976633310318,
      "rewards/margins": -0.030374759808182716,
      "rewards/rejected": -0.012922905385494232,
      "step": 845
    },
    {
      "epoch": 0.4913748039728176,
      "grad_norm": 293.44970703125,
      "learning_rate": 3.771063335270192e-06,
      "logits/chosen": -0.8815056085586548,
      "logits/rejected": -0.8597631454467773,
      "logps/chosen": -69.40690612792969,
      "logps/rejected": -73.88740539550781,
      "loss": 13.5889,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.031432799994945526,
      "rewards/margins": 0.03864104300737381,
      "rewards/rejected": -0.007208243012428284,
      "step": 846
    },
    {
      "epoch": 0.4919556252541093,
      "grad_norm": 293.9603576660156,
      "learning_rate": 3.7696106914584547e-06,
      "logits/chosen": -0.7362099885940552,
      "logits/rejected": -0.7803904414176941,
      "logps/chosen": -74.73085021972656,
      "logps/rejected": -70.69336700439453,
      "loss": 13.1999,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.019457781687378883,
      "rewards/margins": 0.08030074834823608,
      "rewards/rejected": -0.060842953622341156,
      "step": 847
    },
    {
      "epoch": 0.4925364465354011,
      "grad_norm": 317.520751953125,
      "learning_rate": 3.768158047646717e-06,
      "logits/chosen": -0.8254894018173218,
      "logits/rejected": -0.8281943202018738,
      "logps/chosen": -71.23345184326172,
      "logps/rejected": -71.3423080444336,
      "loss": 14.302,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.028336822986602783,
      "rewards/margins": -0.034830138087272644,
      "rewards/rejected": 0.006493322551250458,
      "step": 848
    },
    {
      "epoch": 0.4931172678166928,
      "grad_norm": 339.6095275878906,
      "learning_rate": 3.7667054038349798e-06,
      "logits/chosen": -0.786754846572876,
      "logits/rejected": -0.8182505369186401,
      "logps/chosen": -75.18721771240234,
      "logps/rejected": -77.47499084472656,
      "loss": 14.7391,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06475405395030975,
      "rewards/margins": -0.06797143071889877,
      "rewards/rejected": 0.0032173804938793182,
      "step": 849
    },
    {
      "epoch": 0.49369808909798457,
      "grad_norm": 318.309326171875,
      "learning_rate": 3.7652527600232425e-06,
      "logits/chosen": -0.8893791437149048,
      "logits/rejected": -0.8710716366767883,
      "logps/chosen": -74.41831970214844,
      "logps/rejected": -73.89128875732422,
      "loss": 13.2277,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0054091643542051315,
      "rewards/margins": 0.07398828119039536,
      "rewards/rejected": -0.06857912242412567,
      "step": 850
    },
    {
      "epoch": 0.4942789103792763,
      "grad_norm": 315.2265930175781,
      "learning_rate": 3.7638001162115053e-06,
      "logits/chosen": -0.7621113657951355,
      "logits/rejected": -0.81293123960495,
      "logps/chosen": -78.86296081542969,
      "logps/rejected": -81.82852172851562,
      "loss": 13.244,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03520115092396736,
      "rewards/margins": 0.07237912714481354,
      "rewards/rejected": -0.03717798367142677,
      "step": 851
    },
    {
      "epoch": 0.49485973166056807,
      "grad_norm": 296.0227355957031,
      "learning_rate": 3.7623474723997676e-06,
      "logits/chosen": -0.862707257270813,
      "logits/rejected": -0.7818862795829773,
      "logps/chosen": -72.54582977294922,
      "logps/rejected": -71.32649230957031,
      "loss": 12.8667,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05158460885286331,
      "rewards/margins": 0.11155477911233902,
      "rewards/rejected": -0.05997015908360481,
      "step": 852
    },
    {
      "epoch": 0.4954405529418598,
      "grad_norm": 299.976806640625,
      "learning_rate": 3.7608948285880303e-06,
      "logits/chosen": -0.803802490234375,
      "logits/rejected": -0.9642475843429565,
      "logps/chosen": -71.30755615234375,
      "logps/rejected": -69.78184509277344,
      "loss": 13.1189,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03701017424464226,
      "rewards/margins": 0.11028815805912018,
      "rewards/rejected": -0.07327798753976822,
      "step": 853
    },
    {
      "epoch": 0.4960213742231515,
      "grad_norm": 306.97320556640625,
      "learning_rate": 3.759442184776293e-06,
      "logits/chosen": -0.7374328374862671,
      "logits/rejected": -0.7047148942947388,
      "logps/chosen": -79.32594299316406,
      "logps/rejected": -76.9816665649414,
      "loss": 13.0718,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.031373415142297745,
      "rewards/margins": 0.10228639841079712,
      "rewards/rejected": -0.07091299444437027,
      "step": 854
    },
    {
      "epoch": 0.4966021955044433,
      "grad_norm": 298.43389892578125,
      "learning_rate": 3.7579895409645554e-06,
      "logits/chosen": -0.8767411112785339,
      "logits/rejected": -0.7835893630981445,
      "logps/chosen": -71.76991271972656,
      "logps/rejected": -82.6242904663086,
      "loss": 13.2089,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.024014713242650032,
      "rewards/margins": 0.07497559487819672,
      "rewards/rejected": -0.05096089839935303,
      "step": 855
    },
    {
      "epoch": 0.497183016785735,
      "grad_norm": 297.3446960449219,
      "learning_rate": 3.756536897152818e-06,
      "logits/chosen": -0.7374379634857178,
      "logits/rejected": -0.705323338508606,
      "logps/chosen": -72.24920654296875,
      "logps/rejected": -71.93132019042969,
      "loss": 13.2626,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.018467990681529045,
      "rewards/margins": 0.070315420627594,
      "rewards/rejected": -0.0518474280834198,
      "step": 856
    },
    {
      "epoch": 0.4977638380670268,
      "grad_norm": 330.3368225097656,
      "learning_rate": 3.755084253341081e-06,
      "logits/chosen": -0.7391015291213989,
      "logits/rejected": -0.8123418688774109,
      "logps/chosen": -73.87885284423828,
      "logps/rejected": -79.49480438232422,
      "loss": 13.2954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021732434630393982,
      "rewards/margins": 0.0644746795296669,
      "rewards/rejected": -0.04274224489927292,
      "step": 857
    },
    {
      "epoch": 0.4983446593483185,
      "grad_norm": 311.4170227050781,
      "learning_rate": 3.7536316095293436e-06,
      "logits/chosen": -0.7112471461296082,
      "logits/rejected": -0.743118166923523,
      "logps/chosen": -73.13099670410156,
      "logps/rejected": -73.69615173339844,
      "loss": 13.587,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03787677735090256,
      "rewards/margins": 0.04251720756292343,
      "rewards/rejected": -0.004640430212020874,
      "step": 858
    },
    {
      "epoch": 0.4989254806296103,
      "grad_norm": 285.54547119140625,
      "learning_rate": 3.752178965717606e-06,
      "logits/chosen": -0.805162250995636,
      "logits/rejected": -0.8250443339347839,
      "logps/chosen": -70.72074127197266,
      "logps/rejected": -70.56238555908203,
      "loss": 12.6761,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.048637259751558304,
      "rewards/margins": 0.1373429000377655,
      "rewards/rejected": -0.0887056440114975,
      "step": 859
    },
    {
      "epoch": 0.499506301910902,
      "grad_norm": 294.9131774902344,
      "learning_rate": 3.7507263219058687e-06,
      "logits/chosen": -0.909680962562561,
      "logits/rejected": -0.8498638272285461,
      "logps/chosen": -66.12528991699219,
      "logps/rejected": -68.66703796386719,
      "loss": 13.6062,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03878556936979294,
      "rewards/margins": 0.03481757268309593,
      "rewards/rejected": 0.003967995289713144,
      "step": 860
    },
    {
      "epoch": 0.5000871231921937,
      "grad_norm": 347.707275390625,
      "learning_rate": 3.749273678094132e-06,
      "logits/chosen": -0.8227556943893433,
      "logits/rejected": -0.799028754234314,
      "logps/chosen": -76.69181823730469,
      "logps/rejected": -73.48719787597656,
      "loss": 13.869,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02884422242641449,
      "rewards/margins": 0.014967566356062889,
      "rewards/rejected": -0.04381179064512253,
      "step": 861
    },
    {
      "epoch": 0.5006679444734855,
      "grad_norm": 310.5048828125,
      "learning_rate": 3.7478210342823946e-06,
      "logits/chosen": -0.6707647442817688,
      "logits/rejected": -0.6352334022521973,
      "logps/chosen": -67.49751281738281,
      "logps/rejected": -72.48457336425781,
      "loss": 13.9948,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03386518359184265,
      "rewards/margins": 0.0032462819945067167,
      "rewards/rejected": -0.03711146116256714,
      "step": 862
    },
    {
      "epoch": 0.5012487657547773,
      "grad_norm": 361.6993713378906,
      "learning_rate": 3.746368390470657e-06,
      "logits/chosen": -0.8487448692321777,
      "logits/rejected": -0.7983860373497009,
      "logps/chosen": -70.15946960449219,
      "logps/rejected": -77.11515808105469,
      "loss": 13.3064,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.029017318040132523,
      "rewards/margins": 0.07312078028917313,
      "rewards/rejected": -0.0441034696996212,
      "step": 863
    },
    {
      "epoch": 0.501829587036069,
      "grad_norm": 317.2196960449219,
      "learning_rate": 3.7449157466589197e-06,
      "logits/chosen": -0.693240761756897,
      "logits/rejected": -0.6469998359680176,
      "logps/chosen": -71.18406677246094,
      "logps/rejected": -75.88301086425781,
      "loss": 13.6113,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03340126946568489,
      "rewards/margins": 0.03463831916451454,
      "rewards/rejected": -0.06803958117961884,
      "step": 864
    },
    {
      "epoch": 0.5024104083173607,
      "grad_norm": 314.50555419921875,
      "learning_rate": 3.7434631028471825e-06,
      "logits/chosen": -0.8128454089164734,
      "logits/rejected": -0.7394827604293823,
      "logps/chosen": -70.54247283935547,
      "logps/rejected": -70.06904602050781,
      "loss": 15.048,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.027236783877015114,
      "rewards/margins": -0.09788013994693756,
      "rewards/rejected": 0.070643350481987,
      "step": 865
    },
    {
      "epoch": 0.5029912295986525,
      "grad_norm": 312.5657653808594,
      "learning_rate": 3.742010459035445e-06,
      "logits/chosen": -0.9018377065658569,
      "logits/rejected": -0.9190446138381958,
      "logps/chosen": -69.02962493896484,
      "logps/rejected": -70.04609680175781,
      "loss": 13.2226,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.012089548632502556,
      "rewards/margins": 0.07575584948062897,
      "rewards/rejected": -0.08784539997577667,
      "step": 866
    },
    {
      "epoch": 0.5035720508799443,
      "grad_norm": 330.9731140136719,
      "learning_rate": 3.7405578152237075e-06,
      "logits/chosen": -0.8004295229911804,
      "logits/rejected": -0.8180350065231323,
      "logps/chosen": -73.49920654296875,
      "logps/rejected": -85.6948013305664,
      "loss": 13.8988,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05013388395309448,
      "rewards/margins": 0.010029042139649391,
      "rewards/rejected": -0.060162924230098724,
      "step": 867
    },
    {
      "epoch": 0.5041528721612359,
      "grad_norm": 280.4370422363281,
      "learning_rate": 3.7391051714119703e-06,
      "logits/chosen": -0.9341899752616882,
      "logits/rejected": -0.8376950025558472,
      "logps/chosen": -71.9174575805664,
      "logps/rejected": -73.74215698242188,
      "loss": 13.1955,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.036871034651994705,
      "rewards/margins": 0.07967637479305267,
      "rewards/rejected": -0.04280534014105797,
      "step": 868
    },
    {
      "epoch": 0.5047336934425277,
      "grad_norm": 287.02764892578125,
      "learning_rate": 3.737652527600233e-06,
      "logits/chosen": -0.817496657371521,
      "logits/rejected": -0.7701882123947144,
      "logps/chosen": -76.1856460571289,
      "logps/rejected": -78.75908660888672,
      "loss": 13.4908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014233636669814587,
      "rewards/margins": 0.05255434662103653,
      "rewards/rejected": -0.038320716470479965,
      "step": 869
    },
    {
      "epoch": 0.5053145147238195,
      "grad_norm": 321.85247802734375,
      "learning_rate": 3.7361998837884954e-06,
      "logits/chosen": -0.8572362661361694,
      "logits/rejected": -0.8262847661972046,
      "logps/chosen": -72.18296813964844,
      "logps/rejected": -73.68527221679688,
      "loss": 13.3539,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.023406734690070152,
      "rewards/margins": 0.07013611495494843,
      "rewards/rejected": -0.04672938585281372,
      "step": 870
    },
    {
      "epoch": 0.5058953360051113,
      "grad_norm": 318.94415283203125,
      "learning_rate": 3.734747239976758e-06,
      "logits/chosen": -0.7708402872085571,
      "logits/rejected": -0.8278223276138306,
      "logps/chosen": -75.5167236328125,
      "logps/rejected": -82.86116027832031,
      "loss": 13.5957,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.018369678407907486,
      "rewards/margins": 0.034253206104040146,
      "rewards/rejected": -0.01588352955877781,
      "step": 871
    },
    {
      "epoch": 0.5064761572864029,
      "grad_norm": 315.12384033203125,
      "learning_rate": 3.733294596165021e-06,
      "logits/chosen": -0.8158319592475891,
      "logits/rejected": -0.7617601156234741,
      "logps/chosen": -65.91107940673828,
      "logps/rejected": -74.64680480957031,
      "loss": 13.6166,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.021057626232504845,
      "rewards/margins": 0.0367097444832325,
      "rewards/rejected": -0.057767368853092194,
      "step": 872
    },
    {
      "epoch": 0.5070569785676947,
      "grad_norm": 289.8279113769531,
      "learning_rate": 3.7318419523532836e-06,
      "logits/chosen": -1.12114679813385,
      "logits/rejected": -0.9899064898490906,
      "logps/chosen": -67.73985290527344,
      "logps/rejected": -74.02053833007812,
      "loss": 13.2951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.018153967335820198,
      "rewards/margins": 0.06531564891338348,
      "rewards/rejected": -0.04716167598962784,
      "step": 873
    },
    {
      "epoch": 0.5076377998489865,
      "grad_norm": 291.0061340332031,
      "learning_rate": 3.730389308541546e-06,
      "logits/chosen": -0.8384881019592285,
      "logits/rejected": -0.8821040987968445,
      "logps/chosen": -70.64179992675781,
      "logps/rejected": -74.30955505371094,
      "loss": 13.453,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.009448887780308723,
      "rewards/margins": 0.05563428997993469,
      "rewards/rejected": -0.04618540033698082,
      "step": 874
    },
    {
      "epoch": 0.5082186211302782,
      "grad_norm": 309.33905029296875,
      "learning_rate": 3.7289366647298087e-06,
      "logits/chosen": -0.7903847098350525,
      "logits/rejected": -0.8384620547294617,
      "logps/chosen": -78.0987548828125,
      "logps/rejected": -80.98674011230469,
      "loss": 12.9163,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004122171550989151,
      "rewards/margins": 0.10959751904010773,
      "rewards/rejected": -0.10547534376382828,
      "step": 875
    },
    {
      "epoch": 0.5087994424115699,
      "grad_norm": 297.92266845703125,
      "learning_rate": 3.7274840209180714e-06,
      "logits/chosen": -0.9015482068061829,
      "logits/rejected": -0.8714168667793274,
      "logps/chosen": -73.52940368652344,
      "logps/rejected": -77.55482482910156,
      "loss": 12.4707,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.08103947341442108,
      "rewards/margins": 0.15564516186714172,
      "rewards/rejected": -0.07460571825504303,
      "step": 876
    },
    {
      "epoch": 0.5093802636928617,
      "grad_norm": 269.5519714355469,
      "learning_rate": 3.7260313771063337e-06,
      "logits/chosen": -0.7991993427276611,
      "logits/rejected": -0.7887776494026184,
      "logps/chosen": -70.71697998046875,
      "logps/rejected": -70.16976928710938,
      "loss": 13.1393,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06928080320358276,
      "rewards/margins": 0.08986689150333405,
      "rewards/rejected": -0.02058609202504158,
      "step": 877
    },
    {
      "epoch": 0.5099610849741535,
      "grad_norm": 323.81719970703125,
      "learning_rate": 3.7245787332945965e-06,
      "logits/chosen": -0.6992955803871155,
      "logits/rejected": -0.8329144716262817,
      "logps/chosen": -76.07032012939453,
      "logps/rejected": -72.76537322998047,
      "loss": 13.1797,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.039288222789764404,
      "rewards/margins": 0.08043770492076874,
      "rewards/rejected": -0.11972592771053314,
      "step": 878
    },
    {
      "epoch": 0.5105419062554452,
      "grad_norm": 282.5196228027344,
      "learning_rate": 3.7231260894828592e-06,
      "logits/chosen": -0.8015006184577942,
      "logits/rejected": -0.8797491788864136,
      "logps/chosen": -68.26971435546875,
      "logps/rejected": -71.91568756103516,
      "loss": 12.5433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09241674095392227,
      "rewards/margins": 0.15844053030014038,
      "rewards/rejected": -0.0660238116979599,
      "step": 879
    },
    {
      "epoch": 0.5111227275367369,
      "grad_norm": 313.0185852050781,
      "learning_rate": 3.7216734456711216e-06,
      "logits/chosen": -0.7841866612434387,
      "logits/rejected": -0.7923185229301453,
      "logps/chosen": -76.68905639648438,
      "logps/rejected": -71.82386779785156,
      "loss": 14.0049,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03983437269926071,
      "rewards/margins": -0.002283641602844,
      "rewards/rejected": -0.03755073621869087,
      "step": 880
    },
    {
      "epoch": 0.5117035488180287,
      "grad_norm": 304.0037536621094,
      "learning_rate": 3.7202208018593843e-06,
      "logits/chosen": -0.8578149676322937,
      "logits/rejected": -0.8352439999580383,
      "logps/chosen": -71.64418029785156,
      "logps/rejected": -68.3890151977539,
      "loss": 13.8305,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.005443647503852844,
      "rewards/margins": 0.013478374108672142,
      "rewards/rejected": -0.018922025337815285,
      "step": 881
    },
    {
      "epoch": 0.5122843700993205,
      "grad_norm": 310.70489501953125,
      "learning_rate": 3.718768158047647e-06,
      "logits/chosen": -0.8352943658828735,
      "logits/rejected": -0.8491352200508118,
      "logps/chosen": -75.1880111694336,
      "logps/rejected": -82.05909729003906,
      "loss": 12.9929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.028002599254250526,
      "rewards/margins": 0.104197658598423,
      "rewards/rejected": -0.13220027089118958,
      "step": 882
    },
    {
      "epoch": 0.5128651913806122,
      "grad_norm": 312.11712646484375,
      "learning_rate": 3.71731551423591e-06,
      "logits/chosen": -0.7563623785972595,
      "logits/rejected": -0.8587571978569031,
      "logps/chosen": -74.63130187988281,
      "logps/rejected": -69.7817153930664,
      "loss": 14.3503,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.003069313708692789,
      "rewards/margins": -0.03878450393676758,
      "rewards/rejected": 0.04185382276773453,
      "step": 883
    },
    {
      "epoch": 0.5134460126619039,
      "grad_norm": 317.8841247558594,
      "learning_rate": 3.715862870424172e-06,
      "logits/chosen": -0.8039946556091309,
      "logits/rejected": -0.9972225427627563,
      "logps/chosen": -85.01696014404297,
      "logps/rejected": -66.03048706054688,
      "loss": 14.4587,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07630066573619843,
      "rewards/margins": -0.04431800916790962,
      "rewards/rejected": -0.0319826602935791,
      "step": 884
    },
    {
      "epoch": 0.5140268339431957,
      "grad_norm": 341.647705078125,
      "learning_rate": 3.714410226612435e-06,
      "logits/chosen": -0.8729516863822937,
      "logits/rejected": -0.8806362152099609,
      "logps/chosen": -74.71580505371094,
      "logps/rejected": -74.4959487915039,
      "loss": 14.1933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.062128596007823944,
      "rewards/margins": -0.019561443477869034,
      "rewards/rejected": -0.04256715252995491,
      "step": 885
    },
    {
      "epoch": 0.5146076552244874,
      "grad_norm": 324.4678955078125,
      "learning_rate": 3.7129575828006976e-06,
      "logits/chosen": -0.9463183283805847,
      "logits/rejected": -0.9414758682250977,
      "logps/chosen": -76.0389633178711,
      "logps/rejected": -74.2925033569336,
      "loss": 13.6209,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0420696847140789,
      "rewards/margins": 0.036721937358379364,
      "rewards/rejected": -0.07879161834716797,
      "step": 886
    },
    {
      "epoch": 0.5151884765057791,
      "grad_norm": 350.20831298828125,
      "learning_rate": 3.71150493898896e-06,
      "logits/chosen": -0.7539618015289307,
      "logits/rejected": -0.6907280683517456,
      "logps/chosen": -77.5100326538086,
      "logps/rejected": -68.97132873535156,
      "loss": 14.1455,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0666489452123642,
      "rewards/margins": -0.008004192262887955,
      "rewards/rejected": -0.058644749224185944,
      "step": 887
    },
    {
      "epoch": 0.5157692977870709,
      "grad_norm": 2863.912353515625,
      "learning_rate": 3.7100522951772227e-06,
      "logits/chosen": -0.9735726118087769,
      "logits/rejected": -0.8810579180717468,
      "logps/chosen": -75.67332458496094,
      "logps/rejected": -68.54508209228516,
      "loss": 14.7704,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.038121938705444336,
      "rewards/margins": -0.07279206067323685,
      "rewards/rejected": 0.034670114517211914,
      "step": 888
    },
    {
      "epoch": 0.5163501190683627,
      "grad_norm": 294.0743713378906,
      "learning_rate": 3.7085996513654855e-06,
      "logits/chosen": -0.9168221354484558,
      "logits/rejected": -0.7986600399017334,
      "logps/chosen": -76.08323669433594,
      "logps/rejected": -67.5433120727539,
      "loss": 13.0438,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.014614105224609375,
      "rewards/margins": 0.10053624212741852,
      "rewards/rejected": -0.08592212200164795,
      "step": 889
    },
    {
      "epoch": 0.5169309403496544,
      "grad_norm": 322.785400390625,
      "learning_rate": 3.707147007553748e-06,
      "logits/chosen": -0.814892590045929,
      "logits/rejected": -0.8575268983840942,
      "logps/chosen": -71.9342269897461,
      "logps/rejected": -86.5703125,
      "loss": 13.6283,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.013843230903148651,
      "rewards/margins": 0.03725925087928772,
      "rewards/rejected": -0.05110248178243637,
      "step": 890
    },
    {
      "epoch": 0.5175117616309461,
      "grad_norm": 292.9025573730469,
      "learning_rate": 3.7056943637420105e-06,
      "logits/chosen": -0.7913263440132141,
      "logits/rejected": -0.8443421125411987,
      "logps/chosen": -60.87749481201172,
      "logps/rejected": -78.35881042480469,
      "loss": 13.2476,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04118134453892708,
      "rewards/margins": 0.07499329000711441,
      "rewards/rejected": -0.1161746233701706,
      "step": 891
    },
    {
      "epoch": 0.5180925829122379,
      "grad_norm": 309.80169677734375,
      "learning_rate": 3.7042417199302733e-06,
      "logits/chosen": -1.0392483472824097,
      "logits/rejected": -0.970413088798523,
      "logps/chosen": -74.8720474243164,
      "logps/rejected": -75.41184997558594,
      "loss": 13.5448,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.024524737149477005,
      "rewards/margins": 0.050737909972667694,
      "rewards/rejected": -0.075262650847435,
      "step": 892
    },
    {
      "epoch": 0.5186734041935297,
      "grad_norm": 324.7698059082031,
      "learning_rate": 3.702789076118536e-06,
      "logits/chosen": -0.7926565408706665,
      "logits/rejected": -0.8359763026237488,
      "logps/chosen": -78.66641235351562,
      "logps/rejected": -77.9851303100586,
      "loss": 13.4493,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013259848579764366,
      "rewards/margins": 0.06008158251643181,
      "rewards/rejected": -0.04682173952460289,
      "step": 893
    },
    {
      "epoch": 0.5192542254748214,
      "grad_norm": 311.0304260253906,
      "learning_rate": 3.7013364323067984e-06,
      "logits/chosen": -0.8284494280815125,
      "logits/rejected": -0.8559365272521973,
      "logps/chosen": -76.58866119384766,
      "logps/rejected": -79.98295593261719,
      "loss": 13.8651,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.035661615431308746,
      "rewards/margins": 0.0057185059413313866,
      "rewards/rejected": -0.04138011857867241,
      "step": 894
    },
    {
      "epoch": 0.5198350467561131,
      "grad_norm": 309.0633239746094,
      "learning_rate": 3.699883788495061e-06,
      "logits/chosen": -0.8562793731689453,
      "logits/rejected": -0.7222028970718384,
      "logps/chosen": -72.55845642089844,
      "logps/rejected": -67.55829620361328,
      "loss": 13.8677,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02681596204638481,
      "rewards/margins": 0.007083225063979626,
      "rewards/rejected": -0.03389918431639671,
      "step": 895
    },
    {
      "epoch": 0.5204158680374049,
      "grad_norm": 281.7559509277344,
      "learning_rate": 3.698431144683324e-06,
      "logits/chosen": -0.7869559526443481,
      "logits/rejected": -0.7652812004089355,
      "logps/chosen": -63.31262969970703,
      "logps/rejected": -70.62659454345703,
      "loss": 13.6084,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.033197641372680664,
      "rewards/margins": 0.037535399198532104,
      "rewards/rejected": -0.07073303312063217,
      "step": 896
    },
    {
      "epoch": 0.5209966893186966,
      "grad_norm": 305.7250061035156,
      "learning_rate": 3.696978500871586e-06,
      "logits/chosen": -1.0239300727844238,
      "logits/rejected": -0.8789726495742798,
      "logps/chosen": -70.31292724609375,
      "logps/rejected": -72.78204345703125,
      "loss": 13.9797,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02144424058496952,
      "rewards/margins": -0.005210808012634516,
      "rewards/rejected": 0.0266550425440073,
      "step": 897
    },
    {
      "epoch": 0.5215775105999884,
      "grad_norm": 343.10711669921875,
      "learning_rate": 3.695525857059849e-06,
      "logits/chosen": -0.8156943321228027,
      "logits/rejected": -0.8041397333145142,
      "logps/chosen": -81.42640686035156,
      "logps/rejected": -72.1771469116211,
      "loss": 14.2155,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02531343698501587,
      "rewards/margins": -0.019732346758246422,
      "rewards/rejected": -0.00558108277618885,
      "step": 898
    },
    {
      "epoch": 0.5221583318812801,
      "grad_norm": 316.3569641113281,
      "learning_rate": 3.6940732132481117e-06,
      "logits/chosen": -0.701055645942688,
      "logits/rejected": -0.6751523613929749,
      "logps/chosen": -72.49714660644531,
      "logps/rejected": -70.53977966308594,
      "loss": 13.9347,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.007911900989711285,
      "rewards/margins": 0.0022199698723852634,
      "rewards/rejected": -0.010131875053048134,
      "step": 899
    },
    {
      "epoch": 0.5227391531625719,
      "grad_norm": 352.64306640625,
      "learning_rate": 3.6926205694363744e-06,
      "logits/chosen": -0.7641804814338684,
      "logits/rejected": -0.82763671875,
      "logps/chosen": -77.14866638183594,
      "logps/rejected": -79.8504867553711,
      "loss": 14.8528,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.06686311960220337,
      "rewards/margins": -0.07604999095201492,
      "rewards/rejected": 0.009186875075101852,
      "step": 900
    },
    {
      "epoch": 0.5233199744438636,
      "grad_norm": 315.5428466796875,
      "learning_rate": 3.6911679256246367e-06,
      "logits/chosen": -0.8268402814865112,
      "logits/rejected": -0.7936877608299255,
      "logps/chosen": -75.72247314453125,
      "logps/rejected": -74.36206817626953,
      "loss": 14.1353,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0019198231166228652,
      "rewards/margins": -0.01289793848991394,
      "rewards/rejected": 0.010978116653859615,
      "step": 901
    },
    {
      "epoch": 0.5239007957251554,
      "grad_norm": 303.87799072265625,
      "learning_rate": 3.6897152818128995e-06,
      "logits/chosen": -0.9414458274841309,
      "logits/rejected": -0.8633650541305542,
      "logps/chosen": -73.24238586425781,
      "logps/rejected": -81.83255767822266,
      "loss": 13.4568,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.027762720361351967,
      "rewards/margins": 0.05169368535280228,
      "rewards/rejected": -0.02393096126616001,
      "step": 902
    },
    {
      "epoch": 0.5244816170064471,
      "grad_norm": 336.22467041015625,
      "learning_rate": 3.6882626380011627e-06,
      "logits/chosen": -0.7279499173164368,
      "logits/rejected": -0.9009740948677063,
      "logps/chosen": -81.50945281982422,
      "logps/rejected": -75.34597778320312,
      "loss": 13.8267,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.023870524019002914,
      "rewards/margins": 0.016882585361599922,
      "rewards/rejected": -0.04075310379266739,
      "step": 903
    },
    {
      "epoch": 0.5250624382877389,
      "grad_norm": 308.542724609375,
      "learning_rate": 3.6868099941894254e-06,
      "logits/chosen": -0.8215476274490356,
      "logits/rejected": -0.8264580965042114,
      "logps/chosen": -73.51042175292969,
      "logps/rejected": -70.37979125976562,
      "loss": 14.331,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010108297690749168,
      "rewards/margins": -0.03372166305780411,
      "rewards/rejected": 0.043829962611198425,
      "step": 904
    },
    {
      "epoch": 0.5256432595690306,
      "grad_norm": 374.73370361328125,
      "learning_rate": 3.6853573503776877e-06,
      "logits/chosen": -0.7862215638160706,
      "logits/rejected": -0.6950373649597168,
      "logps/chosen": -76.63065338134766,
      "logps/rejected": -72.22526550292969,
      "loss": 13.453,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02635444700717926,
      "rewards/margins": 0.0567445270717144,
      "rewards/rejected": -0.08309897035360336,
      "step": 905
    },
    {
      "epoch": 0.5262240808503224,
      "grad_norm": 320.5014343261719,
      "learning_rate": 3.6839047065659505e-06,
      "logits/chosen": -0.6966902017593384,
      "logits/rejected": -0.767865002155304,
      "logps/chosen": -73.40306854248047,
      "logps/rejected": -71.08512115478516,
      "loss": 14.5145,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05028662830591202,
      "rewards/margins": -0.047402046620845795,
      "rewards/rejected": -0.002884581685066223,
      "step": 906
    },
    {
      "epoch": 0.5268049021316141,
      "grad_norm": 329.3699951171875,
      "learning_rate": 3.6824520627542132e-06,
      "logits/chosen": -0.7144160270690918,
      "logits/rejected": -0.7741755843162537,
      "logps/chosen": -75.09828186035156,
      "logps/rejected": -78.51341247558594,
      "loss": 14.6293,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06854270398616791,
      "rewards/margins": -0.057167064398527145,
      "rewards/rejected": -0.01137564517557621,
      "step": 907
    },
    {
      "epoch": 0.5273857234129058,
      "grad_norm": 310.02880859375,
      "learning_rate": 3.680999418942476e-06,
      "logits/chosen": -0.8827294111251831,
      "logits/rejected": -0.969725489616394,
      "logps/chosen": -72.49862670898438,
      "logps/rejected": -78.67535400390625,
      "loss": 13.1686,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.014622239395976067,
      "rewards/margins": 0.07751540094614029,
      "rewards/rejected": -0.06289316713809967,
      "step": 908
    },
    {
      "epoch": 0.5279665446941976,
      "grad_norm": 301.2107849121094,
      "learning_rate": 3.6795467751307383e-06,
      "logits/chosen": -0.8012347221374512,
      "logits/rejected": -0.8275817632675171,
      "logps/chosen": -72.11550903320312,
      "logps/rejected": -66.7422103881836,
      "loss": 13.0052,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0879693329334259,
      "rewards/margins": 0.09242668747901917,
      "rewards/rejected": -0.004457362927496433,
      "step": 909
    },
    {
      "epoch": 0.5285473659754893,
      "grad_norm": 327.29351806640625,
      "learning_rate": 3.678094131319001e-06,
      "logits/chosen": -0.8897010684013367,
      "logits/rejected": -0.9195152521133423,
      "logps/chosen": -75.18798828125,
      "logps/rejected": -78.10919952392578,
      "loss": 13.6292,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.00252416729927063,
      "rewards/margins": 0.035282202064991,
      "rewards/rejected": -0.03275803476572037,
      "step": 910
    },
    {
      "epoch": 0.5291281872567811,
      "grad_norm": 572.74853515625,
      "learning_rate": 3.676641487507264e-06,
      "logits/chosen": -0.8815191984176636,
      "logits/rejected": -0.9849497675895691,
      "logps/chosen": -71.17215728759766,
      "logps/rejected": -74.537353515625,
      "loss": 13.3058,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.038278210908174515,
      "rewards/margins": 0.06560848653316498,
      "rewards/rejected": -0.02733028493821621,
      "step": 911
    },
    {
      "epoch": 0.5297090085380728,
      "grad_norm": 365.2557373046875,
      "learning_rate": 3.675188843695526e-06,
      "logits/chosen": -0.9522512555122375,
      "logits/rejected": -1.010914921760559,
      "logps/chosen": -70.09083557128906,
      "logps/rejected": -68.78644561767578,
      "loss": 13.7416,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05300981551408768,
      "rewards/margins": 0.02500464953482151,
      "rewards/rejected": -0.07801447063684464,
      "step": 912
    },
    {
      "epoch": 0.5302898298193646,
      "grad_norm": 328.8595275878906,
      "learning_rate": 3.673736199883789e-06,
      "logits/chosen": -0.9032946825027466,
      "logits/rejected": -0.8389007449150085,
      "logps/chosen": -77.11427307128906,
      "logps/rejected": -67.35809326171875,
      "loss": 13.7302,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.00886593945324421,
      "rewards/margins": 0.030330365523695946,
      "rewards/rejected": -0.03919629380106926,
      "step": 913
    },
    {
      "epoch": 0.5308706511006563,
      "grad_norm": 298.7828369140625,
      "learning_rate": 3.6722835560720516e-06,
      "logits/chosen": -0.6680513620376587,
      "logits/rejected": -0.7470614910125732,
      "logps/chosen": -72.6822280883789,
      "logps/rejected": -72.9788818359375,
      "loss": 13.5136,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010395077057182789,
      "rewards/margins": 0.0404205322265625,
      "rewards/rejected": -0.030025456100702286,
      "step": 914
    },
    {
      "epoch": 0.5314514723819481,
      "grad_norm": 293.72021484375,
      "learning_rate": 3.6708309122603144e-06,
      "logits/chosen": -0.9234855771064758,
      "logits/rejected": -0.9707880020141602,
      "logps/chosen": -73.39376068115234,
      "logps/rejected": -71.17304992675781,
      "loss": 12.8332,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.052253782749176025,
      "rewards/margins": 0.12076272815465927,
      "rewards/rejected": -0.06850893795490265,
      "step": 915
    },
    {
      "epoch": 0.5320322936632398,
      "grad_norm": 309.4018249511719,
      "learning_rate": 3.6693782684485767e-06,
      "logits/chosen": -0.8161689639091492,
      "logits/rejected": -0.9288781881332397,
      "logps/chosen": -82.39122009277344,
      "logps/rejected": -70.08438873291016,
      "loss": 13.7639,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04291496425867081,
      "rewards/margins": 0.02595677599310875,
      "rewards/rejected": -0.06887174397706985,
      "step": 916
    },
    {
      "epoch": 0.5326131149445316,
      "grad_norm": 300.3961181640625,
      "learning_rate": 3.6679256246368394e-06,
      "logits/chosen": -0.877852737903595,
      "logits/rejected": -0.8263446092605591,
      "logps/chosen": -69.59159851074219,
      "logps/rejected": -69.27183532714844,
      "loss": 13.7373,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004416265990585089,
      "rewards/margins": 0.022350173443555832,
      "rewards/rejected": -0.026766439899802208,
      "step": 917
    },
    {
      "epoch": 0.5331939362258233,
      "grad_norm": 326.1161804199219,
      "learning_rate": 3.666472980825102e-06,
      "logits/chosen": -0.7212264537811279,
      "logits/rejected": -0.8631353378295898,
      "logps/chosen": -78.27708435058594,
      "logps/rejected": -80.06709289550781,
      "loss": 13.7463,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0017011367017403245,
      "rewards/margins": 0.020040009170770645,
      "rewards/rejected": -0.018338870257139206,
      "step": 918
    },
    {
      "epoch": 0.533774757507115,
      "grad_norm": 324.24853515625,
      "learning_rate": 3.6650203370133645e-06,
      "logits/chosen": -0.8396110534667969,
      "logits/rejected": -0.9127988815307617,
      "logps/chosen": -79.79960632324219,
      "logps/rejected": -77.63664245605469,
      "loss": 13.6282,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004808626137673855,
      "rewards/margins": 0.03346925228834152,
      "rewards/rejected": -0.028660621494054794,
      "step": 919
    },
    {
      "epoch": 0.5343555787884068,
      "grad_norm": 302.04913330078125,
      "learning_rate": 3.6635676932016273e-06,
      "logits/chosen": -1.0392416715621948,
      "logits/rejected": -1.1793787479400635,
      "logps/chosen": -72.44638061523438,
      "logps/rejected": -72.2593994140625,
      "loss": 12.7915,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028838008642196655,
      "rewards/margins": 0.13549332320690155,
      "rewards/rejected": -0.1066553145647049,
      "step": 920
    },
    {
      "epoch": 0.5349364000696986,
      "grad_norm": 314.06964111328125,
      "learning_rate": 3.66211504938989e-06,
      "logits/chosen": -0.8888261914253235,
      "logits/rejected": -0.761336624622345,
      "logps/chosen": -72.33747863769531,
      "logps/rejected": -73.57784271240234,
      "loss": 13.9558,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0025641447864472866,
      "rewards/margins": 0.002194226486608386,
      "rewards/rejected": -0.004758368246257305,
      "step": 921
    },
    {
      "epoch": 0.5355172213509903,
      "grad_norm": 296.27716064453125,
      "learning_rate": 3.6606624055781528e-06,
      "logits/chosen": -0.7798510789871216,
      "logits/rejected": -0.7086332440376282,
      "logps/chosen": -73.7872543334961,
      "logps/rejected": -67.88629150390625,
      "loss": 13.5935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004376659635454416,
      "rewards/margins": 0.04153291508555412,
      "rewards/rejected": -0.03715625777840614,
      "step": 922
    },
    {
      "epoch": 0.536098042632282,
      "grad_norm": 312.75262451171875,
      "learning_rate": 3.659209761766415e-06,
      "logits/chosen": -0.8951870799064636,
      "logits/rejected": -0.9884964823722839,
      "logps/chosen": -81.04949951171875,
      "logps/rejected": -72.61628723144531,
      "loss": 12.8728,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04099228233098984,
      "rewards/margins": 0.10998284816741943,
      "rewards/rejected": -0.06899057328701019,
      "step": 923
    },
    {
      "epoch": 0.5366788639135738,
      "grad_norm": 335.9742126464844,
      "learning_rate": 3.657757117954678e-06,
      "logits/chosen": -0.9431589841842651,
      "logits/rejected": -1.0438756942749023,
      "logps/chosen": -78.76590728759766,
      "logps/rejected": -71.42301177978516,
      "loss": 13.8252,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0073237232863903046,
      "rewards/margins": 0.013142207637429237,
      "rewards/rejected": -0.020465927198529243,
      "step": 924
    },
    {
      "epoch": 0.5372596851948656,
      "grad_norm": 348.2239685058594,
      "learning_rate": 3.6563044741429406e-06,
      "logits/chosen": -0.6811865568161011,
      "logits/rejected": -0.7321907877922058,
      "logps/chosen": -73.98486328125,
      "logps/rejected": -75.19612121582031,
      "loss": 13.6646,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0041376203298568726,
      "rewards/margins": 0.028655271977186203,
      "rewards/rejected": -0.02451765164732933,
      "step": 925
    },
    {
      "epoch": 0.5378405064761573,
      "grad_norm": 293.9329528808594,
      "learning_rate": 3.654851830331203e-06,
      "logits/chosen": -0.8226318359375,
      "logits/rejected": -0.8354133367538452,
      "logps/chosen": -70.20458984375,
      "logps/rejected": -67.4134521484375,
      "loss": 13.3345,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.002299271058291197,
      "rewards/margins": 0.060788147151470184,
      "rewards/rejected": -0.0584888756275177,
      "step": 926
    },
    {
      "epoch": 0.538421327757449,
      "grad_norm": 310.4035949707031,
      "learning_rate": 3.6533991865194657e-06,
      "logits/chosen": -0.8177189826965332,
      "logits/rejected": -0.9231401681900024,
      "logps/chosen": -69.95651245117188,
      "logps/rejected": -76.14701080322266,
      "loss": 13.749,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0049784379079937935,
      "rewards/margins": 0.020388662815093994,
      "rewards/rejected": -0.025367099791765213,
      "step": 927
    },
    {
      "epoch": 0.5390021490387408,
      "grad_norm": 316.76336669921875,
      "learning_rate": 3.6519465427077284e-06,
      "logits/chosen": -0.8412486910820007,
      "logits/rejected": -0.7791180610656738,
      "logps/chosen": -76.65928649902344,
      "logps/rejected": -79.48634338378906,
      "loss": 13.4202,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.019436553120613098,
      "rewards/margins": 0.060405828058719635,
      "rewards/rejected": -0.04096927493810654,
      "step": 928
    },
    {
      "epoch": 0.5395829703200326,
      "grad_norm": 321.49761962890625,
      "learning_rate": 3.6504938988959907e-06,
      "logits/chosen": -0.8120080828666687,
      "logits/rejected": -0.7900758981704712,
      "logps/chosen": -66.97466278076172,
      "logps/rejected": -77.61859893798828,
      "loss": 14.3962,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02041034772992134,
      "rewards/margins": -0.04176231473684311,
      "rewards/rejected": 0.021351967006921768,
      "step": 929
    },
    {
      "epoch": 0.5401637916013243,
      "grad_norm": 325.2042541503906,
      "learning_rate": 3.6490412550842535e-06,
      "logits/chosen": -0.8616862297058105,
      "logits/rejected": -0.8327882885932922,
      "logps/chosen": -68.14276123046875,
      "logps/rejected": -71.96087646484375,
      "loss": 14.0464,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02421240136027336,
      "rewards/margins": -0.0003652706800494343,
      "rewards/rejected": -0.023847129195928574,
      "step": 930
    },
    {
      "epoch": 0.540744612882616,
      "grad_norm": 323.3883361816406,
      "learning_rate": 3.6475886112725162e-06,
      "logits/chosen": -0.9148539304733276,
      "logits/rejected": -0.8845453262329102,
      "logps/chosen": -75.4246826171875,
      "logps/rejected": -67.24982452392578,
      "loss": 14.2044,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03846925124526024,
      "rewards/margins": -0.011761395260691643,
      "rewards/rejected": -0.026707852259278297,
      "step": 931
    },
    {
      "epoch": 0.5413254341639078,
      "grad_norm": 301.5912780761719,
      "learning_rate": 3.646135967460779e-06,
      "logits/chosen": -0.8207147717475891,
      "logits/rejected": -0.8844156265258789,
      "logps/chosen": -73.19061279296875,
      "logps/rejected": -80.32855987548828,
      "loss": 13.0744,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008339379914104939,
      "rewards/margins": 0.09396232664585114,
      "rewards/rejected": -0.08562295138835907,
      "step": 932
    },
    {
      "epoch": 0.5419062554451995,
      "grad_norm": 313.2552795410156,
      "learning_rate": 3.6446833236490413e-06,
      "logits/chosen": -0.8866413235664368,
      "logits/rejected": -0.8442390561103821,
      "logps/chosen": -71.72423553466797,
      "logps/rejected": -71.9534912109375,
      "loss": 13.7789,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0012622743379324675,
      "rewards/margins": 0.024342460557818413,
      "rewards/rejected": -0.023080188781023026,
      "step": 933
    },
    {
      "epoch": 0.5424870767264912,
      "grad_norm": 381.43170166015625,
      "learning_rate": 3.643230679837304e-06,
      "logits/chosen": -0.8554956316947937,
      "logits/rejected": -0.8630772829055786,
      "logps/chosen": -76.0161361694336,
      "logps/rejected": -74.90440368652344,
      "loss": 13.7078,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.004416673444211483,
      "rewards/margins": 0.028584444895386696,
      "rewards/rejected": -0.024167772382497787,
      "step": 934
    },
    {
      "epoch": 0.543067898007783,
      "grad_norm": 319.60302734375,
      "learning_rate": 3.641778036025567e-06,
      "logits/chosen": -0.7402527928352356,
      "logits/rejected": -0.7208027839660645,
      "logps/chosen": -74.16792297363281,
      "logps/rejected": -76.62369537353516,
      "loss": 13.9184,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.029772579669952393,
      "rewards/margins": 0.016960347071290016,
      "rewards/rejected": -0.04673292487859726,
      "step": 935
    },
    {
      "epoch": 0.5436487192890748,
      "grad_norm": 329.7962951660156,
      "learning_rate": 3.640325392213829e-06,
      "logits/chosen": -0.7540744543075562,
      "logits/rejected": -0.6933923959732056,
      "logps/chosen": -73.45732116699219,
      "logps/rejected": -73.99406433105469,
      "loss": 13.4842,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011500137858092785,
      "rewards/margins": 0.04718298465013504,
      "rewards/rejected": -0.035682838410139084,
      "step": 936
    },
    {
      "epoch": 0.5442295405703665,
      "grad_norm": 293.1716003417969,
      "learning_rate": 3.638872748402092e-06,
      "logits/chosen": -0.7672283053398132,
      "logits/rejected": -0.8011919856071472,
      "logps/chosen": -70.21018981933594,
      "logps/rejected": -73.24788665771484,
      "loss": 13.4906,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01028083823621273,
      "rewards/margins": 0.05888774245977402,
      "rewards/rejected": -0.04860690236091614,
      "step": 937
    },
    {
      "epoch": 0.5448103618516582,
      "grad_norm": 322.726318359375,
      "learning_rate": 3.6374201045903546e-06,
      "logits/chosen": -0.7294805645942688,
      "logits/rejected": -0.6923807859420776,
      "logps/chosen": -73.052734375,
      "logps/rejected": -68.35235595703125,
      "loss": 13.6451,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03901199623942375,
      "rewards/margins": 0.034640949219465256,
      "rewards/rejected": 0.004371042363345623,
      "step": 938
    },
    {
      "epoch": 0.54539118313295,
      "grad_norm": 294.18133544921875,
      "learning_rate": 3.6359674607786174e-06,
      "logits/chosen": -0.7671042680740356,
      "logits/rejected": -0.7237785458564758,
      "logps/chosen": -66.60963439941406,
      "logps/rejected": -75.8608169555664,
      "loss": 13.6033,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.007068338803946972,
      "rewards/margins": 0.04283539205789566,
      "rewards/rejected": -0.04990372806787491,
      "step": 939
    },
    {
      "epoch": 0.5459720044142418,
      "grad_norm": 323.2447204589844,
      "learning_rate": 3.6345148169668797e-06,
      "logits/chosen": -0.7898720502853394,
      "logits/rejected": -0.8044264912605286,
      "logps/chosen": -73.30155944824219,
      "logps/rejected": -74.01371002197266,
      "loss": 15.0194,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.06720665097236633,
      "rewards/margins": -0.09914363920688629,
      "rewards/rejected": 0.031936999410390854,
      "step": 940
    },
    {
      "epoch": 0.5465528256955335,
      "grad_norm": 290.3876953125,
      "learning_rate": 3.6330621731551424e-06,
      "logits/chosen": -0.9731463193893433,
      "logits/rejected": -0.8236031532287598,
      "logps/chosen": -70.91753387451172,
      "logps/rejected": -73.23826599121094,
      "loss": 13.6824,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.024254988878965378,
      "rewards/margins": 0.034400004893541336,
      "rewards/rejected": -0.05865498632192612,
      "step": 941
    },
    {
      "epoch": 0.5471336469768252,
      "grad_norm": 309.3848571777344,
      "learning_rate": 3.631609529343405e-06,
      "logits/chosen": -0.8001748323440552,
      "logits/rejected": -0.8197957873344421,
      "logps/chosen": -71.79235076904297,
      "logps/rejected": -81.0816650390625,
      "loss": 13.3633,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.008796481415629387,
      "rewards/margins": 0.08566837012767792,
      "rewards/rejected": -0.07687188684940338,
      "step": 942
    },
    {
      "epoch": 0.547714468258117,
      "grad_norm": 298.1664123535156,
      "learning_rate": 3.6301568855316675e-06,
      "logits/chosen": -0.8537214398384094,
      "logits/rejected": -0.7996708750724792,
      "logps/chosen": -71.4506607055664,
      "logps/rejected": -71.5552978515625,
      "loss": 13.3384,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.048867061734199524,
      "rewards/margins": 0.07090874016284943,
      "rewards/rejected": -0.022041672840714455,
      "step": 943
    },
    {
      "epoch": 0.5482952895394088,
      "grad_norm": 325.9350891113281,
      "learning_rate": 3.6287042417199303e-06,
      "logits/chosen": -0.7929685115814209,
      "logits/rejected": -0.7167041301727295,
      "logps/chosen": -81.67355346679688,
      "logps/rejected": -79.35840606689453,
      "loss": 13.0682,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04618043079972267,
      "rewards/margins": 0.0885276198387146,
      "rewards/rejected": -0.04234719276428223,
      "step": 944
    },
    {
      "epoch": 0.5488761108207004,
      "grad_norm": 321.2001037597656,
      "learning_rate": 3.6272515979081934e-06,
      "logits/chosen": -0.8102318644523621,
      "logits/rejected": -0.8700863122940063,
      "logps/chosen": -78.3038330078125,
      "logps/rejected": -70.6084213256836,
      "loss": 14.7495,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.053139828145504,
      "rewards/margins": -0.0726703479886055,
      "rewards/rejected": 0.01953052543103695,
      "step": 945
    },
    {
      "epoch": 0.5494569321019922,
      "grad_norm": 317.0191650390625,
      "learning_rate": 3.625798954096456e-06,
      "logits/chosen": -1.0644843578338623,
      "logits/rejected": -1.0164694786071777,
      "logps/chosen": -78.31604766845703,
      "logps/rejected": -71.96430969238281,
      "loss": 14.0692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012915946543216705,
      "rewards/margins": -0.0076404451392591,
      "rewards/rejected": 0.020556394010782242,
      "step": 946
    },
    {
      "epoch": 0.550037753383284,
      "grad_norm": 296.7738342285156,
      "learning_rate": 3.624346310284719e-06,
      "logits/chosen": -0.8858498334884644,
      "logits/rejected": -0.7842674255371094,
      "logps/chosen": -72.47500610351562,
      "logps/rejected": -66.57456970214844,
      "loss": 13.1119,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.06248743459582329,
      "rewards/margins": 0.08361674845218658,
      "rewards/rejected": -0.02112930826842785,
      "step": 947
    },
    {
      "epoch": 0.5506185746645758,
      "grad_norm": 503.9831237792969,
      "learning_rate": 3.6228936664729813e-06,
      "logits/chosen": -0.7140674591064453,
      "logits/rejected": -0.8015111088752747,
      "logps/chosen": -75.12227630615234,
      "logps/rejected": -75.85064697265625,
      "loss": 13.9841,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01585085317492485,
      "rewards/margins": -0.0022057548630982637,
      "rewards/rejected": -0.013645097613334656,
      "step": 948
    },
    {
      "epoch": 0.5511993959458674,
      "grad_norm": 345.8603210449219,
      "learning_rate": 3.621441022661244e-06,
      "logits/chosen": -0.7985360026359558,
      "logits/rejected": -0.9361134767532349,
      "logps/chosen": -81.85668182373047,
      "logps/rejected": -76.9217300415039,
      "loss": 14.4827,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.05369080975651741,
      "rewards/margins": -0.04964115470647812,
      "rewards/rejected": -0.004049652721732855,
      "step": 949
    },
    {
      "epoch": 0.5517802172271592,
      "grad_norm": 360.7843017578125,
      "learning_rate": 3.6199883788495068e-06,
      "logits/chosen": -0.9570215344429016,
      "logits/rejected": -0.978718638420105,
      "logps/chosen": -77.3498764038086,
      "logps/rejected": -78.39435577392578,
      "loss": 15.2509,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.0370364785194397,
      "rewards/margins": -0.12158574908971786,
      "rewards/rejected": 0.08454927057027817,
      "step": 950
    },
    {
      "epoch": 0.552361038508451,
      "grad_norm": 323.8126525878906,
      "learning_rate": 3.618535735037769e-06,
      "logits/chosen": -0.8473807573318481,
      "logits/rejected": -0.8004460334777832,
      "logps/chosen": -72.76786804199219,
      "logps/rejected": -80.35932922363281,
      "loss": 13.0489,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03187683969736099,
      "rewards/margins": 0.0921851173043251,
      "rewards/rejected": -0.060308270156383514,
      "step": 951
    },
    {
      "epoch": 0.5529418597897428,
      "grad_norm": 352.3288269042969,
      "learning_rate": 3.617083091226032e-06,
      "logits/chosen": -0.6603168845176697,
      "logits/rejected": -0.7307640910148621,
      "logps/chosen": -75.92056274414062,
      "logps/rejected": -76.89772033691406,
      "loss": 14.0047,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.00199211691506207,
      "rewards/margins": -0.0017544865841045976,
      "rewards/rejected": -0.0002376347838435322,
      "step": 952
    },
    {
      "epoch": 0.5535226810710344,
      "grad_norm": 304.5995788574219,
      "learning_rate": 3.6156304474142946e-06,
      "logits/chosen": -0.9129480123519897,
      "logits/rejected": -0.8230624198913574,
      "logps/chosen": -77.09077453613281,
      "logps/rejected": -80.87089538574219,
      "loss": 12.8239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.030163967981934547,
      "rewards/margins": 0.12263661623001099,
      "rewards/rejected": -0.09247267246246338,
      "step": 953
    },
    {
      "epoch": 0.5541035023523262,
      "grad_norm": 342.901123046875,
      "learning_rate": 3.614177803602557e-06,
      "logits/chosen": -0.8802892565727234,
      "logits/rejected": -0.9021528959274292,
      "logps/chosen": -73.71620178222656,
      "logps/rejected": -70.15324401855469,
      "loss": 14.2906,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009347392246127129,
      "rewards/margins": -0.028505731374025345,
      "rewards/rejected": 0.019158339127898216,
      "step": 954
    },
    {
      "epoch": 0.554684323633618,
      "grad_norm": 319.2974853515625,
      "learning_rate": 3.6127251597908197e-06,
      "logits/chosen": -0.8257215619087219,
      "logits/rejected": -0.8133133053779602,
      "logps/chosen": -67.73748016357422,
      "logps/rejected": -70.88557434082031,
      "loss": 13.7505,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.01775963604450226,
      "rewards/margins": 0.016628600656986237,
      "rewards/rejected": -0.034388236701488495,
      "step": 955
    },
    {
      "epoch": 0.5552651449149096,
      "grad_norm": 304.05657958984375,
      "learning_rate": 3.6112725159790824e-06,
      "logits/chosen": -0.8870047330856323,
      "logits/rejected": -0.9050837755203247,
      "logps/chosen": -71.76180267333984,
      "logps/rejected": -70.63961791992188,
      "loss": 14.454,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04455003887414932,
      "rewards/margins": -0.05116810277104378,
      "rewards/rejected": 0.00661806296557188,
      "step": 956
    },
    {
      "epoch": 0.5558459661962014,
      "grad_norm": 303.3849182128906,
      "learning_rate": 3.609819872167345e-06,
      "logits/chosen": -0.6691077351570129,
      "logits/rejected": -0.8779782056808472,
      "logps/chosen": -85.62307739257812,
      "logps/rejected": -74.10694885253906,
      "loss": 13.2578,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.017601244151592255,
      "rewards/margins": 0.11997624486684799,
      "rewards/rejected": -0.10237500816583633,
      "step": 957
    },
    {
      "epoch": 0.5564267874774932,
      "grad_norm": 298.76275634765625,
      "learning_rate": 3.6083672283556075e-06,
      "logits/chosen": -0.8502671122550964,
      "logits/rejected": -0.8758736848831177,
      "logps/chosen": -76.85331726074219,
      "logps/rejected": -74.97396087646484,
      "loss": 12.5147,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.07757486402988434,
      "rewards/margins": 0.15835627913475037,
      "rewards/rejected": -0.08078142255544662,
      "step": 958
    },
    {
      "epoch": 0.557007608758785,
      "grad_norm": 287.95330810546875,
      "learning_rate": 3.6069145845438702e-06,
      "logits/chosen": -0.8888352513313293,
      "logits/rejected": -1.0603057146072388,
      "logps/chosen": -67.73649597167969,
      "logps/rejected": -75.5111083984375,
      "loss": 12.9569,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.026487339287996292,
      "rewards/margins": 0.1329462230205536,
      "rewards/rejected": -0.15943357348442078,
      "step": 959
    },
    {
      "epoch": 0.5575884300400766,
      "grad_norm": 306.65069580078125,
      "learning_rate": 3.605461940732133e-06,
      "logits/chosen": -0.7654293775558472,
      "logits/rejected": -0.6960537433624268,
      "logps/chosen": -71.85382080078125,
      "logps/rejected": -71.59233856201172,
      "loss": 13.6787,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02338285744190216,
      "rewards/margins": 0.026502352207899094,
      "rewards/rejected": -0.003119495464488864,
      "step": 960
    },
    {
      "epoch": 0.5581692513213684,
      "grad_norm": 322.8121643066406,
      "learning_rate": 3.6040092969203953e-06,
      "logits/chosen": -0.7408978343009949,
      "logits/rejected": -0.8127990961074829,
      "logps/chosen": -80.58521270751953,
      "logps/rejected": -77.42636108398438,
      "loss": 13.0516,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0336628220975399,
      "rewards/margins": 0.09334491193294525,
      "rewards/rejected": -0.059682078659534454,
      "step": 961
    },
    {
      "epoch": 0.5587500726026602,
      "grad_norm": 312.6292724609375,
      "learning_rate": 3.602556653108658e-06,
      "logits/chosen": -0.9119815826416016,
      "logits/rejected": -0.929103672504425,
      "logps/chosen": -85.22322082519531,
      "logps/rejected": -73.23065948486328,
      "loss": 13.4033,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007932068780064583,
      "rewards/margins": 0.05446599796414375,
      "rewards/rejected": -0.04653392732143402,
      "step": 962
    },
    {
      "epoch": 0.559330893883952,
      "grad_norm": 344.50537109375,
      "learning_rate": 3.601104009296921e-06,
      "logits/chosen": -0.7761534452438354,
      "logits/rejected": -0.73918616771698,
      "logps/chosen": -76.0566177368164,
      "logps/rejected": -76.8918228149414,
      "loss": 13.6649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00900066178292036,
      "rewards/margins": 0.031557004898786545,
      "rewards/rejected": -0.02255634218454361,
      "step": 963
    },
    {
      "epoch": 0.5599117151652436,
      "grad_norm": 297.2245178222656,
      "learning_rate": 3.5996513654851835e-06,
      "logits/chosen": -0.7133857011795044,
      "logits/rejected": -0.7351251840591431,
      "logps/chosen": -71.87410736083984,
      "logps/rejected": -75.77886199951172,
      "loss": 13.3632,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03325660899281502,
      "rewards/margins": 0.06159430742263794,
      "rewards/rejected": -0.09485092014074326,
      "step": 964
    },
    {
      "epoch": 0.5604925364465354,
      "grad_norm": 327.8712463378906,
      "learning_rate": 3.598198721673446e-06,
      "logits/chosen": -0.7299011945724487,
      "logits/rejected": -0.6946900486946106,
      "logps/chosen": -82.22821807861328,
      "logps/rejected": -71.68350982666016,
      "loss": 13.0658,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010126391425728798,
      "rewards/margins": 0.0904078334569931,
      "rewards/rejected": -0.08028144389390945,
      "step": 965
    },
    {
      "epoch": 0.5610733577278272,
      "grad_norm": 314.0105285644531,
      "learning_rate": 3.5967460778617086e-06,
      "logits/chosen": -0.7300828695297241,
      "logits/rejected": -0.7897091507911682,
      "logps/chosen": -72.77555847167969,
      "logps/rejected": -68.70625305175781,
      "loss": 14.111,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02419973351061344,
      "rewards/margins": -0.012854543514549732,
      "rewards/rejected": -0.01134518813341856,
      "step": 966
    },
    {
      "epoch": 0.5616541790091188,
      "grad_norm": 325.6894836425781,
      "learning_rate": 3.5952934340499714e-06,
      "logits/chosen": -0.7582443952560425,
      "logits/rejected": -0.7317585349082947,
      "logps/chosen": -75.26892852783203,
      "logps/rejected": -72.90524291992188,
      "loss": 14.4269,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02938658557832241,
      "rewards/margins": -0.04390256479382515,
      "rewards/rejected": 0.01451596338301897,
      "step": 967
    },
    {
      "epoch": 0.5622350002904106,
      "grad_norm": 282.8306884765625,
      "learning_rate": 3.5938407902382337e-06,
      "logits/chosen": -0.8044571876525879,
      "logits/rejected": -0.8136259317398071,
      "logps/chosen": -63.56464385986328,
      "logps/rejected": -73.00951385498047,
      "loss": 13.2138,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04722351208329201,
      "rewards/margins": 0.08173610270023346,
      "rewards/rejected": -0.034512586891651154,
      "step": 968
    },
    {
      "epoch": 0.5628158215717024,
      "grad_norm": 296.7197570800781,
      "learning_rate": 3.5923881464264964e-06,
      "logits/chosen": -0.8755059242248535,
      "logits/rejected": -0.9272353053092957,
      "logps/chosen": -69.00138092041016,
      "logps/rejected": -73.67031860351562,
      "loss": 13.3606,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04788496717810631,
      "rewards/margins": 0.06125533580780029,
      "rewards/rejected": -0.013370366767048836,
      "step": 969
    },
    {
      "epoch": 0.5633966428529942,
      "grad_norm": 315.6614074707031,
      "learning_rate": 3.590935502614759e-06,
      "logits/chosen": -0.7783223390579224,
      "logits/rejected": -0.8105076551437378,
      "logps/chosen": -70.99641418457031,
      "logps/rejected": -81.84101104736328,
      "loss": 13.0554,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.048261307179927826,
      "rewards/margins": 0.09377080202102661,
      "rewards/rejected": -0.045509494841098785,
      "step": 970
    },
    {
      "epoch": 0.5639774641342858,
      "grad_norm": 324.253662109375,
      "learning_rate": 3.5894828588030215e-06,
      "logits/chosen": -0.714714765548706,
      "logits/rejected": -0.7137161493301392,
      "logps/chosen": -78.5097885131836,
      "logps/rejected": -75.41368103027344,
      "loss": 14.0157,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.07694325596094131,
      "rewards/margins": -0.0059618866071105,
      "rewards/rejected": 0.0829051285982132,
      "step": 971
    },
    {
      "epoch": 0.5645582854155776,
      "grad_norm": 431.720947265625,
      "learning_rate": 3.5880302149912843e-06,
      "logits/chosen": -0.7840239405632019,
      "logits/rejected": -0.8164595365524292,
      "logps/chosen": -80.40445709228516,
      "logps/rejected": -75.40513610839844,
      "loss": 14.0158,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0007454866426996887,
      "rewards/margins": -0.0007643516873940825,
      "rewards/rejected": 1.8867478502215818e-05,
      "step": 972
    },
    {
      "epoch": 0.5651391066968694,
      "grad_norm": 356.0662536621094,
      "learning_rate": 3.586577571179547e-06,
      "logits/chosen": -0.9265009760856628,
      "logits/rejected": -0.7497826814651489,
      "logps/chosen": -72.06420135498047,
      "logps/rejected": -75.36701965332031,
      "loss": 13.7262,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05553603172302246,
      "rewards/margins": 0.020493101328611374,
      "rewards/rejected": 0.035042934119701385,
      "step": 973
    },
    {
      "epoch": 0.5657199279781612,
      "grad_norm": 319.8960266113281,
      "learning_rate": 3.5851249273678098e-06,
      "logits/chosen": -0.8483496904373169,
      "logits/rejected": -0.9086085557937622,
      "logps/chosen": -72.57324981689453,
      "logps/rejected": -82.24463653564453,
      "loss": 13.7945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00978184211999178,
      "rewards/margins": 0.019335050135850906,
      "rewards/rejected": -0.009553213603794575,
      "step": 974
    },
    {
      "epoch": 0.5663007492594528,
      "grad_norm": 331.1341857910156,
      "learning_rate": 3.583672283556072e-06,
      "logits/chosen": -0.9468619227409363,
      "logits/rejected": -0.9842132329940796,
      "logps/chosen": -73.61444854736328,
      "logps/rejected": -69.44871520996094,
      "loss": 14.548,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.002568404423072934,
      "rewards/margins": -0.05364561080932617,
      "rewards/rejected": 0.05107720568776131,
      "step": 975
    },
    {
      "epoch": 0.5668815705407446,
      "grad_norm": 311.6826171875,
      "learning_rate": 3.582219639744335e-06,
      "logits/chosen": -0.8095108866691589,
      "logits/rejected": -0.7686668634414673,
      "logps/chosen": -79.30653381347656,
      "logps/rejected": -73.29191589355469,
      "loss": 13.7811,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.006576309911906719,
      "rewards/margins": 0.019946260377764702,
      "rewards/rejected": -0.026522571220993996,
      "step": 976
    },
    {
      "epoch": 0.5674623918220364,
      "grad_norm": 309.6329345703125,
      "learning_rate": 3.5807669959325976e-06,
      "logits/chosen": -0.7286791205406189,
      "logits/rejected": -0.6476708054542542,
      "logps/chosen": -73.79474639892578,
      "logps/rejected": -64.9814453125,
      "loss": 13.4035,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.009394103661179543,
      "rewards/margins": 0.05736853927373886,
      "rewards/rejected": -0.04797443002462387,
      "step": 977
    },
    {
      "epoch": 0.5680432131033281,
      "grad_norm": 312.59039306640625,
      "learning_rate": 3.57931435212086e-06,
      "logits/chosen": -0.791883111000061,
      "logits/rejected": -0.6957409381866455,
      "logps/chosen": -70.85968017578125,
      "logps/rejected": -69.31065368652344,
      "loss": 13.7411,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.017146646976470947,
      "rewards/margins": 0.02252192422747612,
      "rewards/rejected": -0.039668574929237366,
      "step": 978
    },
    {
      "epoch": 0.5686240343846198,
      "grad_norm": 316.483154296875,
      "learning_rate": 3.5778617083091226e-06,
      "logits/chosen": -0.8095332980155945,
      "logits/rejected": -0.7965534925460815,
      "logps/chosen": -72.15980529785156,
      "logps/rejected": -71.30020904541016,
      "loss": 13.2213,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04437297210097313,
      "rewards/margins": 0.0787133201956749,
      "rewards/rejected": -0.034340351819992065,
      "step": 979
    },
    {
      "epoch": 0.5692048556659116,
      "grad_norm": 283.54541015625,
      "learning_rate": 3.5764090644973854e-06,
      "logits/chosen": -0.8537616729736328,
      "logits/rejected": -0.9587091207504272,
      "logps/chosen": -69.26819610595703,
      "logps/rejected": -72.88671112060547,
      "loss": 13.3589,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.000316973018925637,
      "rewards/margins": 0.06976380944252014,
      "rewards/rejected": -0.07008077204227448,
      "step": 980
    },
    {
      "epoch": 0.5697856769472034,
      "grad_norm": 279.60089111328125,
      "learning_rate": 3.574956420685648e-06,
      "logits/chosen": -0.9589298963546753,
      "logits/rejected": -0.9463101625442505,
      "logps/chosen": -67.99465942382812,
      "logps/rejected": -65.81513977050781,
      "loss": 13.0732,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.013214176520705223,
      "rewards/margins": 0.09724701941013336,
      "rewards/rejected": -0.08403284102678299,
      "step": 981
    },
    {
      "epoch": 0.5703664982284951,
      "grad_norm": 299.99664306640625,
      "learning_rate": 3.5735037768739105e-06,
      "logits/chosen": -0.8253474235534668,
      "logits/rejected": -0.7606749534606934,
      "logps/chosen": -64.20758819580078,
      "logps/rejected": -78.39196014404297,
      "loss": 13.5467,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02975400909781456,
      "rewards/margins": 0.04441283270716667,
      "rewards/rejected": -0.014658820815384388,
      "step": 982
    },
    {
      "epoch": 0.5709473195097868,
      "grad_norm": 316.7391052246094,
      "learning_rate": 3.5720511330621732e-06,
      "logits/chosen": -0.781932532787323,
      "logits/rejected": -0.7325922250747681,
      "logps/chosen": -71.48280334472656,
      "logps/rejected": -72.66695404052734,
      "loss": 13.52,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06907404214143753,
      "rewards/margins": 0.04966479912400246,
      "rewards/rejected": 0.01940922625362873,
      "step": 983
    },
    {
      "epoch": 0.5715281407910786,
      "grad_norm": 305.2508544921875,
      "learning_rate": 3.570598489250436e-06,
      "logits/chosen": -0.7936916947364807,
      "logits/rejected": -0.7673407793045044,
      "logps/chosen": -81.65838623046875,
      "logps/rejected": -74.53465270996094,
      "loss": 13.3595,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.015117776580154896,
      "rewards/margins": 0.06472204625606537,
      "rewards/rejected": -0.04960425943136215,
      "step": 984
    },
    {
      "epoch": 0.5721089620723704,
      "grad_norm": 302.62506103515625,
      "learning_rate": 3.5691458454386983e-06,
      "logits/chosen": -0.765379786491394,
      "logits/rejected": -0.8423225283622742,
      "logps/chosen": -65.9737319946289,
      "logps/rejected": -71.31224060058594,
      "loss": 13.5828,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04746689647436142,
      "rewards/margins": 0.039320915937423706,
      "rewards/rejected": 0.008145982399582863,
      "step": 985
    },
    {
      "epoch": 0.572689783353662,
      "grad_norm": 318.6903991699219,
      "learning_rate": 3.567693201626961e-06,
      "logits/chosen": -0.8056508302688599,
      "logits/rejected": -0.7189480662345886,
      "logps/chosen": -79.517822265625,
      "logps/rejected": -71.33512878417969,
      "loss": 13.5867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021028585731983185,
      "rewards/margins": 0.03351649269461632,
      "rewards/rejected": -0.012487906031310558,
      "step": 986
    },
    {
      "epoch": 0.5732706046349538,
      "grad_norm": 285.9707336425781,
      "learning_rate": 3.5662405578152242e-06,
      "logits/chosen": -0.7602620720863342,
      "logits/rejected": -0.6153804063796997,
      "logps/chosen": -70.01246643066406,
      "logps/rejected": -69.94181823730469,
      "loss": 13.9504,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007558539509773254,
      "rewards/margins": 0.003087778342887759,
      "rewards/rejected": 0.004470758140087128,
      "step": 987
    },
    {
      "epoch": 0.5738514259162456,
      "grad_norm": 322.0240478515625,
      "learning_rate": 3.564787914003487e-06,
      "logits/chosen": -0.7658705115318298,
      "logits/rejected": -0.7216984033584595,
      "logps/chosen": -71.78937530517578,
      "logps/rejected": -70.35456085205078,
      "loss": 14.4603,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00903630256652832,
      "rewards/margins": -0.05042557790875435,
      "rewards/rejected": 0.05946188047528267,
      "step": 988
    },
    {
      "epoch": 0.5744322471975373,
      "grad_norm": 319.0548400878906,
      "learning_rate": 3.5633352701917497e-06,
      "logits/chosen": -0.8453356027603149,
      "logits/rejected": -0.8643606901168823,
      "logps/chosen": -77.3846435546875,
      "logps/rejected": -81.15995788574219,
      "loss": 12.8317,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.031046276912093163,
      "rewards/margins": 0.11689448356628418,
      "rewards/rejected": -0.08584820479154587,
      "step": 989
    },
    {
      "epoch": 0.575013068478829,
      "grad_norm": 347.3050537109375,
      "learning_rate": 3.561882626380012e-06,
      "logits/chosen": -0.647269070148468,
      "logits/rejected": -0.7291135787963867,
      "logps/chosen": -73.54329681396484,
      "logps/rejected": -68.59510040283203,
      "loss": 13.2637,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.07152654230594635,
      "rewards/margins": 0.07307065278291702,
      "rewards/rejected": -0.00154411350376904,
      "step": 990
    },
    {
      "epoch": 0.5755938897601208,
      "grad_norm": 350.6729736328125,
      "learning_rate": 3.5604299825682748e-06,
      "logits/chosen": -0.6177459955215454,
      "logits/rejected": -0.6841322779655457,
      "logps/chosen": -83.19955444335938,
      "logps/rejected": -93.72516632080078,
      "loss": 13.6459,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.006069374270737171,
      "rewards/margins": 0.028904488310217857,
      "rewards/rejected": -0.02283511497080326,
      "step": 991
    },
    {
      "epoch": 0.5761747110414126,
      "grad_norm": 321.6015319824219,
      "learning_rate": 3.5589773387565375e-06,
      "logits/chosen": -0.7162039875984192,
      "logits/rejected": -0.6496952772140503,
      "logps/chosen": -70.39527893066406,
      "logps/rejected": -69.74990844726562,
      "loss": 13.3159,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.10396634042263031,
      "rewards/margins": 0.06428544223308563,
      "rewards/rejected": 0.039680901914834976,
      "step": 992
    },
    {
      "epoch": 0.5767555323227043,
      "grad_norm": 293.7535095214844,
      "learning_rate": 3.5575246949448e-06,
      "logits/chosen": -0.8607357144355774,
      "logits/rejected": -0.8218991160392761,
      "logps/chosen": -66.10880279541016,
      "logps/rejected": -73.78075408935547,
      "loss": 13.5722,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008624534122645855,
      "rewards/margins": 0.03976871818304062,
      "rewards/rejected": -0.03114417754113674,
      "step": 993
    },
    {
      "epoch": 0.577336353603996,
      "grad_norm": 399.10052490234375,
      "learning_rate": 3.5560720511330626e-06,
      "logits/chosen": -0.7579528093338013,
      "logits/rejected": -0.759405791759491,
      "logps/chosen": -73.0143051147461,
      "logps/rejected": -76.09436798095703,
      "loss": 13.904,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0034693810157477856,
      "rewards/margins": 0.00545927882194519,
      "rewards/rejected": -0.001989898504689336,
      "step": 994
    },
    {
      "epoch": 0.5779171748852878,
      "grad_norm": 304.847412109375,
      "learning_rate": 3.5546194073213254e-06,
      "logits/chosen": -0.8246349096298218,
      "logits/rejected": -0.8366926312446594,
      "logps/chosen": -68.1496810913086,
      "logps/rejected": -73.1882095336914,
      "loss": 13.0704,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.10885617882013321,
      "rewards/margins": 0.09571322053670883,
      "rewards/rejected": 0.013142948038876057,
      "step": 995
    },
    {
      "epoch": 0.5784979961665796,
      "grad_norm": 333.4711608886719,
      "learning_rate": 3.553166763509588e-06,
      "logits/chosen": -0.7208290100097656,
      "logits/rejected": -0.6368588209152222,
      "logps/chosen": -72.82388305664062,
      "logps/rejected": -70.48295593261719,
      "loss": 14.5623,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06020081788301468,
      "rewards/margins": -0.06112781912088394,
      "rewards/rejected": 0.0009269967558793724,
      "step": 996
    },
    {
      "epoch": 0.5790788174478713,
      "grad_norm": 333.8407897949219,
      "learning_rate": 3.5517141196978504e-06,
      "logits/chosen": -0.7055472135543823,
      "logits/rejected": -0.8043805956840515,
      "logps/chosen": -73.97242736816406,
      "logps/rejected": -75.32130432128906,
      "loss": 13.9701,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0029317201115190983,
      "rewards/margins": -0.003249152097851038,
      "rewards/rejected": 0.0003174312296323478,
      "step": 997
    },
    {
      "epoch": 0.579659638729163,
      "grad_norm": 283.136962890625,
      "learning_rate": 3.550261475886113e-06,
      "logits/chosen": -0.808709979057312,
      "logits/rejected": -0.8320805430412292,
      "logps/chosen": -70.24005889892578,
      "logps/rejected": -74.91935729980469,
      "loss": 13.5613,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.025156404823064804,
      "rewards/margins": 0.04108768701553345,
      "rewards/rejected": -0.06624408811330795,
      "step": 998
    },
    {
      "epoch": 0.5802404600104548,
      "grad_norm": 301.6640319824219,
      "learning_rate": 3.548808832074376e-06,
      "logits/chosen": -0.8165045976638794,
      "logits/rejected": -0.8338570594787598,
      "logps/chosen": -72.01176452636719,
      "logps/rejected": -72.26456451416016,
      "loss": 13.5339,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.041943442076444626,
      "rewards/margins": 0.04307948797941208,
      "rewards/rejected": -0.0011360436910763383,
      "step": 999
    },
    {
      "epoch": 0.5808212812917465,
      "grad_norm": 609.0323486328125,
      "learning_rate": 3.5473561882626382e-06,
      "logits/chosen": -0.7630084753036499,
      "logits/rejected": -0.7541495561599731,
      "logps/chosen": -70.21868896484375,
      "logps/rejected": -77.51250457763672,
      "loss": 14.8816,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.0775180235505104,
      "rewards/margins": -0.09175113588571548,
      "rewards/rejected": 0.014233121648430824,
      "step": 1000
    },
    {
      "epoch": 0.5814021025730383,
      "grad_norm": 321.739013671875,
      "learning_rate": 3.545903544450901e-06,
      "logits/chosen": -0.8871499300003052,
      "logits/rejected": -0.8623951077461243,
      "logps/chosen": -75.06417083740234,
      "logps/rejected": -74.38804626464844,
      "loss": 13.6185,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007558987941592932,
      "rewards/margins": 0.039406388998031616,
      "rewards/rejected": -0.03184739872813225,
      "step": 1001
    },
    {
      "epoch": 0.58198292385433,
      "grad_norm": 315.44207763671875,
      "learning_rate": 3.5444509006391637e-06,
      "logits/chosen": -0.8708564043045044,
      "logits/rejected": -0.776511549949646,
      "logps/chosen": -82.5244140625,
      "logps/rejected": -74.7889404296875,
      "loss": 13.2792,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00415464024990797,
      "rewards/margins": 0.07237003743648529,
      "rewards/rejected": -0.06821540743112564,
      "step": 1002
    },
    {
      "epoch": 0.5825637451356218,
      "grad_norm": 304.8729248046875,
      "learning_rate": 3.542998256827426e-06,
      "logits/chosen": -0.7839063405990601,
      "logits/rejected": -0.8632082939147949,
      "logps/chosen": -74.94815826416016,
      "logps/rejected": -77.31266784667969,
      "loss": 13.1056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01618318445980549,
      "rewards/margins": 0.08984390646219254,
      "rewards/rejected": -0.0736607238650322,
      "step": 1003
    },
    {
      "epoch": 0.5831445664169135,
      "grad_norm": 321.69281005859375,
      "learning_rate": 3.541545613015689e-06,
      "logits/chosen": -0.8181082010269165,
      "logits/rejected": -1.012582540512085,
      "logps/chosen": -82.91477966308594,
      "logps/rejected": -75.6765365600586,
      "loss": 13.283,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.010467121377587318,
      "rewards/margins": 0.1097029447555542,
      "rewards/rejected": -0.09923581779003143,
      "step": 1004
    },
    {
      "epoch": 0.5837253876982053,
      "grad_norm": 315.97174072265625,
      "learning_rate": 3.5400929692039516e-06,
      "logits/chosen": -0.8757426142692566,
      "logits/rejected": -0.8389043807983398,
      "logps/chosen": -69.27299499511719,
      "logps/rejected": -77.71684265136719,
      "loss": 13.781,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01742733083665371,
      "rewards/margins": 0.013004573993384838,
      "rewards/rejected": 0.0044227601028978825,
      "step": 1005
    },
    {
      "epoch": 0.584306208979497,
      "grad_norm": 312.9149475097656,
      "learning_rate": 3.5386403253922143e-06,
      "logits/chosen": -0.846282958984375,
      "logits/rejected": -0.7437726259231567,
      "logps/chosen": -78.25587463378906,
      "logps/rejected": -73.80956268310547,
      "loss": 13.4853,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.035154685378074646,
      "rewards/margins": 0.04979632794857025,
      "rewards/rejected": -0.014641635119915009,
      "step": 1006
    },
    {
      "epoch": 0.5848870302607888,
      "grad_norm": 316.0489501953125,
      "learning_rate": 3.5371876815804766e-06,
      "logits/chosen": -0.8123098611831665,
      "logits/rejected": -0.7730143070220947,
      "logps/chosen": -71.98994445800781,
      "logps/rejected": -70.49676513671875,
      "loss": 14.0856,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.018845070153474808,
      "rewards/margins": -0.006092634983360767,
      "rewards/rejected": 0.0249377079308033,
      "step": 1007
    },
    {
      "epoch": 0.5854678515420805,
      "grad_norm": 346.1430358886719,
      "learning_rate": 3.5357350377687394e-06,
      "logits/chosen": -0.689911961555481,
      "logits/rejected": -0.7615729570388794,
      "logps/chosen": -84.69590759277344,
      "logps/rejected": -77.64741516113281,
      "loss": 13.412,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02112182416021824,
      "rewards/margins": 0.06356294453144073,
      "rewards/rejected": -0.042441122233867645,
      "step": 1008
    },
    {
      "epoch": 0.5860486728233723,
      "grad_norm": 314.37255859375,
      "learning_rate": 3.534282393957002e-06,
      "logits/chosen": -0.6379407048225403,
      "logits/rejected": -0.815085232257843,
      "logps/chosen": -73.09681701660156,
      "logps/rejected": -69.57933044433594,
      "loss": 13.7977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03233978524804115,
      "rewards/margins": 0.02014748379588127,
      "rewards/rejected": 0.012192296795547009,
      "step": 1009
    },
    {
      "epoch": 0.586629494104664,
      "grad_norm": 301.4626770019531,
      "learning_rate": 3.5328297501452645e-06,
      "logits/chosen": -0.7938534021377563,
      "logits/rejected": -0.6901233196258545,
      "logps/chosen": -78.32262420654297,
      "logps/rejected": -72.402587890625,
      "loss": 13.2066,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0064957900904119015,
      "rewards/margins": 0.08389043807983398,
      "rewards/rejected": -0.09038622677326202,
      "step": 1010
    },
    {
      "epoch": 0.5872103153859557,
      "grad_norm": 306.2314758300781,
      "learning_rate": 3.531377106333527e-06,
      "logits/chosen": -0.6468356251716614,
      "logits/rejected": -0.7668360471725464,
      "logps/chosen": -74.49341583251953,
      "logps/rejected": -68.28553771972656,
      "loss": 13.5847,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0024702679365873337,
      "rewards/margins": 0.04482710734009743,
      "rewards/rejected": -0.04235684499144554,
      "step": 1011
    },
    {
      "epoch": 0.5877911366672475,
      "grad_norm": 290.3107604980469,
      "learning_rate": 3.52992446252179e-06,
      "logits/chosen": -0.7929361462593079,
      "logits/rejected": -0.8646718859672546,
      "logps/chosen": -66.76972961425781,
      "logps/rejected": -67.30169677734375,
      "loss": 13.0828,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.021405553445219994,
      "rewards/margins": 0.09089253097772598,
      "rewards/rejected": -0.06948696821928024,
      "step": 1012
    },
    {
      "epoch": 0.5883719579485392,
      "grad_norm": 299.92913818359375,
      "learning_rate": 3.5284718187100527e-06,
      "logits/chosen": -0.8127965927124023,
      "logits/rejected": -0.877363383769989,
      "logps/chosen": -69.75791931152344,
      "logps/rejected": -74.10560607910156,
      "loss": 13.5382,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01690954528748989,
      "rewards/margins": 0.06993573904037476,
      "rewards/rejected": -0.0868452787399292,
      "step": 1013
    },
    {
      "epoch": 0.588952779229831,
      "grad_norm": 330.538330078125,
      "learning_rate": 3.527019174898315e-06,
      "logits/chosen": -0.6847607493400574,
      "logits/rejected": -0.7449588179588318,
      "logps/chosen": -74.80368041992188,
      "logps/rejected": -77.62113189697266,
      "loss": 13.3523,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02433842420578003,
      "rewards/margins": 0.05932570621371269,
      "rewards/rejected": -0.034987278282642365,
      "step": 1014
    },
    {
      "epoch": 0.5895336005111227,
      "grad_norm": 337.7607116699219,
      "learning_rate": 3.5255665310865778e-06,
      "logits/chosen": -0.7656806707382202,
      "logits/rejected": -0.8294223546981812,
      "logps/chosen": -75.68753814697266,
      "logps/rejected": -78.0618667602539,
      "loss": 13.9191,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.038200318813323975,
      "rewards/margins": 0.011223772540688515,
      "rewards/rejected": 0.02697654999792576,
      "step": 1015
    },
    {
      "epoch": 0.5901144217924145,
      "grad_norm": 533.0556030273438,
      "learning_rate": 3.5241138872748405e-06,
      "logits/chosen": -0.7761787176132202,
      "logits/rejected": -0.8613113164901733,
      "logps/chosen": -74.75994110107422,
      "logps/rejected": -69.85081481933594,
      "loss": 13.8603,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029375886544585228,
      "rewards/margins": 0.01804412342607975,
      "rewards/rejected": 0.011331766843795776,
      "step": 1016
    },
    {
      "epoch": 0.5906952430737062,
      "grad_norm": 311.0755920410156,
      "learning_rate": 3.522661243463103e-06,
      "logits/chosen": -0.8341520428657532,
      "logits/rejected": -0.8708308339118958,
      "logps/chosen": -63.79094314575195,
      "logps/rejected": -72.3699951171875,
      "loss": 14.1461,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.06622471660375595,
      "rewards/margins": -0.021187495440244675,
      "rewards/rejected": -0.045037221163511276,
      "step": 1017
    },
    {
      "epoch": 0.591276064354998,
      "grad_norm": 337.5909729003906,
      "learning_rate": 3.5212085996513656e-06,
      "logits/chosen": -0.6475591063499451,
      "logits/rejected": -0.7518597841262817,
      "logps/chosen": -78.11042785644531,
      "logps/rejected": -76.76148986816406,
      "loss": 14.2729,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.01224745623767376,
      "rewards/margins": -0.022836724296212196,
      "rewards/rejected": 0.035084180533885956,
      "step": 1018
    },
    {
      "epoch": 0.5918568856362897,
      "grad_norm": 332.48895263671875,
      "learning_rate": 3.5197559558396283e-06,
      "logits/chosen": -0.586700975894928,
      "logits/rejected": -0.712017297744751,
      "logps/chosen": -71.53556823730469,
      "logps/rejected": -74.79586029052734,
      "loss": 13.4598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.047216445207595825,
      "rewards/margins": 0.0436665341258049,
      "rewards/rejected": 0.003549909684807062,
      "step": 1019
    },
    {
      "epoch": 0.5924377069175815,
      "grad_norm": 362.25103759765625,
      "learning_rate": 3.5183033120278907e-06,
      "logits/chosen": -0.8641973733901978,
      "logits/rejected": -0.8975082635879517,
      "logps/chosen": -74.8412857055664,
      "logps/rejected": -79.69654846191406,
      "loss": 14.6281,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01472280453890562,
      "rewards/margins": -0.06256036460399628,
      "rewards/rejected": 0.04783756658434868,
      "step": 1020
    },
    {
      "epoch": 0.5930185281988732,
      "grad_norm": 315.7493896484375,
      "learning_rate": 3.5168506682161534e-06,
      "logits/chosen": -0.8684743046760559,
      "logits/rejected": -0.9560597538948059,
      "logps/chosen": -75.23030090332031,
      "logps/rejected": -72.1439437866211,
      "loss": 13.7693,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02732301875948906,
      "rewards/margins": 0.021545682102441788,
      "rewards/rejected": 0.005777344107627869,
      "step": 1021
    },
    {
      "epoch": 0.5935993494801649,
      "grad_norm": 303.66375732421875,
      "learning_rate": 3.515398024404416e-06,
      "logits/chosen": -0.7548766136169434,
      "logits/rejected": -0.8196467161178589,
      "logps/chosen": -62.06414031982422,
      "logps/rejected": -67.89656066894531,
      "loss": 13.0187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.027340680360794067,
      "rewards/margins": 0.0915822833776474,
      "rewards/rejected": -0.06424160301685333,
      "step": 1022
    },
    {
      "epoch": 0.5941801707614567,
      "grad_norm": 339.7117919921875,
      "learning_rate": 3.513945380592679e-06,
      "logits/chosen": -0.7947455644607544,
      "logits/rejected": -0.9344140887260437,
      "logps/chosen": -70.19294738769531,
      "logps/rejected": -75.25838470458984,
      "loss": 13.8368,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011563832871615887,
      "rewards/margins": 0.015606586821377277,
      "rewards/rejected": -0.004042745567858219,
      "step": 1023
    },
    {
      "epoch": 0.5947609920427485,
      "grad_norm": 324.26019287109375,
      "learning_rate": 3.5124927367809412e-06,
      "logits/chosen": -0.7493640780448914,
      "logits/rejected": -0.8248082399368286,
      "logps/chosen": -73.07319641113281,
      "logps/rejected": -74.19573974609375,
      "loss": 13.0593,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.06739606708288193,
      "rewards/margins": 0.0972830206155777,
      "rewards/rejected": -0.029886942356824875,
      "step": 1024
    },
    {
      "epoch": 0.5953418133240402,
      "grad_norm": 284.0368957519531,
      "learning_rate": 3.511040092969204e-06,
      "logits/chosen": -0.8054580688476562,
      "logits/rejected": -1.0106167793273926,
      "logps/chosen": -66.81629943847656,
      "logps/rejected": -74.02056884765625,
      "loss": 11.9232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08729104697704315,
      "rewards/margins": 0.30259591341018677,
      "rewards/rejected": -0.21530482172966003,
      "step": 1025
    },
    {
      "epoch": 0.5959226346053319,
      "grad_norm": 342.6269226074219,
      "learning_rate": 3.5095874491574667e-06,
      "logits/chosen": -0.768220841884613,
      "logits/rejected": -0.7292611598968506,
      "logps/chosen": -81.7083740234375,
      "logps/rejected": -74.98188781738281,
      "loss": 13.6747,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01694565638899803,
      "rewards/margins": 0.0372299961745739,
      "rewards/rejected": -0.020284336060285568,
      "step": 1026
    },
    {
      "epoch": 0.5965034558866237,
      "grad_norm": 334.5113220214844,
      "learning_rate": 3.508134805345729e-06,
      "logits/chosen": -0.7468510270118713,
      "logits/rejected": -0.7208049893379211,
      "logps/chosen": -70.92430114746094,
      "logps/rejected": -66.80095672607422,
      "loss": 13.6899,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.022873494774103165,
      "rewards/margins": 0.03160777688026428,
      "rewards/rejected": -0.008734277449548244,
      "step": 1027
    },
    {
      "epoch": 0.5970842771679155,
      "grad_norm": 321.86279296875,
      "learning_rate": 3.506682161533992e-06,
      "logits/chosen": -0.834597110748291,
      "logits/rejected": -0.9707862734794617,
      "logps/chosen": -71.14798736572266,
      "logps/rejected": -75.78814697265625,
      "loss": 13.2408,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.052504993975162506,
      "rewards/margins": 0.07581616938114166,
      "rewards/rejected": -0.023311173543334007,
      "step": 1028
    },
    {
      "epoch": 0.5976650984492072,
      "grad_norm": 353.73211669921875,
      "learning_rate": 3.505229517722255e-06,
      "logits/chosen": -0.7336338758468628,
      "logits/rejected": -0.7048076391220093,
      "logps/chosen": -68.56995391845703,
      "logps/rejected": -80.11732482910156,
      "loss": 14.0578,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.014443065039813519,
      "rewards/margins": -0.00454490352421999,
      "rewards/rejected": 0.01898796483874321,
      "step": 1029
    },
    {
      "epoch": 0.5982459197304989,
      "grad_norm": 316.87445068359375,
      "learning_rate": 3.5037768739105177e-06,
      "logits/chosen": -0.8267478942871094,
      "logits/rejected": -0.776298999786377,
      "logps/chosen": -78.34546661376953,
      "logps/rejected": -76.53302001953125,
      "loss": 13.2517,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.08723635971546173,
      "rewards/margins": 0.0752679854631424,
      "rewards/rejected": 0.011968366801738739,
      "step": 1030
    },
    {
      "epoch": 0.5988267410117907,
      "grad_norm": 308.5358581542969,
      "learning_rate": 3.5023242300987805e-06,
      "logits/chosen": -0.7111587524414062,
      "logits/rejected": -0.7261529564857483,
      "logps/chosen": -67.70765686035156,
      "logps/rejected": -72.23832702636719,
      "loss": 13.6664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015337610617280006,
      "rewards/margins": 0.033074311912059784,
      "rewards/rejected": -0.017736705020070076,
      "step": 1031
    },
    {
      "epoch": 0.5994075622930825,
      "grad_norm": 317.95379638671875,
      "learning_rate": 3.500871586287043e-06,
      "logits/chosen": -0.9141271710395813,
      "logits/rejected": -0.9438360333442688,
      "logps/chosen": -78.72630310058594,
      "logps/rejected": -68.52434539794922,
      "loss": 13.925,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.010290483012795448,
      "rewards/margins": 0.009686502628028393,
      "rewards/rejected": 0.0006039824220351875,
      "step": 1032
    },
    {
      "epoch": 0.5999883835743741,
      "grad_norm": 327.6629333496094,
      "learning_rate": 3.4994189424753056e-06,
      "logits/chosen": -0.7981666326522827,
      "logits/rejected": -0.828513503074646,
      "logps/chosen": -74.92295837402344,
      "logps/rejected": -77.58927154541016,
      "loss": 14.0699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08681248128414154,
      "rewards/margins": -0.003370219375938177,
      "rewards/rejected": -0.08344225585460663,
      "step": 1033
    },
    {
      "epoch": 0.6005692048556659,
      "grad_norm": 330.2552795410156,
      "learning_rate": 3.4979662986635683e-06,
      "logits/chosen": -0.7577478885650635,
      "logits/rejected": -0.7416873574256897,
      "logps/chosen": -75.49105072021484,
      "logps/rejected": -74.58745574951172,
      "loss": 13.7415,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.028033334761857986,
      "rewards/margins": 0.02812257967889309,
      "rewards/rejected": -8.92441748874262e-05,
      "step": 1034
    },
    {
      "epoch": 0.6011500261369577,
      "grad_norm": 333.61065673828125,
      "learning_rate": 3.4965136548518306e-06,
      "logits/chosen": -0.8367765545845032,
      "logits/rejected": -0.8033145070075989,
      "logps/chosen": -77.6269302368164,
      "logps/rejected": -69.40321350097656,
      "loss": 14.4326,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018094033002853394,
      "rewards/margins": -0.04064077511429787,
      "rewards/rejected": 0.022546743974089622,
      "step": 1035
    },
    {
      "epoch": 0.6017308474182494,
      "grad_norm": 333.8052978515625,
      "learning_rate": 3.4950610110400934e-06,
      "logits/chosen": -0.8207594752311707,
      "logits/rejected": -0.6641306281089783,
      "logps/chosen": -72.93827819824219,
      "logps/rejected": -67.6906509399414,
      "loss": 13.3477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07506252825260162,
      "rewards/margins": 0.06587859243154526,
      "rewards/rejected": 0.00918395072221756,
      "step": 1036
    },
    {
      "epoch": 0.6023116686995411,
      "grad_norm": 292.4611511230469,
      "learning_rate": 3.493608367228356e-06,
      "logits/chosen": -0.9030243754386902,
      "logits/rejected": -0.884412944316864,
      "logps/chosen": -65.49525451660156,
      "logps/rejected": -68.4120101928711,
      "loss": 13.6505,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033711150754243135,
      "rewards/margins": 0.03291068226099014,
      "rewards/rejected": -0.0295395664870739,
      "step": 1037
    },
    {
      "epoch": 0.6028924899808329,
      "grad_norm": 373.8504943847656,
      "learning_rate": 3.492155723416619e-06,
      "logits/chosen": -0.7232368588447571,
      "logits/rejected": -0.7537127137184143,
      "logps/chosen": -71.82563781738281,
      "logps/rejected": -80.13270568847656,
      "loss": 15.0636,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.07963554561138153,
      "rewards/margins": -0.10052303969860077,
      "rewards/rejected": 0.02088748849928379,
      "step": 1038
    },
    {
      "epoch": 0.6034733112621247,
      "grad_norm": 296.366943359375,
      "learning_rate": 3.490703079604881e-06,
      "logits/chosen": -0.8908153772354126,
      "logits/rejected": -0.9026978611946106,
      "logps/chosen": -80.31646728515625,
      "logps/rejected": -72.79670715332031,
      "loss": 14.0552,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04715970531105995,
      "rewards/margins": -0.004522286355495453,
      "rewards/rejected": -0.0426374152302742,
      "step": 1039
    },
    {
      "epoch": 0.6040541325434164,
      "grad_norm": 277.1002502441406,
      "learning_rate": 3.489250435793144e-06,
      "logits/chosen": -0.6462879180908203,
      "logits/rejected": -0.6655235886573792,
      "logps/chosen": -68.87925720214844,
      "logps/rejected": -72.4296646118164,
      "loss": 12.5476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09290014207363129,
      "rewards/margins": 0.1453324854373932,
      "rewards/rejected": -0.0524323508143425,
      "step": 1040
    },
    {
      "epoch": 0.6046349538247081,
      "grad_norm": 309.86962890625,
      "learning_rate": 3.4877977919814067e-06,
      "logits/chosen": -0.9119254350662231,
      "logits/rejected": -0.8515340089797974,
      "logps/chosen": -69.9295425415039,
      "logps/rejected": -74.95957946777344,
      "loss": 13.773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02706020511686802,
      "rewards/margins": 0.031168580055236816,
      "rewards/rejected": -0.0041083842515945435,
      "step": 1041
    },
    {
      "epoch": 0.6052157751059999,
      "grad_norm": 325.9361267089844,
      "learning_rate": 3.486345148169669e-06,
      "logits/chosen": -0.7397585511207581,
      "logits/rejected": -0.9308179020881653,
      "logps/chosen": -73.04594421386719,
      "logps/rejected": -76.44662475585938,
      "loss": 12.9932,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05383353680372238,
      "rewards/margins": 0.12394194304943085,
      "rewards/rejected": -0.07010839879512787,
      "step": 1042
    },
    {
      "epoch": 0.6057965963872917,
      "grad_norm": 320.0553894042969,
      "learning_rate": 3.4848925043579318e-06,
      "logits/chosen": -0.961296558380127,
      "logits/rejected": -0.8340277671813965,
      "logps/chosen": -72.52586364746094,
      "logps/rejected": -76.88352966308594,
      "loss": 14.1283,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004333724267780781,
      "rewards/margins": -0.014794819056987762,
      "rewards/rejected": 0.010461093857884407,
      "step": 1043
    },
    {
      "epoch": 0.6063774176685833,
      "grad_norm": 348.5504150390625,
      "learning_rate": 3.4834398605461945e-06,
      "logits/chosen": -0.8833734393119812,
      "logits/rejected": -0.7459826469421387,
      "logps/chosen": -73.98115539550781,
      "logps/rejected": -68.28959655761719,
      "loss": 14.6744,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10712261497974396,
      "rewards/margins": -0.042904291301965714,
      "rewards/rejected": -0.06421831995248795,
      "step": 1044
    },
    {
      "epoch": 0.6069582389498751,
      "grad_norm": 324.8833312988281,
      "learning_rate": 3.4819872167344573e-06,
      "logits/chosen": -0.7405747771263123,
      "logits/rejected": -0.8007808923721313,
      "logps/chosen": -69.27625274658203,
      "logps/rejected": -76.64971923828125,
      "loss": 13.8301,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04173465073108673,
      "rewards/margins": 0.027003061026334763,
      "rewards/rejected": -0.0687377080321312,
      "step": 1045
    },
    {
      "epoch": 0.6075390602311669,
      "grad_norm": 334.7247009277344,
      "learning_rate": 3.4805345729227196e-06,
      "logits/chosen": -0.8145245313644409,
      "logits/rejected": -0.7577263116836548,
      "logps/chosen": -67.68992614746094,
      "logps/rejected": -66.78765869140625,
      "loss": 14.2135,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.028335655108094215,
      "rewards/margins": -0.01856682077050209,
      "rewards/rejected": -0.009768828749656677,
      "step": 1046
    },
    {
      "epoch": 0.6081198815124587,
      "grad_norm": 306.58026123046875,
      "learning_rate": 3.4790819291109823e-06,
      "logits/chosen": -0.912173867225647,
      "logits/rejected": -0.8691814541816711,
      "logps/chosen": -77.43460845947266,
      "logps/rejected": -77.8487319946289,
      "loss": 13.1494,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.003477914724498987,
      "rewards/margins": 0.08819029480218887,
      "rewards/rejected": -0.09166820347309113,
      "step": 1047
    },
    {
      "epoch": 0.6087007027937503,
      "grad_norm": 304.5981140136719,
      "learning_rate": 3.477629285299245e-06,
      "logits/chosen": -0.79698246717453,
      "logits/rejected": -0.8606882095336914,
      "logps/chosen": -69.93180847167969,
      "logps/rejected": -68.01878356933594,
      "loss": 13.5126,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.018330451101064682,
      "rewards/margins": 0.048832476139068604,
      "rewards/rejected": -0.03050202503800392,
      "step": 1048
    },
    {
      "epoch": 0.6092815240750421,
      "grad_norm": 316.9424743652344,
      "learning_rate": 3.4761766414875074e-06,
      "logits/chosen": -0.628899872303009,
      "logits/rejected": -0.5473764538764954,
      "logps/chosen": -72.062255859375,
      "logps/rejected": -71.70515441894531,
      "loss": 13.2367,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04346921667456627,
      "rewards/margins": 0.07660787552595139,
      "rewards/rejected": -0.03313865512609482,
      "step": 1049
    },
    {
      "epoch": 0.6098623453563339,
      "grad_norm": 311.6398620605469,
      "learning_rate": 3.47472399767577e-06,
      "logits/chosen": -0.8205744624137878,
      "logits/rejected": -0.6652729511260986,
      "logps/chosen": -71.7879409790039,
      "logps/rejected": -84.57965087890625,
      "loss": 13.594,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.019153941422700882,
      "rewards/margins": 0.03847995772957802,
      "rewards/rejected": -0.019326020032167435,
      "step": 1050
    },
    {
      "epoch": 0.6104431666376257,
      "grad_norm": 322.5099792480469,
      "learning_rate": 3.473271353864033e-06,
      "logits/chosen": -0.8064386248588562,
      "logits/rejected": -0.8348105549812317,
      "logps/chosen": -69.54685974121094,
      "logps/rejected": -84.65767669677734,
      "loss": 13.8955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019100148230791092,
      "rewards/margins": 0.006681396160274744,
      "rewards/rejected": -0.025781545788049698,
      "step": 1051
    },
    {
      "epoch": 0.6110239879189173,
      "grad_norm": 318.1051330566406,
      "learning_rate": 3.4718187100522952e-06,
      "logits/chosen": -0.8245420455932617,
      "logits/rejected": -0.785571813583374,
      "logps/chosen": -69.43694305419922,
      "logps/rejected": -64.11416625976562,
      "loss": 13.619,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01868509314954281,
      "rewards/margins": 0.04095892980694771,
      "rewards/rejected": -0.0222738366574049,
      "step": 1052
    },
    {
      "epoch": 0.6116048092002091,
      "grad_norm": 513.671142578125,
      "learning_rate": 3.470366066240558e-06,
      "logits/chosen": -0.8157714605331421,
      "logits/rejected": -0.761914074420929,
      "logps/chosen": -69.4002685546875,
      "logps/rejected": -77.47315979003906,
      "loss": 13.2653,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04124646633863449,
      "rewards/margins": 0.0671718567609787,
      "rewards/rejected": -0.025925392284989357,
      "step": 1053
    },
    {
      "epoch": 0.6121856304815009,
      "grad_norm": 314.2899475097656,
      "learning_rate": 3.4689134224288207e-06,
      "logits/chosen": -0.7699109315872192,
      "logits/rejected": -0.8156440854072571,
      "logps/chosen": -65.8400650024414,
      "logps/rejected": -71.43326568603516,
      "loss": 13.309,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04001081362366676,
      "rewards/margins": 0.06546341627836227,
      "rewards/rejected": -0.025452595204114914,
      "step": 1054
    },
    {
      "epoch": 0.6127664517627925,
      "grad_norm": 315.3565673828125,
      "learning_rate": 3.4674607786170835e-06,
      "logits/chosen": -0.7715741395950317,
      "logits/rejected": -0.8264389038085938,
      "logps/chosen": -77.2141342163086,
      "logps/rejected": -71.24522399902344,
      "loss": 13.2706,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.025661509484052658,
      "rewards/margins": 0.07695712894201279,
      "rewards/rejected": -0.051295626908540726,
      "step": 1055
    },
    {
      "epoch": 0.6133472730440843,
      "grad_norm": 326.30645751953125,
      "learning_rate": 3.466008134805346e-06,
      "logits/chosen": -1.0065734386444092,
      "logits/rejected": -0.9254937171936035,
      "logps/chosen": -80.39340209960938,
      "logps/rejected": -74.71737670898438,
      "loss": 13.7828,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008913049474358559,
      "rewards/margins": 0.025418957695364952,
      "rewards/rejected": -0.03433201089501381,
      "step": 1056
    },
    {
      "epoch": 0.6139280943253761,
      "grad_norm": 322.35064697265625,
      "learning_rate": 3.4645554909936086e-06,
      "logits/chosen": -0.7956243753433228,
      "logits/rejected": -0.7737189531326294,
      "logps/chosen": -71.47244262695312,
      "logps/rejected": -78.56809997558594,
      "loss": 13.6112,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.023699084296822548,
      "rewards/margins": 0.03249689191579819,
      "rewards/rejected": -0.00879780761897564,
      "step": 1057
    },
    {
      "epoch": 0.6145089156066679,
      "grad_norm": 279.84039306640625,
      "learning_rate": 3.4631028471818713e-06,
      "logits/chosen": -0.8014827966690063,
      "logits/rejected": -0.7758724689483643,
      "logps/chosen": -68.9926528930664,
      "logps/rejected": -74.23014831542969,
      "loss": 13.1682,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04919418692588806,
      "rewards/margins": 0.07882948219776154,
      "rewards/rejected": -0.029635295271873474,
      "step": 1058
    },
    {
      "epoch": 0.6150897368879595,
      "grad_norm": 301.5657043457031,
      "learning_rate": 3.4616502033701336e-06,
      "logits/chosen": -0.8164582252502441,
      "logits/rejected": -0.866308867931366,
      "logps/chosen": -74.14891815185547,
      "logps/rejected": -78.6602554321289,
      "loss": 14.076,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.023045312613248825,
      "rewards/margins": -0.009742679074406624,
      "rewards/rejected": -0.013302631676197052,
      "step": 1059
    },
    {
      "epoch": 0.6156705581692513,
      "grad_norm": 320.6341552734375,
      "learning_rate": 3.4601975595583964e-06,
      "logits/chosen": -0.7994133830070496,
      "logits/rejected": -0.8307390213012695,
      "logps/chosen": -68.84827423095703,
      "logps/rejected": -72.86600494384766,
      "loss": 13.5835,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.013884874992072582,
      "rewards/margins": 0.03245928883552551,
      "rewards/rejected": -0.018574412912130356,
      "step": 1060
    },
    {
      "epoch": 0.6162513794505431,
      "grad_norm": 319.4493408203125,
      "learning_rate": 3.458744915746659e-06,
      "logits/chosen": -0.8929045796394348,
      "logits/rejected": -0.8639433979988098,
      "logps/chosen": -71.96308135986328,
      "logps/rejected": -69.4062728881836,
      "loss": 14.8316,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.04168479144573212,
      "rewards/margins": -0.07336924970149994,
      "rewards/rejected": 0.03168446570634842,
      "step": 1061
    },
    {
      "epoch": 0.6168322007318349,
      "grad_norm": 376.7403259277344,
      "learning_rate": 3.457292271934922e-06,
      "logits/chosen": -0.7638422250747681,
      "logits/rejected": -0.8404864072799683,
      "logps/chosen": -71.4857406616211,
      "logps/rejected": -74.17166900634766,
      "loss": 13.4484,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.017488937824964523,
      "rewards/margins": 0.05266827344894409,
      "rewards/rejected": -0.03517933934926987,
      "step": 1062
    },
    {
      "epoch": 0.6174130220131265,
      "grad_norm": 618.029541015625,
      "learning_rate": 3.455839628123184e-06,
      "logits/chosen": -0.7626648545265198,
      "logits/rejected": -0.7984503507614136,
      "logps/chosen": -78.7516098022461,
      "logps/rejected": -72.27967834472656,
      "loss": 13.8324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014249598607420921,
      "rewards/margins": 0.020776888355612755,
      "rewards/rejected": -0.006527288351207972,
      "step": 1063
    },
    {
      "epoch": 0.6179938432944183,
      "grad_norm": 338.4407653808594,
      "learning_rate": 3.454386984311447e-06,
      "logits/chosen": -0.7736749053001404,
      "logits/rejected": -0.7806074023246765,
      "logps/chosen": -76.20438385009766,
      "logps/rejected": -83.10606384277344,
      "loss": 13.5899,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0034310631453990936,
      "rewards/margins": 0.0408799909055233,
      "rewards/rejected": -0.0443110466003418,
      "step": 1064
    },
    {
      "epoch": 0.6185746645757101,
      "grad_norm": 379.8909912109375,
      "learning_rate": 3.4529343404997097e-06,
      "logits/chosen": -0.6248257756233215,
      "logits/rejected": -0.7573126554489136,
      "logps/chosen": -76.0152816772461,
      "logps/rejected": -68.48918151855469,
      "loss": 13.0597,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04882480949163437,
      "rewards/margins": 0.10027780383825302,
      "rewards/rejected": -0.05145301669836044,
      "step": 1065
    },
    {
      "epoch": 0.6191554858570018,
      "grad_norm": 311.7603454589844,
      "learning_rate": 3.451481696687972e-06,
      "logits/chosen": -0.8026106953620911,
      "logits/rejected": -0.9530078768730164,
      "logps/chosen": -71.39447021484375,
      "logps/rejected": -83.27928924560547,
      "loss": 13.8322,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0010744768660515547,
      "rewards/margins": 0.014939280226826668,
      "rewards/rejected": -0.016013754531741142,
      "step": 1066
    },
    {
      "epoch": 0.6197363071382935,
      "grad_norm": 306.9896240234375,
      "learning_rate": 3.4500290528762348e-06,
      "logits/chosen": -0.7450871467590332,
      "logits/rejected": -0.779462993144989,
      "logps/chosen": -68.78193664550781,
      "logps/rejected": -88.88067626953125,
      "loss": 13.2222,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06152988597750664,
      "rewards/margins": 0.07736875861883163,
      "rewards/rejected": -0.0158388689160347,
      "step": 1067
    },
    {
      "epoch": 0.6203171284195853,
      "grad_norm": 316.9907531738281,
      "learning_rate": 3.4485764090644975e-06,
      "logits/chosen": -0.9358032941818237,
      "logits/rejected": -0.8900809288024902,
      "logps/chosen": -74.78514099121094,
      "logps/rejected": -76.52880859375,
      "loss": 13.4457,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03117547370493412,
      "rewards/margins": 0.054793693125247955,
      "rewards/rejected": -0.023618219420313835,
      "step": 1068
    },
    {
      "epoch": 0.6208979497008771,
      "grad_norm": 316.3363037109375,
      "learning_rate": 3.44712376525276e-06,
      "logits/chosen": -0.9358466267585754,
      "logits/rejected": -0.9699243307113647,
      "logps/chosen": -70.09263610839844,
      "logps/rejected": -74.53950500488281,
      "loss": 14.3849,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.039796315133571625,
      "rewards/margins": -0.02795414999127388,
      "rewards/rejected": -0.011842171661555767,
      "step": 1069
    },
    {
      "epoch": 0.6214787709821687,
      "grad_norm": 322.0668029785156,
      "learning_rate": 3.4456711214410226e-06,
      "logits/chosen": -0.6752369999885559,
      "logits/rejected": -0.6133280992507935,
      "logps/chosen": -78.82754516601562,
      "logps/rejected": -77.84974670410156,
      "loss": 14.0816,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0064362711273133755,
      "rewards/margins": -0.008478937670588493,
      "rewards/rejected": 0.002042664447799325,
      "step": 1070
    },
    {
      "epoch": 0.6220595922634605,
      "grad_norm": 382.4546813964844,
      "learning_rate": 3.4442184776292858e-06,
      "logits/chosen": -0.8350537419319153,
      "logits/rejected": -0.8091386556625366,
      "logps/chosen": -71.3176040649414,
      "logps/rejected": -74.65826416015625,
      "loss": 14.12,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0069688535295426846,
      "rewards/margins": -0.013959726318717003,
      "rewards/rejected": 0.006990872323513031,
      "step": 1071
    },
    {
      "epoch": 0.6226404135447523,
      "grad_norm": 300.6478271484375,
      "learning_rate": 3.4427658338175485e-06,
      "logits/chosen": -0.747123122215271,
      "logits/rejected": -0.7096508741378784,
      "logps/chosen": -69.2557373046875,
      "logps/rejected": -74.59153747558594,
      "loss": 13.4692,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0009012311929836869,
      "rewards/margins": 0.05364412069320679,
      "rewards/rejected": -0.05454534292221069,
      "step": 1072
    },
    {
      "epoch": 0.6232212348260441,
      "grad_norm": 306.09912109375,
      "learning_rate": 3.4413131900058113e-06,
      "logits/chosen": -0.8555896878242493,
      "logits/rejected": -0.8579667210578918,
      "logps/chosen": -73.73109436035156,
      "logps/rejected": -70.7890853881836,
      "loss": 13.7861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01904996857047081,
      "rewards/margins": 0.017219513654708862,
      "rewards/rejected": -0.03626948222517967,
      "step": 1073
    },
    {
      "epoch": 0.6238020561073357,
      "grad_norm": 301.705322265625,
      "learning_rate": 3.4398605461940736e-06,
      "logits/chosen": -0.8495151400566101,
      "logits/rejected": -0.8344343900680542,
      "logps/chosen": -68.24090576171875,
      "logps/rejected": -63.2669677734375,
      "loss": 14.0669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02763674035668373,
      "rewards/margins": -0.008190911263227463,
      "rewards/rejected": -0.019445834681391716,
      "step": 1074
    },
    {
      "epoch": 0.6243828773886275,
      "grad_norm": 275.0679016113281,
      "learning_rate": 3.4384079023823363e-06,
      "logits/chosen": -0.8719322085380554,
      "logits/rejected": -0.8970614671707153,
      "logps/chosen": -73.80534362792969,
      "logps/rejected": -70.14080047607422,
      "loss": 12.5039,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.1067904457449913,
      "rewards/margins": 0.15359076857566833,
      "rewards/rejected": -0.046800319105386734,
      "step": 1075
    },
    {
      "epoch": 0.6249636986699193,
      "grad_norm": 304.5047607421875,
      "learning_rate": 3.436955258570599e-06,
      "logits/chosen": -0.7404356002807617,
      "logits/rejected": -0.8264300227165222,
      "logps/chosen": -73.61492919921875,
      "logps/rejected": -72.74659729003906,
      "loss": 13.5417,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.027002831920981407,
      "rewards/margins": 0.04645848646759987,
      "rewards/rejected": -0.019455650821328163,
      "step": 1076
    },
    {
      "epoch": 0.625544519951211,
      "grad_norm": 308.7506408691406,
      "learning_rate": 3.4355026147588614e-06,
      "logits/chosen": -0.6785469055175781,
      "logits/rejected": -0.7987531423568726,
      "logps/chosen": -73.83087158203125,
      "logps/rejected": -67.55155944824219,
      "loss": 13.879,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0036451369524002075,
      "rewards/margins": 0.013614142313599586,
      "rewards/rejected": -0.017259273678064346,
      "step": 1077
    },
    {
      "epoch": 0.6261253412325027,
      "grad_norm": 313.75384521484375,
      "learning_rate": 3.434049970947124e-06,
      "logits/chosen": -0.8155107498168945,
      "logits/rejected": -0.7804557681083679,
      "logps/chosen": -69.31898498535156,
      "logps/rejected": -70.56317138671875,
      "loss": 14.3222,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.001983032329007983,
      "rewards/margins": -0.03253369778394699,
      "rewards/rejected": 0.03055066242814064,
      "step": 1078
    },
    {
      "epoch": 0.6267061625137945,
      "grad_norm": 345.8843994140625,
      "learning_rate": 3.432597327135387e-06,
      "logits/chosen": -0.7614946961402893,
      "logits/rejected": -0.7835288047790527,
      "logps/chosen": -76.03016662597656,
      "logps/rejected": -78.17121124267578,
      "loss": 14.9731,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.04884744808077812,
      "rewards/margins": -0.10027371346950531,
      "rewards/rejected": 0.05142626911401749,
      "step": 1079
    },
    {
      "epoch": 0.6272869837950863,
      "grad_norm": 337.42041015625,
      "learning_rate": 3.4311446833236496e-06,
      "logits/chosen": -0.6408761739730835,
      "logits/rejected": -0.6269320249557495,
      "logps/chosen": -63.5695915222168,
      "logps/rejected": -73.80338287353516,
      "loss": 13.3696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04012690857052803,
      "rewards/margins": 0.060710109770298004,
      "rewards/rejected": -0.020583197474479675,
      "step": 1080
    },
    {
      "epoch": 0.627867805076378,
      "grad_norm": 316.40277099609375,
      "learning_rate": 3.429692039511912e-06,
      "logits/chosen": -0.8447140455245972,
      "logits/rejected": -0.8406610488891602,
      "logps/chosen": -72.83253479003906,
      "logps/rejected": -80.47172546386719,
      "loss": 13.6131,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014124052599072456,
      "rewards/margins": 0.036146778613328934,
      "rewards/rejected": -0.022022727876901627,
      "step": 1081
    },
    {
      "epoch": 0.6284486263576697,
      "grad_norm": 329.9736633300781,
      "learning_rate": 3.4282393957001747e-06,
      "logits/chosen": -0.7001348733901978,
      "logits/rejected": -0.6745010018348694,
      "logps/chosen": -65.6444091796875,
      "logps/rejected": -74.1118392944336,
      "loss": 14.4186,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.049752719700336456,
      "rewards/margins": -0.03884691745042801,
      "rewards/rejected": -0.010905798524618149,
      "step": 1082
    },
    {
      "epoch": 0.6290294476389615,
      "grad_norm": 325.0237731933594,
      "learning_rate": 3.4267867518884375e-06,
      "logits/chosen": -0.7525070309638977,
      "logits/rejected": -0.7167800068855286,
      "logps/chosen": -69.889404296875,
      "logps/rejected": -75.60682678222656,
      "loss": 13.3809,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.014075754210352898,
      "rewards/margins": 0.062196653336286545,
      "rewards/rejected": -0.0762724056839943,
      "step": 1083
    },
    {
      "epoch": 0.6296102689202533,
      "grad_norm": 312.7868957519531,
      "learning_rate": 3.4253341080767e-06,
      "logits/chosen": -0.7102453708648682,
      "logits/rejected": -0.8066665530204773,
      "logps/chosen": -63.25031661987305,
      "logps/rejected": -67.66152954101562,
      "loss": 14.3879,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04945387691259384,
      "rewards/margins": -0.039318956434726715,
      "rewards/rejected": -0.010134927928447723,
      "step": 1084
    },
    {
      "epoch": 0.630191090201545,
      "grad_norm": 296.4964904785156,
      "learning_rate": 3.4238814642649625e-06,
      "logits/chosen": -0.7208272218704224,
      "logits/rejected": -0.7980926036834717,
      "logps/chosen": -66.80009460449219,
      "logps/rejected": -71.90062713623047,
      "loss": 13.4663,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.057175733149051666,
      "rewards/margins": 0.05002796649932861,
      "rewards/rejected": 0.007147759199142456,
      "step": 1085
    },
    {
      "epoch": 0.6307719114828367,
      "grad_norm": 311.69989013671875,
      "learning_rate": 3.4224288204532253e-06,
      "logits/chosen": -0.7312272787094116,
      "logits/rejected": -0.75715172290802,
      "logps/chosen": -73.25116729736328,
      "logps/rejected": -68.67305755615234,
      "loss": 13.5087,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0885586142539978,
      "rewards/margins": 0.05719367414712906,
      "rewards/rejected": 0.03136494383215904,
      "step": 1086
    },
    {
      "epoch": 0.6313527327641285,
      "grad_norm": 342.89666748046875,
      "learning_rate": 3.420976176641488e-06,
      "logits/chosen": -0.7952519059181213,
      "logits/rejected": -0.8625866770744324,
      "logps/chosen": -75.7269287109375,
      "logps/rejected": -75.89814758300781,
      "loss": 13.8974,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.017435293644666672,
      "rewards/margins": 0.009696127846837044,
      "rewards/rejected": 0.007739159278571606,
      "step": 1087
    },
    {
      "epoch": 0.6319335540454202,
      "grad_norm": 301.2434997558594,
      "learning_rate": 3.4195235328297504e-06,
      "logits/chosen": -0.6897446513175964,
      "logits/rejected": -1.0077345371246338,
      "logps/chosen": -68.39761352539062,
      "logps/rejected": -86.28750610351562,
      "loss": 12.6804,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.014521047472953796,
      "rewards/margins": 0.20002944767475128,
      "rewards/rejected": -0.21455049514770508,
      "step": 1088
    },
    {
      "epoch": 0.632514375326712,
      "grad_norm": 320.6565246582031,
      "learning_rate": 3.418070889018013e-06,
      "logits/chosen": -0.7355692386627197,
      "logits/rejected": -0.7012276649475098,
      "logps/chosen": -70.1624984741211,
      "logps/rejected": -77.49390411376953,
      "loss": 13.6464,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01790980063378811,
      "rewards/margins": 0.03169848024845123,
      "rewards/rejected": -0.013788673095405102,
      "step": 1089
    },
    {
      "epoch": 0.6330951966080037,
      "grad_norm": 339.77020263671875,
      "learning_rate": 3.416618245206276e-06,
      "logits/chosen": -0.8961198925971985,
      "logits/rejected": -0.9206236600875854,
      "logps/chosen": -73.21183776855469,
      "logps/rejected": -71.58177185058594,
      "loss": 14.1473,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02303207293152809,
      "rewards/margins": -0.01333695650100708,
      "rewards/rejected": 0.036369021981954575,
      "step": 1090
    },
    {
      "epoch": 0.6336760178892955,
      "grad_norm": 305.9129943847656,
      "learning_rate": 3.415165601394538e-06,
      "logits/chosen": -0.7048637270927429,
      "logits/rejected": -0.7706517577171326,
      "logps/chosen": -76.41432189941406,
      "logps/rejected": -75.1176986694336,
      "loss": 13.6776,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.017997169867157936,
      "rewards/margins": 0.029994655400514603,
      "rewards/rejected": -0.011997489258646965,
      "step": 1091
    },
    {
      "epoch": 0.6342568391705872,
      "grad_norm": 330.4241638183594,
      "learning_rate": 3.413712957582801e-06,
      "logits/chosen": -0.9359704852104187,
      "logits/rejected": -0.9532849192619324,
      "logps/chosen": -72.52864837646484,
      "logps/rejected": -70.50189208984375,
      "loss": 14.8636,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.046659164130687714,
      "rewards/margins": -0.07957638055086136,
      "rewards/rejected": 0.03291721269488335,
      "step": 1092
    },
    {
      "epoch": 0.634837660451879,
      "grad_norm": 304.2950439453125,
      "learning_rate": 3.4122603137710637e-06,
      "logits/chosen": -0.580649733543396,
      "logits/rejected": -0.7970311045646667,
      "logps/chosen": -74.48047637939453,
      "logps/rejected": -70.85575866699219,
      "loss": 13.6111,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.040632303804159164,
      "rewards/margins": 0.04058300331234932,
      "rewards/rejected": 4.9304962885798886e-05,
      "step": 1093
    },
    {
      "epoch": 0.6354184817331707,
      "grad_norm": 401.4617919921875,
      "learning_rate": 3.410807669959326e-06,
      "logits/chosen": -0.7436962127685547,
      "logits/rejected": -0.7610384225845337,
      "logps/chosen": -71.65431213378906,
      "logps/rejected": -77.98550415039062,
      "loss": 14.3401,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.019342470914125443,
      "rewards/margins": -0.04064258188009262,
      "rewards/rejected": 0.021300112828612328,
      "step": 1094
    },
    {
      "epoch": 0.6359993030144625,
      "grad_norm": 310.211669921875,
      "learning_rate": 3.4093550261475888e-06,
      "logits/chosen": -0.880212664604187,
      "logits/rejected": -0.8327865600585938,
      "logps/chosen": -67.9056625366211,
      "logps/rejected": -72.73890686035156,
      "loss": 13.5733,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008872169069945812,
      "rewards/margins": 0.045545946806669235,
      "rewards/rejected": -0.05441810563206673,
      "step": 1095
    },
    {
      "epoch": 0.6365801242957542,
      "grad_norm": 290.6644287109375,
      "learning_rate": 3.4079023823358515e-06,
      "logits/chosen": -0.997941792011261,
      "logits/rejected": -0.9481798410415649,
      "logps/chosen": -67.26060485839844,
      "logps/rejected": -68.79742431640625,
      "loss": 13.2647,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01653924211859703,
      "rewards/margins": 0.06931618601083755,
      "rewards/rejected": -0.052776940166950226,
      "step": 1096
    },
    {
      "epoch": 0.637160945577046,
      "grad_norm": 327.4011535644531,
      "learning_rate": 3.4064497385241143e-06,
      "logits/chosen": -0.795971691608429,
      "logits/rejected": -0.7696972489356995,
      "logps/chosen": -72.01513671875,
      "logps/rejected": -72.37380981445312,
      "loss": 13.2627,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02414756827056408,
      "rewards/margins": 0.06999386847019196,
      "rewards/rejected": -0.04584629833698273,
      "step": 1097
    },
    {
      "epoch": 0.6377417668583377,
      "grad_norm": 564.237548828125,
      "learning_rate": 3.4049970947123766e-06,
      "logits/chosen": -0.9717392921447754,
      "logits/rejected": -0.8530858159065247,
      "logps/chosen": -79.95928192138672,
      "logps/rejected": -80.98189544677734,
      "loss": 13.697,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04589349776506424,
      "rewards/margins": 0.04101797938346863,
      "rewards/rejected": -0.08691148459911346,
      "step": 1098
    },
    {
      "epoch": 0.6383225881396294,
      "grad_norm": 580.7954711914062,
      "learning_rate": 3.4035444509006393e-06,
      "logits/chosen": -0.9145733714103699,
      "logits/rejected": -0.8677693605422974,
      "logps/chosen": -68.48554992675781,
      "logps/rejected": -80.02945709228516,
      "loss": 12.8525,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008354002609848976,
      "rewards/margins": 0.11731596291065216,
      "rewards/rejected": -0.10896197706460953,
      "step": 1099
    },
    {
      "epoch": 0.6389034094209212,
      "grad_norm": 595.4121704101562,
      "learning_rate": 3.402091807088902e-06,
      "logits/chosen": -0.7863033413887024,
      "logits/rejected": -0.733050525188446,
      "logps/chosen": -67.67957305908203,
      "logps/rejected": -71.6598129272461,
      "loss": 14.2907,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.02404143288731575,
      "rewards/margins": -0.030664855614304543,
      "rewards/rejected": 0.05470628663897514,
      "step": 1100
    },
    {
      "epoch": 0.6394842307022129,
      "grad_norm": 309.0567321777344,
      "learning_rate": 3.4006391632771644e-06,
      "logits/chosen": -0.9789560437202454,
      "logits/rejected": -0.9520699381828308,
      "logps/chosen": -75.55183410644531,
      "logps/rejected": -73.79700469970703,
      "loss": 13.6684,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.007935131900012493,
      "rewards/margins": 0.03421130031347275,
      "rewards/rejected": -0.02627617120742798,
      "step": 1101
    },
    {
      "epoch": 0.6400650519835047,
      "grad_norm": 296.9599609375,
      "learning_rate": 3.399186519465427e-06,
      "logits/chosen": -0.8211098909378052,
      "logits/rejected": -0.864301323890686,
      "logps/chosen": -69.31588745117188,
      "logps/rejected": -80.18562316894531,
      "loss": 13.2417,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.054910190403461456,
      "rewards/margins": 0.07895153015851974,
      "rewards/rejected": -0.02404133975505829,
      "step": 1102
    },
    {
      "epoch": 0.6406458732647964,
      "grad_norm": 307.5824890136719,
      "learning_rate": 3.39773387565369e-06,
      "logits/chosen": -0.8680564761161804,
      "logits/rejected": -1.074083924293518,
      "logps/chosen": -73.11421203613281,
      "logps/rejected": -76.54287719726562,
      "loss": 13.4502,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01877606473863125,
      "rewards/margins": 0.16289541125297546,
      "rewards/rejected": -0.14411935210227966,
      "step": 1103
    },
    {
      "epoch": 0.6412266945460882,
      "grad_norm": 367.9165954589844,
      "learning_rate": 3.3962812318419526e-06,
      "logits/chosen": -0.8910163640975952,
      "logits/rejected": -0.838483452796936,
      "logps/chosen": -75.1598129272461,
      "logps/rejected": -79.41618347167969,
      "loss": 14.4498,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.028661027550697327,
      "rewards/margins": -0.04120267927646637,
      "rewards/rejected": 0.012541646137833595,
      "step": 1104
    },
    {
      "epoch": 0.6418075158273799,
      "grad_norm": 312.6385498046875,
      "learning_rate": 3.394828588030215e-06,
      "logits/chosen": -0.8036056756973267,
      "logits/rejected": -0.7834665179252625,
      "logps/chosen": -71.69194793701172,
      "logps/rejected": -73.26815032958984,
      "loss": 13.6329,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.001507095294073224,
      "rewards/margins": 0.03805564343929291,
      "rewards/rejected": -0.03956274315714836,
      "step": 1105
    },
    {
      "epoch": 0.6423883371086717,
      "grad_norm": 311.171142578125,
      "learning_rate": 3.3933759442184777e-06,
      "logits/chosen": -0.7820031642913818,
      "logits/rejected": -0.8056840896606445,
      "logps/chosen": -73.14546203613281,
      "logps/rejected": -70.93217468261719,
      "loss": 13.4964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022750332951545715,
      "rewards/margins": 0.05466002970933914,
      "rewards/rejected": -0.03190969303250313,
      "step": 1106
    },
    {
      "epoch": 0.6429691583899634,
      "grad_norm": 330.8109130859375,
      "learning_rate": 3.3919233004067405e-06,
      "logits/chosen": -0.7964299917221069,
      "logits/rejected": -0.7226386666297913,
      "logps/chosen": -74.08839416503906,
      "logps/rejected": -74.29837036132812,
      "loss": 13.929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.003889267100021243,
      "rewards/margins": 0.004051786847412586,
      "rewards/rejected": -0.007941055111587048,
      "step": 1107
    },
    {
      "epoch": 0.6435499796712552,
      "grad_norm": 312.6083984375,
      "learning_rate": 3.390470656595003e-06,
      "logits/chosen": -0.9366266131401062,
      "logits/rejected": -0.921904444694519,
      "logps/chosen": -73.20463562011719,
      "logps/rejected": -72.86039733886719,
      "loss": 14.2451,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.014935257844626904,
      "rewards/margins": -0.023692576214671135,
      "rewards/rejected": 0.008757324889302254,
      "step": 1108
    },
    {
      "epoch": 0.6441308009525469,
      "grad_norm": 414.65594482421875,
      "learning_rate": 3.3890180127832655e-06,
      "logits/chosen": -0.7851788997650146,
      "logits/rejected": -0.8346858024597168,
      "logps/chosen": -75.82699584960938,
      "logps/rejected": -79.17605590820312,
      "loss": 13.5271,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06471805274486542,
      "rewards/margins": 0.04485418647527695,
      "rewards/rejected": 0.019863862544298172,
      "step": 1109
    },
    {
      "epoch": 0.6447116222338386,
      "grad_norm": 312.5867919921875,
      "learning_rate": 3.3875653689715283e-06,
      "logits/chosen": -0.9736455678939819,
      "logits/rejected": -0.9611663818359375,
      "logps/chosen": -73.07161712646484,
      "logps/rejected": -78.50061798095703,
      "loss": 13.6878,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02146800421178341,
      "rewards/margins": 0.029409591108560562,
      "rewards/rejected": -0.050877589732408524,
      "step": 1110
    },
    {
      "epoch": 0.6452924435151304,
      "grad_norm": 284.8564147949219,
      "learning_rate": 3.386112725159791e-06,
      "logits/chosen": -0.8841636776924133,
      "logits/rejected": -0.7309855222702026,
      "logps/chosen": -77.16401672363281,
      "logps/rejected": -67.12794494628906,
      "loss": 12.5619,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04172271490097046,
      "rewards/margins": 0.1458221971988678,
      "rewards/rejected": -0.10409947484731674,
      "step": 1111
    },
    {
      "epoch": 0.6458732647964222,
      "grad_norm": 336.10302734375,
      "learning_rate": 3.3846600813480534e-06,
      "logits/chosen": -0.8979451060295105,
      "logits/rejected": -0.8460661768913269,
      "logps/chosen": -72.7340316772461,
      "logps/rejected": -77.87813568115234,
      "loss": 14.3766,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.022529829293489456,
      "rewards/margins": -0.03525589033961296,
      "rewards/rejected": 0.01272605825215578,
      "step": 1112
    },
    {
      "epoch": 0.6464540860777139,
      "grad_norm": 312.8513488769531,
      "learning_rate": 3.3832074375363165e-06,
      "logits/chosen": -0.7987014055252075,
      "logits/rejected": -0.7443603873252869,
      "logps/chosen": -77.47074890136719,
      "logps/rejected": -63.526939392089844,
      "loss": 13.7814,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03565699979662895,
      "rewards/margins": 0.018476877361536026,
      "rewards/rejected": 0.017180118709802628,
      "step": 1113
    },
    {
      "epoch": 0.6470349073590056,
      "grad_norm": 317.2981262207031,
      "learning_rate": 3.3817547937245793e-06,
      "logits/chosen": -0.7856286764144897,
      "logits/rejected": -0.696509599685669,
      "logps/chosen": -69.47307586669922,
      "logps/rejected": -72.22679901123047,
      "loss": 13.4791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.058699510991573334,
      "rewards/margins": 0.05481904745101929,
      "rewards/rejected": 0.00388046121224761,
      "step": 1114
    },
    {
      "epoch": 0.6476157286402974,
      "grad_norm": 327.1874694824219,
      "learning_rate": 3.380302149912842e-06,
      "logits/chosen": -0.9234718084335327,
      "logits/rejected": -0.8798847198486328,
      "logps/chosen": -82.38871002197266,
      "logps/rejected": -79.6245346069336,
      "loss": 13.309,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.01846655085682869,
      "rewards/margins": 0.07758887857198715,
      "rewards/rejected": -0.09605542570352554,
      "step": 1115
    },
    {
      "epoch": 0.6481965499215891,
      "grad_norm": 343.098388671875,
      "learning_rate": 3.3788495061011044e-06,
      "logits/chosen": -0.8076757192611694,
      "logits/rejected": -0.8994390368461609,
      "logps/chosen": -81.92533874511719,
      "logps/rejected": -73.13912200927734,
      "loss": 14.8246,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08238311111927032,
      "rewards/margins": -0.06615736335515976,
      "rewards/rejected": -0.016225744038820267,
      "step": 1116
    },
    {
      "epoch": 0.6487773712028809,
      "grad_norm": 326.1660461425781,
      "learning_rate": 3.377396862289367e-06,
      "logits/chosen": -0.8509761691093445,
      "logits/rejected": -0.9029315114021301,
      "logps/chosen": -74.35148620605469,
      "logps/rejected": -86.59319305419922,
      "loss": 13.7825,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04883483424782753,
      "rewards/margins": 0.032496631145477295,
      "rewards/rejected": -0.08133147656917572,
      "step": 1117
    },
    {
      "epoch": 0.6493581924841726,
      "grad_norm": 299.83526611328125,
      "learning_rate": 3.37594421847763e-06,
      "logits/chosen": -0.9672731161117554,
      "logits/rejected": -0.8193171620368958,
      "logps/chosen": -71.75694274902344,
      "logps/rejected": -64.68678283691406,
      "loss": 13.6272,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.001300790929235518,
      "rewards/margins": 0.04346258193254471,
      "rewards/rejected": -0.042161792516708374,
      "step": 1118
    },
    {
      "epoch": 0.6499390137654644,
      "grad_norm": 299.9213562011719,
      "learning_rate": 3.3744915746658926e-06,
      "logits/chosen": -1.042067289352417,
      "logits/rejected": -1.035821795463562,
      "logps/chosen": -64.90753173828125,
      "logps/rejected": -66.659423828125,
      "loss": 13.3194,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.049155447632074356,
      "rewards/margins": 0.06338377296924591,
      "rewards/rejected": -0.01422832626849413,
      "step": 1119
    },
    {
      "epoch": 0.6505198350467561,
      "grad_norm": 309.9093322753906,
      "learning_rate": 3.373038930854155e-06,
      "logits/chosen": -0.8113111257553101,
      "logits/rejected": -0.901233971118927,
      "logps/chosen": -83.24082946777344,
      "logps/rejected": -71.51692199707031,
      "loss": 13.8276,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.048964451998472214,
      "rewards/margins": 0.010548337362706661,
      "rewards/rejected": -0.05951278656721115,
      "step": 1120
    },
    {
      "epoch": 0.6511006563280478,
      "grad_norm": 334.074462890625,
      "learning_rate": 3.3715862870424177e-06,
      "logits/chosen": -0.744590699672699,
      "logits/rejected": -0.7340568900108337,
      "logps/chosen": -75.37425231933594,
      "logps/rejected": -75.96635437011719,
      "loss": 14.1029,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04764125123620033,
      "rewards/margins": -0.012311267666518688,
      "rewards/rejected": -0.03532997891306877,
      "step": 1121
    },
    {
      "epoch": 0.6516814776093396,
      "grad_norm": 311.2660217285156,
      "learning_rate": 3.3701336432306804e-06,
      "logits/chosen": -0.9063982963562012,
      "logits/rejected": -0.8556571006774902,
      "logps/chosen": -81.4077377319336,
      "logps/rejected": -73.09236145019531,
      "loss": 13.9775,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05028345435857773,
      "rewards/margins": 0.006928545422852039,
      "rewards/rejected": -0.05721200257539749,
      "step": 1122
    },
    {
      "epoch": 0.6522622988906314,
      "grad_norm": 306.66705322265625,
      "learning_rate": 3.3686809994189427e-06,
      "logits/chosen": -0.8220956921577454,
      "logits/rejected": -0.8612530827522278,
      "logps/chosen": -73.86631774902344,
      "logps/rejected": -74.8656005859375,
      "loss": 12.8759,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.005287191364914179,
      "rewards/margins": 0.1143055185675621,
      "rewards/rejected": -0.11959271132946014,
      "step": 1123
    },
    {
      "epoch": 0.6528431201719231,
      "grad_norm": 406.51531982421875,
      "learning_rate": 3.3672283556072055e-06,
      "logits/chosen": -0.99102383852005,
      "logits/rejected": -1.0229992866516113,
      "logps/chosen": -81.93040466308594,
      "logps/rejected": -81.38733673095703,
      "loss": 15.0371,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0858633741736412,
      "rewards/margins": -0.08687599003314972,
      "rewards/rejected": 0.0010126233100891113,
      "step": 1124
    },
    {
      "epoch": 0.6534239414532148,
      "grad_norm": 308.5504455566406,
      "learning_rate": 3.3657757117954682e-06,
      "logits/chosen": -0.9116039276123047,
      "logits/rejected": -1.0154939889907837,
      "logps/chosen": -70.75515747070312,
      "logps/rejected": -69.52398681640625,
      "loss": 12.7376,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05423317104578018,
      "rewards/margins": 0.12636461853981018,
      "rewards/rejected": -0.0721314400434494,
      "step": 1125
    },
    {
      "epoch": 0.6540047627345066,
      "grad_norm": 311.83294677734375,
      "learning_rate": 3.3643230679837306e-06,
      "logits/chosen": -0.8607922792434692,
      "logits/rejected": -0.9037091135978699,
      "logps/chosen": -74.27735137939453,
      "logps/rejected": -68.73574829101562,
      "loss": 13.2139,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.021343013271689415,
      "rewards/margins": 0.08003392815589905,
      "rewards/rejected": -0.058690913021564484,
      "step": 1126
    },
    {
      "epoch": 0.6545855840157984,
      "grad_norm": 328.3275451660156,
      "learning_rate": 3.3628704241719933e-06,
      "logits/chosen": -0.7941106557846069,
      "logits/rejected": -0.9193152189254761,
      "logps/chosen": -74.83554077148438,
      "logps/rejected": -77.32106018066406,
      "loss": 13.3681,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.010143203660845757,
      "rewards/margins": 0.06033836677670479,
      "rewards/rejected": -0.050195157527923584,
      "step": 1127
    },
    {
      "epoch": 0.6551664052970901,
      "grad_norm": 418.1154479980469,
      "learning_rate": 3.361417780360256e-06,
      "logits/chosen": -0.926904022693634,
      "logits/rejected": -0.8674629330635071,
      "logps/chosen": -82.01655578613281,
      "logps/rejected": -82.10655975341797,
      "loss": 12.9049,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04920128360390663,
      "rewards/margins": 0.11056945472955704,
      "rewards/rejected": -0.061368174850940704,
      "step": 1128
    },
    {
      "epoch": 0.6557472265783818,
      "grad_norm": 328.4079284667969,
      "learning_rate": 3.359965136548519e-06,
      "logits/chosen": -0.8782418370246887,
      "logits/rejected": -0.930760383605957,
      "logps/chosen": -73.77706146240234,
      "logps/rejected": -77.1947250366211,
      "loss": 13.8319,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03465033322572708,
      "rewards/margins": 0.011911900714039803,
      "rewards/rejected": -0.04656223952770233,
      "step": 1129
    },
    {
      "epoch": 0.6563280478596736,
      "grad_norm": 304.2339782714844,
      "learning_rate": 3.358512492736781e-06,
      "logits/chosen": -0.849596381187439,
      "logits/rejected": -0.9071720242500305,
      "logps/chosen": -74.27670288085938,
      "logps/rejected": -74.19172668457031,
      "loss": 13.5264,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.020483296364545822,
      "rewards/margins": 0.04569234326481819,
      "rewards/rejected": -0.025209054350852966,
      "step": 1130
    },
    {
      "epoch": 0.6569088691409654,
      "grad_norm": 286.1509704589844,
      "learning_rate": 3.357059848925044e-06,
      "logits/chosen": -0.7693842649459839,
      "logits/rejected": -0.96452796459198,
      "logps/chosen": -73.09904479980469,
      "logps/rejected": -75.56321716308594,
      "loss": 13.6971,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07628011703491211,
      "rewards/margins": 0.03913526609539986,
      "rewards/rejected": -0.11541537940502167,
      "step": 1131
    },
    {
      "epoch": 0.657489690422257,
      "grad_norm": 314.5287170410156,
      "learning_rate": 3.3556072051133066e-06,
      "logits/chosen": -0.7937735319137573,
      "logits/rejected": -0.9172106981277466,
      "logps/chosen": -68.91234588623047,
      "logps/rejected": -83.07949829101562,
      "loss": 13.5293,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.006922473199665546,
      "rewards/margins": 0.04392245411872864,
      "rewards/rejected": -0.03699997812509537,
      "step": 1132
    },
    {
      "epoch": 0.6580705117035488,
      "grad_norm": 277.6486511230469,
      "learning_rate": 3.354154561301569e-06,
      "logits/chosen": -0.9479109644889832,
      "logits/rejected": -0.8847756385803223,
      "logps/chosen": -66.35222625732422,
      "logps/rejected": -74.14461517333984,
      "loss": 12.7854,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05040959641337395,
      "rewards/margins": 0.1317688524723053,
      "rewards/rejected": -0.08135926723480225,
      "step": 1133
    },
    {
      "epoch": 0.6586513329848406,
      "grad_norm": 344.2884826660156,
      "learning_rate": 3.3527019174898317e-06,
      "logits/chosen": -0.7381841540336609,
      "logits/rejected": -0.7322576642036438,
      "logps/chosen": -75.91568756103516,
      "logps/rejected": -79.04608154296875,
      "loss": 13.7644,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03366268426179886,
      "rewards/margins": 0.03374585881829262,
      "rewards/rejected": -0.06740853190422058,
      "step": 1134
    },
    {
      "epoch": 0.6592321542661324,
      "grad_norm": 308.2327880859375,
      "learning_rate": 3.3512492736780945e-06,
      "logits/chosen": -0.7495602369308472,
      "logits/rejected": -0.8398062586784363,
      "logps/chosen": -74.60487365722656,
      "logps/rejected": -74.7044906616211,
      "loss": 13.1195,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.009602843783795834,
      "rewards/margins": 0.09286157786846161,
      "rewards/rejected": -0.0832587406039238,
      "step": 1135
    },
    {
      "epoch": 0.659812975547424,
      "grad_norm": 315.85614013671875,
      "learning_rate": 3.349796629866357e-06,
      "logits/chosen": -0.92826908826828,
      "logits/rejected": -0.8133495450019836,
      "logps/chosen": -79.2392807006836,
      "logps/rejected": -67.21851348876953,
      "loss": 14.3548,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029066869989037514,
      "rewards/margins": -0.038375429809093475,
      "rewards/rejected": 0.009308547712862492,
      "step": 1136
    },
    {
      "epoch": 0.6603937968287158,
      "grad_norm": 315.47998046875,
      "learning_rate": 3.3483439860546195e-06,
      "logits/chosen": -0.8024196624755859,
      "logits/rejected": -0.8435264825820923,
      "logps/chosen": -74.04217529296875,
      "logps/rejected": -74.72700500488281,
      "loss": 13.8058,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03207816928625107,
      "rewards/margins": 0.019642286002635956,
      "rewards/rejected": -0.05172046273946762,
      "step": 1137
    },
    {
      "epoch": 0.6609746181100076,
      "grad_norm": 303.85198974609375,
      "learning_rate": 3.3468913422428823e-06,
      "logits/chosen": -0.8626992106437683,
      "logits/rejected": -0.8943392634391785,
      "logps/chosen": -75.42814636230469,
      "logps/rejected": -74.78086853027344,
      "loss": 13.2741,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002096575917676091,
      "rewards/margins": 0.08013061434030533,
      "rewards/rejected": -0.07803403586149216,
      "step": 1138
    },
    {
      "epoch": 0.6615554393912993,
      "grad_norm": 314.4802551269531,
      "learning_rate": 3.345438698431145e-06,
      "logits/chosen": -0.7583316564559937,
      "logits/rejected": -0.8205488324165344,
      "logps/chosen": -79.16755676269531,
      "logps/rejected": -75.33838653564453,
      "loss": 14.128,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02013455331325531,
      "rewards/margins": -0.011084577068686485,
      "rewards/rejected": -0.009049976244568825,
      "step": 1139
    },
    {
      "epoch": 0.662136260672591,
      "grad_norm": 347.2712707519531,
      "learning_rate": 3.3439860546194073e-06,
      "logits/chosen": -0.9793803095817566,
      "logits/rejected": -1.0709476470947266,
      "logps/chosen": -77.69926452636719,
      "logps/rejected": -69.31404113769531,
      "loss": 14.0521,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.009851111099123955,
      "rewards/margins": -0.005331629421561956,
      "rewards/rejected": -0.0045194802805781364,
      "step": 1140
    },
    {
      "epoch": 0.6627170819538828,
      "grad_norm": 319.09906005859375,
      "learning_rate": 3.34253341080767e-06,
      "logits/chosen": -1.0060648918151855,
      "logits/rejected": -0.9142974615097046,
      "logps/chosen": -71.60318756103516,
      "logps/rejected": -66.31644439697266,
      "loss": 13.8988,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01562432199716568,
      "rewards/margins": 0.012615347281098366,
      "rewards/rejected": -0.028239671140909195,
      "step": 1141
    },
    {
      "epoch": 0.6632979032351746,
      "grad_norm": 335.98565673828125,
      "learning_rate": 3.341080766995933e-06,
      "logits/chosen": -0.9369823336601257,
      "logits/rejected": -0.8730899691581726,
      "logps/chosen": -74.31767272949219,
      "logps/rejected": -73.81460571289062,
      "loss": 14.6285,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029477547854185104,
      "rewards/margins": -0.044619448482990265,
      "rewards/rejected": 0.07409698516130447,
      "step": 1142
    },
    {
      "epoch": 0.6638787245164662,
      "grad_norm": 351.40216064453125,
      "learning_rate": 3.339628123184195e-06,
      "logits/chosen": -0.8079739809036255,
      "logits/rejected": -0.8196757435798645,
      "logps/chosen": -77.51908874511719,
      "logps/rejected": -86.69542694091797,
      "loss": 13.0898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03954374045133591,
      "rewards/margins": 0.09758506715297699,
      "rewards/rejected": -0.058041322976350784,
      "step": 1143
    },
    {
      "epoch": 0.664459545797758,
      "grad_norm": 321.00177001953125,
      "learning_rate": 3.338175479372458e-06,
      "logits/chosen": -0.9274177551269531,
      "logits/rejected": -0.9068565368652344,
      "logps/chosen": -72.2402114868164,
      "logps/rejected": -81.47127532958984,
      "loss": 13.9708,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.033435553312301636,
      "rewards/margins": 0.018207602202892303,
      "rewards/rejected": -0.051643144339323044,
      "step": 1144
    },
    {
      "epoch": 0.6650403670790498,
      "grad_norm": 351.00543212890625,
      "learning_rate": 3.3367228355607207e-06,
      "logits/chosen": -0.9931886792182922,
      "logits/rejected": -1.0393340587615967,
      "logps/chosen": -75.20349884033203,
      "logps/rejected": -71.90242767333984,
      "loss": 14.2065,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07046198844909668,
      "rewards/margins": -0.022660408169031143,
      "rewards/rejected": -0.047801584005355835,
      "step": 1145
    },
    {
      "epoch": 0.6656211883603416,
      "grad_norm": 312.2623596191406,
      "learning_rate": 3.3352701917489834e-06,
      "logits/chosen": -0.9696620106697083,
      "logits/rejected": -0.8835649490356445,
      "logps/chosen": -74.25260925292969,
      "logps/rejected": -75.95238494873047,
      "loss": 12.8532,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.026852522045373917,
      "rewards/margins": 0.1164240837097168,
      "rewards/rejected": -0.08957154303789139,
      "step": 1146
    },
    {
      "epoch": 0.6662020096416332,
      "grad_norm": 288.88104248046875,
      "learning_rate": 3.3338175479372457e-06,
      "logits/chosen": -0.8604238629341125,
      "logits/rejected": -1.123032808303833,
      "logps/chosen": -77.32051086425781,
      "logps/rejected": -65.73595428466797,
      "loss": 12.5038,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.029479866847395897,
      "rewards/margins": 0.15638433396816254,
      "rewards/rejected": -0.1269044727087021,
      "step": 1147
    },
    {
      "epoch": 0.666782830922925,
      "grad_norm": 324.7893371582031,
      "learning_rate": 3.3323649041255085e-06,
      "logits/chosen": -0.8295110464096069,
      "logits/rejected": -0.7149649858474731,
      "logps/chosen": -86.52113342285156,
      "logps/rejected": -68.8885269165039,
      "loss": 13.4058,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011997411027550697,
      "rewards/margins": 0.05395049601793289,
      "rewards/rejected": -0.04195307940244675,
      "step": 1148
    },
    {
      "epoch": 0.6673636522042168,
      "grad_norm": 308.4689025878906,
      "learning_rate": 3.3309122603137712e-06,
      "logits/chosen": -0.7572081685066223,
      "logits/rejected": -0.783782422542572,
      "logps/chosen": -81.11930847167969,
      "logps/rejected": -71.57644653320312,
      "loss": 13.4594,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.050214897841215134,
      "rewards/margins": 0.05488036945462227,
      "rewards/rejected": -0.1050952672958374,
      "step": 1149
    },
    {
      "epoch": 0.6679444734855086,
      "grad_norm": 321.4398498535156,
      "learning_rate": 3.3294596165020336e-06,
      "logits/chosen": -0.9056531190872192,
      "logits/rejected": -0.9038206934928894,
      "logps/chosen": -83.10813903808594,
      "logps/rejected": -73.87676239013672,
      "loss": 13.8508,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0317196398973465,
      "rewards/margins": 0.016990767791867256,
      "rewards/rejected": -0.048710405826568604,
      "step": 1150
    },
    {
      "epoch": 0.6685252947668002,
      "grad_norm": 311.9756164550781,
      "learning_rate": 3.3280069726902963e-06,
      "logits/chosen": -1.0237656831741333,
      "logits/rejected": -0.9660177230834961,
      "logps/chosen": -78.77606201171875,
      "logps/rejected": -77.33097076416016,
      "loss": 13.5526,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010856760665774345,
      "rewards/margins": 0.03982258960604668,
      "rewards/rejected": -0.050679344683885574,
      "step": 1151
    },
    {
      "epoch": 0.669106116048092,
      "grad_norm": 318.1045227050781,
      "learning_rate": 3.326554328878559e-06,
      "logits/chosen": -1.1040226221084595,
      "logits/rejected": -0.9501973390579224,
      "logps/chosen": -74.900634765625,
      "logps/rejected": -73.06010437011719,
      "loss": 13.6886,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.018065717071294785,
      "rewards/margins": 0.02897379733622074,
      "rewards/rejected": -0.010908080264925957,
      "step": 1152
    },
    {
      "epoch": 0.6696869373293838,
      "grad_norm": 285.5135498046875,
      "learning_rate": 3.325101685066822e-06,
      "logits/chosen": -0.9260636568069458,
      "logits/rejected": -0.8168819546699524,
      "logps/chosen": -64.40333557128906,
      "logps/rejected": -76.6811294555664,
      "loss": 12.9531,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00420218612998724,
      "rewards/margins": 0.10337436199188232,
      "rewards/rejected": -0.09917217493057251,
      "step": 1153
    },
    {
      "epoch": 0.6702677586106754,
      "grad_norm": 310.67633056640625,
      "learning_rate": 3.323649041255084e-06,
      "logits/chosen": -0.8356355428695679,
      "logits/rejected": -0.7701283097267151,
      "logps/chosen": -68.86634826660156,
      "logps/rejected": -77.45189666748047,
      "loss": 13.3225,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.026029136031866074,
      "rewards/margins": 0.06524486839771271,
      "rewards/rejected": -0.09127400815486908,
      "step": 1154
    },
    {
      "epoch": 0.6708485798919672,
      "grad_norm": 313.2401123046875,
      "learning_rate": 3.3221963974433473e-06,
      "logits/chosen": -1.1406347751617432,
      "logits/rejected": -1.0746700763702393,
      "logps/chosen": -74.25431060791016,
      "logps/rejected": -69.09442901611328,
      "loss": 14.0244,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011160466820001602,
      "rewards/margins": -0.007961602881550789,
      "rewards/rejected": 0.01912207528948784,
      "step": 1155
    },
    {
      "epoch": 0.671429401173259,
      "grad_norm": 315.8984680175781,
      "learning_rate": 3.32074375363161e-06,
      "logits/chosen": -0.9686878323554993,
      "logits/rejected": -1.0331335067749023,
      "logps/chosen": -76.83672332763672,
      "logps/rejected": -93.02784729003906,
      "loss": 14.2047,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.011404897086322308,
      "rewards/margins": -0.010174915194511414,
      "rewards/rejected": -0.001229980611242354,
      "step": 1156
    },
    {
      "epoch": 0.6720102224545508,
      "grad_norm": 304.2946472167969,
      "learning_rate": 3.319291109819873e-06,
      "logits/chosen": -1.0070463418960571,
      "logits/rejected": -0.856772780418396,
      "logps/chosen": -72.10645294189453,
      "logps/rejected": -76.42930603027344,
      "loss": 13.3311,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.030288532376289368,
      "rewards/margins": 0.06094416230916977,
      "rewards/rejected": -0.09123269468545914,
      "step": 1157
    },
    {
      "epoch": 0.6725910437358424,
      "grad_norm": 313.6856384277344,
      "learning_rate": 3.317838466008135e-06,
      "logits/chosen": -0.9823400378227234,
      "logits/rejected": -1.0469614267349243,
      "logps/chosen": -70.67359924316406,
      "logps/rejected": -73.56322479248047,
      "loss": 13.624,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024736547842621803,
      "rewards/margins": 0.03398207575082779,
      "rewards/rejected": -0.009245529770851135,
      "step": 1158
    },
    {
      "epoch": 0.6731718650171342,
      "grad_norm": 341.883056640625,
      "learning_rate": 3.316385822196398e-06,
      "logits/chosen": -0.8451075553894043,
      "logits/rejected": -0.9140728712081909,
      "logps/chosen": -75.22441101074219,
      "logps/rejected": -74.47895050048828,
      "loss": 13.4937,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0378233902156353,
      "rewards/margins": 0.053475283086299896,
      "rewards/rejected": -0.015651885420084,
      "step": 1159
    },
    {
      "epoch": 0.673752686298426,
      "grad_norm": 303.8985290527344,
      "learning_rate": 3.3149331783846606e-06,
      "logits/chosen": -0.9142858386039734,
      "logits/rejected": -0.8438348770141602,
      "logps/chosen": -76.67243194580078,
      "logps/rejected": -76.33660888671875,
      "loss": 13.3926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.045996714383363724,
      "rewards/margins": 0.07819615304470062,
      "rewards/rejected": -0.12419287115335464,
      "step": 1160
    },
    {
      "epoch": 0.6743335075797178,
      "grad_norm": 333.9848937988281,
      "learning_rate": 3.3134805345729234e-06,
      "logits/chosen": -0.9588890075683594,
      "logits/rejected": -0.9215188026428223,
      "logps/chosen": -83.87674713134766,
      "logps/rejected": -82.53553009033203,
      "loss": 13.4186,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02275824174284935,
      "rewards/margins": 0.0674627274274826,
      "rewards/rejected": -0.04470448940992355,
      "step": 1161
    },
    {
      "epoch": 0.6749143288610094,
      "grad_norm": 314.66455078125,
      "learning_rate": 3.3120278907611857e-06,
      "logits/chosen": -0.8470560908317566,
      "logits/rejected": -0.8499711155891418,
      "logps/chosen": -72.74806213378906,
      "logps/rejected": -73.0672836303711,
      "loss": 13.5864,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.022015493363142014,
      "rewards/margins": 0.04026877135038376,
      "rewards/rejected": -0.06228426843881607,
      "step": 1162
    },
    {
      "epoch": 0.6754951501423012,
      "grad_norm": 387.2796630859375,
      "learning_rate": 3.3105752469494484e-06,
      "logits/chosen": -0.9331986308097839,
      "logits/rejected": -0.8966618776321411,
      "logps/chosen": -76.05677032470703,
      "logps/rejected": -76.1444320678711,
      "loss": 14.236,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07434257864952087,
      "rewards/margins": -0.024118661880493164,
      "rewards/rejected": -0.05022391676902771,
      "step": 1163
    },
    {
      "epoch": 0.676075971423593,
      "grad_norm": 311.802490234375,
      "learning_rate": 3.309122603137711e-06,
      "logits/chosen": -0.9647778272628784,
      "logits/rejected": -1.016991138458252,
      "logps/chosen": -74.81549835205078,
      "logps/rejected": -75.46272277832031,
      "loss": 13.395,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02701759897172451,
      "rewards/margins": 0.05372295528650284,
      "rewards/rejected": -0.02670535072684288,
      "step": 1164
    },
    {
      "epoch": 0.6766567927048847,
      "grad_norm": 364.4992370605469,
      "learning_rate": 3.3076699593259735e-06,
      "logits/chosen": -0.892073929309845,
      "logits/rejected": -0.9101592302322388,
      "logps/chosen": -75.46597290039062,
      "logps/rejected": -70.5864028930664,
      "loss": 12.6961,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04906366392970085,
      "rewards/margins": 0.1294187605381012,
      "rewards/rejected": -0.08035509288311005,
      "step": 1165
    },
    {
      "epoch": 0.6772376139861764,
      "grad_norm": 331.6622619628906,
      "learning_rate": 3.3062173155142363e-06,
      "logits/chosen": -0.8724814653396606,
      "logits/rejected": -0.942725658416748,
      "logps/chosen": -84.60514831542969,
      "logps/rejected": -75.0161361694336,
      "loss": 14.3873,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12005509436130524,
      "rewards/margins": -0.03561541438102722,
      "rewards/rejected": -0.08443967998027802,
      "step": 1166
    },
    {
      "epoch": 0.6778184352674682,
      "grad_norm": 317.28472900390625,
      "learning_rate": 3.304764671702499e-06,
      "logits/chosen": -0.9968942403793335,
      "logits/rejected": -1.0448148250579834,
      "logps/chosen": -72.31526947021484,
      "logps/rejected": -73.2225341796875,
      "loss": 13.7419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02112642303109169,
      "rewards/margins": 0.05940958112478256,
      "rewards/rejected": -0.08053599298000336,
      "step": 1167
    },
    {
      "epoch": 0.67839925654876,
      "grad_norm": 287.7674865722656,
      "learning_rate": 3.3033120278907618e-06,
      "logits/chosen": -0.9627317190170288,
      "logits/rejected": -0.8998411893844604,
      "logps/chosen": -71.6623764038086,
      "logps/rejected": -77.93000793457031,
      "loss": 12.8459,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02301429584622383,
      "rewards/margins": 0.12027695029973984,
      "rewards/rejected": -0.0972626656293869,
      "step": 1168
    },
    {
      "epoch": 0.6789800778300517,
      "grad_norm": 302.75921630859375,
      "learning_rate": 3.301859384079024e-06,
      "logits/chosen": -1.0440073013305664,
      "logits/rejected": -1.0617105960845947,
      "logps/chosen": -67.90141296386719,
      "logps/rejected": -71.7067642211914,
      "loss": 13.3234,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0018256537150591612,
      "rewards/margins": 0.06842826306819916,
      "rewards/rejected": -0.0702539011836052,
      "step": 1169
    },
    {
      "epoch": 0.6795608991113434,
      "grad_norm": 300.3533020019531,
      "learning_rate": 3.300406740267287e-06,
      "logits/chosen": -0.8389264345169067,
      "logits/rejected": -0.9770253300666809,
      "logps/chosen": -72.3942642211914,
      "logps/rejected": -74.80130004882812,
      "loss": 13.5891,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.014074298553168774,
      "rewards/margins": 0.04593385383486748,
      "rewards/rejected": -0.03185955062508583,
      "step": 1170
    },
    {
      "epoch": 0.6801417203926352,
      "grad_norm": 333.66729736328125,
      "learning_rate": 3.2989540964555496e-06,
      "logits/chosen": -1.0627646446228027,
      "logits/rejected": -1.105365514755249,
      "logps/chosen": -68.1986312866211,
      "logps/rejected": -77.21229553222656,
      "loss": 13.6637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00904073566198349,
      "rewards/margins": 0.0343346893787384,
      "rewards/rejected": -0.025293949991464615,
      "step": 1171
    },
    {
      "epoch": 0.680722541673927,
      "grad_norm": 291.1269836425781,
      "learning_rate": 3.297501452643812e-06,
      "logits/chosen": -1.032504916191101,
      "logits/rejected": -0.9794846773147583,
      "logps/chosen": -71.5113525390625,
      "logps/rejected": -74.78794860839844,
      "loss": 12.5302,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.10109832137823105,
      "rewards/margins": 0.15338605642318726,
      "rewards/rejected": -0.052287738770246506,
      "step": 1172
    },
    {
      "epoch": 0.6813033629552186,
      "grad_norm": 316.3983459472656,
      "learning_rate": 3.2960488088320747e-06,
      "logits/chosen": -0.8987720608711243,
      "logits/rejected": -0.8584268689155579,
      "logps/chosen": -71.02616882324219,
      "logps/rejected": -73.02992248535156,
      "loss": 14.0606,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01105637475848198,
      "rewards/margins": -0.0024763301480561495,
      "rewards/rejected": -0.008580043911933899,
      "step": 1173
    },
    {
      "epoch": 0.6818841842365104,
      "grad_norm": 289.7396240234375,
      "learning_rate": 3.2945961650203374e-06,
      "logits/chosen": -0.8048896789550781,
      "logits/rejected": -0.8667701482772827,
      "logps/chosen": -69.55760192871094,
      "logps/rejected": -63.885833740234375,
      "loss": 14.2092,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.016201917082071304,
      "rewards/margins": -0.016854044049978256,
      "rewards/rejected": 0.0006521284813061357,
      "step": 1174
    },
    {
      "epoch": 0.6824650055178022,
      "grad_norm": 311.658203125,
      "learning_rate": 3.2931435212085997e-06,
      "logits/chosen": -0.9978891611099243,
      "logits/rejected": -1.0563108921051025,
      "logps/chosen": -73.31373596191406,
      "logps/rejected": -77.0311050415039,
      "loss": 13.9924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.006309698335826397,
      "rewards/margins": 0.0009064801270142198,
      "rewards/rejected": -0.0072161853313446045,
      "step": 1175
    },
    {
      "epoch": 0.6830458267990939,
      "grad_norm": 305.38397216796875,
      "learning_rate": 3.2916908773968625e-06,
      "logits/chosen": -0.918205738067627,
      "logits/rejected": -1.0026493072509766,
      "logps/chosen": -72.55452728271484,
      "logps/rejected": -78.04756164550781,
      "loss": 13.1554,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.00578195508569479,
      "rewards/margins": 0.0799231082201004,
      "rewards/rejected": -0.08570506423711777,
      "step": 1176
    },
    {
      "epoch": 0.6836266480803856,
      "grad_norm": 270.65960693359375,
      "learning_rate": 3.2902382335851252e-06,
      "logits/chosen": -0.9609074592590332,
      "logits/rejected": -1.0622050762176514,
      "logps/chosen": -67.67350769042969,
      "logps/rejected": -76.0232162475586,
      "loss": 12.2427,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.06995700299739838,
      "rewards/margins": 0.180569589138031,
      "rewards/rejected": -0.11061259359121323,
      "step": 1177
    },
    {
      "epoch": 0.6842074693616774,
      "grad_norm": 313.3789367675781,
      "learning_rate": 3.288785589773388e-06,
      "logits/chosen": -1.0121800899505615,
      "logits/rejected": -1.0351585149765015,
      "logps/chosen": -73.86485290527344,
      "logps/rejected": -77.4073257446289,
      "loss": 13.5823,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.025121968239545822,
      "rewards/margins": 0.044621676206588745,
      "rewards/rejected": -0.06974364817142487,
      "step": 1178
    },
    {
      "epoch": 0.6847882906429692,
      "grad_norm": 294.2719421386719,
      "learning_rate": 3.2873329459616503e-06,
      "logits/chosen": -1.0520410537719727,
      "logits/rejected": -0.960279643535614,
      "logps/chosen": -73.86731719970703,
      "logps/rejected": -69.24028015136719,
      "loss": 13.2329,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015290938317775726,
      "rewards/margins": 0.07485704869031906,
      "rewards/rejected": -0.059566110372543335,
      "step": 1179
    },
    {
      "epoch": 0.6853691119242609,
      "grad_norm": 330.83660888671875,
      "learning_rate": 3.285880302149913e-06,
      "logits/chosen": -1.0700443983078003,
      "logits/rejected": -1.0747114419937134,
      "logps/chosen": -74.0168685913086,
      "logps/rejected": -79.26966857910156,
      "loss": 14.3453,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.06104003265500069,
      "rewards/margins": -0.036572180688381195,
      "rewards/rejected": -0.024467846378684044,
      "step": 1180
    },
    {
      "epoch": 0.6859499332055526,
      "grad_norm": 311.9311828613281,
      "learning_rate": 3.284427658338176e-06,
      "logits/chosen": -1.025329828262329,
      "logits/rejected": -0.9475404024124146,
      "logps/chosen": -79.2386703491211,
      "logps/rejected": -76.4250259399414,
      "loss": 13.078,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05809807777404785,
      "rewards/margins": 0.09269105643033981,
      "rewards/rejected": -0.034592971205711365,
      "step": 1181
    },
    {
      "epoch": 0.6865307544868444,
      "grad_norm": 315.7636413574219,
      "learning_rate": 3.282975014526438e-06,
      "logits/chosen": -1.0630699396133423,
      "logits/rejected": -1.1585142612457275,
      "logps/chosen": -72.36962890625,
      "logps/rejected": -73.04064178466797,
      "loss": 13.5534,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07762381434440613,
      "rewards/margins": 0.08811168372631073,
      "rewards/rejected": -0.16573549807071686,
      "step": 1182
    },
    {
      "epoch": 0.6871115757681362,
      "grad_norm": 301.1933898925781,
      "learning_rate": 3.281522370714701e-06,
      "logits/chosen": -0.9292994737625122,
      "logits/rejected": -1.0096980333328247,
      "logps/chosen": -71.71006774902344,
      "logps/rejected": -75.26814270019531,
      "loss": 13.2198,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.030737508088350296,
      "rewards/margins": 0.07256682217121124,
      "rewards/rejected": -0.041829321533441544,
      "step": 1183
    },
    {
      "epoch": 0.6876923970494279,
      "grad_norm": 295.9613952636719,
      "learning_rate": 3.2800697269029636e-06,
      "logits/chosen": -0.8002195358276367,
      "logits/rejected": -0.9209533929824829,
      "logps/chosen": -74.96147155761719,
      "logps/rejected": -60.12278366088867,
      "loss": 14.2883,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.006243336014449596,
      "rewards/margins": -0.03084593452513218,
      "rewards/rejected": 0.0370892696082592,
      "step": 1184
    },
    {
      "epoch": 0.6882732183307196,
      "grad_norm": 383.6124572753906,
      "learning_rate": 3.2786170830912264e-06,
      "logits/chosen": -0.9279994964599609,
      "logits/rejected": -0.8782297372817993,
      "logps/chosen": -69.4768295288086,
      "logps/rejected": -77.15460968017578,
      "loss": 13.1519,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0005604095640592277,
      "rewards/margins": 0.08237127959728241,
      "rewards/rejected": -0.08181087672710419,
      "step": 1185
    },
    {
      "epoch": 0.6888540396120114,
      "grad_norm": 327.428466796875,
      "learning_rate": 3.2771644392794887e-06,
      "logits/chosen": -0.8622177243232727,
      "logits/rejected": -0.8570247888565063,
      "logps/chosen": -68.20109558105469,
      "logps/rejected": -68.4701156616211,
      "loss": 14.3007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03261176869273186,
      "rewards/margins": -0.027772387489676476,
      "rewards/rejected": -0.004839381668716669,
      "step": 1186
    },
    {
      "epoch": 0.6894348608933031,
      "grad_norm": 375.56878662109375,
      "learning_rate": 3.2757117954677514e-06,
      "logits/chosen": -1.067355990409851,
      "logits/rejected": -0.9439983367919922,
      "logps/chosen": -80.35142517089844,
      "logps/rejected": -73.33840942382812,
      "loss": 15.8666,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.1308525949716568,
      "rewards/margins": -0.13905833661556244,
      "rewards/rejected": 0.008205746300518513,
      "step": 1187
    },
    {
      "epoch": 0.6900156821745949,
      "grad_norm": 320.6004333496094,
      "learning_rate": 3.274259151656014e-06,
      "logits/chosen": -0.9834358096122742,
      "logits/rejected": -1.010769009590149,
      "logps/chosen": -71.81704711914062,
      "logps/rejected": -94.93458557128906,
      "loss": 13.6367,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.029428750276565552,
      "rewards/margins": 0.05378251150250435,
      "rewards/rejected": -0.0832112580537796,
      "step": 1188
    },
    {
      "epoch": 0.6905965034558866,
      "grad_norm": 313.9219055175781,
      "learning_rate": 3.2728065078442765e-06,
      "logits/chosen": -1.0372841358184814,
      "logits/rejected": -1.095456600189209,
      "logps/chosen": -70.77440643310547,
      "logps/rejected": -67.47232055664062,
      "loss": 14.279,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.033592257648706436,
      "rewards/margins": -0.023307878524065018,
      "rewards/rejected": -0.010284379124641418,
      "step": 1189
    },
    {
      "epoch": 0.6911773247371784,
      "grad_norm": 307.1705627441406,
      "learning_rate": 3.2713538640325393e-06,
      "logits/chosen": -0.8784843683242798,
      "logits/rejected": -0.8666518330574036,
      "logps/chosen": -67.27767181396484,
      "logps/rejected": -74.40681457519531,
      "loss": 13.8349,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.050198208540678024,
      "rewards/margins": 0.013867275789380074,
      "rewards/rejected": 0.0363309383392334,
      "step": 1190
    },
    {
      "epoch": 0.6917581460184701,
      "grad_norm": 325.02484130859375,
      "learning_rate": 3.269901220220802e-06,
      "logits/chosen": -0.8945049047470093,
      "logits/rejected": -0.8124086260795593,
      "logps/chosen": -77.03472137451172,
      "logps/rejected": -74.22068786621094,
      "loss": 13.7742,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.014175732620060444,
      "rewards/margins": 0.025671545416116714,
      "rewards/rejected": -0.039847277104854584,
      "step": 1191
    },
    {
      "epoch": 0.6923389672997619,
      "grad_norm": 314.4729309082031,
      "learning_rate": 3.2684485764090643e-06,
      "logits/chosen": -0.8569895029067993,
      "logits/rejected": -0.7807025909423828,
      "logps/chosen": -69.51123046875,
      "logps/rejected": -67.6419677734375,
      "loss": 13.8491,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.026939209550619125,
      "rewards/margins": 0.013091212138533592,
      "rewards/rejected": 0.013847997412085533,
      "step": 1192
    },
    {
      "epoch": 0.6929197885810536,
      "grad_norm": 299.4042663574219,
      "learning_rate": 3.266995932597327e-06,
      "logits/chosen": -0.8547220230102539,
      "logits/rejected": -0.9094983339309692,
      "logps/chosen": -71.99398040771484,
      "logps/rejected": -73.09522247314453,
      "loss": 13.2652,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.007436198648065329,
      "rewards/margins": 0.07868284732103348,
      "rewards/rejected": -0.08611904829740524,
      "step": 1193
    },
    {
      "epoch": 0.6935006098623454,
      "grad_norm": 295.1394348144531,
      "learning_rate": 3.26554328878559e-06,
      "logits/chosen": -0.9191268086433411,
      "logits/rejected": -0.9057947993278503,
      "logps/chosen": -71.15728759765625,
      "logps/rejected": -73.70671081542969,
      "loss": 12.9333,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05283800885081291,
      "rewards/margins": 0.10996121168136597,
      "rewards/rejected": -0.05712319537997246,
      "step": 1194
    },
    {
      "epoch": 0.6940814311436371,
      "grad_norm": 313.464111328125,
      "learning_rate": 3.2640906449738526e-06,
      "logits/chosen": -0.9578115344047546,
      "logits/rejected": -1.0164742469787598,
      "logps/chosen": -75.11009216308594,
      "logps/rejected": -75.68206024169922,
      "loss": 12.2113,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.08975354582071304,
      "rewards/margins": 0.1811046302318573,
      "rewards/rejected": -0.09135107696056366,
      "step": 1195
    },
    {
      "epoch": 0.6946622524249288,
      "grad_norm": 296.2933349609375,
      "learning_rate": 3.262638001162115e-06,
      "logits/chosen": -0.9920350313186646,
      "logits/rejected": -1.0618032217025757,
      "logps/chosen": -72.83654022216797,
      "logps/rejected": -74.27031707763672,
      "loss": 13.2457,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01587744429707527,
      "rewards/margins": 0.0722484141588211,
      "rewards/rejected": -0.056370966136455536,
      "step": 1196
    },
    {
      "epoch": 0.6952430737062206,
      "grad_norm": 333.3546142578125,
      "learning_rate": 3.261185357350378e-06,
      "logits/chosen": -1.0486847162246704,
      "logits/rejected": -1.1227794885635376,
      "logps/chosen": -75.2326431274414,
      "logps/rejected": -71.94784545898438,
      "loss": 14.2714,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03368932008743286,
      "rewards/margins": -0.023625794798135757,
      "rewards/rejected": -0.010063527151942253,
      "step": 1197
    },
    {
      "epoch": 0.6958238949875123,
      "grad_norm": 304.2666320800781,
      "learning_rate": 3.259732713538641e-06,
      "logits/chosen": -0.9180057644844055,
      "logits/rejected": -0.9214650392532349,
      "logps/chosen": -71.56143951416016,
      "logps/rejected": -73.12309265136719,
      "loss": 12.8734,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03262796252965927,
      "rewards/margins": 0.12831327319145203,
      "rewards/rejected": -0.09568531811237335,
      "step": 1198
    },
    {
      "epoch": 0.6964047162688041,
      "grad_norm": 334.5450134277344,
      "learning_rate": 3.2582800697269036e-06,
      "logits/chosen": -0.9351629018783569,
      "logits/rejected": -0.8389555215835571,
      "logps/chosen": -74.37076568603516,
      "logps/rejected": -73.4453353881836,
      "loss": 14.321,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.026485230773687363,
      "rewards/margins": -0.0337425172328949,
      "rewards/rejected": 0.007257281336933374,
      "step": 1199
    },
    {
      "epoch": 0.6969855375500958,
      "grad_norm": 342.3309631347656,
      "learning_rate": 3.256827425915166e-06,
      "logits/chosen": -1.0121877193450928,
      "logits/rejected": -1.0378234386444092,
      "logps/chosen": -76.34333038330078,
      "logps/rejected": -77.09635925292969,
      "loss": 14.5164,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05535711720585823,
      "rewards/margins": -0.05440009757876396,
      "rewards/rejected": -0.0009570121765136719,
      "step": 1200
    },
    {
      "epoch": 0.6975663588313876,
      "grad_norm": 299.0294189453125,
      "learning_rate": 3.2553747821034286e-06,
      "logits/chosen": -0.8395780324935913,
      "logits/rejected": -0.8294085264205933,
      "logps/chosen": -73.91521453857422,
      "logps/rejected": -67.51762390136719,
      "loss": 14.2048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012477479875087738,
      "rewards/margins": -0.015345439314842224,
      "rewards/rejected": 0.0028679564129561186,
      "step": 1201
    },
    {
      "epoch": 0.6981471801126793,
      "grad_norm": 293.46697998046875,
      "learning_rate": 3.2539221382916914e-06,
      "logits/chosen": -0.8202503323554993,
      "logits/rejected": -0.7761942744255066,
      "logps/chosen": -68.01689910888672,
      "logps/rejected": -80.66238403320312,
      "loss": 13.3601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004020270891487598,
      "rewards/margins": 0.07202225923538208,
      "rewards/rejected": -0.07604251801967621,
      "step": 1202
    },
    {
      "epoch": 0.6987280013939711,
      "grad_norm": 301.7940673828125,
      "learning_rate": 3.252469494479954e-06,
      "logits/chosen": -0.9627870321273804,
      "logits/rejected": -1.0708242654800415,
      "logps/chosen": -75.9755630493164,
      "logps/rejected": -77.28931427001953,
      "loss": 13.2624,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04609397426247597,
      "rewards/margins": 0.08046616613864899,
      "rewards/rejected": -0.03437219560146332,
      "step": 1203
    },
    {
      "epoch": 0.6993088226752628,
      "grad_norm": 435.3644714355469,
      "learning_rate": 3.2510168506682165e-06,
      "logits/chosen": -0.855689525604248,
      "logits/rejected": -0.9411078691482544,
      "logps/chosen": -81.69770812988281,
      "logps/rejected": -71.78427124023438,
      "loss": 15.3069,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.11791998147964478,
      "rewards/margins": -0.12011977285146713,
      "rewards/rejected": 0.002199783455580473,
      "step": 1204
    },
    {
      "epoch": 0.6998896439565546,
      "grad_norm": 294.9209899902344,
      "learning_rate": 3.2495642068564792e-06,
      "logits/chosen": -1.0104598999023438,
      "logits/rejected": -1.0663111209869385,
      "logps/chosen": -75.40419006347656,
      "logps/rejected": -65.79673767089844,
      "loss": 14.2376,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.01617666520178318,
      "rewards/margins": -0.017444659024477005,
      "rewards/rejected": 0.0012679934734478593,
      "step": 1205
    },
    {
      "epoch": 0.7004704652378463,
      "grad_norm": 291.9886169433594,
      "learning_rate": 3.248111563044742e-06,
      "logits/chosen": -0.8618041276931763,
      "logits/rejected": -0.8567333221435547,
      "logps/chosen": -68.65798950195312,
      "logps/rejected": -77.28287506103516,
      "loss": 13.5089,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02400004304945469,
      "rewards/margins": 0.04948422685265541,
      "rewards/rejected": -0.07348427176475525,
      "step": 1206
    },
    {
      "epoch": 0.7010512865191381,
      "grad_norm": 290.79443359375,
      "learning_rate": 3.2466589192330043e-06,
      "logits/chosen": -0.8943878412246704,
      "logits/rejected": -0.9081516265869141,
      "logps/chosen": -65.51563262939453,
      "logps/rejected": -69.14381408691406,
      "loss": 13.6643,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04395107552409172,
      "rewards/margins": 0.03300014138221741,
      "rewards/rejected": -0.07695120573043823,
      "step": 1207
    },
    {
      "epoch": 0.7016321078004298,
      "grad_norm": 324.72216796875,
      "learning_rate": 3.245206275421267e-06,
      "logits/chosen": -0.8826707005500793,
      "logits/rejected": -0.8771419525146484,
      "logps/chosen": -70.04827880859375,
      "logps/rejected": -77.23084259033203,
      "loss": 14.3103,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024362996220588684,
      "rewards/margins": -0.024887358769774437,
      "rewards/rejected": 0.04925035685300827,
      "step": 1208
    },
    {
      "epoch": 0.7022129290817215,
      "grad_norm": 304.3308410644531,
      "learning_rate": 3.2437536316095298e-06,
      "logits/chosen": -0.9273678660392761,
      "logits/rejected": -0.9473909139633179,
      "logps/chosen": -74.80255126953125,
      "logps/rejected": -74.05912780761719,
      "loss": 13.0948,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.018330611288547516,
      "rewards/margins": 0.09634025394916534,
      "rewards/rejected": -0.07800963521003723,
      "step": 1209
    },
    {
      "epoch": 0.7027937503630133,
      "grad_norm": 307.7835693359375,
      "learning_rate": 3.2423009877977925e-06,
      "logits/chosen": -0.8647984266281128,
      "logits/rejected": -0.8359298706054688,
      "logps/chosen": -66.73087310791016,
      "logps/rejected": -72.38240051269531,
      "loss": 13.4688,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06543731689453125,
      "rewards/margins": 0.051355160772800446,
      "rewards/rejected": 0.014082154259085655,
      "step": 1210
    },
    {
      "epoch": 0.7033745716443051,
      "grad_norm": 326.6808166503906,
      "learning_rate": 3.240848343986055e-06,
      "logits/chosen": -0.840550422668457,
      "logits/rejected": -0.9085513949394226,
      "logps/chosen": -71.64811706542969,
      "logps/rejected": -69.97435760498047,
      "loss": 14.124,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02114766277372837,
      "rewards/margins": -0.009716121479868889,
      "rewards/rejected": -0.011431543156504631,
      "step": 1211
    },
    {
      "epoch": 0.7039553929255968,
      "grad_norm": 322.2374267578125,
      "learning_rate": 3.2393957001743176e-06,
      "logits/chosen": -0.9925252199172974,
      "logits/rejected": -1.0222442150115967,
      "logps/chosen": -75.80982971191406,
      "logps/rejected": -64.89044189453125,
      "loss": 13.8639,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02768515609204769,
      "rewards/margins": 0.02285769209265709,
      "rewards/rejected": -0.05054285004734993,
      "step": 1212
    },
    {
      "epoch": 0.7045362142068885,
      "grad_norm": 309.2051086425781,
      "learning_rate": 3.2379430563625804e-06,
      "logits/chosen": -0.8814622163772583,
      "logits/rejected": -0.8693148493766785,
      "logps/chosen": -78.34906768798828,
      "logps/rejected": -79.20980834960938,
      "loss": 12.9492,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02256811037659645,
      "rewards/margins": 0.11201903969049454,
      "rewards/rejected": -0.08945093303918839,
      "step": 1213
    },
    {
      "epoch": 0.7051170354881803,
      "grad_norm": 306.2784423828125,
      "learning_rate": 3.2364904125508427e-06,
      "logits/chosen": -0.9752852320671082,
      "logits/rejected": -0.8968347311019897,
      "logps/chosen": -69.79313659667969,
      "logps/rejected": -75.76102447509766,
      "loss": 13.2644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0007508553680963814,
      "rewards/margins": 0.08302091062068939,
      "rewards/rejected": -0.08227004110813141,
      "step": 1214
    },
    {
      "epoch": 0.705697856769472,
      "grad_norm": 317.89794921875,
      "learning_rate": 3.2350377687391054e-06,
      "logits/chosen": -0.7320946455001831,
      "logits/rejected": -0.7363893985748291,
      "logps/chosen": -71.71336364746094,
      "logps/rejected": -73.88663482666016,
      "loss": 14.9091,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.03206131234765053,
      "rewards/margins": -0.07985241711139679,
      "rewards/rejected": 0.04779110103845596,
      "step": 1215
    },
    {
      "epoch": 0.7062786780507638,
      "grad_norm": 308.9206848144531,
      "learning_rate": 3.233585124927368e-06,
      "logits/chosen": -0.8405545949935913,
      "logits/rejected": -0.8385445475578308,
      "logps/chosen": -68.86060333251953,
      "logps/rejected": -80.04986572265625,
      "loss": 13.7195,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017079973593354225,
      "rewards/margins": 0.021983934566378593,
      "rewards/rejected": -0.03906390443444252,
      "step": 1216
    },
    {
      "epoch": 0.7068594993320555,
      "grad_norm": 316.7115478515625,
      "learning_rate": 3.232132481115631e-06,
      "logits/chosen": -0.9487533569335938,
      "logits/rejected": -0.9738863110542297,
      "logps/chosen": -68.33602142333984,
      "logps/rejected": -74.55831146240234,
      "loss": 13.2224,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0484335832297802,
      "rewards/margins": 0.07291674613952637,
      "rewards/rejected": -0.02448316477239132,
      "step": 1217
    },
    {
      "epoch": 0.7074403206133473,
      "grad_norm": 316.3020324707031,
      "learning_rate": 3.2306798373038933e-06,
      "logits/chosen": -1.0268185138702393,
      "logits/rejected": -1.0504379272460938,
      "logps/chosen": -81.70768737792969,
      "logps/rejected": -87.26634216308594,
      "loss": 13.3512,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0006445225444622338,
      "rewards/margins": 0.07782299816608429,
      "rewards/rejected": -0.07846752554178238,
      "step": 1218
    },
    {
      "epoch": 0.708021141894639,
      "grad_norm": 339.1500549316406,
      "learning_rate": 3.229227193492156e-06,
      "logits/chosen": -0.8876482248306274,
      "logits/rejected": -0.8302451968193054,
      "logps/chosen": -85.76036834716797,
      "logps/rejected": -70.45575714111328,
      "loss": 13.7043,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05848958343267441,
      "rewards/margins": 0.024620074778795242,
      "rewards/rejected": -0.08310966193675995,
      "step": 1219
    },
    {
      "epoch": 0.7086019631759307,
      "grad_norm": 326.1057434082031,
      "learning_rate": 3.2277745496804187e-06,
      "logits/chosen": -1.015522837638855,
      "logits/rejected": -1.013179898262024,
      "logps/chosen": -70.21974182128906,
      "logps/rejected": -80.85169219970703,
      "loss": 13.5872,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.001294175162911415,
      "rewards/margins": 0.03975391015410423,
      "rewards/rejected": -0.0410480871796608,
      "step": 1220
    },
    {
      "epoch": 0.7091827844572225,
      "grad_norm": 3547.16796875,
      "learning_rate": 3.226321905868681e-06,
      "logits/chosen": -0.9397285580635071,
      "logits/rejected": -0.928897500038147,
      "logps/chosen": -74.49229431152344,
      "logps/rejected": -66.12333679199219,
      "loss": 13.9887,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.028601933270692825,
      "rewards/margins": -0.0020258903969079256,
      "rewards/rejected": -0.02657604217529297,
      "step": 1221
    },
    {
      "epoch": 0.7097636057385143,
      "grad_norm": 282.8453063964844,
      "learning_rate": 3.224869262056944e-06,
      "logits/chosen": -0.996697723865509,
      "logits/rejected": -1.0278717279434204,
      "logps/chosen": -68.15303039550781,
      "logps/rejected": -71.10795593261719,
      "loss": 12.931,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.021135643124580383,
      "rewards/margins": 0.11395503580570221,
      "rewards/rejected": -0.1350906789302826,
      "step": 1222
    },
    {
      "epoch": 0.710344427019806,
      "grad_norm": 310.6962585449219,
      "learning_rate": 3.2234166182452066e-06,
      "logits/chosen": -1.0512793064117432,
      "logits/rejected": -1.0435963869094849,
      "logps/chosen": -74.12106323242188,
      "logps/rejected": -74.80326843261719,
      "loss": 13.3912,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03831661492586136,
      "rewards/margins": 0.06471587717533112,
      "rewards/rejected": -0.026399265974760056,
      "step": 1223
    },
    {
      "epoch": 0.7109252483010977,
      "grad_norm": 320.367431640625,
      "learning_rate": 3.221963974433469e-06,
      "logits/chosen": -0.8990233540534973,
      "logits/rejected": -0.8427421450614929,
      "logps/chosen": -72.86744689941406,
      "logps/rejected": -73.25343322753906,
      "loss": 13.8253,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03275548666715622,
      "rewards/margins": 0.021472405642271042,
      "rewards/rejected": 0.011283071711659431,
      "step": 1224
    },
    {
      "epoch": 0.7115060695823895,
      "grad_norm": 305.998291015625,
      "learning_rate": 3.2205113306217316e-06,
      "logits/chosen": -1.018647313117981,
      "logits/rejected": -1.0316414833068848,
      "logps/chosen": -74.46012878417969,
      "logps/rejected": -75.38187408447266,
      "loss": 12.8475,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06711751222610474,
      "rewards/margins": 0.1182142049074173,
      "rewards/rejected": -0.05109669640660286,
      "step": 1225
    },
    {
      "epoch": 0.7120868908636813,
      "grad_norm": 318.189697265625,
      "learning_rate": 3.2190586868099944e-06,
      "logits/chosen": -0.899054229259491,
      "logits/rejected": -0.9100243449211121,
      "logps/chosen": -72.66035461425781,
      "logps/rejected": -70.22972869873047,
      "loss": 14.2404,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04246487095952034,
      "rewards/margins": -0.023234158754348755,
      "rewards/rejected": -0.019230708479881287,
      "step": 1226
    },
    {
      "epoch": 0.712667712144973,
      "grad_norm": 312.01116943359375,
      "learning_rate": 3.217606042998257e-06,
      "logits/chosen": -0.9111840128898621,
      "logits/rejected": -0.926331639289856,
      "logps/chosen": -73.4972152709961,
      "logps/rejected": -79.52141571044922,
      "loss": 13.7016,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03973189368844032,
      "rewards/margins": 0.027790825814008713,
      "rewards/rejected": 0.011941070668399334,
      "step": 1227
    },
    {
      "epoch": 0.7132485334262647,
      "grad_norm": 315.48468017578125,
      "learning_rate": 3.2161533991865195e-06,
      "logits/chosen": -0.9904237985610962,
      "logits/rejected": -0.9717411994934082,
      "logps/chosen": -75.15087890625,
      "logps/rejected": -80.14024353027344,
      "loss": 13.5057,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04499629884958267,
      "rewards/margins": 0.05106980353593826,
      "rewards/rejected": -0.006073503755033016,
      "step": 1228
    },
    {
      "epoch": 0.7138293547075565,
      "grad_norm": 300.9493713378906,
      "learning_rate": 3.2147007553747822e-06,
      "logits/chosen": -1.084952712059021,
      "logits/rejected": -1.0708225965499878,
      "logps/chosen": -73.69815063476562,
      "logps/rejected": -71.60685729980469,
      "loss": 12.7981,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.020317967981100082,
      "rewards/margins": 0.12214857339859009,
      "rewards/rejected": -0.14246656000614166,
      "step": 1229
    },
    {
      "epoch": 0.7144101759888483,
      "grad_norm": 287.88763427734375,
      "learning_rate": 3.213248111563045e-06,
      "logits/chosen": -1.0959196090698242,
      "logits/rejected": -1.0423675775527954,
      "logps/chosen": -77.42252349853516,
      "logps/rejected": -70.7403564453125,
      "loss": 13.1465,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.053702641278505325,
      "rewards/margins": 0.08882278949022293,
      "rewards/rejected": -0.035120148211717606,
      "step": 1230
    },
    {
      "epoch": 0.7149909972701399,
      "grad_norm": 317.7434387207031,
      "learning_rate": 3.2117954677513073e-06,
      "logits/chosen": -1.050706386566162,
      "logits/rejected": -0.9100669622421265,
      "logps/chosen": -77.49993896484375,
      "logps/rejected": -68.96791076660156,
      "loss": 14.098,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01931348443031311,
      "rewards/margins": 0.0012981354957446456,
      "rewards/rejected": 0.01801534928381443,
      "step": 1231
    },
    {
      "epoch": 0.7155718185514317,
      "grad_norm": 305.69171142578125,
      "learning_rate": 3.21034282393957e-06,
      "logits/chosen": -0.9822510480880737,
      "logits/rejected": -0.8831478357315063,
      "logps/chosen": -71.09330749511719,
      "logps/rejected": -67.7369384765625,
      "loss": 13.9734,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04790610074996948,
      "rewards/margins": -0.0017275720601901412,
      "rewards/rejected": -0.046178530901670456,
      "step": 1232
    },
    {
      "epoch": 0.7161526398327235,
      "grad_norm": 303.33258056640625,
      "learning_rate": 3.2088901801278328e-06,
      "logits/chosen": -0.9934064149856567,
      "logits/rejected": -0.9465386271476746,
      "logps/chosen": -68.63888549804688,
      "logps/rejected": -78.98455047607422,
      "loss": 13.0203,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.041070565581321716,
      "rewards/margins": 0.09636726975440979,
      "rewards/rejected": -0.05529669672250748,
      "step": 1233
    },
    {
      "epoch": 0.7167334611140153,
      "grad_norm": 350.7889099121094,
      "learning_rate": 3.2074375363160955e-06,
      "logits/chosen": -1.109919548034668,
      "logits/rejected": -1.0614010095596313,
      "logps/chosen": -80.05448913574219,
      "logps/rejected": -79.92615509033203,
      "loss": 14.5732,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.07071670144796371,
      "rewards/margins": -0.05340186506509781,
      "rewards/rejected": -0.017314845696091652,
      "step": 1234
    },
    {
      "epoch": 0.7173142823953069,
      "grad_norm": 334.0653991699219,
      "learning_rate": 3.205984892504358e-06,
      "logits/chosen": -0.8801966905593872,
      "logits/rejected": -0.8142051696777344,
      "logps/chosen": -73.92015075683594,
      "logps/rejected": -67.52606964111328,
      "loss": 14.604,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.005336902104318142,
      "rewards/margins": -0.05028069019317627,
      "rewards/rejected": 0.05561758950352669,
      "step": 1235
    },
    {
      "epoch": 0.7178951036765987,
      "grad_norm": 319.0472717285156,
      "learning_rate": 3.2045322486926206e-06,
      "logits/chosen": -0.7527344822883606,
      "logits/rejected": -0.8575745820999146,
      "logps/chosen": -70.49696350097656,
      "logps/rejected": -77.09193420410156,
      "loss": 13.7637,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0880822017788887,
      "rewards/margins": 0.02064911648631096,
      "rewards/rejected": 0.06743309646844864,
      "step": 1236
    },
    {
      "epoch": 0.7184759249578905,
      "grad_norm": 314.6608581542969,
      "learning_rate": 3.2030796048808834e-06,
      "logits/chosen": -1.0314970016479492,
      "logits/rejected": -1.0729162693023682,
      "logps/chosen": -75.82598114013672,
      "logps/rejected": -78.76053619384766,
      "loss": 13.0635,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06382361799478531,
      "rewards/margins": 0.10007448494434357,
      "rewards/rejected": -0.03625086694955826,
      "step": 1237
    },
    {
      "epoch": 0.7190567462391823,
      "grad_norm": 314.0679626464844,
      "learning_rate": 3.2016269610691457e-06,
      "logits/chosen": -0.7936090230941772,
      "logits/rejected": -0.8422238230705261,
      "logps/chosen": -75.70396423339844,
      "logps/rejected": -72.00861358642578,
      "loss": 13.9709,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02658630535006523,
      "rewards/margins": -0.0011108957696706057,
      "rewards/rejected": 0.02769719436764717,
      "step": 1238
    },
    {
      "epoch": 0.7196375675204739,
      "grad_norm": 335.3764953613281,
      "learning_rate": 3.200174317257409e-06,
      "logits/chosen": -0.9401591420173645,
      "logits/rejected": -1.030762791633606,
      "logps/chosen": -70.6431655883789,
      "logps/rejected": -71.48957061767578,
      "loss": 13.6414,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.040466949343681335,
      "rewards/margins": 0.043619535863399506,
      "rewards/rejected": -0.08408647030591965,
      "step": 1239
    },
    {
      "epoch": 0.7202183888017657,
      "grad_norm": 322.8601379394531,
      "learning_rate": 3.1987216734456716e-06,
      "logits/chosen": -1.184298038482666,
      "logits/rejected": -1.2170809507369995,
      "logps/chosen": -82.28440856933594,
      "logps/rejected": -73.7635726928711,
      "loss": 14.3871,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03017536923289299,
      "rewards/margins": -0.036898013204336166,
      "rewards/rejected": 0.006722640246152878,
      "step": 1240
    },
    {
      "epoch": 0.7207992100830575,
      "grad_norm": 347.85748291015625,
      "learning_rate": 3.1972690296339343e-06,
      "logits/chosen": -1.074629783630371,
      "logits/rejected": -1.0130137205123901,
      "logps/chosen": -74.544677734375,
      "logps/rejected": -81.3597640991211,
      "loss": 13.9095,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07186390459537506,
      "rewards/margins": 0.011945419013500214,
      "rewards/rejected": -0.08380932360887527,
      "step": 1241
    },
    {
      "epoch": 0.7213800313643491,
      "grad_norm": 309.85186767578125,
      "learning_rate": 3.195816385822197e-06,
      "logits/chosen": -0.8857796788215637,
      "logits/rejected": -0.9163784980773926,
      "logps/chosen": -73.91609191894531,
      "logps/rejected": -74.36446380615234,
      "loss": 13.286,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015781346708536148,
      "rewards/margins": 0.07729706913232803,
      "rewards/rejected": -0.06151571124792099,
      "step": 1242
    },
    {
      "epoch": 0.7219608526456409,
      "grad_norm": 375.2605895996094,
      "learning_rate": 3.1943637420104594e-06,
      "logits/chosen": -1.1714117527008057,
      "logits/rejected": -1.1711294651031494,
      "logps/chosen": -88.12577819824219,
      "logps/rejected": -75.9127197265625,
      "loss": 15.1098,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.051274098455905914,
      "rewards/margins": -0.10958840698003769,
      "rewards/rejected": 0.058314334601163864,
      "step": 1243
    },
    {
      "epoch": 0.7225416739269327,
      "grad_norm": 329.9234619140625,
      "learning_rate": 3.192911098198722e-06,
      "logits/chosen": -0.8440088033676147,
      "logits/rejected": -0.8402212262153625,
      "logps/chosen": -74.02096557617188,
      "logps/rejected": -78.38151550292969,
      "loss": 13.8329,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03374292701482773,
      "rewards/margins": 0.01769348978996277,
      "rewards/rejected": 0.01604943349957466,
      "step": 1244
    },
    {
      "epoch": 0.7231224952082245,
      "grad_norm": 307.36962890625,
      "learning_rate": 3.191458454386985e-06,
      "logits/chosen": -0.9148715734481812,
      "logits/rejected": -0.8585114479064941,
      "logps/chosen": -76.9505844116211,
      "logps/rejected": -70.01712036132812,
      "loss": 13.1411,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.043477851897478104,
      "rewards/margins": 0.08735030889511108,
      "rewards/rejected": -0.04387245327234268,
      "step": 1245
    },
    {
      "epoch": 0.7237033164895161,
      "grad_norm": 303.0053405761719,
      "learning_rate": 3.1900058105752472e-06,
      "logits/chosen": -0.981308102607727,
      "logits/rejected": -1.0617504119873047,
      "logps/chosen": -75.29297637939453,
      "logps/rejected": -74.94789123535156,
      "loss": 13.1593,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.011156311258673668,
      "rewards/margins": 0.11750247329473495,
      "rewards/rejected": -0.10634617507457733,
      "step": 1246
    },
    {
      "epoch": 0.7242841377708079,
      "grad_norm": 305.6419982910156,
      "learning_rate": 3.18855316676351e-06,
      "logits/chosen": -0.9547861814498901,
      "logits/rejected": -0.9406298398971558,
      "logps/chosen": -74.48167419433594,
      "logps/rejected": -80.66239166259766,
      "loss": 13.3056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005810488946735859,
      "rewards/margins": 0.07631019502878189,
      "rewards/rejected": -0.07049970328807831,
      "step": 1247
    },
    {
      "epoch": 0.7248649590520997,
      "grad_norm": 322.7685241699219,
      "learning_rate": 3.1871005229517727e-06,
      "logits/chosen": -0.8207041621208191,
      "logits/rejected": -0.9001065492630005,
      "logps/chosen": -75.53369140625,
      "logps/rejected": -78.82037353515625,
      "loss": 13.5941,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.022084522992372513,
      "rewards/margins": 0.040417276322841644,
      "rewards/rejected": -0.06250180304050446,
      "step": 1248
    },
    {
      "epoch": 0.7254457803333915,
      "grad_norm": 296.7803649902344,
      "learning_rate": 3.185647879140035e-06,
      "logits/chosen": -0.892406165599823,
      "logits/rejected": -0.8686800003051758,
      "logps/chosen": -67.49946594238281,
      "logps/rejected": -72.9912338256836,
      "loss": 13.7522,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03343478590250015,
      "rewards/margins": 0.025259148329496384,
      "rewards/rejected": -0.058693934231996536,
      "step": 1249
    },
    {
      "epoch": 0.7260266016146831,
      "grad_norm": 302.85687255859375,
      "learning_rate": 3.184195235328298e-06,
      "logits/chosen": -0.9714136123657227,
      "logits/rejected": -1.0543344020843506,
      "logps/chosen": -69.90233612060547,
      "logps/rejected": -66.53976440429688,
      "loss": 13.0513,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007732037454843521,
      "rewards/margins": 0.09551627188920975,
      "rewards/rejected": -0.08778424561023712,
      "step": 1250
    },
    {
      "epoch": 0.7266074228959749,
      "grad_norm": 386.9495544433594,
      "learning_rate": 3.1827425915165606e-06,
      "logits/chosen": -0.9501975774765015,
      "logits/rejected": -0.8718827962875366,
      "logps/chosen": -71.18165588378906,
      "logps/rejected": -73.68255615234375,
      "loss": 13.3055,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.00821919646114111,
      "rewards/margins": 0.06419769674539566,
      "rewards/rejected": -0.05597849562764168,
      "step": 1251
    },
    {
      "epoch": 0.7271882441772667,
      "grad_norm": 349.3519592285156,
      "learning_rate": 3.1812899477048233e-06,
      "logits/chosen": -1.033111333847046,
      "logits/rejected": -0.9466876983642578,
      "logps/chosen": -78.17009735107422,
      "logps/rejected": -76.4998550415039,
      "loss": 14.2267,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02998080477118492,
      "rewards/margins": -0.023897850885987282,
      "rewards/rejected": -0.006082954350858927,
      "step": 1252
    },
    {
      "epoch": 0.7277690654585584,
      "grad_norm": 315.51739501953125,
      "learning_rate": 3.1798373038930856e-06,
      "logits/chosen": -0.7700793147087097,
      "logits/rejected": -0.7328799962997437,
      "logps/chosen": -70.72773742675781,
      "logps/rejected": -78.58648681640625,
      "loss": 13.1893,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006938972510397434,
      "rewards/margins": 0.08656276762485504,
      "rewards/rejected": -0.0935017392039299,
      "step": 1253
    },
    {
      "epoch": 0.7283498867398501,
      "grad_norm": 343.72174072265625,
      "learning_rate": 3.1783846600813484e-06,
      "logits/chosen": -1.0412442684173584,
      "logits/rejected": -1.0394227504730225,
      "logps/chosen": -73.90029907226562,
      "logps/rejected": -79.7289810180664,
      "loss": 13.9338,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.037149734795093536,
      "rewards/margins": 0.005179956555366516,
      "rewards/rejected": -0.04232969135046005,
      "step": 1254
    },
    {
      "epoch": 0.7289307080211419,
      "grad_norm": 295.3687438964844,
      "learning_rate": 3.176932016269611e-06,
      "logits/chosen": -0.7562029957771301,
      "logits/rejected": -0.8874691724777222,
      "logps/chosen": -65.7853012084961,
      "logps/rejected": -70.8729248046875,
      "loss": 12.6619,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.12159298360347748,
      "rewards/margins": 0.1388033777475357,
      "rewards/rejected": -0.01721039041876793,
      "step": 1255
    },
    {
      "epoch": 0.7295115293024337,
      "grad_norm": 323.8238220214844,
      "learning_rate": 3.1754793724578735e-06,
      "logits/chosen": -1.0097589492797852,
      "logits/rejected": -0.9334095120429993,
      "logps/chosen": -72.65444946289062,
      "logps/rejected": -76.5811538696289,
      "loss": 14.3614,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02453356608748436,
      "rewards/margins": -0.04125526174902916,
      "rewards/rejected": 0.01672169752418995,
      "step": 1256
    },
    {
      "epoch": 0.7300923505837253,
      "grad_norm": 350.0208740234375,
      "learning_rate": 3.174026728646136e-06,
      "logits/chosen": -0.7854102253913879,
      "logits/rejected": -0.7936570048332214,
      "logps/chosen": -71.28059387207031,
      "logps/rejected": -73.92378997802734,
      "loss": 13.9144,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.020718097686767578,
      "rewards/margins": 0.01618293486535549,
      "rewards/rejected": 0.004535162355750799,
      "step": 1257
    },
    {
      "epoch": 0.7306731718650171,
      "grad_norm": 294.0679931640625,
      "learning_rate": 3.172574084834399e-06,
      "logits/chosen": -0.9716030955314636,
      "logits/rejected": -0.9176068305969238,
      "logps/chosen": -65.84822082519531,
      "logps/rejected": -62.31769943237305,
      "loss": 13.7181,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0477416105568409,
      "rewards/margins": 0.02603049948811531,
      "rewards/rejected": -0.07377209514379501,
      "step": 1258
    },
    {
      "epoch": 0.7312539931463089,
      "grad_norm": 320.6388854980469,
      "learning_rate": 3.1711214410226617e-06,
      "logits/chosen": -1.0247890949249268,
      "logits/rejected": -0.8913010358810425,
      "logps/chosen": -67.58940124511719,
      "logps/rejected": -71.38215637207031,
      "loss": 13.7051,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0041189719922840595,
      "rewards/margins": 0.029431123286485672,
      "rewards/rejected": -0.0253121517598629,
      "step": 1259
    },
    {
      "epoch": 0.7318348144276007,
      "grad_norm": 333.4161071777344,
      "learning_rate": 3.169668797210924e-06,
      "logits/chosen": -0.9546328783035278,
      "logits/rejected": -1.0250972509384155,
      "logps/chosen": -67.36685943603516,
      "logps/rejected": -69.79907989501953,
      "loss": 15.0047,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09877635538578033,
      "rewards/margins": -0.0952109694480896,
      "rewards/rejected": -0.003565377090126276,
      "step": 1260
    },
    {
      "epoch": 0.7324156357088923,
      "grad_norm": 313.89764404296875,
      "learning_rate": 3.1682161533991868e-06,
      "logits/chosen": -1.0455704927444458,
      "logits/rejected": -1.1571590900421143,
      "logps/chosen": -72.12377166748047,
      "logps/rejected": -83.68880462646484,
      "loss": 13.8536,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04903135448694229,
      "rewards/margins": 0.029356136918067932,
      "rewards/rejected": -0.07838748395442963,
      "step": 1261
    },
    {
      "epoch": 0.7329964569901841,
      "grad_norm": 473.7555847167969,
      "learning_rate": 3.1667635095874495e-06,
      "logits/chosen": -0.9349552989006042,
      "logits/rejected": -0.9804345965385437,
      "logps/chosen": -79.89446258544922,
      "logps/rejected": -78.34078216552734,
      "loss": 13.4535,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006529764737933874,
      "rewards/margins": 0.0674697533249855,
      "rewards/rejected": -0.07399950921535492,
      "step": 1262
    },
    {
      "epoch": 0.7335772782714759,
      "grad_norm": 323.3653869628906,
      "learning_rate": 3.165310865775712e-06,
      "logits/chosen": -0.82118159532547,
      "logits/rejected": -0.908584713935852,
      "logps/chosen": -78.5911865234375,
      "logps/rejected": -70.24919128417969,
      "loss": 13.8618,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.008846777491271496,
      "rewards/margins": 0.015095236711204052,
      "rewards/rejected": -0.006248453166335821,
      "step": 1263
    },
    {
      "epoch": 0.7341580995527676,
      "grad_norm": 301.5028381347656,
      "learning_rate": 3.1638582219639746e-06,
      "logits/chosen": -0.9459626078605652,
      "logits/rejected": -0.9003406763076782,
      "logps/chosen": -78.91160583496094,
      "logps/rejected": -68.72669219970703,
      "loss": 13.6504,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.013523939065635204,
      "rewards/margins": 0.0413726270198822,
      "rewards/rejected": -0.02784869447350502,
      "step": 1264
    },
    {
      "epoch": 0.7347389208340593,
      "grad_norm": 321.7524719238281,
      "learning_rate": 3.1624055781522373e-06,
      "logits/chosen": -0.9552342295646667,
      "logits/rejected": -1.019533634185791,
      "logps/chosen": -76.62181091308594,
      "logps/rejected": -78.6225357055664,
      "loss": 13.6056,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005645562428981066,
      "rewards/margins": 0.11532227694988251,
      "rewards/rejected": -0.12096782773733139,
      "step": 1265
    },
    {
      "epoch": 0.7353197421153511,
      "grad_norm": 306.43109130859375,
      "learning_rate": 3.1609529343404997e-06,
      "logits/chosen": -0.8540971875190735,
      "logits/rejected": -0.968962550163269,
      "logps/chosen": -67.66834259033203,
      "logps/rejected": -80.2847900390625,
      "loss": 13.3406,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.020695503801107407,
      "rewards/margins": 0.0667974203824997,
      "rewards/rejected": -0.046101927757263184,
      "step": 1266
    },
    {
      "epoch": 0.7359005633966429,
      "grad_norm": 323.08331298828125,
      "learning_rate": 3.1595002905287624e-06,
      "logits/chosen": -0.8610451817512512,
      "logits/rejected": -0.8618380427360535,
      "logps/chosen": -70.77239990234375,
      "logps/rejected": -73.03816986083984,
      "loss": 14.5571,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.008128318004310131,
      "rewards/margins": -0.05933644250035286,
      "rewards/rejected": 0.05120812729001045,
      "step": 1267
    },
    {
      "epoch": 0.7364813846779346,
      "grad_norm": 335.77569580078125,
      "learning_rate": 3.158047646717025e-06,
      "logits/chosen": -0.8688570857048035,
      "logits/rejected": -0.8984639048576355,
      "logps/chosen": -71.42537689208984,
      "logps/rejected": -81.7527084350586,
      "loss": 13.9077,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.09310576319694519,
      "rewards/margins": 0.011070755310356617,
      "rewards/rejected": -0.10417652130126953,
      "step": 1268
    },
    {
      "epoch": 0.7370622059592263,
      "grad_norm": 355.16046142578125,
      "learning_rate": 3.156595002905288e-06,
      "logits/chosen": -0.9459040760993958,
      "logits/rejected": -0.9788221120834351,
      "logps/chosen": -78.80431365966797,
      "logps/rejected": -76.61771392822266,
      "loss": 13.4248,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.054968319833278656,
      "rewards/margins": 0.06222357600927353,
      "rewards/rejected": -0.007255258969962597,
      "step": 1269
    },
    {
      "epoch": 0.7376430272405181,
      "grad_norm": 324.6065673828125,
      "learning_rate": 3.1551423590935502e-06,
      "logits/chosen": -0.9320200681686401,
      "logits/rejected": -0.9266518354415894,
      "logps/chosen": -73.32524108886719,
      "logps/rejected": -71.88298034667969,
      "loss": 13.2304,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.017843326553702354,
      "rewards/margins": 0.07853879034519196,
      "rewards/rejected": -0.06069546937942505,
      "step": 1270
    },
    {
      "epoch": 0.7382238485218099,
      "grad_norm": 290.72308349609375,
      "learning_rate": 3.153689715281813e-06,
      "logits/chosen": -0.9279607534408569,
      "logits/rejected": -0.9869905710220337,
      "logps/chosen": -69.19869995117188,
      "logps/rejected": -69.46534729003906,
      "loss": 13.7807,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.001915226923301816,
      "rewards/margins": 0.028262412175536156,
      "rewards/rejected": -0.030177637934684753,
      "step": 1271
    },
    {
      "epoch": 0.7388046698031016,
      "grad_norm": 322.3456115722656,
      "learning_rate": 3.1522370714700757e-06,
      "logits/chosen": -0.9604189991950989,
      "logits/rejected": -0.9729669690132141,
      "logps/chosen": -77.07878112792969,
      "logps/rejected": -76.24308776855469,
      "loss": 13.9018,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03510335087776184,
      "rewards/margins": 0.009103739634156227,
      "rewards/rejected": -0.04420708492398262,
      "step": 1272
    },
    {
      "epoch": 0.7393854910843933,
      "grad_norm": 290.1760559082031,
      "learning_rate": 3.150784427658338e-06,
      "logits/chosen": -0.7609505653381348,
      "logits/rejected": -0.940750777721405,
      "logps/chosen": -73.50611877441406,
      "logps/rejected": -84.53253173828125,
      "loss": 12.1886,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0856599435210228,
      "rewards/margins": 0.26221829652786255,
      "rewards/rejected": -0.17655836045742035,
      "step": 1273
    },
    {
      "epoch": 0.7399663123656851,
      "grad_norm": 287.99951171875,
      "learning_rate": 3.149331783846601e-06,
      "logits/chosen": -0.8309062123298645,
      "logits/rejected": -0.8096998929977417,
      "logps/chosen": -73.81452178955078,
      "logps/rejected": -65.81900787353516,
      "loss": 12.7097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08087469637393951,
      "rewards/margins": 0.13408292829990387,
      "rewards/rejected": -0.053208231925964355,
      "step": 1274
    },
    {
      "epoch": 0.7405471336469768,
      "grad_norm": 474.6933288574219,
      "learning_rate": 3.1478791400348636e-06,
      "logits/chosen": -1.0732654333114624,
      "logits/rejected": -1.0369174480438232,
      "logps/chosen": -69.6240234375,
      "logps/rejected": -77.09357452392578,
      "loss": 14.4602,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.01463285367935896,
      "rewards/margins": -0.03189690411090851,
      "rewards/rejected": 0.017264049500226974,
      "step": 1275
    },
    {
      "epoch": 0.7411279549282686,
      "grad_norm": 306.6213073730469,
      "learning_rate": 3.1464264962231263e-06,
      "logits/chosen": -0.7501953840255737,
      "logits/rejected": -0.7591122984886169,
      "logps/chosen": -76.5811767578125,
      "logps/rejected": -71.42620849609375,
      "loss": 13.1374,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.027137190103530884,
      "rewards/margins": 0.08468880504369736,
      "rewards/rejected": -0.057551611214876175,
      "step": 1276
    },
    {
      "epoch": 0.7417087762095603,
      "grad_norm": 335.8228454589844,
      "learning_rate": 3.1449738524113886e-06,
      "logits/chosen": -0.8871307373046875,
      "logits/rejected": -0.8877555727958679,
      "logps/chosen": -79.09004974365234,
      "logps/rejected": -66.54139709472656,
      "loss": 14.4295,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0550902895629406,
      "rewards/margins": -0.04246683418750763,
      "rewards/rejected": -0.012623448856174946,
      "step": 1277
    },
    {
      "epoch": 0.7422895974908521,
      "grad_norm": 306.65771484375,
      "learning_rate": 3.1435212085996514e-06,
      "logits/chosen": -0.9728385806083679,
      "logits/rejected": -0.9518416523933411,
      "logps/chosen": -69.57067108154297,
      "logps/rejected": -71.71105194091797,
      "loss": 14.227,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.047601040452718735,
      "rewards/margins": -0.023971671238541603,
      "rewards/rejected": -0.02362937107682228,
      "step": 1278
    },
    {
      "epoch": 0.7428704187721438,
      "grad_norm": 327.6387023925781,
      "learning_rate": 3.142068564787914e-06,
      "logits/chosen": -0.8745107650756836,
      "logits/rejected": -0.8107272982597351,
      "logps/chosen": -75.21369934082031,
      "logps/rejected": -72.1344223022461,
      "loss": 14.316,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04476819187402725,
      "rewards/margins": -0.03526737540960312,
      "rewards/rejected": -0.009500814601778984,
      "step": 1279
    },
    {
      "epoch": 0.7434512400534355,
      "grad_norm": 300.59722900390625,
      "learning_rate": 3.1406159209761765e-06,
      "logits/chosen": -0.9880915880203247,
      "logits/rejected": -0.9610216021537781,
      "logps/chosen": -67.36884307861328,
      "logps/rejected": -79.30865478515625,
      "loss": 13.4605,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.048171158879995346,
      "rewards/margins": 0.061096321791410446,
      "rewards/rejected": -0.012925168499350548,
      "step": 1280
    },
    {
      "epoch": 0.7440320613347273,
      "grad_norm": 314.7283020019531,
      "learning_rate": 3.1391632771644396e-06,
      "logits/chosen": -0.9206606149673462,
      "logits/rejected": -0.9953149557113647,
      "logps/chosen": -70.48161315917969,
      "logps/rejected": -77.36676025390625,
      "loss": 13.5039,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02788415551185608,
      "rewards/margins": 0.06114567443728447,
      "rewards/rejected": -0.03326151892542839,
      "step": 1281
    },
    {
      "epoch": 0.7446128826160191,
      "grad_norm": 397.0846252441406,
      "learning_rate": 3.1377106333527024e-06,
      "logits/chosen": -0.9257551431655884,
      "logits/rejected": -0.9087467193603516,
      "logps/chosen": -81.15228271484375,
      "logps/rejected": -76.59354400634766,
      "loss": 14.4729,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0699133574962616,
      "rewards/margins": -0.04452745243906975,
      "rewards/rejected": -0.02538590505719185,
      "step": 1282
    },
    {
      "epoch": 0.7451937038973108,
      "grad_norm": 317.1257629394531,
      "learning_rate": 3.136257989540965e-06,
      "logits/chosen": -0.9509184956550598,
      "logits/rejected": -0.8939563632011414,
      "logps/chosen": -81.9539794921875,
      "logps/rejected": -71.4426498413086,
      "loss": 14.2237,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01727578043937683,
      "rewards/margins": -0.011873170733451843,
      "rewards/rejected": -0.005402614828199148,
      "step": 1283
    },
    {
      "epoch": 0.7457745251786025,
      "grad_norm": 306.5256042480469,
      "learning_rate": 3.134805345729228e-06,
      "logits/chosen": -0.7918493151664734,
      "logits/rejected": -0.8878865242004395,
      "logps/chosen": -70.88008880615234,
      "logps/rejected": -72.70407104492188,
      "loss": 13.7775,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012965632602572441,
      "rewards/margins": 0.02363957278430462,
      "rewards/rejected": -0.03660520538687706,
      "step": 1284
    },
    {
      "epoch": 0.7463553464598943,
      "grad_norm": 383.96746826171875,
      "learning_rate": 3.13335270191749e-06,
      "logits/chosen": -0.9462077021598816,
      "logits/rejected": -1.135608434677124,
      "logps/chosen": -76.05490112304688,
      "logps/rejected": -66.54570007324219,
      "loss": 12.7749,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.013959522359073162,
      "rewards/margins": 0.13498705625534058,
      "rewards/rejected": -0.12102754414081573,
      "step": 1285
    },
    {
      "epoch": 0.746936167741186,
      "grad_norm": 301.2254333496094,
      "learning_rate": 3.131900058105753e-06,
      "logits/chosen": -0.9826423525810242,
      "logits/rejected": -0.9145171046257019,
      "logps/chosen": -69.3663330078125,
      "logps/rejected": -73.3055419921875,
      "loss": 13.4906,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03162767365574837,
      "rewards/margins": 0.05837094783782959,
      "rewards/rejected": -0.026743274182081223,
      "step": 1286
    },
    {
      "epoch": 0.7475169890224778,
      "grad_norm": 281.2210388183594,
      "learning_rate": 3.1304474142940157e-06,
      "logits/chosen": -0.936673641204834,
      "logits/rejected": -0.9934436082839966,
      "logps/chosen": -68.77691650390625,
      "logps/rejected": -69.91971588134766,
      "loss": 13.2363,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05332762002944946,
      "rewards/margins": 0.07535450160503387,
      "rewards/rejected": -0.022026890888810158,
      "step": 1287
    },
    {
      "epoch": 0.7480978103037695,
      "grad_norm": 329.33441162109375,
      "learning_rate": 3.128994770482278e-06,
      "logits/chosen": -0.9381875991821289,
      "logits/rejected": -0.877325713634491,
      "logps/chosen": -74.95464324951172,
      "logps/rejected": -74.71601104736328,
      "loss": 14.3683,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.009458988904953003,
      "rewards/margins": -0.0346088781952858,
      "rewards/rejected": 0.0440678708255291,
      "step": 1288
    },
    {
      "epoch": 0.7486786315850613,
      "grad_norm": 308.08905029296875,
      "learning_rate": 3.1275421266705408e-06,
      "logits/chosen": -0.8872352838516235,
      "logits/rejected": -0.9058473706245422,
      "logps/chosen": -76.67219543457031,
      "logps/rejected": -70.76875305175781,
      "loss": 13.5716,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.011845914646983147,
      "rewards/margins": 0.03853718191385269,
      "rewards/rejected": -0.026691267266869545,
      "step": 1289
    },
    {
      "epoch": 0.749259452866353,
      "grad_norm": 325.87884521484375,
      "learning_rate": 3.1260894828588035e-06,
      "logits/chosen": -0.9273878931999207,
      "logits/rejected": -0.9734488725662231,
      "logps/chosen": -76.02559661865234,
      "logps/rejected": -70.01312255859375,
      "loss": 14.3236,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03266528248786926,
      "rewards/margins": -0.03003951907157898,
      "rewards/rejected": -0.0026257620193064213,
      "step": 1290
    },
    {
      "epoch": 0.7498402741476448,
      "grad_norm": 297.24737548828125,
      "learning_rate": 3.1246368390470663e-06,
      "logits/chosen": -0.8632006645202637,
      "logits/rejected": -0.8015877604484558,
      "logps/chosen": -68.97894287109375,
      "logps/rejected": -81.93291473388672,
      "loss": 13.5402,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03800540044903755,
      "rewards/margins": 0.05160471051931381,
      "rewards/rejected": -0.013599300757050514,
      "step": 1291
    },
    {
      "epoch": 0.7504210954289365,
      "grad_norm": 302.7586364746094,
      "learning_rate": 3.1231841952353286e-06,
      "logits/chosen": -1.1155800819396973,
      "logits/rejected": -1.0501878261566162,
      "logps/chosen": -68.69818878173828,
      "logps/rejected": -66.80998992919922,
      "loss": 13.624,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.04426661878824234,
      "rewards/margins": 0.033911194652318954,
      "rewards/rejected": 0.010355427861213684,
      "step": 1292
    },
    {
      "epoch": 0.7510019167102283,
      "grad_norm": 368.1701965332031,
      "learning_rate": 3.1217315514235913e-06,
      "logits/chosen": -0.7901303768157959,
      "logits/rejected": -0.7890164852142334,
      "logps/chosen": -67.72219848632812,
      "logps/rejected": -70.45135498046875,
      "loss": 13.9425,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004095485899597406,
      "rewards/margins": 0.004590289201587439,
      "rewards/rejected": -0.0004948042333126068,
      "step": 1293
    },
    {
      "epoch": 0.75158273799152,
      "grad_norm": 293.4873962402344,
      "learning_rate": 3.120278907611854e-06,
      "logits/chosen": -0.8267465829849243,
      "logits/rejected": -0.8374005556106567,
      "logps/chosen": -64.80427551269531,
      "logps/rejected": -66.46602630615234,
      "loss": 14.0721,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005284186918288469,
      "rewards/margins": -0.009990280494093895,
      "rewards/rejected": 0.015274465084075928,
      "step": 1294
    },
    {
      "epoch": 0.7521635592728118,
      "grad_norm": 305.2198486328125,
      "learning_rate": 3.1188262638001164e-06,
      "logits/chosen": -0.944267749786377,
      "logits/rejected": -1.0252511501312256,
      "logps/chosen": -70.48851013183594,
      "logps/rejected": -68.64114379882812,
      "loss": 13.9951,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0024089247453957796,
      "rewards/margins": -0.005520508624613285,
      "rewards/rejected": 0.0031115845777094364,
      "step": 1295
    },
    {
      "epoch": 0.7527443805541035,
      "grad_norm": 295.1757507324219,
      "learning_rate": 3.117373619988379e-06,
      "logits/chosen": -0.768851637840271,
      "logits/rejected": -0.8648530840873718,
      "logps/chosen": -72.7383041381836,
      "logps/rejected": -61.505218505859375,
      "loss": 12.7695,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.02676945924758911,
      "rewards/margins": 0.12528559565544128,
      "rewards/rejected": -0.09851614385843277,
      "step": 1296
    },
    {
      "epoch": 0.7533252018353952,
      "grad_norm": 293.5204162597656,
      "learning_rate": 3.115920976176642e-06,
      "logits/chosen": -1.0298740863800049,
      "logits/rejected": -1.0123705863952637,
      "logps/chosen": -74.05130767822266,
      "logps/rejected": -68.51252746582031,
      "loss": 13.2703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.018832139670848846,
      "rewards/margins": 0.07043415307998657,
      "rewards/rejected": -0.051602013409137726,
      "step": 1297
    },
    {
      "epoch": 0.753906023116687,
      "grad_norm": 329.7220153808594,
      "learning_rate": 3.1144683323649042e-06,
      "logits/chosen": -0.8656724095344543,
      "logits/rejected": -0.9276447296142578,
      "logps/chosen": -78.12594604492188,
      "logps/rejected": -76.8603286743164,
      "loss": 13.8096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0003139674663543701,
      "rewards/margins": 0.01961832121014595,
      "rewards/rejected": -0.01930435374379158,
      "step": 1298
    },
    {
      "epoch": 0.7544868443979788,
      "grad_norm": 300.7325439453125,
      "learning_rate": 3.113015688553167e-06,
      "logits/chosen": -0.7556174397468567,
      "logits/rejected": -0.8270395398139954,
      "logps/chosen": -66.18405151367188,
      "logps/rejected": -72.25556945800781,
      "loss": 13.4483,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04187128320336342,
      "rewards/margins": 0.05631660297513008,
      "rewards/rejected": -0.014445314183831215,
      "step": 1299
    },
    {
      "epoch": 0.7550676656792705,
      "grad_norm": 332.7188415527344,
      "learning_rate": 3.1115630447414297e-06,
      "logits/chosen": -0.9498234987258911,
      "logits/rejected": -0.9386661648750305,
      "logps/chosen": -79.1153793334961,
      "logps/rejected": -73.41973876953125,
      "loss": 13.5749,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0045986296609044075,
      "rewards/margins": 0.04127994924783707,
      "rewards/rejected": -0.0458785817027092,
      "step": 1300
    },
    {
      "epoch": 0.7556484869605622,
      "grad_norm": 297.5984191894531,
      "learning_rate": 3.1101104009296925e-06,
      "logits/chosen": -0.8738743662834167,
      "logits/rejected": -0.9288954734802246,
      "logps/chosen": -76.1825180053711,
      "logps/rejected": -70.42452239990234,
      "loss": 12.6535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04402433708310127,
      "rewards/margins": 0.14227306842803955,
      "rewards/rejected": -0.09824874997138977,
      "step": 1301
    },
    {
      "epoch": 0.756229308241854,
      "grad_norm": 313.9720458984375,
      "learning_rate": 3.108657757117955e-06,
      "logits/chosen": -0.9596983194351196,
      "logits/rejected": -0.8854199647903442,
      "logps/chosen": -74.87777709960938,
      "logps/rejected": -68.69590759277344,
      "loss": 13.8843,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.026798080652952194,
      "rewards/margins": 0.010439385659992695,
      "rewards/rejected": -0.03723746910691261,
      "step": 1302
    },
    {
      "epoch": 0.7568101295231457,
      "grad_norm": 322.28472900390625,
      "learning_rate": 3.1072051133062175e-06,
      "logits/chosen": -0.710699737071991,
      "logits/rejected": -0.6639520525932312,
      "logps/chosen": -72.1385498046875,
      "logps/rejected": -77.3552017211914,
      "loss": 14.0005,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00393889332190156,
      "rewards/margins": 0.004963460378348827,
      "rewards/rejected": -0.008902350440621376,
      "step": 1303
    },
    {
      "epoch": 0.7573909508044375,
      "grad_norm": 301.8449401855469,
      "learning_rate": 3.1057524694944803e-06,
      "logits/chosen": -0.8113482594490051,
      "logits/rejected": -0.8424245715141296,
      "logps/chosen": -67.51692199707031,
      "logps/rejected": -73.65216827392578,
      "loss": 13.8924,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04349323362112045,
      "rewards/margins": 0.015273170545697212,
      "rewards/rejected": -0.058766402304172516,
      "step": 1304
    },
    {
      "epoch": 0.7579717720857292,
      "grad_norm": 325.9951477050781,
      "learning_rate": 3.1042998256827426e-06,
      "logits/chosen": -0.8810790777206421,
      "logits/rejected": -0.9614898562431335,
      "logps/chosen": -72.43753814697266,
      "logps/rejected": -76.32234191894531,
      "loss": 14.3891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0030550980009138584,
      "rewards/margins": -0.03376467153429985,
      "rewards/rejected": 0.03681976720690727,
      "step": 1305
    },
    {
      "epoch": 0.758552593367021,
      "grad_norm": 413.15533447265625,
      "learning_rate": 3.1028471818710054e-06,
      "logits/chosen": -1.0327026844024658,
      "logits/rejected": -0.9508744478225708,
      "logps/chosen": -71.23845672607422,
      "logps/rejected": -69.53571319580078,
      "loss": 14.3083,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.02447402849793434,
      "rewards/margins": -0.03209930285811424,
      "rewards/rejected": 0.007625264581292868,
      "step": 1306
    },
    {
      "epoch": 0.7591334146483127,
      "grad_norm": 291.16741943359375,
      "learning_rate": 3.101394538059268e-06,
      "logits/chosen": -0.9580721855163574,
      "logits/rejected": -1.1341726779937744,
      "logps/chosen": -78.12037658691406,
      "logps/rejected": -80.23886108398438,
      "loss": 12.1551,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.09904322773218155,
      "rewards/margins": 0.24864165484905243,
      "rewards/rejected": -0.14959844946861267,
      "step": 1307
    },
    {
      "epoch": 0.7597142359296044,
      "grad_norm": 306.8900146484375,
      "learning_rate": 3.099941894247531e-06,
      "logits/chosen": -0.8082634210586548,
      "logits/rejected": -0.9796327352523804,
      "logps/chosen": -71.74502563476562,
      "logps/rejected": -69.88240814208984,
      "loss": 13.7265,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03651000186800957,
      "rewards/margins": 0.0696808323264122,
      "rewards/rejected": -0.10619081556797028,
      "step": 1308
    },
    {
      "epoch": 0.7602950572108962,
      "grad_norm": 320.541748046875,
      "learning_rate": 3.098489250435793e-06,
      "logits/chosen": -0.7988053560256958,
      "logits/rejected": -0.9913209080696106,
      "logps/chosen": -74.5243911743164,
      "logps/rejected": -86.07827758789062,
      "loss": 14.2169,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.08174944669008255,
      "rewards/margins": -0.002851475030183792,
      "rewards/rejected": -0.07889796793460846,
      "step": 1309
    },
    {
      "epoch": 0.760875878492188,
      "grad_norm": 292.3503723144531,
      "learning_rate": 3.097036606624056e-06,
      "logits/chosen": -0.8505334854125977,
      "logits/rejected": -0.8206362724304199,
      "logps/chosen": -66.17271423339844,
      "logps/rejected": -72.32818603515625,
      "loss": 13.3208,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03991877660155296,
      "rewards/margins": 0.07060422748327255,
      "rewards/rejected": -0.030685454607009888,
      "step": 1310
    },
    {
      "epoch": 0.7614566997734797,
      "grad_norm": 312.9989013671875,
      "learning_rate": 3.0955839628123187e-06,
      "logits/chosen": -0.8575171232223511,
      "logits/rejected": -0.9133882522583008,
      "logps/chosen": -73.03935241699219,
      "logps/rejected": -71.58662414550781,
      "loss": 14.1034,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0011013038456439972,
      "rewards/margins": -0.0046140579506754875,
      "rewards/rejected": 0.003512748284265399,
      "step": 1311
    },
    {
      "epoch": 0.7620375210547714,
      "grad_norm": 313.8850402832031,
      "learning_rate": 3.094131319000581e-06,
      "logits/chosen": -0.8145445585250854,
      "logits/rejected": -0.7335568070411682,
      "logps/chosen": -72.54698944091797,
      "logps/rejected": -73.94429016113281,
      "loss": 13.9135,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009393749758601189,
      "rewards/margins": 0.009464768692851067,
      "rewards/rejected": -7.101949449861422e-05,
      "step": 1312
    },
    {
      "epoch": 0.7626183423360632,
      "grad_norm": 299.4548645019531,
      "learning_rate": 3.0926786751888438e-06,
      "logits/chosen": -0.8695998191833496,
      "logits/rejected": -0.895071804523468,
      "logps/chosen": -74.72267150878906,
      "logps/rejected": -73.9729995727539,
      "loss": 13.605,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02649582549929619,
      "rewards/margins": 0.040801845490932465,
      "rewards/rejected": -0.01430602092295885,
      "step": 1313
    },
    {
      "epoch": 0.763199163617355,
      "grad_norm": 329.5212707519531,
      "learning_rate": 3.0912260313771065e-06,
      "logits/chosen": -0.7945253252983093,
      "logits/rejected": -0.8804613947868347,
      "logps/chosen": -76.73372650146484,
      "logps/rejected": -86.13858032226562,
      "loss": 14.4551,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.11028039455413818,
      "rewards/margins": -0.04522523283958435,
      "rewards/rejected": -0.06505516171455383,
      "step": 1314
    },
    {
      "epoch": 0.7637799848986467,
      "grad_norm": 318.5284729003906,
      "learning_rate": 3.089773387565369e-06,
      "logits/chosen": -0.9539583921432495,
      "logits/rejected": -0.9986858367919922,
      "logps/chosen": -69.06396484375,
      "logps/rejected": -77.87904357910156,
      "loss": 13.5426,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015168903395533562,
      "rewards/margins": 0.042577486485242844,
      "rewards/rejected": -0.02740858867764473,
      "step": 1315
    },
    {
      "epoch": 0.7643608061799384,
      "grad_norm": 288.8753967285156,
      "learning_rate": 3.0883207437536316e-06,
      "logits/chosen": -0.9640684127807617,
      "logits/rejected": -0.9261250495910645,
      "logps/chosen": -70.24547576904297,
      "logps/rejected": -69.94730377197266,
      "loss": 13.2532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.060040153563022614,
      "rewards/margins": 0.07011136412620544,
      "rewards/rejected": -0.010071211494505405,
      "step": 1316
    },
    {
      "epoch": 0.7649416274612302,
      "grad_norm": 298.02410888671875,
      "learning_rate": 3.0868680999418943e-06,
      "logits/chosen": -0.9826720952987671,
      "logits/rejected": -1.0072005987167358,
      "logps/chosen": -67.46409606933594,
      "logps/rejected": -77.1575698852539,
      "loss": 13.7659,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009459376335144043,
      "rewards/margins": 0.017017792910337448,
      "rewards/rejected": -0.026477163657546043,
      "step": 1317
    },
    {
      "epoch": 0.765522448742522,
      "grad_norm": 321.3743591308594,
      "learning_rate": 3.085415456130157e-06,
      "logits/chosen": -1.0327075719833374,
      "logits/rejected": -1.1174871921539307,
      "logps/chosen": -72.16029357910156,
      "logps/rejected": -73.67405700683594,
      "loss": 14.3089,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.07403494417667389,
      "rewards/margins": 0.001424503279849887,
      "rewards/rejected": -0.07545944303274155,
      "step": 1318
    },
    {
      "epoch": 0.7661032700238136,
      "grad_norm": 307.7449035644531,
      "learning_rate": 3.0839628123184194e-06,
      "logits/chosen": -0.9941368103027344,
      "logits/rejected": -1.006868839263916,
      "logps/chosen": -71.70137023925781,
      "logps/rejected": -70.49494934082031,
      "loss": 13.1374,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0033687218092381954,
      "rewards/margins": 0.0898309126496315,
      "rewards/rejected": -0.09319963306188583,
      "step": 1319
    },
    {
      "epoch": 0.7666840913051054,
      "grad_norm": 320.2607421875,
      "learning_rate": 3.082510168506682e-06,
      "logits/chosen": -0.9001753926277161,
      "logits/rejected": -0.9374383091926575,
      "logps/chosen": -80.38785552978516,
      "logps/rejected": -78.30015563964844,
      "loss": 13.4005,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.020780716091394424,
      "rewards/margins": 0.059726834297180176,
      "rewards/rejected": -0.03894611448049545,
      "step": 1320
    },
    {
      "epoch": 0.7672649125863972,
      "grad_norm": 339.5959777832031,
      "learning_rate": 3.081057524694945e-06,
      "logits/chosen": -0.7729434370994568,
      "logits/rejected": -0.7935265898704529,
      "logps/chosen": -68.46189880371094,
      "logps/rejected": -73.17298889160156,
      "loss": 14.9243,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04347784072160721,
      "rewards/margins": -0.09590767323970795,
      "rewards/rejected": 0.05242983624339104,
      "step": 1321
    },
    {
      "epoch": 0.767845733867689,
      "grad_norm": 319.3999328613281,
      "learning_rate": 3.0796048808832072e-06,
      "logits/chosen": -0.9267279505729675,
      "logits/rejected": -0.9965450167655945,
      "logps/chosen": -74.07978820800781,
      "logps/rejected": -79.91683197021484,
      "loss": 14.0976,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.013504219241440296,
      "rewards/margins": -0.014489811845123768,
      "rewards/rejected": 0.027994031086564064,
      "step": 1322
    },
    {
      "epoch": 0.7684265551489806,
      "grad_norm": 322.2994079589844,
      "learning_rate": 3.0781522370714704e-06,
      "logits/chosen": -1.0502088069915771,
      "logits/rejected": -1.0726592540740967,
      "logps/chosen": -83.62413024902344,
      "logps/rejected": -73.68818664550781,
      "loss": 13.8526,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0060726734809577465,
      "rewards/margins": 0.012974309735000134,
      "rewards/rejected": -0.006901636719703674,
      "step": 1323
    },
    {
      "epoch": 0.7690073764302724,
      "grad_norm": 329.8638916015625,
      "learning_rate": 3.076699593259733e-06,
      "logits/chosen": -0.9872323870658875,
      "logits/rejected": -0.9380897283554077,
      "logps/chosen": -69.95539855957031,
      "logps/rejected": -71.13307189941406,
      "loss": 13.9479,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003933420870453119,
      "rewards/margins": 0.011535009369254112,
      "rewards/rejected": -0.007601586170494556,
      "step": 1324
    },
    {
      "epoch": 0.7695881977115642,
      "grad_norm": 301.33099365234375,
      "learning_rate": 3.075246949447996e-06,
      "logits/chosen": -0.8701971769332886,
      "logits/rejected": -0.9738820791244507,
      "logps/chosen": -65.2311019897461,
      "logps/rejected": -71.92955017089844,
      "loss": 13.9764,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.030279258266091347,
      "rewards/margins": -0.0048055145889520645,
      "rewards/rejected": -0.025473738089203835,
      "step": 1325
    },
    {
      "epoch": 0.770169018992856,
      "grad_norm": 320.4786376953125,
      "learning_rate": 3.0737943056362586e-06,
      "logits/chosen": -0.9620093107223511,
      "logits/rejected": -0.9475566744804382,
      "logps/chosen": -66.34928894042969,
      "logps/rejected": -71.0966567993164,
      "loss": 13.9339,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018435748293995857,
      "rewards/margins": -0.0015368014574050903,
      "rewards/rejected": -0.016898948699235916,
      "step": 1326
    },
    {
      "epoch": 0.7707498402741476,
      "grad_norm": 451.9235534667969,
      "learning_rate": 3.072341661824521e-06,
      "logits/chosen": -0.946884036064148,
      "logits/rejected": -0.8728286027908325,
      "logps/chosen": -73.80119323730469,
      "logps/rejected": -90.36488342285156,
      "loss": 13.3669,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.017404107376933098,
      "rewards/margins": 0.05732819437980652,
      "rewards/rejected": -0.03992409259080887,
      "step": 1327
    },
    {
      "epoch": 0.7713306615554394,
      "grad_norm": 329.6585998535156,
      "learning_rate": 3.0708890180127837e-06,
      "logits/chosen": -1.070633888244629,
      "logits/rejected": -0.9621335864067078,
      "logps/chosen": -80.99287414550781,
      "logps/rejected": -79.79194641113281,
      "loss": 14.0768,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03282466530799866,
      "rewards/margins": 0.0002280652552144602,
      "rewards/rejected": 0.032596610486507416,
      "step": 1328
    },
    {
      "epoch": 0.7719114828367312,
      "grad_norm": 327.6514587402344,
      "learning_rate": 3.0694363742010465e-06,
      "logits/chosen": -0.9293088912963867,
      "logits/rejected": -0.8865857124328613,
      "logps/chosen": -74.66876983642578,
      "logps/rejected": -67.32837677001953,
      "loss": 15.0054,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0507052056491375,
      "rewards/margins": -0.09132341295480728,
      "rewards/rejected": 0.04061821475625038,
      "step": 1329
    },
    {
      "epoch": 0.7724923041180228,
      "grad_norm": 336.0072937011719,
      "learning_rate": 3.067983730389309e-06,
      "logits/chosen": -0.9453585743904114,
      "logits/rejected": -0.9577397108078003,
      "logps/chosen": -84.74609375,
      "logps/rejected": -69.93091583251953,
      "loss": 13.3923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005408868193626404,
      "rewards/margins": 0.05733504146337509,
      "rewards/rejected": -0.051926176995038986,
      "step": 1330
    },
    {
      "epoch": 0.7730731253993146,
      "grad_norm": 323.63134765625,
      "learning_rate": 3.0665310865775715e-06,
      "logits/chosen": -0.9443836212158203,
      "logits/rejected": -0.8413750529289246,
      "logps/chosen": -66.75128173828125,
      "logps/rejected": -74.71034240722656,
      "loss": 14.0526,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.026594584807753563,
      "rewards/margins": -0.005962553434073925,
      "rewards/rejected": 0.03255714476108551,
      "step": 1331
    },
    {
      "epoch": 0.7736539466806064,
      "grad_norm": 315.0282287597656,
      "learning_rate": 3.0650784427658343e-06,
      "logits/chosen": -1.0176613330841064,
      "logits/rejected": -1.000049352645874,
      "logps/chosen": -83.16175842285156,
      "logps/rejected": -84.27059173583984,
      "loss": 12.5816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03647014871239662,
      "rewards/margins": 0.15192298591136932,
      "rewards/rejected": -0.1154528483748436,
      "step": 1332
    },
    {
      "epoch": 0.7742347679618982,
      "grad_norm": 307.80889892578125,
      "learning_rate": 3.063625798954097e-06,
      "logits/chosen": -0.8602831959724426,
      "logits/rejected": -0.8878147006034851,
      "logps/chosen": -75.41206359863281,
      "logps/rejected": -71.3490219116211,
      "loss": 13.5843,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03422687575221062,
      "rewards/margins": 0.0405036062002182,
      "rewards/rejected": -0.006276731379330158,
      "step": 1333
    },
    {
      "epoch": 0.7748155892431898,
      "grad_norm": 349.5753479003906,
      "learning_rate": 3.0621731551423594e-06,
      "logits/chosen": -0.8943806886672974,
      "logits/rejected": -0.7531766891479492,
      "logps/chosen": -73.33009338378906,
      "logps/rejected": -71.51890563964844,
      "loss": 13.7964,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0026279359590262175,
      "rewards/margins": 0.014700889587402344,
      "rewards/rejected": -0.01732882484793663,
      "step": 1334
    },
    {
      "epoch": 0.7753964105244816,
      "grad_norm": 661.4951171875,
      "learning_rate": 3.060720511330622e-06,
      "logits/chosen": -1.0936822891235352,
      "logits/rejected": -1.012609839439392,
      "logps/chosen": -71.6592025756836,
      "logps/rejected": -73.36859130859375,
      "loss": 13.7073,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.017734484747052193,
      "rewards/margins": 0.025820275768637657,
      "rewards/rejected": -0.04355476424098015,
      "step": 1335
    },
    {
      "epoch": 0.7759772318057734,
      "grad_norm": 314.4260559082031,
      "learning_rate": 3.059267867518885e-06,
      "logits/chosen": -0.8542010188102722,
      "logits/rejected": -0.9453112483024597,
      "logps/chosen": -75.16136932373047,
      "logps/rejected": -73.34740447998047,
      "loss": 13.9912,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0639212355017662,
      "rewards/margins": -0.00419047474861145,
      "rewards/rejected": -0.05973076820373535,
      "step": 1336
    },
    {
      "epoch": 0.7765580530870652,
      "grad_norm": 325.88604736328125,
      "learning_rate": 3.057815223707147e-06,
      "logits/chosen": -0.8881939649581909,
      "logits/rejected": -0.9086592793464661,
      "logps/chosen": -68.64024353027344,
      "logps/rejected": -76.59254455566406,
      "loss": 13.2217,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03400281071662903,
      "rewards/margins": 0.07575125992298126,
      "rewards/rejected": -0.04174845293164253,
      "step": 1337
    },
    {
      "epoch": 0.7771388743683568,
      "grad_norm": 320.7890625,
      "learning_rate": 3.05636257989541e-06,
      "logits/chosen": -0.9593494534492493,
      "logits/rejected": -0.9191232919692993,
      "logps/chosen": -76.92513275146484,
      "logps/rejected": -73.51411437988281,
      "loss": 14.091,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014224298298358917,
      "rewards/margins": -0.008345646783709526,
      "rewards/rejected": 0.022569945082068443,
      "step": 1338
    },
    {
      "epoch": 0.7777196956496486,
      "grad_norm": 325.1307067871094,
      "learning_rate": 3.0549099360836727e-06,
      "logits/chosen": -0.8813505172729492,
      "logits/rejected": -0.8474019765853882,
      "logps/chosen": -68.29415893554688,
      "logps/rejected": -66.09028625488281,
      "loss": 13.6048,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.019740797579288483,
      "rewards/margins": 0.04052061587572098,
      "rewards/rejected": -0.020779822021722794,
      "step": 1339
    },
    {
      "epoch": 0.7783005169309404,
      "grad_norm": 350.1361999511719,
      "learning_rate": 3.0534572922719354e-06,
      "logits/chosen": -0.9261456727981567,
      "logits/rejected": -0.9206829071044922,
      "logps/chosen": -75.91891479492188,
      "logps/rejected": -71.4757308959961,
      "loss": 14.0848,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04015408083796501,
      "rewards/margins": 0.0006739646196365356,
      "rewards/rejected": -0.04082803800702095,
      "step": 1340
    },
    {
      "epoch": 0.778881338212232,
      "grad_norm": 291.70867919921875,
      "learning_rate": 3.0520046484601977e-06,
      "logits/chosen": -0.8435354232788086,
      "logits/rejected": -0.8543268442153931,
      "logps/chosen": -72.85694885253906,
      "logps/rejected": -68.74168395996094,
      "loss": 13.2644,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.027489816769957542,
      "rewards/margins": 0.07461129873991013,
      "rewards/rejected": -0.047121480107307434,
      "step": 1341
    },
    {
      "epoch": 0.7794621594935238,
      "grad_norm": 331.1748352050781,
      "learning_rate": 3.0505520046484605e-06,
      "logits/chosen": -0.8756608963012695,
      "logits/rejected": -0.8256868124008179,
      "logps/chosen": -74.95640563964844,
      "logps/rejected": -66.36515045166016,
      "loss": 13.3304,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.030013050884008408,
      "rewards/margins": 0.0678246021270752,
      "rewards/rejected": -0.03781154006719589,
      "step": 1342
    },
    {
      "epoch": 0.7800429807748156,
      "grad_norm": 326.93756103515625,
      "learning_rate": 3.0490993608367232e-06,
      "logits/chosen": -0.8186312913894653,
      "logits/rejected": -0.81517493724823,
      "logps/chosen": -65.97537994384766,
      "logps/rejected": -80.20524597167969,
      "loss": 13.7043,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01899949088692665,
      "rewards/margins": 0.027245566248893738,
      "rewards/rejected": -0.008246068842709064,
      "step": 1343
    },
    {
      "epoch": 0.7806238020561074,
      "grad_norm": 312.2550048828125,
      "learning_rate": 3.0476467170249856e-06,
      "logits/chosen": -0.8998020887374878,
      "logits/rejected": -1.1422725915908813,
      "logps/chosen": -74.52033996582031,
      "logps/rejected": -71.10977172851562,
      "loss": 12.9752,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.07996420562267303,
      "rewards/margins": 0.14556726813316345,
      "rewards/rejected": -0.06560306251049042,
      "step": 1344
    },
    {
      "epoch": 0.781204623337399,
      "grad_norm": 310.1502380371094,
      "learning_rate": 3.0461940732132483e-06,
      "logits/chosen": -0.830686092376709,
      "logits/rejected": -0.9051626324653625,
      "logps/chosen": -69.29808044433594,
      "logps/rejected": -69.40185546875,
      "loss": 14.2198,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05074814707040787,
      "rewards/margins": -0.005796412937343121,
      "rewards/rejected": -0.044951725751161575,
      "step": 1345
    },
    {
      "epoch": 0.7817854446186908,
      "grad_norm": 318.5868835449219,
      "learning_rate": 3.044741429401511e-06,
      "logits/chosen": -0.8136617541313171,
      "logits/rejected": -0.8709491491317749,
      "logps/chosen": -72.2114486694336,
      "logps/rejected": -78.18385314941406,
      "loss": 13.3512,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.07459510862827301,
      "rewards/margins": 0.06512270867824554,
      "rewards/rejected": 0.009472398087382317,
      "step": 1346
    },
    {
      "epoch": 0.7823662658999826,
      "grad_norm": 291.2342529296875,
      "learning_rate": 3.0432887855897734e-06,
      "logits/chosen": -0.9937755465507507,
      "logits/rejected": -0.94462651014328,
      "logps/chosen": -77.37869262695312,
      "logps/rejected": -72.7966537475586,
      "loss": 13.2368,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.019733846187591553,
      "rewards/margins": 0.07204891741275787,
      "rewards/rejected": -0.05231507495045662,
      "step": 1347
    },
    {
      "epoch": 0.7829470871812744,
      "grad_norm": 313.6431579589844,
      "learning_rate": 3.041836141778036e-06,
      "logits/chosen": -0.8202837109565735,
      "logits/rejected": -0.8769274950027466,
      "logps/chosen": -73.77352905273438,
      "logps/rejected": -76.7472152709961,
      "loss": 13.3779,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028761591762304306,
      "rewards/margins": 0.05612437054514885,
      "rewards/rejected": -0.027362775057554245,
      "step": 1348
    },
    {
      "epoch": 0.783527908462566,
      "grad_norm": 340.0185241699219,
      "learning_rate": 3.040383497966299e-06,
      "logits/chosen": -0.8248658180236816,
      "logits/rejected": -0.871644139289856,
      "logps/chosen": -73.95661926269531,
      "logps/rejected": -71.35751342773438,
      "loss": 14.2664,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03927985206246376,
      "rewards/margins": -0.03121361695230007,
      "rewards/rejected": -0.008066235110163689,
      "step": 1349
    },
    {
      "epoch": 0.7841087297438578,
      "grad_norm": 337.849365234375,
      "learning_rate": 3.0389308541545616e-06,
      "logits/chosen": -0.8870008587837219,
      "logits/rejected": -0.8619573712348938,
      "logps/chosen": -78.1021499633789,
      "logps/rejected": -71.81489562988281,
      "loss": 14.2105,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0021542080212384462,
      "rewards/margins": -0.025884348899126053,
      "rewards/rejected": 0.023730134591460228,
      "step": 1350
    },
    {
      "epoch": 0.7846895510251496,
      "grad_norm": 306.9468994140625,
      "learning_rate": 3.037478210342824e-06,
      "logits/chosen": -0.9687013626098633,
      "logits/rejected": -0.9463915824890137,
      "logps/chosen": -71.2443618774414,
      "logps/rejected": -70.51518249511719,
      "loss": 13.5972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0010921467328444123,
      "rewards/margins": 0.036162327975034714,
      "rewards/rejected": -0.035070180892944336,
      "step": 1351
    },
    {
      "epoch": 0.7852703723064413,
      "grad_norm": 1007.7548217773438,
      "learning_rate": 3.0360255665310867e-06,
      "logits/chosen": -0.915471076965332,
      "logits/rejected": -0.8410797119140625,
      "logps/chosen": -74.26629638671875,
      "logps/rejected": -70.72218322753906,
      "loss": 15.7603,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.11322425305843353,
      "rewards/margins": -0.1384105682373047,
      "rewards/rejected": 0.02518630027770996,
      "step": 1352
    },
    {
      "epoch": 0.785851193587733,
      "grad_norm": 311.32769775390625,
      "learning_rate": 3.0345729227193495e-06,
      "logits/chosen": -0.7496891021728516,
      "logits/rejected": -0.7368720769882202,
      "logps/chosen": -79.49574279785156,
      "logps/rejected": -75.98058319091797,
      "loss": 13.6301,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.041738640516996384,
      "rewards/margins": 0.033691614866256714,
      "rewards/rejected": 0.008047039620578289,
      "step": 1353
    },
    {
      "epoch": 0.7864320148690248,
      "grad_norm": 288.973388671875,
      "learning_rate": 3.0331202789076118e-06,
      "logits/chosen": -1.0128023624420166,
      "logits/rejected": -0.9074182510375977,
      "logps/chosen": -69.997314453125,
      "logps/rejected": -67.61974334716797,
      "loss": 14.0068,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04602381959557533,
      "rewards/margins": 0.0015851057833060622,
      "rewards/rejected": 0.04443871229887009,
      "step": 1354
    },
    {
      "epoch": 0.7870128361503166,
      "grad_norm": 311.2589416503906,
      "learning_rate": 3.0316676350958745e-06,
      "logits/chosen": -0.860263466835022,
      "logits/rejected": -1.0293904542922974,
      "logps/chosen": -79.18681335449219,
      "logps/rejected": -73.83085632324219,
      "loss": 12.7251,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.031762413680553436,
      "rewards/margins": 0.15182720124721527,
      "rewards/rejected": -0.12006479501724243,
      "step": 1355
    },
    {
      "epoch": 0.7875936574316083,
      "grad_norm": 328.4019775390625,
      "learning_rate": 3.0302149912841373e-06,
      "logits/chosen": -0.6678148508071899,
      "logits/rejected": -0.744316816329956,
      "logps/chosen": -76.285888671875,
      "logps/rejected": -74.52369689941406,
      "loss": 14.6096,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.043553780764341354,
      "rewards/margins": -0.060098446905612946,
      "rewards/rejected": 0.01654466614127159,
      "step": 1356
    },
    {
      "epoch": 0.7881744787129,
      "grad_norm": 329.6540222167969,
      "learning_rate": 3.0287623474724e-06,
      "logits/chosen": -0.9145728945732117,
      "logits/rejected": -0.8466494679450989,
      "logps/chosen": -78.23568725585938,
      "logps/rejected": -76.97039031982422,
      "loss": 14.6969,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04897257685661316,
      "rewards/margins": -0.06480839103460312,
      "rewards/rejected": 0.015835819765925407,
      "step": 1357
    },
    {
      "epoch": 0.7887552999941918,
      "grad_norm": 297.9798278808594,
      "learning_rate": 3.0273097036606624e-06,
      "logits/chosen": -0.8726884722709656,
      "logits/rejected": -0.8678280711174011,
      "logps/chosen": -72.16580200195312,
      "logps/rejected": -69.71571350097656,
      "loss": 13.3646,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00622510863468051,
      "rewards/margins": 0.06236076354980469,
      "rewards/rejected": -0.06858587265014648,
      "step": 1358
    },
    {
      "epoch": 0.7893361212754836,
      "grad_norm": 313.60687255859375,
      "learning_rate": 3.025857059848925e-06,
      "logits/chosen": -0.8595548868179321,
      "logits/rejected": -0.8038026690483093,
      "logps/chosen": -70.47291564941406,
      "logps/rejected": -68.73007202148438,
      "loss": 13.9345,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.019606666639447212,
      "rewards/margins": 0.019154131412506104,
      "rewards/rejected": 0.0004525393305812031,
      "step": 1359
    },
    {
      "epoch": 0.7899169425567752,
      "grad_norm": 291.983154296875,
      "learning_rate": 3.024404416037188e-06,
      "logits/chosen": -0.8659041523933411,
      "logits/rejected": -0.8374651074409485,
      "logps/chosen": -69.31880187988281,
      "logps/rejected": -72.99031066894531,
      "loss": 12.9454,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.008247283287346363,
      "rewards/margins": 0.111447274684906,
      "rewards/rejected": -0.1196945458650589,
      "step": 1360
    },
    {
      "epoch": 0.790497763838067,
      "grad_norm": 298.4236755371094,
      "learning_rate": 3.02295177222545e-06,
      "logits/chosen": -0.7991964817047119,
      "logits/rejected": -0.9084981083869934,
      "logps/chosen": -68.1712646484375,
      "logps/rejected": -64.58174133300781,
      "loss": 13.5036,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03998025134205818,
      "rewards/margins": 0.04308316111564636,
      "rewards/rejected": -0.003102910937741399,
      "step": 1361
    },
    {
      "epoch": 0.7910785851193588,
      "grad_norm": 317.5739440917969,
      "learning_rate": 3.021499128413713e-06,
      "logits/chosen": -0.8296391367912292,
      "logits/rejected": -0.8046083450317383,
      "logps/chosen": -63.57970428466797,
      "logps/rejected": -74.22569274902344,
      "loss": 13.5866,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05577964708209038,
      "rewards/margins": 0.03943698853254318,
      "rewards/rejected": 0.01634264923632145,
      "step": 1362
    },
    {
      "epoch": 0.7916594064006505,
      "grad_norm": 328.33489990234375,
      "learning_rate": 3.0200464846019757e-06,
      "logits/chosen": -0.8937657475471497,
      "logits/rejected": -0.7614256143569946,
      "logps/chosen": -78.67537689208984,
      "logps/rejected": -73.56626892089844,
      "loss": 14.1702,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.011616593226790428,
      "rewards/margins": -0.013118448667228222,
      "rewards/rejected": 0.02473502978682518,
      "step": 1363
    },
    {
      "epoch": 0.7922402276819422,
      "grad_norm": 307.547119140625,
      "learning_rate": 3.018593840790238e-06,
      "logits/chosen": -0.9623318910598755,
      "logits/rejected": -0.8778507113456726,
      "logps/chosen": -68.7757339477539,
      "logps/rejected": -76.71199798583984,
      "loss": 13.36,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.023236006498336792,
      "rewards/margins": 0.06152491644024849,
      "rewards/rejected": -0.0382889062166214,
      "step": 1364
    },
    {
      "epoch": 0.792821048963234,
      "grad_norm": 324.67230224609375,
      "learning_rate": 3.0171411969785016e-06,
      "logits/chosen": -1.1191600561141968,
      "logits/rejected": -1.02480947971344,
      "logps/chosen": -75.34868621826172,
      "logps/rejected": -80.51942443847656,
      "loss": 14.0121,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0019715665839612484,
      "rewards/margins": 0.002730574458837509,
      "rewards/rejected": -0.0007590114837512374,
      "step": 1365
    },
    {
      "epoch": 0.7934018702445258,
      "grad_norm": 345.59210205078125,
      "learning_rate": 3.015688553166764e-06,
      "logits/chosen": -0.7707211375236511,
      "logits/rejected": -0.8302076458930969,
      "logps/chosen": -77.45845031738281,
      "logps/rejected": -73.25044250488281,
      "loss": 14.078,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04284262657165527,
      "rewards/margins": -0.00815762672573328,
      "rewards/rejected": -0.03468500077724457,
      "step": 1366
    },
    {
      "epoch": 0.7939826915258175,
      "grad_norm": 303.2406921386719,
      "learning_rate": 3.0142359093550267e-06,
      "logits/chosen": -0.9011403322219849,
      "logits/rejected": -0.9611363410949707,
      "logps/chosen": -64.09588623046875,
      "logps/rejected": -68.89429473876953,
      "loss": 14.2597,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04039134085178375,
      "rewards/margins": -0.03255877643823624,
      "rewards/rejected": -0.00783256720751524,
      "step": 1367
    },
    {
      "epoch": 0.7945635128071092,
      "grad_norm": 284.2441101074219,
      "learning_rate": 3.0127832655432894e-06,
      "logits/chosen": -0.9885448217391968,
      "logits/rejected": -1.032010793685913,
      "logps/chosen": -71.85505676269531,
      "logps/rejected": -74.30782318115234,
      "loss": 13.0534,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0135162603110075,
      "rewards/margins": 0.125481978058815,
      "rewards/rejected": -0.13899824023246765,
      "step": 1368
    },
    {
      "epoch": 0.795144334088401,
      "grad_norm": 315.44305419921875,
      "learning_rate": 3.0113306217315517e-06,
      "logits/chosen": -0.8221847414970398,
      "logits/rejected": -0.7739245891571045,
      "logps/chosen": -74.33781433105469,
      "logps/rejected": -65.6959457397461,
      "loss": 13.691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0006410256028175354,
      "rewards/margins": 0.022573314607143402,
      "rewards/rejected": -0.021932287141680717,
      "step": 1369
    },
    {
      "epoch": 0.7957251553696928,
      "grad_norm": 310.6796875,
      "learning_rate": 3.0098779779198145e-06,
      "logits/chosen": -0.8911339044570923,
      "logits/rejected": -1.0014220476150513,
      "logps/chosen": -73.21330261230469,
      "logps/rejected": -69.49861145019531,
      "loss": 14.2187,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021617893129587173,
      "rewards/margins": -0.020058469846844673,
      "rewards/rejected": -0.001559421420097351,
      "step": 1370
    },
    {
      "epoch": 0.7963059766509845,
      "grad_norm": 322.3150634765625,
      "learning_rate": 3.0084253341080772e-06,
      "logits/chosen": -0.8698734045028687,
      "logits/rejected": -0.8341646194458008,
      "logps/chosen": -70.78841400146484,
      "logps/rejected": -76.29813385009766,
      "loss": 14.1758,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04047202691435814,
      "rewards/margins": -0.018879977986216545,
      "rewards/rejected": -0.021592045202851295,
      "step": 1371
    },
    {
      "epoch": 0.7968867979322762,
      "grad_norm": 325.392333984375,
      "learning_rate": 3.0069726902963396e-06,
      "logits/chosen": -0.8510764241218567,
      "logits/rejected": -0.9262604713439941,
      "logps/chosen": -72.66088104248047,
      "logps/rejected": -66.18681335449219,
      "loss": 13.0238,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0028306760359555483,
      "rewards/margins": 0.09332326054573059,
      "rewards/rejected": -0.09615393728017807,
      "step": 1372
    },
    {
      "epoch": 0.797467619213568,
      "grad_norm": 310.9512023925781,
      "learning_rate": 3.0055200464846023e-06,
      "logits/chosen": -0.9182130694389343,
      "logits/rejected": -0.9772962331771851,
      "logps/chosen": -77.55223083496094,
      "logps/rejected": -72.72220611572266,
      "loss": 12.7803,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05390697717666626,
      "rewards/margins": 0.13290613889694214,
      "rewards/rejected": -0.07899917662143707,
      "step": 1373
    },
    {
      "epoch": 0.7980484404948597,
      "grad_norm": 351.2005920410156,
      "learning_rate": 3.004067402672865e-06,
      "logits/chosen": -0.9052426218986511,
      "logits/rejected": -0.8361685872077942,
      "logps/chosen": -71.26953887939453,
      "logps/rejected": -79.6133804321289,
      "loss": 13.698,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02061326429247856,
      "rewards/margins": 0.027254462242126465,
      "rewards/rejected": -0.00664120027795434,
      "step": 1374
    },
    {
      "epoch": 0.7986292617761515,
      "grad_norm": 344.826416015625,
      "learning_rate": 3.002614758861128e-06,
      "logits/chosen": -0.7878270149230957,
      "logits/rejected": -0.7709068655967712,
      "logps/chosen": -73.96141052246094,
      "logps/rejected": -71.29468536376953,
      "loss": 13.745,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.012798281386494637,
      "rewards/margins": 0.02574135735630989,
      "rewards/rejected": -0.038539640605449677,
      "step": 1375
    },
    {
      "epoch": 0.7992100830574432,
      "grad_norm": 331.14666748046875,
      "learning_rate": 3.00116211504939e-06,
      "logits/chosen": -0.8426164388656616,
      "logits/rejected": -0.9244447946548462,
      "logps/chosen": -75.20966339111328,
      "logps/rejected": -73.4683609008789,
      "loss": 13.6653,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01234610565006733,
      "rewards/margins": 0.02751067280769348,
      "rewards/rejected": -0.03985677659511566,
      "step": 1376
    },
    {
      "epoch": 0.799790904338735,
      "grad_norm": 293.5313415527344,
      "learning_rate": 2.999709471237653e-06,
      "logits/chosen": -1.044769287109375,
      "logits/rejected": -1.266304612159729,
      "logps/chosen": -78.04954528808594,
      "logps/rejected": -75.4638671875,
      "loss": 12.5882,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03527790307998657,
      "rewards/margins": 0.15076994895935059,
      "rewards/rejected": -0.11549206078052521,
      "step": 1377
    },
    {
      "epoch": 0.8003717256200267,
      "grad_norm": 289.6751708984375,
      "learning_rate": 2.9982568274259156e-06,
      "logits/chosen": -0.9611708521842957,
      "logits/rejected": -1.0750725269317627,
      "logps/chosen": -74.12474822998047,
      "logps/rejected": -89.33628845214844,
      "loss": 12.1874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11065836995840073,
      "rewards/margins": 0.19909630715847015,
      "rewards/rejected": -0.08843793720006943,
      "step": 1378
    },
    {
      "epoch": 0.8009525469013185,
      "grad_norm": 390.03192138671875,
      "learning_rate": 2.996804183614178e-06,
      "logits/chosen": -1.0337811708450317,
      "logits/rejected": -0.9826676249504089,
      "logps/chosen": -73.61744689941406,
      "logps/rejected": -73.42425537109375,
      "loss": 13.8009,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0021939887665212154,
      "rewards/margins": 0.01644139736890793,
      "rewards/rejected": -0.01863538846373558,
      "step": 1379
    },
    {
      "epoch": 0.8015333681826102,
      "grad_norm": 334.7769470214844,
      "learning_rate": 2.9953515398024407e-06,
      "logits/chosen": -0.8718684315681458,
      "logits/rejected": -0.7975374460220337,
      "logps/chosen": -84.67412567138672,
      "logps/rejected": -74.53395080566406,
      "loss": 13.9407,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04416005685925484,
      "rewards/margins": 0.011963719502091408,
      "rewards/rejected": -0.0561237707734108,
      "step": 1380
    },
    {
      "epoch": 0.802114189463902,
      "grad_norm": 297.78509521484375,
      "learning_rate": 2.9938988959907034e-06,
      "logits/chosen": -0.9542218446731567,
      "logits/rejected": -0.9323599934577942,
      "logps/chosen": -70.81591796875,
      "logps/rejected": -66.9459228515625,
      "loss": 12.9819,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05045939236879349,
      "rewards/margins": 0.10639425367116928,
      "rewards/rejected": -0.05593486875295639,
      "step": 1381
    },
    {
      "epoch": 0.8026950107451937,
      "grad_norm": 313.09698486328125,
      "learning_rate": 2.992446252178966e-06,
      "logits/chosen": -0.7120882272720337,
      "logits/rejected": -0.7091692686080933,
      "logps/chosen": -69.15006256103516,
      "logps/rejected": -69.49944305419922,
      "loss": 13.6383,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.001335333799943328,
      "rewards/margins": 0.034621793776750565,
      "rewards/rejected": -0.03328645974397659,
      "step": 1382
    },
    {
      "epoch": 0.8032758320264854,
      "grad_norm": 298.8887634277344,
      "learning_rate": 2.9909936083672285e-06,
      "logits/chosen": -0.8668020367622375,
      "logits/rejected": -0.9281014204025269,
      "logps/chosen": -68.71341705322266,
      "logps/rejected": -79.57211303710938,
      "loss": 12.104,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.056803785264492035,
      "rewards/margins": 0.19968357682228088,
      "rewards/rejected": -0.14287981390953064,
      "step": 1383
    },
    {
      "epoch": 0.8038566533077772,
      "grad_norm": 303.8555908203125,
      "learning_rate": 2.9895409645554913e-06,
      "logits/chosen": -0.9737163782119751,
      "logits/rejected": -0.8915233612060547,
      "logps/chosen": -65.64739990234375,
      "logps/rejected": -77.95571899414062,
      "loss": 13.3841,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0005382836097851396,
      "rewards/margins": 0.05962703377008438,
      "rewards/rejected": -0.06016530841588974,
      "step": 1384
    },
    {
      "epoch": 0.8044374745890689,
      "grad_norm": 301.3564147949219,
      "learning_rate": 2.988088320743754e-06,
      "logits/chosen": -0.9112704992294312,
      "logits/rejected": -0.9425870776176453,
      "logps/chosen": -76.31121826171875,
      "logps/rejected": -77.12882995605469,
      "loss": 12.855,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0437435656785965,
      "rewards/margins": 0.12112482637166977,
      "rewards/rejected": -0.07738125324249268,
      "step": 1385
    },
    {
      "epoch": 0.8050182958703607,
      "grad_norm": 322.3490295410156,
      "learning_rate": 2.9866356769320163e-06,
      "logits/chosen": -0.8554973602294922,
      "logits/rejected": -0.9562565684318542,
      "logps/chosen": -76.09308624267578,
      "logps/rejected": -71.78053283691406,
      "loss": 13.4199,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.004631166812032461,
      "rewards/margins": 0.05598657205700874,
      "rewards/rejected": -0.05135541036725044,
      "step": 1386
    },
    {
      "epoch": 0.8055991171516524,
      "grad_norm": 337.444091796875,
      "learning_rate": 2.985183033120279e-06,
      "logits/chosen": -0.807202160358429,
      "logits/rejected": -0.8988969922065735,
      "logps/chosen": -70.60244750976562,
      "logps/rejected": -75.38993835449219,
      "loss": 14.3192,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.028507042676210403,
      "rewards/margins": -0.030303260311484337,
      "rewards/rejected": 0.0017962135607376695,
      "step": 1387
    },
    {
      "epoch": 0.8061799384329442,
      "grad_norm": 306.1874084472656,
      "learning_rate": 2.983730389308542e-06,
      "logits/chosen": -0.8419469594955444,
      "logits/rejected": -0.9874483346939087,
      "logps/chosen": -69.77263641357422,
      "logps/rejected": -73.81204986572266,
      "loss": 13.9882,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0005180264124646783,
      "rewards/margins": -0.0012780275428667665,
      "rewards/rejected": 0.0017960546538233757,
      "step": 1388
    },
    {
      "epoch": 0.8067607597142359,
      "grad_norm": 311.86865234375,
      "learning_rate": 2.982277745496804e-06,
      "logits/chosen": -0.8463094830513,
      "logits/rejected": -0.8941611051559448,
      "logps/chosen": -76.22781372070312,
      "logps/rejected": -78.11372375488281,
      "loss": 13.6727,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.006520853843539953,
      "rewards/margins": 0.035121574997901917,
      "rewards/rejected": -0.028600722551345825,
      "step": 1389
    },
    {
      "epoch": 0.8073415809955277,
      "grad_norm": 318.3154296875,
      "learning_rate": 2.980825101685067e-06,
      "logits/chosen": -0.781771183013916,
      "logits/rejected": -0.8518495559692383,
      "logps/chosen": -75.18206787109375,
      "logps/rejected": -79.75123596191406,
      "loss": 14.1264,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.042932551354169846,
      "rewards/margins": -0.0072795734740793705,
      "rewards/rejected": -0.03565298020839691,
      "step": 1390
    },
    {
      "epoch": 0.8079224022768194,
      "grad_norm": 337.6727600097656,
      "learning_rate": 2.9793724578733297e-06,
      "logits/chosen": -0.9709503054618835,
      "logits/rejected": -0.9282326698303223,
      "logps/chosen": -74.23689270019531,
      "logps/rejected": -75.7239990234375,
      "loss": 14.4094,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.02538228966295719,
      "rewards/margins": -0.04640632122755051,
      "rewards/rejected": 0.021024029701948166,
      "step": 1391
    },
    {
      "epoch": 0.8085032235581112,
      "grad_norm": 301.5517883300781,
      "learning_rate": 2.9779198140615924e-06,
      "logits/chosen": -0.8642464876174927,
      "logits/rejected": -0.8270998001098633,
      "logps/chosen": -80.44632720947266,
      "logps/rejected": -70.0821762084961,
      "loss": 13.4707,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014341345056891441,
      "rewards/margins": 0.05069577693939209,
      "rewards/rejected": -0.0363544337451458,
      "step": 1392
    },
    {
      "epoch": 0.8090840448394029,
      "grad_norm": 288.4425048828125,
      "learning_rate": 2.9764671702498547e-06,
      "logits/chosen": -0.9424541592597961,
      "logits/rejected": -0.808376133441925,
      "logps/chosen": -69.18309020996094,
      "logps/rejected": -67.02249908447266,
      "loss": 13.4446,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012684144079685211,
      "rewards/margins": 0.053430359810590744,
      "rewards/rejected": -0.040746212005615234,
      "step": 1393
    },
    {
      "epoch": 0.8096648661206947,
      "grad_norm": 314.92498779296875,
      "learning_rate": 2.9750145264381175e-06,
      "logits/chosen": -0.8285082578659058,
      "logits/rejected": -0.8708974123001099,
      "logps/chosen": -65.28099822998047,
      "logps/rejected": -74.25511169433594,
      "loss": 13.5233,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.003757838159799576,
      "rewards/margins": 0.042324211448431015,
      "rewards/rejected": -0.04608204588294029,
      "step": 1394
    },
    {
      "epoch": 0.8102456874019864,
      "grad_norm": 300.69091796875,
      "learning_rate": 2.9735618826263802e-06,
      "logits/chosen": -0.9099591374397278,
      "logits/rejected": -1.00680410861969,
      "logps/chosen": -74.21427154541016,
      "logps/rejected": -75.23872375488281,
      "loss": 12.9511,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04937635362148285,
      "rewards/margins": 0.1410948783159256,
      "rewards/rejected": -0.09171853214502335,
      "step": 1395
    },
    {
      "epoch": 0.8108265086832781,
      "grad_norm": 301.8944091796875,
      "learning_rate": 2.9721092388146426e-06,
      "logits/chosen": -0.9558166265487671,
      "logits/rejected": -0.9425419569015503,
      "logps/chosen": -65.77293395996094,
      "logps/rejected": -73.72007751464844,
      "loss": 13.1303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.006804055068641901,
      "rewards/margins": 0.09389863908290863,
      "rewards/rejected": -0.10070270299911499,
      "step": 1396
    },
    {
      "epoch": 0.8114073299645699,
      "grad_norm": 329.2503967285156,
      "learning_rate": 2.9706565950029053e-06,
      "logits/chosen": -0.7545236945152283,
      "logits/rejected": -0.7351499199867249,
      "logps/chosen": -82.5606918334961,
      "logps/rejected": -69.96710205078125,
      "loss": 14.3537,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.08269495517015457,
      "rewards/margins": -0.033942438662052155,
      "rewards/rejected": -0.048752523958683014,
      "step": 1397
    },
    {
      "epoch": 0.8119881512458617,
      "grad_norm": 307.9728698730469,
      "learning_rate": 2.969203951191168e-06,
      "logits/chosen": -0.8963637351989746,
      "logits/rejected": -0.967424750328064,
      "logps/chosen": -74.98493957519531,
      "logps/rejected": -77.74474334716797,
      "loss": 13.6997,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.014015071094036102,
      "rewards/margins": 0.03292082995176315,
      "rewards/rejected": -0.046935904771089554,
      "step": 1398
    },
    {
      "epoch": 0.8125689725271534,
      "grad_norm": 331.90692138671875,
      "learning_rate": 2.967751307379431e-06,
      "logits/chosen": -0.8685439825057983,
      "logits/rejected": -0.7758145332336426,
      "logps/chosen": -84.01808166503906,
      "logps/rejected": -66.17570495605469,
      "loss": 14.4558,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.048323314636945724,
      "rewards/margins": -0.045513082295656204,
      "rewards/rejected": -0.002810239791870117,
      "step": 1399
    },
    {
      "epoch": 0.8131497938084451,
      "grad_norm": 295.3406677246094,
      "learning_rate": 2.966298663567693e-06,
      "logits/chosen": -0.9515465497970581,
      "logits/rejected": -0.9444233179092407,
      "logps/chosen": -72.34852600097656,
      "logps/rejected": -76.84788513183594,
      "loss": 12.8631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05914708226919174,
      "rewards/margins": 0.11663947999477386,
      "rewards/rejected": -0.05749241262674332,
      "step": 1400
    },
    {
      "epoch": 0.8137306150897369,
      "grad_norm": 323.6412353515625,
      "learning_rate": 2.964846019755956e-06,
      "logits/chosen": -0.877872109413147,
      "logits/rejected": -0.9010562896728516,
      "logps/chosen": -75.88202667236328,
      "logps/rejected": -70.96057891845703,
      "loss": 13.7134,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.022446606308221817,
      "rewards/margins": 0.0358426570892334,
      "rewards/rejected": -0.05828927829861641,
      "step": 1401
    },
    {
      "epoch": 0.8143114363710287,
      "grad_norm": 376.44122314453125,
      "learning_rate": 2.9633933759442186e-06,
      "logits/chosen": -0.9167495965957642,
      "logits/rejected": -0.9177714586257935,
      "logps/chosen": -77.82963562011719,
      "logps/rejected": -74.28784942626953,
      "loss": 13.7398,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022596869617700577,
      "rewards/margins": 0.023731734603643417,
      "rewards/rejected": -0.046328600496053696,
      "step": 1402
    },
    {
      "epoch": 0.8148922576523204,
      "grad_norm": 361.0466003417969,
      "learning_rate": 2.961940732132481e-06,
      "logits/chosen": -0.7660300135612488,
      "logits/rejected": -0.8450537919998169,
      "logps/chosen": -79.71549987792969,
      "logps/rejected": -78.52848815917969,
      "loss": 14.0703,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.029864352196455002,
      "rewards/margins": -0.0030341260135173798,
      "rewards/rejected": -0.026830222457647324,
      "step": 1403
    },
    {
      "epoch": 0.8154730789336121,
      "grad_norm": 290.4995422363281,
      "learning_rate": 2.9604880883207437e-06,
      "logits/chosen": -0.9871827960014343,
      "logits/rejected": -1.0963327884674072,
      "logps/chosen": -67.67317199707031,
      "logps/rejected": -68.42801666259766,
      "loss": 14.1438,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0310450941324234,
      "rewards/margins": -0.01996983215212822,
      "rewards/rejected": -0.01107526570558548,
      "step": 1404
    },
    {
      "epoch": 0.8160539002149039,
      "grad_norm": 337.4749755859375,
      "learning_rate": 2.9590354445090064e-06,
      "logits/chosen": -0.8815649747848511,
      "logits/rejected": -0.9569485783576965,
      "logps/chosen": -79.97685241699219,
      "logps/rejected": -77.7378158569336,
      "loss": 14.4737,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.05383127182722092,
      "rewards/margins": -0.04738280922174454,
      "rewards/rejected": -0.0064484672620892525,
      "step": 1405
    },
    {
      "epoch": 0.8166347214961956,
      "grad_norm": 298.2803039550781,
      "learning_rate": 2.957582800697269e-06,
      "logits/chosen": -0.9669227600097656,
      "logits/rejected": -0.932029128074646,
      "logps/chosen": -70.30120849609375,
      "logps/rejected": -65.73628234863281,
      "loss": 13.782,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00205295835621655,
      "rewards/margins": 0.015735972672700882,
      "rewards/rejected": -0.013683013617992401,
      "step": 1406
    },
    {
      "epoch": 0.8172155427774873,
      "grad_norm": 313.683349609375,
      "learning_rate": 2.9561301568855324e-06,
      "logits/chosen": -0.9539187550544739,
      "logits/rejected": -0.9567904472351074,
      "logps/chosen": -66.65169525146484,
      "logps/rejected": -77.30280303955078,
      "loss": 13.8461,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.019451383501291275,
      "rewards/margins": 0.02249503880739212,
      "rewards/rejected": -0.0419464185833931,
      "step": 1407
    },
    {
      "epoch": 0.8177963640587791,
      "grad_norm": 324.68841552734375,
      "learning_rate": 2.9546775130737947e-06,
      "logits/chosen": -0.8528417348861694,
      "logits/rejected": -0.8416927456855774,
      "logps/chosen": -72.50676727294922,
      "logps/rejected": -70.4826889038086,
      "loss": 13.8105,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06990914046764374,
      "rewards/margins": 0.019082918763160706,
      "rewards/rejected": -0.08899206668138504,
      "step": 1408
    },
    {
      "epoch": 0.8183771853400709,
      "grad_norm": 350.4996032714844,
      "learning_rate": 2.9532248692620574e-06,
      "logits/chosen": -0.8740476369857788,
      "logits/rejected": -0.8426333665847778,
      "logps/chosen": -73.0979995727539,
      "logps/rejected": -78.89864349365234,
      "loss": 13.5108,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008096178993582726,
      "rewards/margins": 0.058543860912323,
      "rewards/rejected": -0.05044768005609512,
      "step": 1409
    },
    {
      "epoch": 0.8189580066213626,
      "grad_norm": 300.794921875,
      "learning_rate": 2.95177222545032e-06,
      "logits/chosen": -1.023101568222046,
      "logits/rejected": -0.9668956995010376,
      "logps/chosen": -68.93318939208984,
      "logps/rejected": -79.46096801757812,
      "loss": 14.2038,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.05931961536407471,
      "rewards/margins": -0.02067120373249054,
      "rewards/rejected": -0.03864841163158417,
      "step": 1410
    },
    {
      "epoch": 0.8195388279026543,
      "grad_norm": 298.94775390625,
      "learning_rate": 2.9503195816385825e-06,
      "logits/chosen": -0.7894174456596375,
      "logits/rejected": -0.7799513936042786,
      "logps/chosen": -65.86666870117188,
      "logps/rejected": -72.9088134765625,
      "loss": 13.3772,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.037196483463048935,
      "rewards/margins": 0.06542030721902847,
      "rewards/rejected": -0.028223831206560135,
      "step": 1411
    },
    {
      "epoch": 0.8201196491839461,
      "grad_norm": 327.1647033691406,
      "learning_rate": 2.9488669378268453e-06,
      "logits/chosen": -0.9241153001785278,
      "logits/rejected": -1.0121660232543945,
      "logps/chosen": -74.90141296386719,
      "logps/rejected": -74.28089904785156,
      "loss": 13.8113,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.021306011825799942,
      "rewards/margins": 0.018331706523895264,
      "rewards/rejected": 0.002974300878122449,
      "step": 1412
    },
    {
      "epoch": 0.8207004704652379,
      "grad_norm": 321.0006408691406,
      "learning_rate": 2.947414294015108e-06,
      "logits/chosen": -0.9207015037536621,
      "logits/rejected": -0.9460538625717163,
      "logps/chosen": -63.860313415527344,
      "logps/rejected": -74.50765228271484,
      "loss": 14.1264,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07574287801980972,
      "rewards/margins": -0.01194755733013153,
      "rewards/rejected": -0.0637953132390976,
      "step": 1413
    },
    {
      "epoch": 0.8212812917465296,
      "grad_norm": 331.079345703125,
      "learning_rate": 2.9459616502033708e-06,
      "logits/chosen": -1.0412019491195679,
      "logits/rejected": -0.9624441862106323,
      "logps/chosen": -83.08404541015625,
      "logps/rejected": -72.27083587646484,
      "loss": 14.8622,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.07116438448429108,
      "rewards/margins": -0.08456090092658997,
      "rewards/rejected": 0.01339652854949236,
      "step": 1414
    },
    {
      "epoch": 0.8218621130278213,
      "grad_norm": 297.1318664550781,
      "learning_rate": 2.944509006391633e-06,
      "logits/chosen": -0.8080541491508484,
      "logits/rejected": -0.9105886220932007,
      "logps/chosen": -73.99314880371094,
      "logps/rejected": -70.49571990966797,
      "loss": 13.7404,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018458887934684753,
      "rewards/margins": 0.023319009691476822,
      "rewards/rejected": -0.041777901351451874,
      "step": 1415
    },
    {
      "epoch": 0.8224429343091131,
      "grad_norm": 311.7056579589844,
      "learning_rate": 2.943056362579896e-06,
      "logits/chosen": -0.9526283144950867,
      "logits/rejected": -0.9018238186836243,
      "logps/chosen": -77.3647232055664,
      "logps/rejected": -74.00312805175781,
      "loss": 14.124,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.014940363354980946,
      "rewards/margins": -0.005972468759864569,
      "rewards/rejected": -0.008967895992100239,
      "step": 1416
    },
    {
      "epoch": 0.8230237555904049,
      "grad_norm": 320.6746520996094,
      "learning_rate": 2.9416037187681586e-06,
      "logits/chosen": -0.8339821100234985,
      "logits/rejected": -0.8510110974311829,
      "logps/chosen": -73.5208740234375,
      "logps/rejected": -70.19255065917969,
      "loss": 13.9567,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04577286168932915,
      "rewards/margins": 0.0035650632344186306,
      "rewards/rejected": -0.04933793097734451,
      "step": 1417
    },
    {
      "epoch": 0.8236045768716965,
      "grad_norm": 322.4148254394531,
      "learning_rate": 2.940151074956421e-06,
      "logits/chosen": -0.7864097356796265,
      "logits/rejected": -0.8227651715278625,
      "logps/chosen": -77.32823181152344,
      "logps/rejected": -78.11876678466797,
      "loss": 12.9272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01557239331305027,
      "rewards/margins": 0.1051282063126564,
      "rewards/rejected": -0.08955581486225128,
      "step": 1418
    },
    {
      "epoch": 0.8241853981529883,
      "grad_norm": 287.50152587890625,
      "learning_rate": 2.9386984311446837e-06,
      "logits/chosen": -1.1195625066757202,
      "logits/rejected": -1.123811960220337,
      "logps/chosen": -68.20274353027344,
      "logps/rejected": -71.1021957397461,
      "loss": 13.94,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.027792204171419144,
      "rewards/margins": 0.005404070019721985,
      "rewards/rejected": -0.03319627791643143,
      "step": 1419
    },
    {
      "epoch": 0.8247662194342801,
      "grad_norm": 348.2055969238281,
      "learning_rate": 2.9372457873329464e-06,
      "logits/chosen": -0.8875170946121216,
      "logits/rejected": -0.9901360273361206,
      "logps/chosen": -85.96516418457031,
      "logps/rejected": -66.6903076171875,
      "loss": 14.5797,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.10675116628408432,
      "rewards/margins": -0.058211199939250946,
      "rewards/rejected": -0.04853995889425278,
      "step": 1420
    },
    {
      "epoch": 0.8253470407155719,
      "grad_norm": 417.23162841796875,
      "learning_rate": 2.9357931435212087e-06,
      "logits/chosen": -1.0550312995910645,
      "logits/rejected": -0.996401309967041,
      "logps/chosen": -71.97113037109375,
      "logps/rejected": -76.80352783203125,
      "loss": 13.1061,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.005310612730681896,
      "rewards/margins": 0.09801144897937775,
      "rewards/rejected": -0.10332205146551132,
      "step": 1421
    },
    {
      "epoch": 0.8259278619968635,
      "grad_norm": 302.4499206542969,
      "learning_rate": 2.9343404997094715e-06,
      "logits/chosen": -1.032568335533142,
      "logits/rejected": -0.8247776031494141,
      "logps/chosen": -73.15726470947266,
      "logps/rejected": -72.27852630615234,
      "loss": 12.6442,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.06986106187105179,
      "rewards/margins": 0.13212266564369202,
      "rewards/rejected": -0.06226159259676933,
      "step": 1422
    },
    {
      "epoch": 0.8265086832781553,
      "grad_norm": 288.7543640136719,
      "learning_rate": 2.9328878558977342e-06,
      "logits/chosen": -0.9890671968460083,
      "logits/rejected": -0.9611910581588745,
      "logps/chosen": -69.77980041503906,
      "logps/rejected": -72.4341049194336,
      "loss": 13.6315,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.024638932198286057,
      "rewards/margins": 0.04840928316116333,
      "rewards/rejected": -0.023770350962877274,
      "step": 1423
    },
    {
      "epoch": 0.8270895045594471,
      "grad_norm": 345.2132263183594,
      "learning_rate": 2.931435212085997e-06,
      "logits/chosen": -0.7981261014938354,
      "logits/rejected": -0.8269329071044922,
      "logps/chosen": -71.18927001953125,
      "logps/rejected": -77.73841857910156,
      "loss": 13.5076,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01264518965035677,
      "rewards/margins": 0.04303009808063507,
      "rewards/rejected": -0.030384909361600876,
      "step": 1424
    },
    {
      "epoch": 0.8276703258407389,
      "grad_norm": 309.0422668457031,
      "learning_rate": 2.9299825682742593e-06,
      "logits/chosen": -1.059066891670227,
      "logits/rejected": -1.0526816844940186,
      "logps/chosen": -76.29167175292969,
      "logps/rejected": -77.83811950683594,
      "loss": 13.0896,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.006864713039249182,
      "rewards/margins": 0.08848677575588226,
      "rewards/rejected": -0.08162206411361694,
      "step": 1425
    },
    {
      "epoch": 0.8282511471220305,
      "grad_norm": 347.056396484375,
      "learning_rate": 2.928529924462522e-06,
      "logits/chosen": -0.8492454290390015,
      "logits/rejected": -0.9330434799194336,
      "logps/chosen": -66.89024353027344,
      "logps/rejected": -75.56095886230469,
      "loss": 13.597,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.040436942130327225,
      "rewards/margins": 0.04871886968612671,
      "rewards/rejected": -0.08915580809116364,
      "step": 1426
    },
    {
      "epoch": 0.8288319684033223,
      "grad_norm": 347.6139221191406,
      "learning_rate": 2.927077280650785e-06,
      "logits/chosen": -0.8413177728652954,
      "logits/rejected": -0.866229236125946,
      "logps/chosen": -65.55363464355469,
      "logps/rejected": -72.32474517822266,
      "loss": 14.4387,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.054840099066495895,
      "rewards/margins": -0.04780929163098335,
      "rewards/rejected": -0.007030805107206106,
      "step": 1427
    },
    {
      "epoch": 0.8294127896846141,
      "grad_norm": 313.5008239746094,
      "learning_rate": 2.925624636839047e-06,
      "logits/chosen": -1.1590675115585327,
      "logits/rejected": -1.1246588230133057,
      "logps/chosen": -70.39925384521484,
      "logps/rejected": -72.60347747802734,
      "loss": 13.9254,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.00591286551207304,
      "rewards/margins": 0.003524102969095111,
      "rewards/rejected": 0.002388761844485998,
      "step": 1428
    },
    {
      "epoch": 0.8299936109659057,
      "grad_norm": 328.209228515625,
      "learning_rate": 2.92417199302731e-06,
      "logits/chosen": -1.1466877460479736,
      "logits/rejected": -1.0815269947052002,
      "logps/chosen": -76.59765625,
      "logps/rejected": -75.53077697753906,
      "loss": 13.4079,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.011960210278630257,
      "rewards/margins": 0.06420392543077469,
      "rewards/rejected": -0.052243709564208984,
      "step": 1429
    },
    {
      "epoch": 0.8305744322471975,
      "grad_norm": 316.848876953125,
      "learning_rate": 2.9227193492155726e-06,
      "logits/chosen": -0.8794231414794922,
      "logits/rejected": -0.93865966796875,
      "logps/chosen": -68.74443817138672,
      "logps/rejected": -73.03550720214844,
      "loss": 12.6436,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03573547676205635,
      "rewards/margins": 0.13287413120269775,
      "rewards/rejected": -0.09713868796825409,
      "step": 1430
    },
    {
      "epoch": 0.8311552535284893,
      "grad_norm": 308.3152160644531,
      "learning_rate": 2.9212667054038354e-06,
      "logits/chosen": -0.9460890889167786,
      "logits/rejected": -0.990256667137146,
      "logps/chosen": -75.53749084472656,
      "logps/rejected": -78.43303680419922,
      "loss": 12.8778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008117609657347202,
      "rewards/margins": 0.11438294500112534,
      "rewards/rejected": -0.10626532882452011,
      "step": 1431
    },
    {
      "epoch": 0.8317360748097811,
      "grad_norm": 324.6167297363281,
      "learning_rate": 2.9198140615920977e-06,
      "logits/chosen": -0.8981647491455078,
      "logits/rejected": -0.8859692811965942,
      "logps/chosen": -71.46989440917969,
      "logps/rejected": -78.45552062988281,
      "loss": 14.0206,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04371855780482292,
      "rewards/margins": 0.0002274781436426565,
      "rewards/rejected": -0.0439460389316082,
      "step": 1432
    },
    {
      "epoch": 0.8323168960910727,
      "grad_norm": 308.68634033203125,
      "learning_rate": 2.9183614177803604e-06,
      "logits/chosen": -0.9657155871391296,
      "logits/rejected": -1.0007869005203247,
      "logps/chosen": -73.20808410644531,
      "logps/rejected": -75.70503997802734,
      "loss": 13.3747,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05411189794540405,
      "rewards/margins": 0.061391375958919525,
      "rewards/rejected": -0.11550327390432358,
      "step": 1433
    },
    {
      "epoch": 0.8328977173723645,
      "grad_norm": 308.5299072265625,
      "learning_rate": 2.916908773968623e-06,
      "logits/chosen": -0.8470290899276733,
      "logits/rejected": -0.8813098073005676,
      "logps/chosen": -74.58438873291016,
      "logps/rejected": -68.4504623413086,
      "loss": 13.6615,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01930718496441841,
      "rewards/margins": 0.02946723997592926,
      "rewards/rejected": -0.04877442494034767,
      "step": 1434
    },
    {
      "epoch": 0.8334785386536563,
      "grad_norm": 299.2701721191406,
      "learning_rate": 2.9154561301568855e-06,
      "logits/chosen": -0.9995430707931519,
      "logits/rejected": -0.8818809390068054,
      "logps/chosen": -74.71501159667969,
      "logps/rejected": -71.56771087646484,
      "loss": 13.3951,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0019346639746800065,
      "rewards/margins": 0.0627359002828598,
      "rewards/rejected": -0.06467055529356003,
      "step": 1435
    },
    {
      "epoch": 0.8340593599349481,
      "grad_norm": 395.1256408691406,
      "learning_rate": 2.9140034863451483e-06,
      "logits/chosen": -0.8388514518737793,
      "logits/rejected": -0.9286414384841919,
      "logps/chosen": -70.10887145996094,
      "logps/rejected": -75.85285949707031,
      "loss": 13.5973,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03658416494727135,
      "rewards/margins": 0.03549937531352043,
      "rewards/rejected": -0.07208352535963058,
      "step": 1436
    },
    {
      "epoch": 0.8346401812162397,
      "grad_norm": 336.1195983886719,
      "learning_rate": 2.912550842533411e-06,
      "logits/chosen": -0.9179220199584961,
      "logits/rejected": -0.9435710906982422,
      "logps/chosen": -77.24826049804688,
      "logps/rejected": -90.61140441894531,
      "loss": 13.0167,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05517718195915222,
      "rewards/margins": 0.10005372762680054,
      "rewards/rejected": -0.15523092448711395,
      "step": 1437
    },
    {
      "epoch": 0.8352210024975315,
      "grad_norm": 315.1141052246094,
      "learning_rate": 2.9110981987216733e-06,
      "logits/chosen": -1.0482752323150635,
      "logits/rejected": -0.9595603942871094,
      "logps/chosen": -71.0394058227539,
      "logps/rejected": -66.7883071899414,
      "loss": 13.7938,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.035364214330911636,
      "rewards/margins": 0.015838205814361572,
      "rewards/rejected": -0.051202427595853806,
      "step": 1438
    },
    {
      "epoch": 0.8358018237788233,
      "grad_norm": 306.5217590332031,
      "learning_rate": 2.909645554909936e-06,
      "logits/chosen": -0.9053821563720703,
      "logits/rejected": -0.9982587695121765,
      "logps/chosen": -78.62854766845703,
      "logps/rejected": -76.83899688720703,
      "loss": 12.9262,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02452382817864418,
      "rewards/margins": 0.11406532675027847,
      "rewards/rejected": -0.089541494846344,
      "step": 1439
    },
    {
      "epoch": 0.836382645060115,
      "grad_norm": 325.85809326171875,
      "learning_rate": 2.908192911098199e-06,
      "logits/chosen": -1.0166677236557007,
      "logits/rejected": -1.0623500347137451,
      "logps/chosen": -76.91658020019531,
      "logps/rejected": -81.35865020751953,
      "loss": 13.5881,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03841046243906021,
      "rewards/margins": 0.040515996515750885,
      "rewards/rejected": -0.0789264589548111,
      "step": 1440
    },
    {
      "epoch": 0.8369634663414067,
      "grad_norm": 306.7456359863281,
      "learning_rate": 2.9067402672864616e-06,
      "logits/chosen": -0.9058912396430969,
      "logits/rejected": -0.955280601978302,
      "logps/chosen": -69.76827239990234,
      "logps/rejected": -80.06465148925781,
      "loss": 13.8793,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03792574629187584,
      "rewards/margins": 0.011397892609238625,
      "rewards/rejected": -0.049323637038469315,
      "step": 1441
    },
    {
      "epoch": 0.8375442876226985,
      "grad_norm": 316.8137512207031,
      "learning_rate": 2.905287623474724e-06,
      "logits/chosen": -0.9326680898666382,
      "logits/rejected": -0.9002841114997864,
      "logps/chosen": -64.26530456542969,
      "logps/rejected": -72.42106628417969,
      "loss": 13.421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.023713741451501846,
      "rewards/margins": 0.05455396696925163,
      "rewards/rejected": -0.03084021434187889,
      "step": 1442
    },
    {
      "epoch": 0.8381251089039903,
      "grad_norm": 330.69598388671875,
      "learning_rate": 2.9038349796629866e-06,
      "logits/chosen": -0.9995059967041016,
      "logits/rejected": -0.9997901916503906,
      "logps/chosen": -68.09516143798828,
      "logps/rejected": -78.32083892822266,
      "loss": 13.0762,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028431197628378868,
      "rewards/margins": 0.08672399818897247,
      "rewards/rejected": -0.05829279497265816,
      "step": 1443
    },
    {
      "epoch": 0.838705930185282,
      "grad_norm": 309.0249938964844,
      "learning_rate": 2.9023823358512494e-06,
      "logits/chosen": -0.9827935099601746,
      "logits/rejected": -0.906152606010437,
      "logps/chosen": -75.81268310546875,
      "logps/rejected": -70.94896697998047,
      "loss": 13.5151,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04596186429262161,
      "rewards/margins": 0.052903175354003906,
      "rewards/rejected": -0.09886502474546432,
      "step": 1444
    },
    {
      "epoch": 0.8392867514665737,
      "grad_norm": 343.3897399902344,
      "learning_rate": 2.9009296920395117e-06,
      "logits/chosen": -1.070989966392517,
      "logits/rejected": -0.9897557497024536,
      "logps/chosen": -77.44450378417969,
      "logps/rejected": -75.32661437988281,
      "loss": 14.1568,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04107704386115074,
      "rewards/margins": -0.002468069549649954,
      "rewards/rejected": -0.03860897943377495,
      "step": 1445
    },
    {
      "epoch": 0.8398675727478655,
      "grad_norm": 321.6297607421875,
      "learning_rate": 2.8994770482277745e-06,
      "logits/chosen": -1.0299676656723022,
      "logits/rejected": -0.9927219152450562,
      "logps/chosen": -78.69172668457031,
      "logps/rejected": -86.01963806152344,
      "loss": 14.1958,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.106023870408535,
      "rewards/margins": -0.01636986993253231,
      "rewards/rejected": -0.08965399861335754,
      "step": 1446
    },
    {
      "epoch": 0.8404483940291573,
      "grad_norm": 488.09423828125,
      "learning_rate": 2.8980244044160372e-06,
      "logits/chosen": -1.02532958984375,
      "logits/rejected": -1.01516592502594,
      "logps/chosen": -66.1051025390625,
      "logps/rejected": -72.0055923461914,
      "loss": 13.6172,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03778787702322006,
      "rewards/margins": 0.036361563950777054,
      "rewards/rejected": 0.001426316099241376,
      "step": 1447
    },
    {
      "epoch": 0.8410292153104489,
      "grad_norm": 312.9158935546875,
      "learning_rate": 2.8965717606043e-06,
      "logits/chosen": -0.9840737581253052,
      "logits/rejected": -0.9164690971374512,
      "logps/chosen": -76.03489685058594,
      "logps/rejected": -73.18302917480469,
      "loss": 14.261,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.005087234079837799,
      "rewards/margins": -0.01781270280480385,
      "rewards/rejected": 0.012725469656288624,
      "step": 1448
    },
    {
      "epoch": 0.8416100365917407,
      "grad_norm": 304.1776123046875,
      "learning_rate": 2.895119116792563e-06,
      "logits/chosen": -0.9928930997848511,
      "logits/rejected": -0.9349110722541809,
      "logps/chosen": -71.4068832397461,
      "logps/rejected": -80.73994445800781,
      "loss": 13.1757,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0036755844485014677,
      "rewards/margins": 0.08609313517808914,
      "rewards/rejected": -0.08241754025220871,
      "step": 1449
    },
    {
      "epoch": 0.8421908578730325,
      "grad_norm": 318.96539306640625,
      "learning_rate": 2.8936664729808255e-06,
      "logits/chosen": -0.9667062759399414,
      "logits/rejected": -0.9356053471565247,
      "logps/chosen": -72.20743560791016,
      "logps/rejected": -75.35475158691406,
      "loss": 13.3513,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005824977066367865,
      "rewards/margins": 0.06783638894557953,
      "rewards/rejected": -0.06201140955090523,
      "step": 1450
    },
    {
      "epoch": 0.8427716791543242,
      "grad_norm": 287.2841491699219,
      "learning_rate": 2.8922138291690882e-06,
      "logits/chosen": -1.0728602409362793,
      "logits/rejected": -1.0354301929473877,
      "logps/chosen": -66.82252502441406,
      "logps/rejected": -69.35397338867188,
      "loss": 13.2312,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.032742682844400406,
      "rewards/margins": 0.0718207135796547,
      "rewards/rejected": -0.1045634001493454,
      "step": 1451
    },
    {
      "epoch": 0.8433525004356159,
      "grad_norm": 308.7333679199219,
      "learning_rate": 2.890761185357351e-06,
      "logits/chosen": -0.8575905561447144,
      "logits/rejected": -0.7339831590652466,
      "logps/chosen": -67.56846618652344,
      "logps/rejected": -75.03273010253906,
      "loss": 13.8363,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05015332251787186,
      "rewards/margins": 0.01076444797217846,
      "rewards/rejected": -0.060917772352695465,
      "step": 1452
    },
    {
      "epoch": 0.8439333217169077,
      "grad_norm": 445.4147644042969,
      "learning_rate": 2.8893085415456133e-06,
      "logits/chosen": -1.0251576900482178,
      "logits/rejected": -1.0761711597442627,
      "logps/chosen": -76.17536926269531,
      "logps/rejected": -70.47728729248047,
      "loss": 14.9522,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09727099537849426,
      "rewards/margins": -0.09437744319438934,
      "rewards/rejected": -0.002893555909395218,
      "step": 1453
    },
    {
      "epoch": 0.8445141429981995,
      "grad_norm": 300.1446533203125,
      "learning_rate": 2.887855897733876e-06,
      "logits/chosen": -1.00319504737854,
      "logits/rejected": -1.0032870769500732,
      "logps/chosen": -64.68209838867188,
      "logps/rejected": -72.49250793457031,
      "loss": 13.7014,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0016712173819541931,
      "rewards/margins": 0.0283278226852417,
      "rewards/rejected": -0.02665659785270691,
      "step": 1454
    },
    {
      "epoch": 0.8450949642794912,
      "grad_norm": 301.9476623535156,
      "learning_rate": 2.8864032539221388e-06,
      "logits/chosen": -1.200361967086792,
      "logits/rejected": -1.1305880546569824,
      "logps/chosen": -68.9275131225586,
      "logps/rejected": -69.94751739501953,
      "loss": 13.3525,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01948600821197033,
      "rewards/margins": 0.06596256792545319,
      "rewards/rejected": -0.04647655412554741,
      "step": 1455
    },
    {
      "epoch": 0.8456757855607829,
      "grad_norm": 302.3983459472656,
      "learning_rate": 2.8849506101104015e-06,
      "logits/chosen": -1.1100473403930664,
      "logits/rejected": -1.3247666358947754,
      "logps/chosen": -68.0783462524414,
      "logps/rejected": -83.70719146728516,
      "loss": 13.2909,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.04055756703019142,
      "rewards/margins": 0.11079002916812897,
      "rewards/rejected": -0.1513475775718689,
      "step": 1456
    },
    {
      "epoch": 0.8462566068420747,
      "grad_norm": 294.5367431640625,
      "learning_rate": 2.883497966298664e-06,
      "logits/chosen": -0.9274812936782837,
      "logits/rejected": -0.8486302495002747,
      "logps/chosen": -68.12483215332031,
      "logps/rejected": -68.38768768310547,
      "loss": 13.8271,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.001477489247918129,
      "rewards/margins": 0.011916229501366615,
      "rewards/rejected": -0.013393716886639595,
      "step": 1457
    },
    {
      "epoch": 0.8468374281233665,
      "grad_norm": 381.0744934082031,
      "learning_rate": 2.8820453224869266e-06,
      "logits/chosen": -1.015917420387268,
      "logits/rejected": -1.0308749675750732,
      "logps/chosen": -82.45670318603516,
      "logps/rejected": -65.22205352783203,
      "loss": 13.8406,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06658829748630524,
      "rewards/margins": 0.03000468574464321,
      "rewards/rejected": -0.0965929850935936,
      "step": 1458
    },
    {
      "epoch": 0.8474182494046582,
      "grad_norm": 284.975830078125,
      "learning_rate": 2.8805926786751894e-06,
      "logits/chosen": -0.8940141797065735,
      "logits/rejected": -0.7934109568595886,
      "logps/chosen": -73.81705474853516,
      "logps/rejected": -81.38023376464844,
      "loss": 12.7466,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.027592455968260765,
      "rewards/margins": 0.13420981168746948,
      "rewards/rejected": -0.10661735385656357,
      "step": 1459
    },
    {
      "epoch": 0.8479990706859499,
      "grad_norm": 354.286376953125,
      "learning_rate": 2.8791400348634517e-06,
      "logits/chosen": -0.8523277044296265,
      "logits/rejected": -0.8773848414421082,
      "logps/chosen": -83.81401062011719,
      "logps/rejected": -77.32756042480469,
      "loss": 14.9811,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.11302118003368378,
      "rewards/margins": -0.0927332416176796,
      "rewards/rejected": -0.02028796449303627,
      "step": 1460
    },
    {
      "epoch": 0.8485798919672417,
      "grad_norm": 314.47808837890625,
      "learning_rate": 2.8776873910517144e-06,
      "logits/chosen": -1.027400016784668,
      "logits/rejected": -1.0183266401290894,
      "logps/chosen": -75.68716430664062,
      "logps/rejected": -76.20285034179688,
      "loss": 14.3653,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.08116734027862549,
      "rewards/margins": -0.03845272585749626,
      "rewards/rejected": -0.04271461069583893,
      "step": 1461
    },
    {
      "epoch": 0.8491607132485334,
      "grad_norm": 298.08343505859375,
      "learning_rate": 2.876234747239977e-06,
      "logits/chosen": -0.9866994023323059,
      "logits/rejected": -0.9981306195259094,
      "logps/chosen": -70.24420166015625,
      "logps/rejected": -74.43152618408203,
      "loss": 13.5533,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07787815481424332,
      "rewards/margins": 0.046763062477111816,
      "rewards/rejected": -0.12464121729135513,
      "step": 1462
    },
    {
      "epoch": 0.8497415345298251,
      "grad_norm": 309.0505676269531,
      "learning_rate": 2.87478210342824e-06,
      "logits/chosen": -1.0744378566741943,
      "logits/rejected": -1.095942735671997,
      "logps/chosen": -67.72260284423828,
      "logps/rejected": -75.03489685058594,
      "loss": 13.4294,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02707652375102043,
      "rewards/margins": 0.0543799102306366,
      "rewards/rejected": -0.027303392067551613,
      "step": 1463
    },
    {
      "epoch": 0.8503223558111169,
      "grad_norm": 308.20086669921875,
      "learning_rate": 2.8733294596165022e-06,
      "logits/chosen": -1.0495295524597168,
      "logits/rejected": -1.0689680576324463,
      "logps/chosen": -71.0096435546875,
      "logps/rejected": -72.75456237792969,
      "loss": 13.748,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04317242652177811,
      "rewards/margins": 0.024141186848282814,
      "rewards/rejected": -0.06731361895799637,
      "step": 1464
    },
    {
      "epoch": 0.8509031770924087,
      "grad_norm": 619.3375854492188,
      "learning_rate": 2.871876815804765e-06,
      "logits/chosen": -0.8766723871231079,
      "logits/rejected": -0.9712117314338684,
      "logps/chosen": -78.13890075683594,
      "logps/rejected": -78.69232177734375,
      "loss": 13.7055,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03211286664009094,
      "rewards/margins": 0.030408170074224472,
      "rewards/rejected": -0.06252104043960571,
      "step": 1465
    },
    {
      "epoch": 0.8514839983737004,
      "grad_norm": 308.6454162597656,
      "learning_rate": 2.8704241719930277e-06,
      "logits/chosen": -1.1278132200241089,
      "logits/rejected": -1.1916383504867554,
      "logps/chosen": -74.99970245361328,
      "logps/rejected": -75.26470184326172,
      "loss": 13.6449,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.022476574406027794,
      "rewards/margins": 0.03218165412545204,
      "rewards/rejected": -0.05465823411941528,
      "step": 1466
    },
    {
      "epoch": 0.8520648196549921,
      "grad_norm": 316.8673095703125,
      "learning_rate": 2.86897152818129e-06,
      "logits/chosen": -0.8103786706924438,
      "logits/rejected": -0.7996621131896973,
      "logps/chosen": -72.07603454589844,
      "logps/rejected": -80.98165893554688,
      "loss": 13.3186,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05856870487332344,
      "rewards/margins": 0.062176547944545746,
      "rewards/rejected": -0.12074526399374008,
      "step": 1467
    },
    {
      "epoch": 0.8526456409362839,
      "grad_norm": 340.2526550292969,
      "learning_rate": 2.867518884369553e-06,
      "logits/chosen": -0.7877839803695679,
      "logits/rejected": -0.7645701766014099,
      "logps/chosen": -70.13272094726562,
      "logps/rejected": -70.92051696777344,
      "loss": 12.9132,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03563921898603439,
      "rewards/margins": 0.10783687978982925,
      "rewards/rejected": -0.07219766080379486,
      "step": 1468
    },
    {
      "epoch": 0.8532264622175757,
      "grad_norm": 313.46856689453125,
      "learning_rate": 2.8660662405578156e-06,
      "logits/chosen": -0.9348775148391724,
      "logits/rejected": -0.9333136677742004,
      "logps/chosen": -68.47344207763672,
      "logps/rejected": -73.52359008789062,
      "loss": 13.9446,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.026959236711263657,
      "rewards/margins": 0.003736050333827734,
      "rewards/rejected": -0.03069528378546238,
      "step": 1469
    },
    {
      "epoch": 0.8538072834988674,
      "grad_norm": 310.25128173828125,
      "learning_rate": 2.864613596746078e-06,
      "logits/chosen": -1.1887505054473877,
      "logits/rejected": -1.1720144748687744,
      "logps/chosen": -76.692626953125,
      "logps/rejected": -69.181396484375,
      "loss": 13.9905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.030578309670090675,
      "rewards/margins": -0.001125065959058702,
      "rewards/rejected": -0.02945324219763279,
      "step": 1470
    },
    {
      "epoch": 0.8543881047801591,
      "grad_norm": 320.1770935058594,
      "learning_rate": 2.8631609529343406e-06,
      "logits/chosen": -0.9049227833747864,
      "logits/rejected": -0.9818126559257507,
      "logps/chosen": -72.09139251708984,
      "logps/rejected": -76.15870666503906,
      "loss": 13.1158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.023979365825653076,
      "rewards/margins": 0.08431822806596756,
      "rewards/rejected": -0.10829760134220123,
      "step": 1471
    },
    {
      "epoch": 0.8549689260614509,
      "grad_norm": 301.6313781738281,
      "learning_rate": 2.8617083091226034e-06,
      "logits/chosen": -1.2870230674743652,
      "logits/rejected": -1.2937277555465698,
      "logps/chosen": -77.83735656738281,
      "logps/rejected": -83.67927551269531,
      "loss": 12.2098,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.029856283217668533,
      "rewards/margins": 0.18860362470149994,
      "rewards/rejected": -0.1587473452091217,
      "step": 1472
    },
    {
      "epoch": 0.8555497473427426,
      "grad_norm": 300.3925476074219,
      "learning_rate": 2.860255665310866e-06,
      "logits/chosen": -1.0691075325012207,
      "logits/rejected": -1.2402117252349854,
      "logps/chosen": -76.41740417480469,
      "logps/rejected": -68.46692657470703,
      "loss": 13.1559,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05581715703010559,
      "rewards/margins": 0.08918066322803497,
      "rewards/rejected": -0.14499780535697937,
      "step": 1473
    },
    {
      "epoch": 0.8561305686240344,
      "grad_norm": 321.46917724609375,
      "learning_rate": 2.8588030214991285e-06,
      "logits/chosen": -1.0409283638000488,
      "logits/rejected": -1.0422083139419556,
      "logps/chosen": -81.19866180419922,
      "logps/rejected": -76.9925537109375,
      "loss": 14.0133,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.11208139359951019,
      "rewards/margins": 0.011363961733877659,
      "rewards/rejected": -0.12344535440206528,
      "step": 1474
    },
    {
      "epoch": 0.8567113899053261,
      "grad_norm": 339.6994934082031,
      "learning_rate": 2.857350377687391e-06,
      "logits/chosen": -0.9717720150947571,
      "logits/rejected": -0.9631099700927734,
      "logps/chosen": -68.03271484375,
      "logps/rejected": -75.67191314697266,
      "loss": 13.3001,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02387564815580845,
      "rewards/margins": 0.06487865000963211,
      "rewards/rejected": -0.08875429630279541,
      "step": 1475
    },
    {
      "epoch": 0.8572922111866179,
      "grad_norm": 314.8028869628906,
      "learning_rate": 2.855897733875654e-06,
      "logits/chosen": -1.1045962572097778,
      "logits/rejected": -1.0674214363098145,
      "logps/chosen": -76.48600006103516,
      "logps/rejected": -78.5645751953125,
      "loss": 13.7583,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.031141653656959534,
      "rewards/margins": 0.024553081020712852,
      "rewards/rejected": -0.055694736540317535,
      "step": 1476
    },
    {
      "epoch": 0.8578730324679096,
      "grad_norm": 332.6769714355469,
      "learning_rate": 2.8544450900639163e-06,
      "logits/chosen": -1.0836639404296875,
      "logits/rejected": -1.0983717441558838,
      "logps/chosen": -71.63214111328125,
      "logps/rejected": -79.2393798828125,
      "loss": 14.0886,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10234622657299042,
      "rewards/margins": -0.012714147567749023,
      "rewards/rejected": -0.0896320790052414,
      "step": 1477
    },
    {
      "epoch": 0.8584538537492014,
      "grad_norm": 283.8846740722656,
      "learning_rate": 2.852992446252179e-06,
      "logits/chosen": -0.9009078145027161,
      "logits/rejected": -0.9883058667182922,
      "logps/chosen": -73.82437896728516,
      "logps/rejected": -73.50035858154297,
      "loss": 12.4806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03795861080288887,
      "rewards/margins": 0.15546271204948425,
      "rewards/rejected": -0.11750410497188568,
      "step": 1478
    },
    {
      "epoch": 0.8590346750304931,
      "grad_norm": 287.013427734375,
      "learning_rate": 2.8515398024404418e-06,
      "logits/chosen": -1.077418565750122,
      "logits/rejected": -1.118657112121582,
      "logps/chosen": -76.33918762207031,
      "logps/rejected": -78.89106750488281,
      "loss": 12.687,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02672363445162773,
      "rewards/margins": 0.13711220026016235,
      "rewards/rejected": -0.11038856208324432,
      "step": 1479
    },
    {
      "epoch": 0.8596154963117849,
      "grad_norm": 304.09521484375,
      "learning_rate": 2.8500871586287045e-06,
      "logits/chosen": -1.0066934823989868,
      "logits/rejected": -0.9893206357955933,
      "logps/chosen": -71.27021789550781,
      "logps/rejected": -75.72471618652344,
      "loss": 13.5684,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0031398951541632414,
      "rewards/margins": 0.05370044708251953,
      "rewards/rejected": -0.05056055262684822,
      "step": 1480
    },
    {
      "epoch": 0.8601963175930766,
      "grad_norm": 320.46783447265625,
      "learning_rate": 2.848634514816967e-06,
      "logits/chosen": -1.1885395050048828,
      "logits/rejected": -1.0513757467269897,
      "logps/chosen": -77.12410736083984,
      "logps/rejected": -73.9107894897461,
      "loss": 13.3804,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0005124583840370178,
      "rewards/margins": 0.0726466029882431,
      "rewards/rejected": -0.07315906137228012,
      "step": 1481
    },
    {
      "epoch": 0.8607771388743684,
      "grad_norm": 326.2270202636719,
      "learning_rate": 2.8471818710052296e-06,
      "logits/chosen": -1.0655676126480103,
      "logits/rejected": -0.9947828054428101,
      "logps/chosen": -68.42686462402344,
      "logps/rejected": -86.8310775756836,
      "loss": 13.1214,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04280921071767807,
      "rewards/margins": 0.09633222967386246,
      "rewards/rejected": -0.13914144039154053,
      "step": 1482
    },
    {
      "epoch": 0.8613579601556601,
      "grad_norm": 336.5929260253906,
      "learning_rate": 2.8457292271934923e-06,
      "logits/chosen": -1.029888391494751,
      "logits/rejected": -0.9574079513549805,
      "logps/chosen": -73.42161560058594,
      "logps/rejected": -73.63734436035156,
      "loss": 15.088,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.06948383152484894,
      "rewards/margins": -0.10173497349023819,
      "rewards/rejected": 0.03225115314126015,
      "step": 1483
    },
    {
      "epoch": 0.8619387814369518,
      "grad_norm": 319.0052795410156,
      "learning_rate": 2.8442765833817547e-06,
      "logits/chosen": -1.1239211559295654,
      "logits/rejected": -1.1674638986587524,
      "logps/chosen": -68.56141662597656,
      "logps/rejected": -82.07525634765625,
      "loss": 13.5989,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05102337524294853,
      "rewards/margins": 0.04584553837776184,
      "rewards/rejected": -0.09686891734600067,
      "step": 1484
    },
    {
      "epoch": 0.8625196027182436,
      "grad_norm": 323.8281555175781,
      "learning_rate": 2.8428239395700174e-06,
      "logits/chosen": -0.9768516421318054,
      "logits/rejected": -1.0456371307373047,
      "logps/chosen": -70.48323822021484,
      "logps/rejected": -73.61297607421875,
      "loss": 13.7821,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04092666506767273,
      "rewards/margins": 0.013883814215660095,
      "rewards/rejected": -0.05481048300862312,
      "step": 1485
    },
    {
      "epoch": 0.8631004239995353,
      "grad_norm": 289.0621032714844,
      "learning_rate": 2.84137129575828e-06,
      "logits/chosen": -0.9260729551315308,
      "logits/rejected": -0.9236747622489929,
      "logps/chosen": -71.13074493408203,
      "logps/rejected": -67.59986877441406,
      "loss": 12.8563,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02574968710541725,
      "rewards/margins": 0.11963015794754028,
      "rewards/rejected": -0.09388046711683273,
      "step": 1486
    },
    {
      "epoch": 0.8636812452808271,
      "grad_norm": 329.6697998046875,
      "learning_rate": 2.8399186519465425e-06,
      "logits/chosen": -1.1566098928451538,
      "logits/rejected": -1.0534679889678955,
      "logps/chosen": -75.1569595336914,
      "logps/rejected": -80.20098876953125,
      "loss": 13.7328,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04923470690846443,
      "rewards/margins": 0.03127080947160721,
      "rewards/rejected": -0.08050551265478134,
      "step": 1487
    },
    {
      "epoch": 0.8642620665621188,
      "grad_norm": 316.08087158203125,
      "learning_rate": 2.8384660081348052e-06,
      "logits/chosen": -0.796632707118988,
      "logits/rejected": -0.8546813130378723,
      "logps/chosen": -72.66250610351562,
      "logps/rejected": -68.24896240234375,
      "loss": 13.9145,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04194372147321701,
      "rewards/margins": 0.01639500819146633,
      "rewards/rejected": -0.05833873152732849,
      "step": 1488
    },
    {
      "epoch": 0.8648428878434106,
      "grad_norm": 390.2724609375,
      "learning_rate": 2.837013364323068e-06,
      "logits/chosen": -1.1093589067459106,
      "logits/rejected": -0.9831436276435852,
      "logps/chosen": -82.72193145751953,
      "logps/rejected": -76.21109771728516,
      "loss": 14.6968,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.1354256421327591,
      "rewards/margins": -0.07078886777162552,
      "rewards/rejected": -0.06463678181171417,
      "step": 1489
    },
    {
      "epoch": 0.8654237091247023,
      "grad_norm": 558.0623168945312,
      "learning_rate": 2.8355607205113307e-06,
      "logits/chosen": -1.0331647396087646,
      "logits/rejected": -1.060380220413208,
      "logps/chosen": -67.2535400390625,
      "logps/rejected": -74.99746704101562,
      "loss": 13.9305,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.003717402694746852,
      "rewards/margins": 0.0061567798256874084,
      "rewards/rejected": -0.00987417995929718,
      "step": 1490
    },
    {
      "epoch": 0.8660045304059941,
      "grad_norm": 329.20257568359375,
      "learning_rate": 2.834108076699594e-06,
      "logits/chosen": -1.1107969284057617,
      "logits/rejected": -1.2981940507888794,
      "logps/chosen": -83.60453796386719,
      "logps/rejected": -76.19818115234375,
      "loss": 13.4898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.057194728404283524,
      "rewards/margins": 0.10175208747386932,
      "rewards/rejected": -0.15894684195518494,
      "step": 1491
    },
    {
      "epoch": 0.8665853516872858,
      "grad_norm": 343.27349853515625,
      "learning_rate": 2.8326554328878562e-06,
      "logits/chosen": -1.012703537940979,
      "logits/rejected": -1.0172144174575806,
      "logps/chosen": -73.91706085205078,
      "logps/rejected": -76.37930297851562,
      "loss": 14.4629,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.024712469428777695,
      "rewards/margins": -0.04650706797838211,
      "rewards/rejected": 0.021794596686959267,
      "step": 1492
    },
    {
      "epoch": 0.8671661729685776,
      "grad_norm": 315.9669189453125,
      "learning_rate": 2.831202789076119e-06,
      "logits/chosen": -1.0503861904144287,
      "logits/rejected": -0.9793065786361694,
      "logps/chosen": -77.78846740722656,
      "logps/rejected": -71.60701751708984,
      "loss": 13.431,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.005364026874303818,
      "rewards/margins": 0.054167795926332474,
      "rewards/rejected": -0.05953182652592659,
      "step": 1493
    },
    {
      "epoch": 0.8677469942498693,
      "grad_norm": 302.8532409667969,
      "learning_rate": 2.8297501452643817e-06,
      "logits/chosen": -1.0804574489593506,
      "logits/rejected": -1.0893501043319702,
      "logps/chosen": -66.95835876464844,
      "logps/rejected": -75.90795135498047,
      "loss": 13.0723,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.01469604391604662,
      "rewards/margins": 0.09421535581350327,
      "rewards/rejected": -0.07951931655406952,
      "step": 1494
    },
    {
      "epoch": 0.868327815531161,
      "grad_norm": 292.158203125,
      "learning_rate": 2.828297501452644e-06,
      "logits/chosen": -0.9565943479537964,
      "logits/rejected": -0.9601955413818359,
      "logps/chosen": -69.89134216308594,
      "logps/rejected": -68.9963150024414,
      "loss": 13.7521,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.038061775267124176,
      "rewards/margins": 0.026355329900979996,
      "rewards/rejected": -0.06441710889339447,
      "step": 1495
    },
    {
      "epoch": 0.8689086368124528,
      "grad_norm": 343.9333801269531,
      "learning_rate": 2.826844857640907e-06,
      "logits/chosen": -1.0213762521743774,
      "logits/rejected": -1.0055370330810547,
      "logps/chosen": -83.21332550048828,
      "logps/rejected": -72.47955322265625,
      "loss": 13.6176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014096224680542946,
      "rewards/margins": 0.03957110643386841,
      "rewards/rejected": -0.025474881753325462,
      "step": 1496
    },
    {
      "epoch": 0.8694894580937446,
      "grad_norm": 349.282470703125,
      "learning_rate": 2.8253922138291696e-06,
      "logits/chosen": -1.0412659645080566,
      "logits/rejected": -1.0009477138519287,
      "logps/chosen": -76.58332061767578,
      "logps/rejected": -81.25813293457031,
      "loss": 13.648,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.034825343638658524,
      "rewards/margins": 0.03631298243999481,
      "rewards/rejected": -0.07113832235336304,
      "step": 1497
    },
    {
      "epoch": 0.8700702793750363,
      "grad_norm": 285.526123046875,
      "learning_rate": 2.8239395700174323e-06,
      "logits/chosen": -1.0425231456756592,
      "logits/rejected": -1.000209927558899,
      "logps/chosen": -73.44279479980469,
      "logps/rejected": -68.46315002441406,
      "loss": 13.6916,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.061227064579725266,
      "rewards/margins": 0.029174108058214188,
      "rewards/rejected": -0.09040118008852005,
      "step": 1498
    },
    {
      "epoch": 0.870651100656328,
      "grad_norm": 306.06683349609375,
      "learning_rate": 2.8224869262056946e-06,
      "logits/chosen": -1.0404552221298218,
      "logits/rejected": -1.0073459148406982,
      "logps/chosen": -73.07347106933594,
      "logps/rejected": -74.87454223632812,
      "loss": 13.8479,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07166783511638641,
      "rewards/margins": 0.009706784971058369,
      "rewards/rejected": -0.08137460798025131,
      "step": 1499
    },
    {
      "epoch": 0.8712319219376198,
      "grad_norm": 329.17132568359375,
      "learning_rate": 2.8210342823939574e-06,
      "logits/chosen": -1.040482759475708,
      "logits/rejected": -1.052018165588379,
      "logps/chosen": -73.73755645751953,
      "logps/rejected": -75.89900970458984,
      "loss": 14.0054,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07770119607448578,
      "rewards/margins": 0.0005748653784394264,
      "rewards/rejected": -0.07827606052160263,
      "step": 1500
    },
    {
      "epoch": 0.8718127432189116,
      "grad_norm": 348.2522888183594,
      "learning_rate": 2.81958163858222e-06,
      "logits/chosen": -1.0787187814712524,
      "logits/rejected": -1.0543690919876099,
      "logps/chosen": -79.14573669433594,
      "logps/rejected": -76.54524230957031,
      "loss": 13.9058,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.05225607007741928,
      "rewards/margins": 0.01397562213242054,
      "rewards/rejected": -0.06623169779777527,
      "step": 1501
    },
    {
      "epoch": 0.8723935645002033,
      "grad_norm": 312.89569091796875,
      "learning_rate": 2.8181289947704825e-06,
      "logits/chosen": -0.9827702641487122,
      "logits/rejected": -0.9704543352127075,
      "logps/chosen": -80.55365753173828,
      "logps/rejected": -79.46866607666016,
      "loss": 13.1548,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04625701159238815,
      "rewards/margins": 0.13537849485874176,
      "rewards/rejected": -0.1816355139017105,
      "step": 1502
    },
    {
      "epoch": 0.872974385781495,
      "grad_norm": 283.2939453125,
      "learning_rate": 2.816676350958745e-06,
      "logits/chosen": -1.069556713104248,
      "logits/rejected": -1.1777994632720947,
      "logps/chosen": -73.52742767333984,
      "logps/rejected": -66.3348617553711,
      "loss": 13.0545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0282745398581028,
      "rewards/margins": 0.107033871114254,
      "rewards/rejected": -0.1353084146976471,
      "step": 1503
    },
    {
      "epoch": 0.8735552070627868,
      "grad_norm": 344.55621337890625,
      "learning_rate": 2.815223707147008e-06,
      "logits/chosen": -1.1189879179000854,
      "logits/rejected": -1.147080421447754,
      "logps/chosen": -85.13226318359375,
      "logps/rejected": -75.75519561767578,
      "loss": 14.8115,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.08230282366275787,
      "rewards/margins": -0.080418661236763,
      "rewards/rejected": -0.0018841437995433807,
      "step": 1504
    },
    {
      "epoch": 0.8741360283440786,
      "grad_norm": 328.4111022949219,
      "learning_rate": 2.8137710633352707e-06,
      "logits/chosen": -0.9595583081245422,
      "logits/rejected": -0.9176647067070007,
      "logps/chosen": -76.80940246582031,
      "logps/rejected": -72.98136901855469,
      "loss": 13.9418,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03471698611974716,
      "rewards/margins": 0.0032317906152457,
      "rewards/rejected": -0.03794877976179123,
      "step": 1505
    },
    {
      "epoch": 0.8747168496253702,
      "grad_norm": 327.2943115234375,
      "learning_rate": 2.812318419523533e-06,
      "logits/chosen": -1.0511448383331299,
      "logits/rejected": -1.1165978908538818,
      "logps/chosen": -71.33417510986328,
      "logps/rejected": -81.13162994384766,
      "loss": 13.8795,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09488337486982346,
      "rewards/margins": 0.012004921212792397,
      "rewards/rejected": -0.1068882942199707,
      "step": 1506
    },
    {
      "epoch": 0.875297670906662,
      "grad_norm": 327.3268737792969,
      "learning_rate": 2.8108657757117958e-06,
      "logits/chosen": -1.1129240989685059,
      "logits/rejected": -1.0243065357208252,
      "logps/chosen": -69.58003234863281,
      "logps/rejected": -75.43324279785156,
      "loss": 14.2412,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05431095510721207,
      "rewards/margins": -0.02518748864531517,
      "rewards/rejected": -0.02912346087396145,
      "step": 1507
    },
    {
      "epoch": 0.8758784921879538,
      "grad_norm": 316.2397766113281,
      "learning_rate": 2.8094131319000585e-06,
      "logits/chosen": -1.0628914833068848,
      "logits/rejected": -1.055234432220459,
      "logps/chosen": -76.38618469238281,
      "logps/rejected": -75.1251220703125,
      "loss": 12.1304,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04954692721366882,
      "rewards/margins": 0.1964379847049713,
      "rewards/rejected": -0.1468910425901413,
      "step": 1508
    },
    {
      "epoch": 0.8764593134692455,
      "grad_norm": 325.85516357421875,
      "learning_rate": 2.807960488088321e-06,
      "logits/chosen": -1.1783287525177002,
      "logits/rejected": -1.0815140008926392,
      "logps/chosen": -72.00505065917969,
      "logps/rejected": -71.20240020751953,
      "loss": 13.7431,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.045400962233543396,
      "rewards/margins": 0.023116644471883774,
      "rewards/rejected": -0.06851761043071747,
      "step": 1509
    },
    {
      "epoch": 0.8770401347505372,
      "grad_norm": 329.67681884765625,
      "learning_rate": 2.8065078442765836e-06,
      "logits/chosen": -1.2273672819137573,
      "logits/rejected": -1.2378545999526978,
      "logps/chosen": -76.3592300415039,
      "logps/rejected": -77.40835571289062,
      "loss": 14.0931,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07257591933012009,
      "rewards/margins": -0.0125619787722826,
      "rewards/rejected": -0.06001394987106323,
      "step": 1510
    },
    {
      "epoch": 0.877620956031829,
      "grad_norm": 351.07305908203125,
      "learning_rate": 2.8050552004648463e-06,
      "logits/chosen": -0.9000270962715149,
      "logits/rejected": -0.8484228253364563,
      "logps/chosen": -80.53620910644531,
      "logps/rejected": -83.14128112792969,
      "loss": 14.1951,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07437893003225327,
      "rewards/margins": -0.0010452487040311098,
      "rewards/rejected": -0.07333368062973022,
      "step": 1511
    },
    {
      "epoch": 0.8782017773131208,
      "grad_norm": 309.98126220703125,
      "learning_rate": 2.8036025566531087e-06,
      "logits/chosen": -1.0538280010223389,
      "logits/rejected": -1.109566330909729,
      "logps/chosen": -74.8058853149414,
      "logps/rejected": -79.24540710449219,
      "loss": 14.146,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0626085177063942,
      "rewards/margins": -0.014944806694984436,
      "rewards/rejected": -0.047663696110248566,
      "step": 1512
    },
    {
      "epoch": 0.8787825985944125,
      "grad_norm": 337.3796081542969,
      "learning_rate": 2.8021499128413714e-06,
      "logits/chosen": -0.9413551092147827,
      "logits/rejected": -0.9596421122550964,
      "logps/chosen": -75.41625213623047,
      "logps/rejected": -77.13274383544922,
      "loss": 13.9717,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01829986833035946,
      "rewards/margins": -0.0025507272221148014,
      "rewards/rejected": -0.015749141573905945,
      "step": 1513
    },
    {
      "epoch": 0.8793634198757042,
      "grad_norm": 296.00714111328125,
      "learning_rate": 2.800697269029634e-06,
      "logits/chosen": -1.2827659845352173,
      "logits/rejected": -1.2931110858917236,
      "logps/chosen": -68.38859558105469,
      "logps/rejected": -71.1988754272461,
      "loss": 12.7008,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.026534665375947952,
      "rewards/margins": 0.1583460420370102,
      "rewards/rejected": -0.13181138038635254,
      "step": 1514
    },
    {
      "epoch": 0.879944241156996,
      "grad_norm": 290.5866394042969,
      "learning_rate": 2.799244625217897e-06,
      "logits/chosen": -1.0109509229660034,
      "logits/rejected": -1.0494440793991089,
      "logps/chosen": -68.14818572998047,
      "logps/rejected": -75.16905975341797,
      "loss": 13.0992,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004838766064494848,
      "rewards/margins": 0.11755865812301636,
      "rewards/rejected": -0.11271987110376358,
      "step": 1515
    },
    {
      "epoch": 0.8805250624382878,
      "grad_norm": 313.7828369140625,
      "learning_rate": 2.7977919814061592e-06,
      "logits/chosen": -1.1633681058883667,
      "logits/rejected": -1.117737054824829,
      "logps/chosen": -76.01618957519531,
      "logps/rejected": -83.85478973388672,
      "loss": 13.4189,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.042233385145664215,
      "rewards/margins": 0.056312453001737595,
      "rewards/rejected": -0.09854583442211151,
      "step": 1516
    },
    {
      "epoch": 0.8811058837195794,
      "grad_norm": 358.4498291015625,
      "learning_rate": 2.796339337594422e-06,
      "logits/chosen": -0.9746224284172058,
      "logits/rejected": -1.0602543354034424,
      "logps/chosen": -88.80683135986328,
      "logps/rejected": -70.98249816894531,
      "loss": 12.6696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01634419523179531,
      "rewards/margins": 0.13881227374076843,
      "rewards/rejected": -0.12246807664632797,
      "step": 1517
    },
    {
      "epoch": 0.8816867050008712,
      "grad_norm": 307.6180725097656,
      "learning_rate": 2.7948866937826847e-06,
      "logits/chosen": -1.0281355381011963,
      "logits/rejected": -1.1118850708007812,
      "logps/chosen": -71.7519302368164,
      "logps/rejected": -75.54393768310547,
      "loss": 12.6761,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.057506006211042404,
      "rewards/margins": 0.14585106074810028,
      "rewards/rejected": -0.08834506571292877,
      "step": 1518
    },
    {
      "epoch": 0.882267526282163,
      "grad_norm": 306.2071228027344,
      "learning_rate": 2.793434049970947e-06,
      "logits/chosen": -0.8931095004081726,
      "logits/rejected": -0.8693500757217407,
      "logps/chosen": -73.35682678222656,
      "logps/rejected": -73.05211639404297,
      "loss": 13.6748,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.019168265163898468,
      "rewards/margins": 0.03217936307191849,
      "rewards/rejected": -0.05134762451052666,
      "step": 1519
    },
    {
      "epoch": 0.8828483475634548,
      "grad_norm": 292.9280700683594,
      "learning_rate": 2.79198140615921e-06,
      "logits/chosen": -1.0760751962661743,
      "logits/rejected": -1.1053167581558228,
      "logps/chosen": -76.94102478027344,
      "logps/rejected": -73.51872253417969,
      "loss": 12.8502,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03917863219976425,
      "rewards/margins": 0.11818484961986542,
      "rewards/rejected": -0.07900620996952057,
      "step": 1520
    },
    {
      "epoch": 0.8834291688447464,
      "grad_norm": 333.2906188964844,
      "learning_rate": 2.7905287623474726e-06,
      "logits/chosen": -0.9828292727470398,
      "logits/rejected": -1.0600343942642212,
      "logps/chosen": -73.73872375488281,
      "logps/rejected": -80.23456573486328,
      "loss": 13.5049,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03052309714257717,
      "rewards/margins": 0.05165810510516167,
      "rewards/rejected": -0.08218120038509369,
      "step": 1521
    },
    {
      "epoch": 0.8840099901260382,
      "grad_norm": 340.60162353515625,
      "learning_rate": 2.7890761185357353e-06,
      "logits/chosen": -0.9410535097122192,
      "logits/rejected": -1.07439386844635,
      "logps/chosen": -78.42755126953125,
      "logps/rejected": -82.86685180664062,
      "loss": 13.4182,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04844370484352112,
      "rewards/margins": 0.07036168873310089,
      "rewards/rejected": -0.11880539357662201,
      "step": 1522
    },
    {
      "epoch": 0.88459081140733,
      "grad_norm": 313.5581970214844,
      "learning_rate": 2.7876234747239976e-06,
      "logits/chosen": -0.9727820158004761,
      "logits/rejected": -0.9957553744316101,
      "logps/chosen": -74.45868682861328,
      "logps/rejected": -76.96858978271484,
      "loss": 13.4248,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.004863498732447624,
      "rewards/margins": 0.06017541140317917,
      "rewards/rejected": -0.05531191825866699,
      "step": 1523
    },
    {
      "epoch": 0.8851716326886218,
      "grad_norm": 340.4576416015625,
      "learning_rate": 2.7861708309122604e-06,
      "logits/chosen": -1.1561301946640015,
      "logits/rejected": -1.0398067235946655,
      "logps/chosen": -83.97417449951172,
      "logps/rejected": -73.97357177734375,
      "loss": 14.3515,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.038643233478069305,
      "rewards/margins": -0.03368345648050308,
      "rewards/rejected": -0.00495978444814682,
      "step": 1524
    },
    {
      "epoch": 0.8857524539699134,
      "grad_norm": 332.86517333984375,
      "learning_rate": 2.784718187100523e-06,
      "logits/chosen": -0.8696984052658081,
      "logits/rejected": -1.0832719802856445,
      "logps/chosen": -77.92228698730469,
      "logps/rejected": -73.15290832519531,
      "loss": 13.4269,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07254264503717422,
      "rewards/margins": 0.1103074923157692,
      "rewards/rejected": -0.18285015225410461,
      "step": 1525
    },
    {
      "epoch": 0.8863332752512052,
      "grad_norm": 304.9974365234375,
      "learning_rate": 2.7832655432887854e-06,
      "logits/chosen": -0.852143406867981,
      "logits/rejected": -0.8875290155410767,
      "logps/chosen": -70.97572326660156,
      "logps/rejected": -73.21073150634766,
      "loss": 12.9774,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008584086783230305,
      "rewards/margins": 0.10297232866287231,
      "rewards/rejected": -0.09438825398683548,
      "step": 1526
    },
    {
      "epoch": 0.886914096532497,
      "grad_norm": 327.7203369140625,
      "learning_rate": 2.781812899477048e-06,
      "logits/chosen": -1.0457631349563599,
      "logits/rejected": -1.0966553688049316,
      "logps/chosen": -76.08070373535156,
      "logps/rejected": -71.44209289550781,
      "loss": 14.2675,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07447735965251923,
      "rewards/margins": -0.02885119616985321,
      "rewards/rejected": -0.045626163482666016,
      "step": 1527
    },
    {
      "epoch": 0.8874949178137886,
      "grad_norm": 335.9598388671875,
      "learning_rate": 2.780360255665311e-06,
      "logits/chosen": -1.1053820848464966,
      "logits/rejected": -1.1497929096221924,
      "logps/chosen": -76.91766357421875,
      "logps/rejected": -76.04541778564453,
      "loss": 13.3454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06218590587377548,
      "rewards/margins": 0.06706562638282776,
      "rewards/rejected": -0.12925153970718384,
      "step": 1528
    },
    {
      "epoch": 0.8880757390950804,
      "grad_norm": 466.4450378417969,
      "learning_rate": 2.7789076118535737e-06,
      "logits/chosen": -1.1102619171142578,
      "logits/rejected": -1.0646207332611084,
      "logps/chosen": -68.73627471923828,
      "logps/rejected": -76.07550048828125,
      "loss": 13.3774,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.022596340626478195,
      "rewards/margins": 0.06255432218313217,
      "rewards/rejected": -0.03995798900723457,
      "step": 1529
    },
    {
      "epoch": 0.8886565603763722,
      "grad_norm": 317.5010070800781,
      "learning_rate": 2.777454968041836e-06,
      "logits/chosen": -1.0609385967254639,
      "logits/rejected": -1.048516035079956,
      "logps/chosen": -75.3376693725586,
      "logps/rejected": -72.77949523925781,
      "loss": 13.8612,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02821994759142399,
      "rewards/margins": 0.02019057795405388,
      "rewards/rejected": -0.04841053485870361,
      "step": 1530
    },
    {
      "epoch": 0.889237381657664,
      "grad_norm": 319.053955078125,
      "learning_rate": 2.7760023242300988e-06,
      "logits/chosen": -1.0589898824691772,
      "logits/rejected": -1.110540509223938,
      "logps/chosen": -70.92110443115234,
      "logps/rejected": -74.95904541015625,
      "loss": 14.0485,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06374859809875488,
      "rewards/margins": -0.001369261764921248,
      "rewards/rejected": -0.06237933784723282,
      "step": 1531
    },
    {
      "epoch": 0.8898182029389556,
      "grad_norm": 317.98980712890625,
      "learning_rate": 2.7745496804183615e-06,
      "logits/chosen": -0.9375941157341003,
      "logits/rejected": -0.8811028599739075,
      "logps/chosen": -74.5658950805664,
      "logps/rejected": -68.92941284179688,
      "loss": 14.5105,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.07548018544912338,
      "rewards/margins": -0.05086630582809448,
      "rewards/rejected": -0.0246138758957386,
      "step": 1532
    },
    {
      "epoch": 0.8903990242202474,
      "grad_norm": 303.7709045410156,
      "learning_rate": 2.7730970366066247e-06,
      "logits/chosen": -1.0005435943603516,
      "logits/rejected": -0.9476556777954102,
      "logps/chosen": -71.90227508544922,
      "logps/rejected": -73.4381103515625,
      "loss": 13.7999,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03456462174654007,
      "rewards/margins": 0.013041317462921143,
      "rewards/rejected": -0.04760593920946121,
      "step": 1533
    },
    {
      "epoch": 0.8909798455015392,
      "grad_norm": 305.30987548828125,
      "learning_rate": 2.771644392794887e-06,
      "logits/chosen": -1.0671470165252686,
      "logits/rejected": -1.0722938776016235,
      "logps/chosen": -71.49290466308594,
      "logps/rejected": -74.4961166381836,
      "loss": 13.8259,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.126017227768898,
      "rewards/margins": 0.020876895636320114,
      "rewards/rejected": -0.14689412713050842,
      "step": 1534
    },
    {
      "epoch": 0.891560666782831,
      "grad_norm": 327.72906494140625,
      "learning_rate": 2.7701917489831498e-06,
      "logits/chosen": -1.0457624197006226,
      "logits/rejected": -1.1075061559677124,
      "logps/chosen": -74.98625183105469,
      "logps/rejected": -75.95701599121094,
      "loss": 14.385,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0718541294336319,
      "rewards/margins": -0.035516977310180664,
      "rewards/rejected": -0.036337144672870636,
      "step": 1535
    },
    {
      "epoch": 0.8921414880641226,
      "grad_norm": 336.9986877441406,
      "learning_rate": 2.7687391051714125e-06,
      "logits/chosen": -1.0766903162002563,
      "logits/rejected": -1.0336723327636719,
      "logps/chosen": -80.77274322509766,
      "logps/rejected": -88.4638442993164,
      "loss": 12.9422,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03478812426328659,
      "rewards/margins": 0.1052180752158165,
      "rewards/rejected": -0.14000621438026428,
      "step": 1536
    },
    {
      "epoch": 0.8927223093454144,
      "grad_norm": 306.76885986328125,
      "learning_rate": 2.7672864613596753e-06,
      "logits/chosen": -1.1064250469207764,
      "logits/rejected": -1.1780726909637451,
      "logps/chosen": -76.44205474853516,
      "logps/rejected": -77.57832336425781,
      "loss": 13.4555,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09355898946523666,
      "rewards/margins": 0.05021384358406067,
      "rewards/rejected": -0.14377282559871674,
      "step": 1537
    },
    {
      "epoch": 0.8933031306267062,
      "grad_norm": 308.50164794921875,
      "learning_rate": 2.7658338175479376e-06,
      "logits/chosen": -1.1411300897598267,
      "logits/rejected": -1.088274359703064,
      "logps/chosen": -74.40532684326172,
      "logps/rejected": -68.87134552001953,
      "loss": 13.8145,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03937583044171333,
      "rewards/margins": 0.01634802110493183,
      "rewards/rejected": -0.055723853409290314,
      "step": 1538
    },
    {
      "epoch": 0.893883951907998,
      "grad_norm": 329.44708251953125,
      "learning_rate": 2.7643811737362003e-06,
      "logits/chosen": -0.8624466061592102,
      "logits/rejected": -0.9080654978752136,
      "logps/chosen": -73.55668640136719,
      "logps/rejected": -73.4706802368164,
      "loss": 14.1385,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07051645219326019,
      "rewards/margins": -0.019844913855195045,
      "rewards/rejected": -0.05067153647542,
      "step": 1539
    },
    {
      "epoch": 0.8944647731892896,
      "grad_norm": 307.167236328125,
      "learning_rate": 2.762928529924463e-06,
      "logits/chosen": -0.9339910745620728,
      "logits/rejected": -0.9897225499153137,
      "logps/chosen": -67.60140991210938,
      "logps/rejected": -70.32308959960938,
      "loss": 14.2207,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03770753741264343,
      "rewards/margins": -0.02606254816055298,
      "rewards/rejected": -0.011644983664155006,
      "step": 1540
    },
    {
      "epoch": 0.8950455944705814,
      "grad_norm": 309.9949951171875,
      "learning_rate": 2.7614758861127254e-06,
      "logits/chosen": -1.2669035196304321,
      "logits/rejected": -1.2563022375106812,
      "logps/chosen": -70.59803009033203,
      "logps/rejected": -74.32190704345703,
      "loss": 12.8892,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0034139647614210844,
      "rewards/margins": 0.11826606094837189,
      "rewards/rejected": -0.12168002128601074,
      "step": 1541
    },
    {
      "epoch": 0.8956264157518732,
      "grad_norm": 376.1979064941406,
      "learning_rate": 2.760023242300988e-06,
      "logits/chosen": -1.109890103340149,
      "logits/rejected": -1.1250545978546143,
      "logps/chosen": -75.4137191772461,
      "logps/rejected": -86.34205627441406,
      "loss": 12.9027,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029204901307821274,
      "rewards/margins": 0.1107078343629837,
      "rewards/rejected": -0.13991273939609528,
      "step": 1542
    },
    {
      "epoch": 0.8962072370331648,
      "grad_norm": 320.7944641113281,
      "learning_rate": 2.758570598489251e-06,
      "logits/chosen": -1.0295648574829102,
      "logits/rejected": -0.9855254292488098,
      "logps/chosen": -76.03120422363281,
      "logps/rejected": -84.80982971191406,
      "loss": 14.1721,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.08011921495199203,
      "rewards/margins": -0.014234116300940514,
      "rewards/rejected": -0.06588510423898697,
      "step": 1543
    },
    {
      "epoch": 0.8967880583144566,
      "grad_norm": 321.6352844238281,
      "learning_rate": 2.7571179546775132e-06,
      "logits/chosen": -1.0978248119354248,
      "logits/rejected": -1.0874483585357666,
      "logps/chosen": -67.959716796875,
      "logps/rejected": -72.90054321289062,
      "loss": 12.7312,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.020734911784529686,
      "rewards/margins": 0.18873177468776703,
      "rewards/rejected": -0.1679968684911728,
      "step": 1544
    },
    {
      "epoch": 0.8973688795957484,
      "grad_norm": 289.0782470703125,
      "learning_rate": 2.755665310865776e-06,
      "logits/chosen": -1.039381742477417,
      "logits/rejected": -1.1132748126983643,
      "logps/chosen": -65.71759796142578,
      "logps/rejected": -71.22647857666016,
      "loss": 12.6239,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.028374116867780685,
      "rewards/margins": 0.13511744141578674,
      "rewards/rejected": -0.16349157691001892,
      "step": 1545
    },
    {
      "epoch": 0.8979497008770402,
      "grad_norm": 311.4884948730469,
      "learning_rate": 2.7542126670540387e-06,
      "logits/chosen": -1.0354833602905273,
      "logits/rejected": -1.056183099746704,
      "logps/chosen": -76.19552612304688,
      "logps/rejected": -65.21043395996094,
      "loss": 13.8962,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0006060108426027,
      "rewards/margins": 0.01374263595789671,
      "rewards/rejected": -0.013136625289916992,
      "step": 1546
    },
    {
      "epoch": 0.8985305221583318,
      "grad_norm": 332.51678466796875,
      "learning_rate": 2.7527600232423015e-06,
      "logits/chosen": -1.043365240097046,
      "logits/rejected": -1.0468759536743164,
      "logps/chosen": -80.89611053466797,
      "logps/rejected": -77.58426666259766,
      "loss": 13.895,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07452279329299927,
      "rewards/margins": 0.01256744284182787,
      "rewards/rejected": -0.08709023892879486,
      "step": 1547
    },
    {
      "epoch": 0.8991113434396236,
      "grad_norm": 303.91790771484375,
      "learning_rate": 2.751307379430564e-06,
      "logits/chosen": -1.1453994512557983,
      "logits/rejected": -1.1467690467834473,
      "logps/chosen": -72.29669189453125,
      "logps/rejected": -71.61175537109375,
      "loss": 13.0957,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03583524003624916,
      "rewards/margins": 0.09328620135784149,
      "rewards/rejected": -0.05745095759630203,
      "step": 1548
    },
    {
      "epoch": 0.8996921647209154,
      "grad_norm": 288.9295654296875,
      "learning_rate": 2.7498547356188265e-06,
      "logits/chosen": -1.0072671175003052,
      "logits/rejected": -0.9869436025619507,
      "logps/chosen": -70.14595794677734,
      "logps/rejected": -75.24882507324219,
      "loss": 13.1812,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0006589032709598541,
      "rewards/margins": 0.09968983381986618,
      "rewards/rejected": -0.09903092682361603,
      "step": 1549
    },
    {
      "epoch": 0.9002729860022072,
      "grad_norm": 338.63238525390625,
      "learning_rate": 2.7484020918070893e-06,
      "logits/chosen": -0.9667510986328125,
      "logits/rejected": -0.7641538381576538,
      "logps/chosen": -72.39374542236328,
      "logps/rejected": -80.44822692871094,
      "loss": 14.4049,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.10608190298080444,
      "rewards/margins": -0.03980861231684685,
      "rewards/rejected": -0.0662732869386673,
      "step": 1550
    },
    {
      "epoch": 0.9008538072834988,
      "grad_norm": 851.591552734375,
      "learning_rate": 2.7469494479953516e-06,
      "logits/chosen": -1.0545737743377686,
      "logits/rejected": -0.9808444976806641,
      "logps/chosen": -76.17280578613281,
      "logps/rejected": -80.02181243896484,
      "loss": 13.5107,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003184771630913019,
      "rewards/margins": 0.05705474689602852,
      "rewards/rejected": -0.053869981318712234,
      "step": 1551
    },
    {
      "epoch": 0.9014346285647906,
      "grad_norm": 281.4333190917969,
      "learning_rate": 2.7454968041836144e-06,
      "logits/chosen": -0.9128431081771851,
      "logits/rejected": -0.9650856852531433,
      "logps/chosen": -72.76704406738281,
      "logps/rejected": -73.77500915527344,
      "loss": 12.4123,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06197698041796684,
      "rewards/margins": 0.16426773369312286,
      "rewards/rejected": -0.10229077190160751,
      "step": 1552
    },
    {
      "epoch": 0.9020154498460824,
      "grad_norm": 328.09417724609375,
      "learning_rate": 2.744044160371877e-06,
      "logits/chosen": -0.8784101605415344,
      "logits/rejected": -0.962557315826416,
      "logps/chosen": -73.94979858398438,
      "logps/rejected": -79.94593811035156,
      "loss": 13.7407,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.026542458683252335,
      "rewards/margins": 0.028787728399038315,
      "rewards/rejected": -0.002245277166366577,
      "step": 1553
    },
    {
      "epoch": 0.9025962711273741,
      "grad_norm": 317.3074645996094,
      "learning_rate": 2.74259151656014e-06,
      "logits/chosen": -0.974891185760498,
      "logits/rejected": -0.9492782354354858,
      "logps/chosen": -71.26887512207031,
      "logps/rejected": -69.10708618164062,
      "loss": 14.3659,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0672605037689209,
      "rewards/margins": -0.027857232838869095,
      "rewards/rejected": -0.0394032783806324,
      "step": 1554
    },
    {
      "epoch": 0.9031770924086658,
      "grad_norm": 320.19232177734375,
      "learning_rate": 2.741138872748402e-06,
      "logits/chosen": -1.0460035800933838,
      "logits/rejected": -1.0370324850082397,
      "logps/chosen": -71.09989166259766,
      "logps/rejected": -73.34397888183594,
      "loss": 14.4139,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0712951272726059,
      "rewards/margins": -0.039732128381729126,
      "rewards/rejected": -0.03156300634145737,
      "step": 1555
    },
    {
      "epoch": 0.9037579136899576,
      "grad_norm": 304.53253173828125,
      "learning_rate": 2.739686228936665e-06,
      "logits/chosen": -0.8934981226921082,
      "logits/rejected": -0.9283573031425476,
      "logps/chosen": -70.9642333984375,
      "logps/rejected": -70.85687255859375,
      "loss": 13.3816,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.006696996279060841,
      "rewards/margins": 0.06989070028066635,
      "rewards/rejected": -0.07658769190311432,
      "step": 1556
    },
    {
      "epoch": 0.9043387349712494,
      "grad_norm": 318.5834655761719,
      "learning_rate": 2.7382335851249277e-06,
      "logits/chosen": -1.1270331144332886,
      "logits/rejected": -1.241042137145996,
      "logps/chosen": -79.31059265136719,
      "logps/rejected": -67.57523345947266,
      "loss": 14.1368,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07185395807027817,
      "rewards/margins": -0.0023391663562506437,
      "rewards/rejected": -0.06951479613780975,
      "step": 1557
    },
    {
      "epoch": 0.9049195562525411,
      "grad_norm": 316.7093811035156,
      "learning_rate": 2.73678094131319e-06,
      "logits/chosen": -0.8947698473930359,
      "logits/rejected": -0.9548945426940918,
      "logps/chosen": -74.25798797607422,
      "logps/rejected": -75.74564361572266,
      "loss": 12.2241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008803398348391056,
      "rewards/margins": 0.22689393162727356,
      "rewards/rejected": -0.21809053421020508,
      "step": 1558
    },
    {
      "epoch": 0.9055003775338328,
      "grad_norm": 301.3543701171875,
      "learning_rate": 2.7353282975014528e-06,
      "logits/chosen": -0.8594738245010376,
      "logits/rejected": -0.7915431261062622,
      "logps/chosen": -73.03811645507812,
      "logps/rejected": -81.76316833496094,
      "loss": 13.9018,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.052253615111112595,
      "rewards/margins": 0.01153495442122221,
      "rewards/rejected": -0.06378856301307678,
      "step": 1559
    },
    {
      "epoch": 0.9060811988151246,
      "grad_norm": 327.1970520019531,
      "learning_rate": 2.7338756536897155e-06,
      "logits/chosen": -1.1117613315582275,
      "logits/rejected": -1.0234920978546143,
      "logps/chosen": -70.29054260253906,
      "logps/rejected": -67.37655639648438,
      "loss": 14.3895,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.02677222527563572,
      "rewards/margins": -0.037412893027067184,
      "rewards/rejected": 0.010640670545399189,
      "step": 1560
    },
    {
      "epoch": 0.9066620200964164,
      "grad_norm": 356.2079162597656,
      "learning_rate": 2.732423009877978e-06,
      "logits/chosen": -1.1316090822219849,
      "logits/rejected": -1.1205003261566162,
      "logps/chosen": -75.7646713256836,
      "logps/rejected": -74.67683410644531,
      "loss": 14.284,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.020529666915535927,
      "rewards/margins": -0.031136516481637955,
      "rewards/rejected": 0.010606837458908558,
      "step": 1561
    },
    {
      "epoch": 0.907242841377708,
      "grad_norm": 326.6283874511719,
      "learning_rate": 2.7309703660662406e-06,
      "logits/chosen": -1.1408169269561768,
      "logits/rejected": -1.1834619045257568,
      "logps/chosen": -77.70711517333984,
      "logps/rejected": -70.83873748779297,
      "loss": 12.8239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007504454348236322,
      "rewards/margins": 0.11551402509212494,
      "rewards/rejected": -0.12301848828792572,
      "step": 1562
    },
    {
      "epoch": 0.9078236626589998,
      "grad_norm": 309.9686584472656,
      "learning_rate": 2.7295177222545033e-06,
      "logits/chosen": -0.9643760919570923,
      "logits/rejected": -0.9090847969055176,
      "logps/chosen": -74.62329864501953,
      "logps/rejected": -76.15159606933594,
      "loss": 14.0489,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10272586345672607,
      "rewards/margins": 0.008718031458556652,
      "rewards/rejected": -0.11144387722015381,
      "step": 1563
    },
    {
      "epoch": 0.9084044839402916,
      "grad_norm": 426.8013610839844,
      "learning_rate": 2.728065078442766e-06,
      "logits/chosen": -0.9545722007751465,
      "logits/rejected": -0.9323128461837769,
      "logps/chosen": -78.92039489746094,
      "logps/rejected": -72.99091339111328,
      "loss": 14.1113,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04323782026767731,
      "rewards/margins": -0.013293745927512646,
      "rewards/rejected": -0.029944077134132385,
      "step": 1564
    },
    {
      "epoch": 0.9089853052215833,
      "grad_norm": 393.0836486816406,
      "learning_rate": 2.7266124346310284e-06,
      "logits/chosen": -1.1172763109207153,
      "logits/rejected": -1.112242579460144,
      "logps/chosen": -80.32032775878906,
      "logps/rejected": -78.71788024902344,
      "loss": 14.0178,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.020738277584314346,
      "rewards/margins": 0.0024770349264144897,
      "rewards/rejected": -0.023215312510728836,
      "step": 1565
    },
    {
      "epoch": 0.909566126502875,
      "grad_norm": 311.92926025390625,
      "learning_rate": 2.725159790819291e-06,
      "logits/chosen": -1.1338139772415161,
      "logits/rejected": -0.9636766314506531,
      "logps/chosen": -77.21595001220703,
      "logps/rejected": -74.08401489257812,
      "loss": 13.7945,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00015135519788600504,
      "rewards/margins": 0.0294949971139431,
      "rewards/rejected": -0.029646355658769608,
      "step": 1566
    },
    {
      "epoch": 0.9101469477841668,
      "grad_norm": 309.5121154785156,
      "learning_rate": 2.723707147007554e-06,
      "logits/chosen": -1.0614473819732666,
      "logits/rejected": -1.0921483039855957,
      "logps/chosen": -62.976707458496094,
      "logps/rejected": -70.13724517822266,
      "loss": 13.7547,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04165538772940636,
      "rewards/margins": 0.022319577634334564,
      "rewards/rejected": -0.06397496908903122,
      "step": 1567
    },
    {
      "epoch": 0.9107277690654586,
      "grad_norm": 328.0290222167969,
      "learning_rate": 2.7222545031958162e-06,
      "logits/chosen": -1.06927490234375,
      "logits/rejected": -1.007405400276184,
      "logps/chosen": -74.2555160522461,
      "logps/rejected": -75.97761535644531,
      "loss": 13.5073,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.031740374863147736,
      "rewards/margins": 0.049203962087631226,
      "rewards/rejected": -0.08094432950019836,
      "step": 1568
    },
    {
      "epoch": 0.9113085903467503,
      "grad_norm": 350.5167541503906,
      "learning_rate": 2.720801859384079e-06,
      "logits/chosen": -0.9914863705635071,
      "logits/rejected": -1.0328764915466309,
      "logps/chosen": -87.16015625,
      "logps/rejected": -75.11557006835938,
      "loss": 14.5459,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1052887886762619,
      "rewards/margins": -0.05007118731737137,
      "rewards/rejected": -0.05521758645772934,
      "step": 1569
    },
    {
      "epoch": 0.911889411628042,
      "grad_norm": 341.8493347167969,
      "learning_rate": 2.7193492155723417e-06,
      "logits/chosen": -0.9765124320983887,
      "logits/rejected": -1.0000501871109009,
      "logps/chosen": -84.66938781738281,
      "logps/rejected": -83.18614959716797,
      "loss": 13.694,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02267245389521122,
      "rewards/margins": 0.029958084225654602,
      "rewards/rejected": -0.05263054370880127,
      "step": 1570
    },
    {
      "epoch": 0.9124702329093338,
      "grad_norm": 349.04559326171875,
      "learning_rate": 2.7178965717606045e-06,
      "logits/chosen": -0.9269010424613953,
      "logits/rejected": -0.8979067802429199,
      "logps/chosen": -91.25633239746094,
      "logps/rejected": -73.48929595947266,
      "loss": 14.2045,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04949615150690079,
      "rewards/margins": -0.019208669662475586,
      "rewards/rejected": -0.03028748370707035,
      "step": 1571
    },
    {
      "epoch": 0.9130510541906256,
      "grad_norm": 325.1496887207031,
      "learning_rate": 2.716443927948867e-06,
      "logits/chosen": -1.224446415901184,
      "logits/rejected": -1.0582832098007202,
      "logps/chosen": -80.11524963378906,
      "logps/rejected": -70.09358215332031,
      "loss": 14.6293,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08207925409078598,
      "rewards/margins": -0.05909162759780884,
      "rewards/rejected": -0.022987637668848038,
      "step": 1572
    },
    {
      "epoch": 0.9136318754719173,
      "grad_norm": 304.30810546875,
      "learning_rate": 2.7149912841371295e-06,
      "logits/chosen": -0.8877927660942078,
      "logits/rejected": -0.8836013674736023,
      "logps/chosen": -76.41388702392578,
      "logps/rejected": -82.39083862304688,
      "loss": 13.1625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01563407853245735,
      "rewards/margins": 0.08687739074230194,
      "rewards/rejected": -0.07124332338571548,
      "step": 1573
    },
    {
      "epoch": 0.914212696753209,
      "grad_norm": 308.4176025390625,
      "learning_rate": 2.7135386403253923e-06,
      "logits/chosen": -0.9067532420158386,
      "logits/rejected": -0.9132000207901001,
      "logps/chosen": -77.9535903930664,
      "logps/rejected": -71.26287841796875,
      "loss": 13.5629,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011409558355808258,
      "rewards/margins": 0.04753192141652107,
      "rewards/rejected": -0.058941490948200226,
      "step": 1574
    },
    {
      "epoch": 0.9147935180345008,
      "grad_norm": 298.0390319824219,
      "learning_rate": 2.7120859965136555e-06,
      "logits/chosen": -1.0085927248001099,
      "logits/rejected": -0.9561688303947449,
      "logps/chosen": -72.54247283935547,
      "logps/rejected": -69.73148345947266,
      "loss": 13.3398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03178802877664566,
      "rewards/margins": 0.06776704639196396,
      "rewards/rejected": -0.09955506771802902,
      "step": 1575
    },
    {
      "epoch": 0.9153743393157925,
      "grad_norm": 320.9405212402344,
      "learning_rate": 2.7106333527019178e-06,
      "logits/chosen": -0.8656220436096191,
      "logits/rejected": -0.9310104250907898,
      "logps/chosen": -69.6420669555664,
      "logps/rejected": -73.74166870117188,
      "loss": 13.1833,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.025753051042556763,
      "rewards/margins": 0.08832113444805145,
      "rewards/rejected": -0.06256809085607529,
      "step": 1576
    },
    {
      "epoch": 0.9159551605970843,
      "grad_norm": 312.3841552734375,
      "learning_rate": 2.7091807088901805e-06,
      "logits/chosen": -1.0169761180877686,
      "logits/rejected": -1.0056474208831787,
      "logps/chosen": -77.76615142822266,
      "logps/rejected": -70.25587463378906,
      "loss": 14.0031,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08563989400863647,
      "rewards/margins": 0.007304549217224121,
      "rewards/rejected": -0.0929444432258606,
      "step": 1577
    },
    {
      "epoch": 0.916535981878376,
      "grad_norm": 300.8182373046875,
      "learning_rate": 2.7077280650784433e-06,
      "logits/chosen": -1.0372730493545532,
      "logits/rejected": -1.0177842378616333,
      "logps/chosen": -72.30513000488281,
      "logps/rejected": -83.52772521972656,
      "loss": 13.0234,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.018783967941999435,
      "rewards/margins": 0.10926733165979385,
      "rewards/rejected": -0.09048337489366531,
      "step": 1578
    },
    {
      "epoch": 0.9171168031596678,
      "grad_norm": 320.063232421875,
      "learning_rate": 2.706275421266706e-06,
      "logits/chosen": -1.0391422510147095,
      "logits/rejected": -1.1998745203018188,
      "logps/chosen": -78.41818237304688,
      "logps/rejected": -76.33103942871094,
      "loss": 13.2447,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03698853403329849,
      "rewards/margins": 0.08352704346179962,
      "rewards/rejected": -0.12051556259393692,
      "step": 1579
    },
    {
      "epoch": 0.9176976244409595,
      "grad_norm": 309.7467346191406,
      "learning_rate": 2.7048227774549684e-06,
      "logits/chosen": -0.8094549179077148,
      "logits/rejected": -0.9255477786064148,
      "logps/chosen": -67.97891998291016,
      "logps/rejected": -76.82670593261719,
      "loss": 12.6973,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.009428349323570728,
      "rewards/margins": 0.14061811566352844,
      "rewards/rejected": -0.13118976354599,
      "step": 1580
    },
    {
      "epoch": 0.9182784457222513,
      "grad_norm": 323.81304931640625,
      "learning_rate": 2.703370133643231e-06,
      "logits/chosen": -1.1210846900939941,
      "logits/rejected": -1.122443437576294,
      "logps/chosen": -76.90458679199219,
      "logps/rejected": -75.1322021484375,
      "loss": 13.2782,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027773786336183548,
      "rewards/margins": 0.06942864507436752,
      "rewards/rejected": -0.041654862463474274,
      "step": 1581
    },
    {
      "epoch": 0.918859267003543,
      "grad_norm": 359.33868408203125,
      "learning_rate": 2.701917489831494e-06,
      "logits/chosen": -0.9813796877861023,
      "logits/rejected": -0.8498438596725464,
      "logps/chosen": -82.3498764038086,
      "logps/rejected": -69.68416595458984,
      "loss": 14.7191,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0988243967294693,
      "rewards/margins": -0.061216533184051514,
      "rewards/rejected": -0.03760785609483719,
      "step": 1582
    },
    {
      "epoch": 0.9194400882848348,
      "grad_norm": 304.72637939453125,
      "learning_rate": 2.700464846019756e-06,
      "logits/chosen": -0.9884494543075562,
      "logits/rejected": -0.9771407842636108,
      "logps/chosen": -71.66685485839844,
      "logps/rejected": -71.19456481933594,
      "loss": 13.7853,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04275290668010712,
      "rewards/margins": 0.02688916027545929,
      "rewards/rejected": -0.0696420669555664,
      "step": 1583
    },
    {
      "epoch": 0.9200209095661265,
      "grad_norm": 318.5897521972656,
      "learning_rate": 2.699012202208019e-06,
      "logits/chosen": -1.0953223705291748,
      "logits/rejected": -1.0901046991348267,
      "logps/chosen": -73.74061584472656,
      "logps/rejected": -68.30780792236328,
      "loss": 14.2367,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.024284830316901207,
      "rewards/margins": -0.019015511497855186,
      "rewards/rejected": -0.0052693248726427555,
      "step": 1584
    },
    {
      "epoch": 0.9206017308474183,
      "grad_norm": 349.8983154296875,
      "learning_rate": 2.6975595583962817e-06,
      "logits/chosen": -0.9590209722518921,
      "logits/rejected": -0.9498333930969238,
      "logps/chosen": -76.97708129882812,
      "logps/rejected": -71.45216369628906,
      "loss": 14.1988,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.015870697796344757,
      "rewards/margins": -0.02084464393556118,
      "rewards/rejected": 0.004973948001861572,
      "step": 1585
    },
    {
      "epoch": 0.92118255212871,
      "grad_norm": 294.58349609375,
      "learning_rate": 2.6961069145845444e-06,
      "logits/chosen": -1.043100357055664,
      "logits/rejected": -1.083487868309021,
      "logps/chosen": -67.35298919677734,
      "logps/rejected": -72.49873352050781,
      "loss": 13.4668,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04845147579908371,
      "rewards/margins": 0.04783755540847778,
      "rewards/rejected": 0.0006139159086160362,
      "step": 1586
    },
    {
      "epoch": 0.9217633734100017,
      "grad_norm": 300.7452087402344,
      "learning_rate": 2.6946542707728067e-06,
      "logits/chosen": -0.9788883328437805,
      "logits/rejected": -1.011438250541687,
      "logps/chosen": -73.98201751708984,
      "logps/rejected": -71.40184783935547,
      "loss": 13.3078,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03575678542256355,
      "rewards/margins": 0.07184077799320221,
      "rewards/rejected": -0.10759756714105606,
      "step": 1587
    },
    {
      "epoch": 0.9223441946912935,
      "grad_norm": 308.70892333984375,
      "learning_rate": 2.6932016269610695e-06,
      "logits/chosen": -1.1238864660263062,
      "logits/rejected": -1.0784916877746582,
      "logps/chosen": -72.8819808959961,
      "logps/rejected": -80.44390869140625,
      "loss": 12.9709,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.018401149660348892,
      "rewards/margins": 0.10786961019039154,
      "rewards/rejected": -0.12627077102661133,
      "step": 1588
    },
    {
      "epoch": 0.9229250159725852,
      "grad_norm": 311.8727111816406,
      "learning_rate": 2.6917489831493322e-06,
      "logits/chosen": -0.942405104637146,
      "logits/rejected": -1.0860522985458374,
      "logps/chosen": -78.456787109375,
      "logps/rejected": -65.52259826660156,
      "loss": 13.6529,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.047195903956890106,
      "rewards/margins": 0.054706789553165436,
      "rewards/rejected": -0.10190270096063614,
      "step": 1589
    },
    {
      "epoch": 0.923505837253877,
      "grad_norm": 312.83709716796875,
      "learning_rate": 2.6902963393375946e-06,
      "logits/chosen": -0.9841133952140808,
      "logits/rejected": -1.0482124090194702,
      "logps/chosen": -71.87995910644531,
      "logps/rejected": -71.43251037597656,
      "loss": 13.8283,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03615868464112282,
      "rewards/margins": 0.017513807862997055,
      "rewards/rejected": -0.05367249250411987,
      "step": 1590
    },
    {
      "epoch": 0.9240866585351687,
      "grad_norm": 292.1714782714844,
      "learning_rate": 2.6888436955258573e-06,
      "logits/chosen": -0.9446626901626587,
      "logits/rejected": -0.987004280090332,
      "logps/chosen": -80.11546325683594,
      "logps/rejected": -74.48158264160156,
      "loss": 12.7609,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.005090351216495037,
      "rewards/margins": 0.13897670805454254,
      "rewards/rejected": -0.13388636708259583,
      "step": 1591
    },
    {
      "epoch": 0.9246674798164605,
      "grad_norm": 316.179443359375,
      "learning_rate": 2.68739105171412e-06,
      "logits/chosen": -0.8669527769088745,
      "logits/rejected": -0.8660700917243958,
      "logps/chosen": -76.60184478759766,
      "logps/rejected": -72.5396728515625,
      "loss": 13.8222,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07705489546060562,
      "rewards/margins": 0.029973501339554787,
      "rewards/rejected": -0.10702840238809586,
      "step": 1592
    },
    {
      "epoch": 0.9252483010977522,
      "grad_norm": 359.2261962890625,
      "learning_rate": 2.6859384079023824e-06,
      "logits/chosen": -1.035829782485962,
      "logits/rejected": -1.0491011142730713,
      "logps/chosen": -76.83370208740234,
      "logps/rejected": -65.81129455566406,
      "loss": 14.1763,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.023300159722566605,
      "rewards/margins": -0.013147483579814434,
      "rewards/rejected": -0.010152682662010193,
      "step": 1593
    },
    {
      "epoch": 0.925829122379044,
      "grad_norm": 330.5743713378906,
      "learning_rate": 2.684485764090645e-06,
      "logits/chosen": -1.0941197872161865,
      "logits/rejected": -1.1699881553649902,
      "logps/chosen": -83.11845397949219,
      "logps/rejected": -79.29049682617188,
      "loss": 14.2467,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08977533876895905,
      "rewards/margins": -0.02421572431921959,
      "rewards/rejected": -0.06555961072444916,
      "step": 1594
    },
    {
      "epoch": 0.9264099436603357,
      "grad_norm": 331.95660400390625,
      "learning_rate": 2.683033120278908e-06,
      "logits/chosen": -1.0440118312835693,
      "logits/rejected": -1.0916916131973267,
      "logps/chosen": -73.2425308227539,
      "logps/rejected": -84.84503173828125,
      "loss": 13.1941,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04397767782211304,
      "rewards/margins": 0.0771600753068924,
      "rewards/rejected": -0.12113772332668304,
      "step": 1595
    },
    {
      "epoch": 0.9269907649416275,
      "grad_norm": 319.1327819824219,
      "learning_rate": 2.6815804764671706e-06,
      "logits/chosen": -1.0137518644332886,
      "logits/rejected": -1.0327105522155762,
      "logps/chosen": -80.38282775878906,
      "logps/rejected": -77.029541015625,
      "loss": 13.0025,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.035895176231861115,
      "rewards/margins": 0.10854997485876083,
      "rewards/rejected": -0.14444515109062195,
      "step": 1596
    },
    {
      "epoch": 0.9275715862229192,
      "grad_norm": 300.7707214355469,
      "learning_rate": 2.680127832655433e-06,
      "logits/chosen": -1.1969159841537476,
      "logits/rejected": -1.2382760047912598,
      "logps/chosen": -70.50984191894531,
      "logps/rejected": -75.9458236694336,
      "loss": 12.5108,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.003427135292440653,
      "rewards/margins": 0.22601857781410217,
      "rewards/rejected": -0.22259142994880676,
      "step": 1597
    },
    {
      "epoch": 0.9281524075042109,
      "grad_norm": 306.5569763183594,
      "learning_rate": 2.6786751888436957e-06,
      "logits/chosen": -0.8149584531784058,
      "logits/rejected": -0.9376369714736938,
      "logps/chosen": -78.22328186035156,
      "logps/rejected": -74.53474426269531,
      "loss": 13.7458,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0640365406870842,
      "rewards/margins": 0.02214570716023445,
      "rewards/rejected": -0.08618225157260895,
      "step": 1598
    },
    {
      "epoch": 0.9287332287855027,
      "grad_norm": 304.4878234863281,
      "learning_rate": 2.6772225450319585e-06,
      "logits/chosen": -0.9616473913192749,
      "logits/rejected": -1.0493296384811401,
      "logps/chosen": -78.84361267089844,
      "logps/rejected": -72.33888244628906,
      "loss": 12.9432,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02687544748187065,
      "rewards/margins": 0.11200690269470215,
      "rewards/rejected": -0.0851314514875412,
      "step": 1599
    },
    {
      "epoch": 0.9293140500667945,
      "grad_norm": 347.7174377441406,
      "learning_rate": 2.6757699012202208e-06,
      "logits/chosen": -1.1099224090576172,
      "logits/rejected": -1.0037864446640015,
      "logps/chosen": -68.02949523925781,
      "logps/rejected": -71.78794860839844,
      "loss": 15.2831,
      "rewards/accuracies": 0.15000000596046448,
      "rewards/chosen": -0.10204082727432251,
      "rewards/margins": -0.13043563067913055,
      "rewards/rejected": 0.0283947940915823,
      "step": 1600
    },
    {
      "epoch": 0.9298948713480862,
      "grad_norm": 299.0332336425781,
      "learning_rate": 2.6743172574084835e-06,
      "logits/chosen": -1.0257774591445923,
      "logits/rejected": -0.9751164317131042,
      "logps/chosen": -71.59093475341797,
      "logps/rejected": -72.38304138183594,
      "loss": 13.4701,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.007627165410667658,
      "rewards/margins": 0.046560898423194885,
      "rewards/rejected": -0.05418806150555611,
      "step": 1601
    },
    {
      "epoch": 0.9304756926293779,
      "grad_norm": 313.7255554199219,
      "learning_rate": 2.6728646135967463e-06,
      "logits/chosen": -1.1090632677078247,
      "logits/rejected": -1.094504952430725,
      "logps/chosen": -78.38826751708984,
      "logps/rejected": -69.06160736083984,
      "loss": 13.8859,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04326648637652397,
      "rewards/margins": 0.005500602535903454,
      "rewards/rejected": -0.04876708984375,
      "step": 1602
    },
    {
      "epoch": 0.9310565139106697,
      "grad_norm": 313.8421325683594,
      "learning_rate": 2.671411969785009e-06,
      "logits/chosen": -0.8764586448669434,
      "logits/rejected": -0.9077078700065613,
      "logps/chosen": -69.64143371582031,
      "logps/rejected": -76.69544982910156,
      "loss": 13.2255,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03643161803483963,
      "rewards/margins": 0.08787351846694946,
      "rewards/rejected": -0.12430514395236969,
      "step": 1603
    },
    {
      "epoch": 0.9316373351919615,
      "grad_norm": 322.4040832519531,
      "learning_rate": 2.6699593259732714e-06,
      "logits/chosen": -1.0882867574691772,
      "logits/rejected": -1.0712209939956665,
      "logps/chosen": -72.6526107788086,
      "logps/rejected": -73.7510986328125,
      "loss": 13.9615,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.09960527718067169,
      "rewards/margins": 0.012370998971164227,
      "rewards/rejected": -0.11197628080844879,
      "step": 1604
    },
    {
      "epoch": 0.9322181564732532,
      "grad_norm": 310.5035400390625,
      "learning_rate": 2.668506682161534e-06,
      "logits/chosen": -0.9501513242721558,
      "logits/rejected": -0.9882256388664246,
      "logps/chosen": -76.9458999633789,
      "logps/rejected": -76.29025268554688,
      "loss": 13.7632,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02920551598072052,
      "rewards/margins": 0.020055226981639862,
      "rewards/rejected": -0.04926074296236038,
      "step": 1605
    },
    {
      "epoch": 0.9327989777545449,
      "grad_norm": 300.83197021484375,
      "learning_rate": 2.667054038349797e-06,
      "logits/chosen": -1.0857970714569092,
      "logits/rejected": -0.9955571889877319,
      "logps/chosen": -65.53767395019531,
      "logps/rejected": -73.99423217773438,
      "loss": 13.0487,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04984000325202942,
      "rewards/margins": 0.09770919382572174,
      "rewards/rejected": -0.14754922688007355,
      "step": 1606
    },
    {
      "epoch": 0.9333797990358367,
      "grad_norm": 317.34686279296875,
      "learning_rate": 2.665601394538059e-06,
      "logits/chosen": -0.8815418481826782,
      "logits/rejected": -0.9183310270309448,
      "logps/chosen": -76.18892669677734,
      "logps/rejected": -75.93521118164062,
      "loss": 14.4527,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07652492821216583,
      "rewards/margins": -0.04064531996846199,
      "rewards/rejected": -0.03587961196899414,
      "step": 1607
    },
    {
      "epoch": 0.9339606203171285,
      "grad_norm": 403.3301086425781,
      "learning_rate": 2.664148750726322e-06,
      "logits/chosen": -0.9464950561523438,
      "logits/rejected": -1.032447099685669,
      "logps/chosen": -75.31007385253906,
      "logps/rejected": -71.3287124633789,
      "loss": 13.7649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05783992260694504,
      "rewards/margins": 0.032148055732250214,
      "rewards/rejected": -0.08998798578977585,
      "step": 1608
    },
    {
      "epoch": 0.9345414415984201,
      "grad_norm": 317.0668640136719,
      "learning_rate": 2.6626961069145847e-06,
      "logits/chosen": -1.023955225944519,
      "logits/rejected": -1.1011669635772705,
      "logps/chosen": -77.45109558105469,
      "logps/rejected": -74.42662811279297,
      "loss": 14.0911,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.002137671457603574,
      "rewards/margins": -0.005251598544418812,
      "rewards/rejected": 0.003113922430202365,
      "step": 1609
    },
    {
      "epoch": 0.9351222628797119,
      "grad_norm": 285.6341857910156,
      "learning_rate": 2.661243463102847e-06,
      "logits/chosen": -0.8273960947990417,
      "logits/rejected": -0.8372589945793152,
      "logps/chosen": -73.46778106689453,
      "logps/rejected": -73.86982727050781,
      "loss": 13.1531,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.013894091360270977,
      "rewards/margins": 0.08530426770448685,
      "rewards/rejected": -0.0714101791381836,
      "step": 1610
    },
    {
      "epoch": 0.9357030841610037,
      "grad_norm": 292.4166564941406,
      "learning_rate": 2.6597908192911097e-06,
      "logits/chosen": -0.9895612597465515,
      "logits/rejected": -1.0841071605682373,
      "logps/chosen": -75.07707214355469,
      "logps/rejected": -69.95777893066406,
      "loss": 12.3541,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00758351618424058,
      "rewards/margins": 0.18220645189285278,
      "rewards/rejected": -0.17462292313575745,
      "step": 1611
    },
    {
      "epoch": 0.9362839054422954,
      "grad_norm": 332.04998779296875,
      "learning_rate": 2.6583381754793725e-06,
      "logits/chosen": -0.8807666897773743,
      "logits/rejected": -0.9918161630630493,
      "logps/chosen": -74.91699981689453,
      "logps/rejected": -71.86626434326172,
      "loss": 14.9629,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09208264201879501,
      "rewards/margins": -0.08915847539901733,
      "rewards/rejected": -0.0029241659212857485,
      "step": 1612
    },
    {
      "epoch": 0.9368647267235871,
      "grad_norm": 298.9432678222656,
      "learning_rate": 2.6568855316676352e-06,
      "logits/chosen": -1.006432056427002,
      "logits/rejected": -0.8920149803161621,
      "logps/chosen": -68.74695587158203,
      "logps/rejected": -80.55778503417969,
      "loss": 13.4857,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.052919406443834305,
      "rewards/margins": 0.04934057593345642,
      "rewards/rejected": -0.10225997120141983,
      "step": 1613
    },
    {
      "epoch": 0.9374455480048789,
      "grad_norm": 321.1860656738281,
      "learning_rate": 2.6554328878558976e-06,
      "logits/chosen": -0.9797677993774414,
      "logits/rejected": -0.9094613194465637,
      "logps/chosen": -68.5245590209961,
      "logps/rejected": -69.34654998779297,
      "loss": 14.6291,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.0931304469704628,
      "rewards/margins": -0.06554857641458511,
      "rewards/rejected": -0.027581870555877686,
      "step": 1614
    },
    {
      "epoch": 0.9380263692861707,
      "grad_norm": 319.1537780761719,
      "learning_rate": 2.6539802440441603e-06,
      "logits/chosen": -1.0659425258636475,
      "logits/rejected": -1.097715139389038,
      "logps/chosen": -75.71224212646484,
      "logps/rejected": -79.82614135742188,
      "loss": 13.7336,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009594631381332874,
      "rewards/margins": 0.03794409707188606,
      "rewards/rejected": -0.04753873124718666,
      "step": 1615
    },
    {
      "epoch": 0.9386071905674624,
      "grad_norm": 340.1188049316406,
      "learning_rate": 2.652527600232423e-06,
      "logits/chosen": -0.9111448526382446,
      "logits/rejected": -1.014478087425232,
      "logps/chosen": -72.21406555175781,
      "logps/rejected": -87.39029693603516,
      "loss": 14.0704,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07805314660072327,
      "rewards/margins": -0.008613268844783306,
      "rewards/rejected": -0.06943987309932709,
      "step": 1616
    },
    {
      "epoch": 0.9391880118487541,
      "grad_norm": 324.6610107421875,
      "learning_rate": 2.6510749564206862e-06,
      "logits/chosen": -0.9898998141288757,
      "logits/rejected": -1.0052530765533447,
      "logps/chosen": -74.08876037597656,
      "logps/rejected": -74.66224670410156,
      "loss": 13.7696,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06431504338979721,
      "rewards/margins": 0.02580626867711544,
      "rewards/rejected": -0.0901213139295578,
      "step": 1617
    },
    {
      "epoch": 0.9397688331300459,
      "grad_norm": 309.86962890625,
      "learning_rate": 2.6496223126089486e-06,
      "logits/chosen": -1.055121660232544,
      "logits/rejected": -1.0201737880706787,
      "logps/chosen": -68.85248565673828,
      "logps/rejected": -73.47175598144531,
      "loss": 13.9124,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.019904637709259987,
      "rewards/margins": 0.0063825249671936035,
      "rewards/rejected": -0.02628716453909874,
      "step": 1618
    },
    {
      "epoch": 0.9403496544113377,
      "grad_norm": 370.7422180175781,
      "learning_rate": 2.6481696687972113e-06,
      "logits/chosen": -1.0947623252868652,
      "logits/rejected": -1.2657474279403687,
      "logps/chosen": -72.32966613769531,
      "logps/rejected": -78.00166320800781,
      "loss": 12.7809,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.026739204302430153,
      "rewards/margins": 0.19895856082439423,
      "rewards/rejected": -0.22569775581359863,
      "step": 1619
    },
    {
      "epoch": 0.9409304756926293,
      "grad_norm": 332.30718994140625,
      "learning_rate": 2.646717024985474e-06,
      "logits/chosen": -0.9388984441757202,
      "logits/rejected": -0.880285382270813,
      "logps/chosen": -81.50289154052734,
      "logps/rejected": -86.61189270019531,
      "loss": 13.8417,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08244191110134125,
      "rewards/margins": 0.026695813983678818,
      "rewards/rejected": -0.10913772881031036,
      "step": 1620
    },
    {
      "epoch": 0.9415112969739211,
      "grad_norm": 302.078369140625,
      "learning_rate": 2.645264381173737e-06,
      "logits/chosen": -0.9218322038650513,
      "logits/rejected": -1.005524754524231,
      "logps/chosen": -73.56781005859375,
      "logps/rejected": -73.6583023071289,
      "loss": 13.7027,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05132985860109329,
      "rewards/margins": 0.0359496995806694,
      "rewards/rejected": -0.0872795507311821,
      "step": 1621
    },
    {
      "epoch": 0.9420921182552129,
      "grad_norm": 304.9523010253906,
      "learning_rate": 2.643811737361999e-06,
      "logits/chosen": -0.9903371930122375,
      "logits/rejected": -0.8510664105415344,
      "logps/chosen": -78.66585540771484,
      "logps/rejected": -74.52757263183594,
      "loss": 13.1046,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.012658503837883472,
      "rewards/margins": 0.09112439304590225,
      "rewards/rejected": -0.10378290712833405,
      "step": 1622
    },
    {
      "epoch": 0.9426729395365047,
      "grad_norm": 363.7208251953125,
      "learning_rate": 2.642359093550262e-06,
      "logits/chosen": -0.9037588238716125,
      "logits/rejected": -0.9354912638664246,
      "logps/chosen": -75.28627014160156,
      "logps/rejected": -77.58480072021484,
      "loss": 14.1373,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.08841550350189209,
      "rewards/margins": -0.01985004171729088,
      "rewards/rejected": -0.06856545805931091,
      "step": 1623
    },
    {
      "epoch": 0.9432537608177963,
      "grad_norm": 595.0357055664062,
      "learning_rate": 2.6409064497385246e-06,
      "logits/chosen": -1.0555496215820312,
      "logits/rejected": -1.1325217485427856,
      "logps/chosen": -73.49629974365234,
      "logps/rejected": -81.2973861694336,
      "loss": 12.3747,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0019625455606728792,
      "rewards/margins": 0.1695987582206726,
      "rewards/rejected": -0.1715613305568695,
      "step": 1624
    },
    {
      "epoch": 0.9438345820990881,
      "grad_norm": 326.1748962402344,
      "learning_rate": 2.639453805926787e-06,
      "logits/chosen": -0.9975989460945129,
      "logits/rejected": -1.0014257431030273,
      "logps/chosen": -73.63597869873047,
      "logps/rejected": -78.60856628417969,
      "loss": 13.4802,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005343839526176453,
      "rewards/margins": 0.05836004763841629,
      "rewards/rejected": -0.05301620811223984,
      "step": 1625
    },
    {
      "epoch": 0.9444154033803799,
      "grad_norm": 301.3356628417969,
      "learning_rate": 2.6380011621150497e-06,
      "logits/chosen": -0.9281686544418335,
      "logits/rejected": -0.9274351000785828,
      "logps/chosen": -71.00225067138672,
      "logps/rejected": -69.36186218261719,
      "loss": 13.3989,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.018090274184942245,
      "rewards/margins": 0.05907534435391426,
      "rewards/rejected": -0.040985070168972015,
      "step": 1626
    },
    {
      "epoch": 0.9449962246616717,
      "grad_norm": 297.7870178222656,
      "learning_rate": 2.6365485183033124e-06,
      "logits/chosen": -1.1305437088012695,
      "logits/rejected": -1.130078911781311,
      "logps/chosen": -67.68418884277344,
      "logps/rejected": -78.84674835205078,
      "loss": 13.8481,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.036765143275260925,
      "rewards/margins": 0.01842624321579933,
      "rewards/rejected": -0.05519137904047966,
      "step": 1627
    },
    {
      "epoch": 0.9455770459429633,
      "grad_norm": 310.2338562011719,
      "learning_rate": 2.635095874491575e-06,
      "logits/chosen": -1.004183053970337,
      "logits/rejected": -1.048419713973999,
      "logps/chosen": -76.5638198852539,
      "logps/rejected": -88.52049255371094,
      "loss": 12.7393,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.016577254980802536,
      "rewards/margins": 0.1453401744365692,
      "rewards/rejected": -0.16191743314266205,
      "step": 1628
    },
    {
      "epoch": 0.9461578672242551,
      "grad_norm": 311.5450134277344,
      "learning_rate": 2.6336432306798375e-06,
      "logits/chosen": -0.9774214625358582,
      "logits/rejected": -0.9302505254745483,
      "logps/chosen": -66.8978042602539,
      "logps/rejected": -72.2725601196289,
      "loss": 13.3443,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03544515371322632,
      "rewards/margins": 0.0618865080177784,
      "rewards/rejected": -0.09733164310455322,
      "step": 1629
    },
    {
      "epoch": 0.9467386885055469,
      "grad_norm": 282.8682861328125,
      "learning_rate": 2.6321905868681003e-06,
      "logits/chosen": -0.961567223072052,
      "logits/rejected": -1.0476669073104858,
      "logps/chosen": -70.57254028320312,
      "logps/rejected": -79.38347625732422,
      "loss": 12.7302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.022200727835297585,
      "rewards/margins": 0.1265060007572174,
      "rewards/rejected": -0.14870671927928925,
      "step": 1630
    },
    {
      "epoch": 0.9473195097868385,
      "grad_norm": 345.95550537109375,
      "learning_rate": 2.630737943056363e-06,
      "logits/chosen": -1.0288885831832886,
      "logits/rejected": -1.0817986726760864,
      "logps/chosen": -80.58564758300781,
      "logps/rejected": -80.86302947998047,
      "loss": 14.3336,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.12510161101818085,
      "rewards/margins": -0.03208181634545326,
      "rewards/rejected": -0.09301979839801788,
      "step": 1631
    },
    {
      "epoch": 0.9479003310681303,
      "grad_norm": 319.99188232421875,
      "learning_rate": 2.6292852992446253e-06,
      "logits/chosen": -1.1356427669525146,
      "logits/rejected": -1.0835113525390625,
      "logps/chosen": -71.69888305664062,
      "logps/rejected": -75.07586669921875,
      "loss": 13.3202,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02006458304822445,
      "rewards/margins": 0.07214848697185516,
      "rewards/rejected": -0.09221307933330536,
      "step": 1632
    },
    {
      "epoch": 0.9484811523494221,
      "grad_norm": 313.5886535644531,
      "learning_rate": 2.627832655432888e-06,
      "logits/chosen": -1.0788170099258423,
      "logits/rejected": -1.0514395236968994,
      "logps/chosen": -73.72484588623047,
      "logps/rejected": -74.73408508300781,
      "loss": 13.6372,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022092115134000778,
      "rewards/margins": 0.04484707862138748,
      "rewards/rejected": -0.06693919003009796,
      "step": 1633
    },
    {
      "epoch": 0.9490619736307139,
      "grad_norm": 305.9249267578125,
      "learning_rate": 2.626380011621151e-06,
      "logits/chosen": -1.0812559127807617,
      "logits/rejected": -1.174479365348816,
      "logps/chosen": -72.90000915527344,
      "logps/rejected": -78.41686248779297,
      "loss": 13.3336,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03430159389972687,
      "rewards/margins": 0.06872163712978363,
      "rewards/rejected": -0.1030232310295105,
      "step": 1634
    },
    {
      "epoch": 0.9496427949120055,
      "grad_norm": 359.0313415527344,
      "learning_rate": 2.6249273678094136e-06,
      "logits/chosen": -0.967424213886261,
      "logits/rejected": -0.9965407252311707,
      "logps/chosen": -73.54337310791016,
      "logps/rejected": -76.62731170654297,
      "loss": 14.0575,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04725205525755882,
      "rewards/margins": -0.002465140772983432,
      "rewards/rejected": -0.04478692263364792,
      "step": 1635
    },
    {
      "epoch": 0.9502236161932973,
      "grad_norm": 309.18121337890625,
      "learning_rate": 2.623474723997676e-06,
      "logits/chosen": -1.1406584978103638,
      "logits/rejected": -1.104427695274353,
      "logps/chosen": -75.27462768554688,
      "logps/rejected": -72.25857543945312,
      "loss": 13.1751,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.020470213145017624,
      "rewards/margins": 0.08202612400054932,
      "rewards/rejected": -0.10249634087085724,
      "step": 1636
    },
    {
      "epoch": 0.9508044374745891,
      "grad_norm": 316.88043212890625,
      "learning_rate": 2.6220220801859387e-06,
      "logits/chosen": -1.1179893016815186,
      "logits/rejected": -1.1213700771331787,
      "logps/chosen": -76.03439331054688,
      "logps/rejected": -75.00591278076172,
      "loss": 13.9533,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.048169057816267014,
      "rewards/margins": 0.0091823386028409,
      "rewards/rejected": -0.05735139176249504,
      "step": 1637
    },
    {
      "epoch": 0.9513852587558809,
      "grad_norm": 314.9276123046875,
      "learning_rate": 2.6205694363742014e-06,
      "logits/chosen": -1.09652841091156,
      "logits/rejected": -1.1022472381591797,
      "logps/chosen": -73.7913818359375,
      "logps/rejected": -69.4230728149414,
      "loss": 14.6184,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.07484273612499237,
      "rewards/margins": -0.05537329241633415,
      "rewards/rejected": -0.01946944370865822,
      "step": 1638
    },
    {
      "epoch": 0.9519660800371725,
      "grad_norm": 348.3751525878906,
      "learning_rate": 2.6191167925624637e-06,
      "logits/chosen": -0.9402965307235718,
      "logits/rejected": -0.918109118938446,
      "logps/chosen": -78.03495025634766,
      "logps/rejected": -80.16193389892578,
      "loss": 14.0145,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07915445417165756,
      "rewards/margins": -0.001969304634258151,
      "rewards/rejected": -0.07718515396118164,
      "step": 1639
    },
    {
      "epoch": 0.9525469013184643,
      "grad_norm": 698.2963256835938,
      "learning_rate": 2.6176641487507265e-06,
      "logits/chosen": -1.1015129089355469,
      "logits/rejected": -1.128722906112671,
      "logps/chosen": -76.55816650390625,
      "logps/rejected": -89.9703598022461,
      "loss": 12.9666,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06669773906469345,
      "rewards/margins": 0.10916386544704437,
      "rewards/rejected": -0.17586161196231842,
      "step": 1640
    },
    {
      "epoch": 0.9531277225997561,
      "grad_norm": 336.7152404785156,
      "learning_rate": 2.6162115049389892e-06,
      "logits/chosen": -1.0806176662445068,
      "logits/rejected": -1.0479252338409424,
      "logps/chosen": -73.31221008300781,
      "logps/rejected": -73.0496826171875,
      "loss": 13.5804,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.036660175770521164,
      "rewards/margins": 0.03841117396950722,
      "rewards/rejected": -0.0017509929602965713,
      "step": 1641
    },
    {
      "epoch": 0.9537085438810478,
      "grad_norm": 473.8951416015625,
      "learning_rate": 2.6147588611272516e-06,
      "logits/chosen": -0.9855740666389465,
      "logits/rejected": -1.040444254875183,
      "logps/chosen": -70.26768493652344,
      "logps/rejected": -74.44651794433594,
      "loss": 14.8604,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.10035822540521622,
      "rewards/margins": -0.09087743610143661,
      "rewards/rejected": -0.009480791166424751,
      "step": 1642
    },
    {
      "epoch": 0.9542893651623395,
      "grad_norm": 327.31109619140625,
      "learning_rate": 2.6133062173155143e-06,
      "logits/chosen": -1.0069469213485718,
      "logits/rejected": -0.9739433526992798,
      "logps/chosen": -76.33607482910156,
      "logps/rejected": -74.96504211425781,
      "loss": 14.1413,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.036099713295698166,
      "rewards/margins": -0.01753252185881138,
      "rewards/rejected": -0.018567193299531937,
      "step": 1643
    },
    {
      "epoch": 0.9548701864436313,
      "grad_norm": 281.3694152832031,
      "learning_rate": 2.611853573503777e-06,
      "logits/chosen": -1.066506266593933,
      "logits/rejected": -1.081606388092041,
      "logps/chosen": -74.58091735839844,
      "logps/rejected": -79.44447326660156,
      "loss": 12.7016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008866215124726295,
      "rewards/margins": 0.1444592922925949,
      "rewards/rejected": -0.13559308648109436,
      "step": 1644
    },
    {
      "epoch": 0.9554510077249231,
      "grad_norm": 307.878662109375,
      "learning_rate": 2.61040092969204e-06,
      "logits/chosen": -1.0650668144226074,
      "logits/rejected": -1.0968126058578491,
      "logps/chosen": -72.53206634521484,
      "logps/rejected": -79.52193450927734,
      "loss": 13.8802,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.04437771812081337,
      "rewards/margins": 0.00487914914265275,
      "rewards/rejected": -0.04925686866044998,
      "step": 1645
    },
    {
      "epoch": 0.9560318290062148,
      "grad_norm": 324.9822082519531,
      "learning_rate": 2.608948285880302e-06,
      "logits/chosen": -1.005570650100708,
      "logits/rejected": -1.02733314037323,
      "logps/chosen": -66.51573181152344,
      "logps/rejected": -69.79955291748047,
      "loss": 13.339,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07535625994205475,
      "rewards/margins": 0.06180157512426376,
      "rewards/rejected": -0.13715782761573792,
      "step": 1646
    },
    {
      "epoch": 0.9566126502875065,
      "grad_norm": 304.5415344238281,
      "learning_rate": 2.607495642068565e-06,
      "logits/chosen": -1.0655357837677002,
      "logits/rejected": -1.089559555053711,
      "logps/chosen": -69.73716735839844,
      "logps/rejected": -70.25821685791016,
      "loss": 12.7139,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012229815125465393,
      "rewards/margins": 0.1376718133687973,
      "rewards/rejected": -0.1254419982433319,
      "step": 1647
    },
    {
      "epoch": 0.9571934715687983,
      "grad_norm": 283.9061584472656,
      "learning_rate": 2.6060429982568276e-06,
      "logits/chosen": -1.0645396709442139,
      "logits/rejected": -1.0818393230438232,
      "logps/chosen": -73.74522399902344,
      "logps/rejected": -70.01896667480469,
      "loss": 12.9035,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015870463103055954,
      "rewards/margins": 0.11696214973926544,
      "rewards/rejected": -0.101091668009758,
      "step": 1648
    },
    {
      "epoch": 0.9577742928500901,
      "grad_norm": 321.0312805175781,
      "learning_rate": 2.60459035444509e-06,
      "logits/chosen": -0.931603729724884,
      "logits/rejected": -0.9701339602470398,
      "logps/chosen": -67.54893493652344,
      "logps/rejected": -80.33594512939453,
      "loss": 13.4542,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019261101260781288,
      "rewards/margins": 0.05602710321545601,
      "rewards/rejected": -0.07528820633888245,
      "step": 1649
    },
    {
      "epoch": 0.9583551141313817,
      "grad_norm": 286.6662292480469,
      "learning_rate": 2.6031377106333527e-06,
      "logits/chosen": -0.9944067001342773,
      "logits/rejected": -1.2817535400390625,
      "logps/chosen": -76.48405456542969,
      "logps/rejected": -88.2748794555664,
      "loss": 11.8039,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.023223085328936577,
      "rewards/margins": 0.3373262584209442,
      "rewards/rejected": -0.3141031563282013,
      "step": 1650
    },
    {
      "epoch": 0.9589359354126735,
      "grad_norm": 329.37713623046875,
      "learning_rate": 2.6016850668216154e-06,
      "logits/chosen": -0.922402560710907,
      "logits/rejected": -0.8550537824630737,
      "logps/chosen": -76.62870788574219,
      "logps/rejected": -80.34883880615234,
      "loss": 13.668,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09244881570339203,
      "rewards/margins": 0.02698623016476631,
      "rewards/rejected": -0.11943503469228745,
      "step": 1651
    },
    {
      "epoch": 0.9595167566939653,
      "grad_norm": 310.3904724121094,
      "learning_rate": 2.600232423009878e-06,
      "logits/chosen": -1.2607991695404053,
      "logits/rejected": -1.077472448348999,
      "logps/chosen": -70.1756820678711,
      "logps/rejected": -74.11396789550781,
      "loss": 14.292,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04567917436361313,
      "rewards/margins": -0.03175630047917366,
      "rewards/rejected": -0.013922872953116894,
      "step": 1652
    },
    {
      "epoch": 0.960097577975257,
      "grad_norm": 310.15460205078125,
      "learning_rate": 2.5987797791981405e-06,
      "logits/chosen": -1.0908806324005127,
      "logits/rejected": -1.0850508213043213,
      "logps/chosen": -74.20774841308594,
      "logps/rejected": -73.19671630859375,
      "loss": 14.331,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06622732430696487,
      "rewards/margins": -0.025598257780075073,
      "rewards/rejected": -0.0406290739774704,
      "step": 1653
    },
    {
      "epoch": 0.9606783992565487,
      "grad_norm": 314.9224548339844,
      "learning_rate": 2.5973271353864033e-06,
      "logits/chosen": -1.1002670526504517,
      "logits/rejected": -1.0931947231292725,
      "logps/chosen": -74.95215606689453,
      "logps/rejected": -77.90531921386719,
      "loss": 14.3312,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.04412032663822174,
      "rewards/margins": -0.024371739476919174,
      "rewards/rejected": -0.019748590886592865,
      "step": 1654
    },
    {
      "epoch": 0.9612592205378405,
      "grad_norm": 335.9540100097656,
      "learning_rate": 2.595874491574666e-06,
      "logits/chosen": -1.0642696619033813,
      "logits/rejected": -0.8969653248786926,
      "logps/chosen": -69.37012481689453,
      "logps/rejected": -70.08804321289062,
      "loss": 14.0696,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07831525802612305,
      "rewards/margins": -0.006942230276763439,
      "rewards/rejected": -0.07137302309274673,
      "step": 1655
    },
    {
      "epoch": 0.9618400418191323,
      "grad_norm": 303.8190612792969,
      "learning_rate": 2.5944218477629283e-06,
      "logits/chosen": -1.232759714126587,
      "logits/rejected": -1.1754319667816162,
      "logps/chosen": -77.84904479980469,
      "logps/rejected": -70.85216522216797,
      "loss": 13.4052,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.039317332208156586,
      "rewards/margins": 0.0675087720155716,
      "rewards/rejected": -0.10682611167430878,
      "step": 1656
    },
    {
      "epoch": 0.962420863100424,
      "grad_norm": 305.9680480957031,
      "learning_rate": 2.592969203951191e-06,
      "logits/chosen": -1.0391418933868408,
      "logits/rejected": -1.109973669052124,
      "logps/chosen": -72.61897277832031,
      "logps/rejected": -78.79949188232422,
      "loss": 13.4044,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05770915746688843,
      "rewards/margins": 0.05846037343144417,
      "rewards/rejected": -0.1161695346236229,
      "step": 1657
    },
    {
      "epoch": 0.9630016843817157,
      "grad_norm": 290.8988342285156,
      "learning_rate": 2.591516560139454e-06,
      "logits/chosen": -1.0394976139068604,
      "logits/rejected": -1.0156317949295044,
      "logps/chosen": -66.53820037841797,
      "logps/rejected": -64.29721069335938,
      "loss": 14.0649,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04657341539859772,
      "rewards/margins": -0.010933073237538338,
      "rewards/rejected": -0.03564033657312393,
      "step": 1658
    },
    {
      "epoch": 0.9635825056630075,
      "grad_norm": 315.23895263671875,
      "learning_rate": 2.590063916327717e-06,
      "logits/chosen": -0.9935659170150757,
      "logits/rejected": -1.0015393495559692,
      "logps/chosen": -68.51786804199219,
      "logps/rejected": -69.16304016113281,
      "loss": 14.9079,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.13074307143688202,
      "rewards/margins": -0.09582234919071198,
      "rewards/rejected": -0.034920718520879745,
      "step": 1659
    },
    {
      "epoch": 0.9641633269442993,
      "grad_norm": 317.5841064453125,
      "learning_rate": 2.5886112725159798e-06,
      "logits/chosen": -1.1780986785888672,
      "logits/rejected": -1.1185228824615479,
      "logps/chosen": -80.13496398925781,
      "logps/rejected": -79.49234771728516,
      "loss": 13.4738,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029046854004263878,
      "rewards/margins": 0.06537636369466782,
      "rewards/rejected": -0.09442321956157684,
      "step": 1660
    },
    {
      "epoch": 0.964744148225591,
      "grad_norm": 313.482666015625,
      "learning_rate": 2.587158628704242e-06,
      "logits/chosen": -1.0593732595443726,
      "logits/rejected": -0.9994481205940247,
      "logps/chosen": -75.6661148071289,
      "logps/rejected": -75.5912094116211,
      "loss": 13.7113,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05127748101949692,
      "rewards/margins": 0.027803445234894753,
      "rewards/rejected": -0.07908091694116592,
      "step": 1661
    },
    {
      "epoch": 0.9653249695068827,
      "grad_norm": 316.85107421875,
      "learning_rate": 2.585705984892505e-06,
      "logits/chosen": -1.2219678163528442,
      "logits/rejected": -1.133565902709961,
      "logps/chosen": -79.41874694824219,
      "logps/rejected": -80.63514709472656,
      "loss": 13.4792,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03214912489056587,
      "rewards/margins": 0.057010721415281296,
      "rewards/rejected": -0.08915985375642776,
      "step": 1662
    },
    {
      "epoch": 0.9659057907881745,
      "grad_norm": 328.014404296875,
      "learning_rate": 2.5842533410807676e-06,
      "logits/chosen": -1.0830070972442627,
      "logits/rejected": -1.1433053016662598,
      "logps/chosen": -79.09217071533203,
      "logps/rejected": -74.84022521972656,
      "loss": 14.2003,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.08502332121133804,
      "rewards/margins": 0.01838522218167782,
      "rewards/rejected": -0.10340853780508041,
      "step": 1663
    },
    {
      "epoch": 0.9664866120694662,
      "grad_norm": 338.3645324707031,
      "learning_rate": 2.58280069726903e-06,
      "logits/chosen": -1.0388402938842773,
      "logits/rejected": -1.0141454935073853,
      "logps/chosen": -72.27071380615234,
      "logps/rejected": -82.36555480957031,
      "loss": 14.5886,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.13550472259521484,
      "rewards/margins": -0.05550233647227287,
      "rewards/rejected": -0.08000236749649048,
      "step": 1664
    },
    {
      "epoch": 0.967067433350758,
      "grad_norm": 338.9569396972656,
      "learning_rate": 2.5813480534572926e-06,
      "logits/chosen": -1.1725468635559082,
      "logits/rejected": -1.143462896347046,
      "logps/chosen": -74.5365219116211,
      "logps/rejected": -72.63186645507812,
      "loss": 14.5354,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1107749193906784,
      "rewards/margins": -0.05176005885004997,
      "rewards/rejected": -0.059014856815338135,
      "step": 1665
    },
    {
      "epoch": 0.9676482546320497,
      "grad_norm": 346.3096618652344,
      "learning_rate": 2.5798954096455554e-06,
      "logits/chosen": -0.8586591482162476,
      "logits/rejected": -0.9516043663024902,
      "logps/chosen": -70.58949279785156,
      "logps/rejected": -71.14974212646484,
      "loss": 13.1937,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0421382412314415,
      "rewards/margins": 0.0767202377319336,
      "rewards/rejected": -0.1188584715127945,
      "step": 1666
    },
    {
      "epoch": 0.9682290759133415,
      "grad_norm": 309.0997619628906,
      "learning_rate": 2.5784427658338177e-06,
      "logits/chosen": -0.8756416440010071,
      "logits/rejected": -0.8066015243530273,
      "logps/chosen": -67.46591186523438,
      "logps/rejected": -72.75110626220703,
      "loss": 13.9135,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08465950936079025,
      "rewards/margins": 0.004559764172881842,
      "rewards/rejected": -0.08921927213668823,
      "step": 1667
    },
    {
      "epoch": 0.9688098971946332,
      "grad_norm": 313.6292724609375,
      "learning_rate": 2.5769901220220805e-06,
      "logits/chosen": -1.0599316358566284,
      "logits/rejected": -1.0807254314422607,
      "logps/chosen": -78.5136489868164,
      "logps/rejected": -73.09068298339844,
      "loss": 13.7649,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09171977639198303,
      "rewards/margins": 0.022846868261694908,
      "rewards/rejected": -0.11456664651632309,
      "step": 1668
    },
    {
      "epoch": 0.969390718475925,
      "grad_norm": 358.3715515136719,
      "learning_rate": 2.5755374782103432e-06,
      "logits/chosen": -1.1112323999404907,
      "logits/rejected": -1.0399501323699951,
      "logps/chosen": -83.4367446899414,
      "logps/rejected": -83.8092041015625,
      "loss": 13.4541,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07526461035013199,
      "rewards/margins": 0.05893157795071602,
      "rewards/rejected": -0.1341961920261383,
      "step": 1669
    },
    {
      "epoch": 0.9699715397572167,
      "grad_norm": 290.758544921875,
      "learning_rate": 2.574084834398606e-06,
      "logits/chosen": -1.232049822807312,
      "logits/rejected": -1.0986875295639038,
      "logps/chosen": -70.46229553222656,
      "logps/rejected": -73.52723693847656,
      "loss": 12.8435,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.045075662434101105,
      "rewards/margins": 0.12104110419750214,
      "rewards/rejected": -0.16611677408218384,
      "step": 1670
    },
    {
      "epoch": 0.9705523610385085,
      "grad_norm": 408.366943359375,
      "learning_rate": 2.5726321905868683e-06,
      "logits/chosen": -0.9108990430831909,
      "logits/rejected": -0.9383079409599304,
      "logps/chosen": -82.34011840820312,
      "logps/rejected": -80.4591064453125,
      "loss": 13.6321,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08920159935951233,
      "rewards/margins": 0.09411170333623886,
      "rewards/rejected": -0.1833132952451706,
      "step": 1671
    },
    {
      "epoch": 0.9711331823198002,
      "grad_norm": 308.26629638671875,
      "learning_rate": 2.571179546775131e-06,
      "logits/chosen": -1.0400654077529907,
      "logits/rejected": -1.1011639833450317,
      "logps/chosen": -69.07063293457031,
      "logps/rejected": -71.2532730102539,
      "loss": 14.0999,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07324647158384323,
      "rewards/margins": -0.006299248896539211,
      "rewards/rejected": -0.06694721430540085,
      "step": 1672
    },
    {
      "epoch": 0.971714003601092,
      "grad_norm": 423.69134521484375,
      "learning_rate": 2.5697269029633938e-06,
      "logits/chosen": -1.1098686456680298,
      "logits/rejected": -1.100983738899231,
      "logps/chosen": -70.90492248535156,
      "logps/rejected": -68.08970642089844,
      "loss": 14.1874,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07784977555274963,
      "rewards/margins": -0.00046813784865662456,
      "rewards/rejected": -0.07738164812326431,
      "step": 1673
    },
    {
      "epoch": 0.9722948248823837,
      "grad_norm": 326.7045593261719,
      "learning_rate": 2.568274259151656e-06,
      "logits/chosen": -1.0890681743621826,
      "logits/rejected": -1.1120952367782593,
      "logps/chosen": -72.4934310913086,
      "logps/rejected": -72.59373474121094,
      "loss": 13.9302,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04118148982524872,
      "rewards/margins": 0.005435384809970856,
      "rewards/rejected": -0.046616874635219574,
      "step": 1674
    },
    {
      "epoch": 0.9728756461636754,
      "grad_norm": 303.94873046875,
      "learning_rate": 2.566821615339919e-06,
      "logits/chosen": -1.1346571445465088,
      "logits/rejected": -1.205708384513855,
      "logps/chosen": -73.45823669433594,
      "logps/rejected": -67.34284973144531,
      "loss": 13.8327,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.011705625802278519,
      "rewards/margins": 0.013669428415596485,
      "rewards/rejected": -0.001963794929906726,
      "step": 1675
    },
    {
      "epoch": 0.9734564674449672,
      "grad_norm": 340.87005615234375,
      "learning_rate": 2.5653689715281816e-06,
      "logits/chosen": -1.0697561502456665,
      "logits/rejected": -1.1431515216827393,
      "logps/chosen": -74.03730773925781,
      "logps/rejected": -84.31428527832031,
      "loss": 14.1895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.107114776968956,
      "rewards/margins": -0.021280210465192795,
      "rewards/rejected": -0.0858345553278923,
      "step": 1676
    },
    {
      "epoch": 0.9740372887262589,
      "grad_norm": 300.523681640625,
      "learning_rate": 2.5639163277164444e-06,
      "logits/chosen": -1.0482876300811768,
      "logits/rejected": -0.9724637866020203,
      "logps/chosen": -67.77294921875,
      "logps/rejected": -68.99544525146484,
      "loss": 13.2654,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.056502826511859894,
      "rewards/margins": 0.07467863708734512,
      "rewards/rejected": -0.13118146359920502,
      "step": 1677
    },
    {
      "epoch": 0.9746181100075507,
      "grad_norm": 344.5461730957031,
      "learning_rate": 2.5624636839047067e-06,
      "logits/chosen": -1.010778784751892,
      "logits/rejected": -1.0810010433197021,
      "logps/chosen": -82.27607727050781,
      "logps/rejected": -73.64949035644531,
      "loss": 13.8788,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06360437721014023,
      "rewards/margins": 0.01501272339373827,
      "rewards/rejected": -0.07861708849668503,
      "step": 1678
    },
    {
      "epoch": 0.9751989312888424,
      "grad_norm": 304.4271545410156,
      "learning_rate": 2.5610110400929694e-06,
      "logits/chosen": -1.026784896850586,
      "logits/rejected": -1.1491081714630127,
      "logps/chosen": -71.38555145263672,
      "logps/rejected": -73.12422943115234,
      "loss": 13.4462,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.041596684604883194,
      "rewards/margins": 0.12548679113388062,
      "rewards/rejected": -0.1670834720134735,
      "step": 1679
    },
    {
      "epoch": 0.9757797525701342,
      "grad_norm": 323.35614013671875,
      "learning_rate": 2.559558396281232e-06,
      "logits/chosen": -1.07467782497406,
      "logits/rejected": -1.025925874710083,
      "logps/chosen": -74.79801940917969,
      "logps/rejected": -66.38459777832031,
      "loss": 14.0058,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.08111674338579178,
      "rewards/margins": 0.006164426915347576,
      "rewards/rejected": -0.08728117495775223,
      "step": 1680
    },
    {
      "epoch": 0.9763605738514259,
      "grad_norm": 348.9921875,
      "learning_rate": 2.5581057524694945e-06,
      "logits/chosen": -1.064215898513794,
      "logits/rejected": -1.0668865442276,
      "logps/chosen": -75.97596740722656,
      "logps/rejected": -79.41194152832031,
      "loss": 14.4345,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07904750108718872,
      "rewards/margins": -0.04122578352689743,
      "rewards/rejected": -0.03782171756029129,
      "step": 1681
    },
    {
      "epoch": 0.9769413951327177,
      "grad_norm": 323.9701232910156,
      "learning_rate": 2.5566531086577573e-06,
      "logits/chosen": -1.131103277206421,
      "logits/rejected": -1.123769998550415,
      "logps/chosen": -67.6153564453125,
      "logps/rejected": -73.26863098144531,
      "loss": 14.366,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.07523433864116669,
      "rewards/margins": -0.037893153727054596,
      "rewards/rejected": -0.037341196089982986,
      "step": 1682
    },
    {
      "epoch": 0.9775222164140094,
      "grad_norm": 296.8529052734375,
      "learning_rate": 2.55520046484602e-06,
      "logits/chosen": -1.2083275318145752,
      "logits/rejected": -1.1943024396896362,
      "logps/chosen": -71.191650390625,
      "logps/rejected": -75.9756088256836,
      "loss": 12.9872,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02410920336842537,
      "rewards/margins": 0.12185738235712051,
      "rewards/rejected": -0.09774816036224365,
      "step": 1683
    },
    {
      "epoch": 0.9781030376953012,
      "grad_norm": 325.1348571777344,
      "learning_rate": 2.5537478210342823e-06,
      "logits/chosen": -0.9783321619033813,
      "logits/rejected": -1.0359395742416382,
      "logps/chosen": -78.31156921386719,
      "logps/rejected": -69.6674575805664,
      "loss": 13.9924,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.059808146208524704,
      "rewards/margins": -0.0001798242301447317,
      "rewards/rejected": -0.05962831899523735,
      "step": 1684
    },
    {
      "epoch": 0.9786838589765929,
      "grad_norm": 287.490234375,
      "learning_rate": 2.552295177222545e-06,
      "logits/chosen": -1.0006749629974365,
      "logits/rejected": -1.102601170539856,
      "logps/chosen": -75.44730377197266,
      "logps/rejected": -81.95941162109375,
      "loss": 12.6821,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06169646978378296,
      "rewards/margins": 0.13653148710727692,
      "rewards/rejected": -0.19822795689105988,
      "step": 1685
    },
    {
      "epoch": 0.9792646802578846,
      "grad_norm": 293.2078857421875,
      "learning_rate": 2.550842533410808e-06,
      "logits/chosen": -0.8583377599716187,
      "logits/rejected": -0.8753503561019897,
      "logps/chosen": -69.07522583007812,
      "logps/rejected": -70.85563659667969,
      "loss": 13.8437,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0760253518819809,
      "rewards/margins": 0.01626693829894066,
      "rewards/rejected": -0.09229229390621185,
      "step": 1686
    },
    {
      "epoch": 0.9798455015391764,
      "grad_norm": 293.9259033203125,
      "learning_rate": 2.5493898895990706e-06,
      "logits/chosen": -0.9493265151977539,
      "logits/rejected": -1.0210487842559814,
      "logps/chosen": -67.49405670166016,
      "logps/rejected": -79.09819030761719,
      "loss": 13.4273,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0075791822746396065,
      "rewards/margins": 0.053987205028533936,
      "rewards/rejected": -0.046408019959926605,
      "step": 1687
    },
    {
      "epoch": 0.9804263228204682,
      "grad_norm": 311.675537109375,
      "learning_rate": 2.547937245787333e-06,
      "logits/chosen": -1.0591866970062256,
      "logits/rejected": -1.2229483127593994,
      "logps/chosen": -78.27244567871094,
      "logps/rejected": -93.38812255859375,
      "loss": 12.432,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.005819785408675671,
      "rewards/margins": 0.16608306765556335,
      "rewards/rejected": -0.1602632701396942,
      "step": 1688
    },
    {
      "epoch": 0.9810071441017599,
      "grad_norm": 331.8810119628906,
      "learning_rate": 2.5464846019755956e-06,
      "logits/chosen": -1.0761051177978516,
      "logits/rejected": -1.0155199766159058,
      "logps/chosen": -80.69140625,
      "logps/rejected": -71.69276428222656,
      "loss": 13.7616,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04481067508459091,
      "rewards/margins": 0.022894708439707756,
      "rewards/rejected": -0.06770537793636322,
      "step": 1689
    },
    {
      "epoch": 0.9815879653830516,
      "grad_norm": 324.2416076660156,
      "learning_rate": 2.5450319581638584e-06,
      "logits/chosen": -0.9733175039291382,
      "logits/rejected": -0.9295781254768372,
      "logps/chosen": -70.86591339111328,
      "logps/rejected": -65.49756622314453,
      "loss": 13.6746,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02684326097369194,
      "rewards/margins": 0.03324420005083084,
      "rewards/rejected": -0.006400938145816326,
      "step": 1690
    },
    {
      "epoch": 0.9821687866643434,
      "grad_norm": 321.970703125,
      "learning_rate": 2.5435793143521207e-06,
      "logits/chosen": -0.9246444702148438,
      "logits/rejected": -0.8594695925712585,
      "logps/chosen": -75.39814758300781,
      "logps/rejected": -75.02884674072266,
      "loss": 13.8478,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06917383521795273,
      "rewards/margins": 0.023299330845475197,
      "rewards/rejected": -0.09247316420078278,
      "step": 1691
    },
    {
      "epoch": 0.9827496079456352,
      "grad_norm": 313.23828125,
      "learning_rate": 2.5421266705403835e-06,
      "logits/chosen": -0.9367784261703491,
      "logits/rejected": -0.9928357005119324,
      "logps/chosen": -70.94462585449219,
      "logps/rejected": -70.25473022460938,
      "loss": 13.5832,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.006189526524394751,
      "rewards/margins": 0.03919747844338417,
      "rewards/rejected": -0.04538700729608536,
      "step": 1692
    },
    {
      "epoch": 0.9833304292269269,
      "grad_norm": 309.0666198730469,
      "learning_rate": 2.5406740267286462e-06,
      "logits/chosen": -1.0375077724456787,
      "logits/rejected": -1.2075139284133911,
      "logps/chosen": -79.04317474365234,
      "logps/rejected": -84.03590393066406,
      "loss": 13.2263,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05505346134305,
      "rewards/margins": 0.08161797374486923,
      "rewards/rejected": -0.13667142391204834,
      "step": 1693
    },
    {
      "epoch": 0.9839112505082186,
      "grad_norm": 324.5203857421875,
      "learning_rate": 2.539221382916909e-06,
      "logits/chosen": -0.9552820324897766,
      "logits/rejected": -1.0778672695159912,
      "logps/chosen": -81.62934875488281,
      "logps/rejected": -73.92164611816406,
      "loss": 13.7568,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06554460525512695,
      "rewards/margins": 0.01730671152472496,
      "rewards/rejected": -0.08285132050514221,
      "step": 1694
    },
    {
      "epoch": 0.9844920717895104,
      "grad_norm": 303.24884033203125,
      "learning_rate": 2.5377687391051713e-06,
      "logits/chosen": -0.8851510882377625,
      "logits/rejected": -0.9082107543945312,
      "logps/chosen": -71.15213775634766,
      "logps/rejected": -76.115478515625,
      "loss": 13.1769,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06722573190927505,
      "rewards/margins": 0.08196327835321426,
      "rewards/rejected": -0.14918899536132812,
      "step": 1695
    },
    {
      "epoch": 0.9850728930708021,
      "grad_norm": 365.0985107421875,
      "learning_rate": 2.536316095293434e-06,
      "logits/chosen": -1.0518224239349365,
      "logits/rejected": -1.0206438302993774,
      "logps/chosen": -86.0371322631836,
      "logps/rejected": -70.37875366210938,
      "loss": 14.6223,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.10060302913188934,
      "rewards/margins": -0.06344429403543472,
      "rewards/rejected": -0.03715873882174492,
      "step": 1696
    },
    {
      "epoch": 0.9856537143520938,
      "grad_norm": 320.1251220703125,
      "learning_rate": 2.5348634514816968e-06,
      "logits/chosen": -0.9742853045463562,
      "logits/rejected": -1.027646780014038,
      "logps/chosen": -71.27460479736328,
      "logps/rejected": -76.10606384277344,
      "loss": 13.7345,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.054471857845783234,
      "rewards/margins": 0.02452598698437214,
      "rewards/rejected": -0.07899783551692963,
      "step": 1697
    },
    {
      "epoch": 0.9862345356333856,
      "grad_norm": 332.81097412109375,
      "learning_rate": 2.533410807669959e-06,
      "logits/chosen": -0.9751776456832886,
      "logits/rejected": -1.018146276473999,
      "logps/chosen": -72.87306213378906,
      "logps/rejected": -83.01241302490234,
      "loss": 14.4578,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.07500861585140228,
      "rewards/margins": -0.044311076402664185,
      "rewards/rejected": -0.0306975357234478,
      "step": 1698
    },
    {
      "epoch": 0.9868153569146774,
      "grad_norm": 330.2864074707031,
      "learning_rate": 2.531958163858222e-06,
      "logits/chosen": -0.930493175983429,
      "logits/rejected": -0.9279171228408813,
      "logps/chosen": -71.18681335449219,
      "logps/rejected": -80.98843383789062,
      "loss": 13.2138,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04132339358329773,
      "rewards/margins": 0.08029758185148239,
      "rewards/rejected": -0.12162097543478012,
      "step": 1699
    },
    {
      "epoch": 0.9873961781959691,
      "grad_norm": 275.1678771972656,
      "learning_rate": 2.5305055200464846e-06,
      "logits/chosen": -0.9828249216079712,
      "logits/rejected": -1.0559133291244507,
      "logps/chosen": -74.69087219238281,
      "logps/rejected": -77.69873046875,
      "loss": 12.3374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02543102204799652,
      "rewards/margins": 0.18530091643333435,
      "rewards/rejected": -0.15986987948417664,
      "step": 1700
    },
    {
      "epoch": 0.9879769994772608,
      "grad_norm": 330.0083312988281,
      "learning_rate": 2.5290528762347478e-06,
      "logits/chosen": -0.9956077337265015,
      "logits/rejected": -1.0499070882797241,
      "logps/chosen": -75.12333679199219,
      "logps/rejected": -82.81266021728516,
      "loss": 13.7205,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.060855261981487274,
      "rewards/margins": 0.024372128769755363,
      "rewards/rejected": -0.08522740006446838,
      "step": 1701
    },
    {
      "epoch": 0.9885578207585526,
      "grad_norm": 301.10809326171875,
      "learning_rate": 2.5276002324230105e-06,
      "logits/chosen": -1.0844744443893433,
      "logits/rejected": -1.0837066173553467,
      "logps/chosen": -70.41144561767578,
      "logps/rejected": -80.79979705810547,
      "loss": 13.7769,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11004139482975006,
      "rewards/margins": 0.03207280859351158,
      "rewards/rejected": -0.14211422204971313,
      "step": 1702
    },
    {
      "epoch": 0.9891386420398444,
      "grad_norm": 289.4908142089844,
      "learning_rate": 2.526147588611273e-06,
      "logits/chosen": -0.9275791049003601,
      "logits/rejected": -1.0820963382720947,
      "logps/chosen": -68.19480895996094,
      "logps/rejected": -66.89730072021484,
      "loss": 13.5915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08118017762899399,
      "rewards/margins": 0.042484525591135025,
      "rewards/rejected": -0.12366469949483871,
      "step": 1703
    },
    {
      "epoch": 0.9897194633211361,
      "grad_norm": 333.2899475097656,
      "learning_rate": 2.5246949447995356e-06,
      "logits/chosen": -1.262130856513977,
      "logits/rejected": -1.1707360744476318,
      "logps/chosen": -74.81425476074219,
      "logps/rejected": -81.31537628173828,
      "loss": 14.3077,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08035866916179657,
      "rewards/margins": -0.034176699817180634,
      "rewards/rejected": -0.046181973069906235,
      "step": 1704
    },
    {
      "epoch": 0.9903002846024278,
      "grad_norm": 315.7989196777344,
      "learning_rate": 2.5232423009877983e-06,
      "logits/chosen": -1.0200614929199219,
      "logits/rejected": -1.0713000297546387,
      "logps/chosen": -78.96744537353516,
      "logps/rejected": -71.71560668945312,
      "loss": 13.947,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09802685678005219,
      "rewards/margins": 0.01535411924123764,
      "rewards/rejected": -0.11338096857070923,
      "step": 1705
    },
    {
      "epoch": 0.9908811058837196,
      "grad_norm": 353.2605895996094,
      "learning_rate": 2.5217896571760607e-06,
      "logits/chosen": -1.1126110553741455,
      "logits/rejected": -1.1421066522598267,
      "logps/chosen": -84.41290283203125,
      "logps/rejected": -73.45205688476562,
      "loss": 14.2306,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0743161290884018,
      "rewards/margins": -0.02221987023949623,
      "rewards/rejected": -0.05209627002477646,
      "step": 1706
    },
    {
      "epoch": 0.9914619271650114,
      "grad_norm": 333.77435302734375,
      "learning_rate": 2.5203370133643234e-06,
      "logits/chosen": -1.0462347269058228,
      "logits/rejected": -1.1154237985610962,
      "logps/chosen": -77.18974304199219,
      "logps/rejected": -79.68231201171875,
      "loss": 13.4857,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.019268883392214775,
      "rewards/margins": 0.04767680913209915,
      "rewards/rejected": -0.06694569438695908,
      "step": 1707
    },
    {
      "epoch": 0.992042748446303,
      "grad_norm": 330.7596435546875,
      "learning_rate": 2.518884369552586e-06,
      "logits/chosen": -0.9167941212654114,
      "logits/rejected": -0.8993526697158813,
      "logps/chosen": -79.42576599121094,
      "logps/rejected": -74.48179626464844,
      "loss": 13.9793,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08656813204288483,
      "rewards/margins": 0.010369491763412952,
      "rewards/rejected": -0.0969376340508461,
      "step": 1708
    },
    {
      "epoch": 0.9926235697275948,
      "grad_norm": 349.20379638671875,
      "learning_rate": 2.517431725740849e-06,
      "logits/chosen": -1.0866729021072388,
      "logits/rejected": -1.1045340299606323,
      "logps/chosen": -76.00537109375,
      "logps/rejected": -72.83673858642578,
      "loss": 14.5475,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.038225430995225906,
      "rewards/margins": -0.05270807817578316,
      "rewards/rejected": 0.014482642523944378,
      "step": 1709
    },
    {
      "epoch": 0.9932043910088866,
      "grad_norm": 325.26934814453125,
      "learning_rate": 2.5159790819291112e-06,
      "logits/chosen": -0.9479598999023438,
      "logits/rejected": -1.1353987455368042,
      "logps/chosen": -81.82806396484375,
      "logps/rejected": -74.009765625,
      "loss": 13.9247,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11259279400110245,
      "rewards/margins": 0.008973972871899605,
      "rewards/rejected": -0.1215667575597763,
      "step": 1710
    },
    {
      "epoch": 0.9937852122901784,
      "grad_norm": 345.9144592285156,
      "learning_rate": 2.514526438117374e-06,
      "logits/chosen": -0.9086335897445679,
      "logits/rejected": -0.8605769276618958,
      "logps/chosen": -80.50118255615234,
      "logps/rejected": -84.93008422851562,
      "loss": 14.529,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0473504364490509,
      "rewards/margins": -0.05777702480554581,
      "rewards/rejected": 0.010426589287817478,
      "step": 1711
    },
    {
      "epoch": 0.99436603357147,
      "grad_norm": 363.84906005859375,
      "learning_rate": 2.5130737943056367e-06,
      "logits/chosen": -1.154592514038086,
      "logits/rejected": -1.107348084449768,
      "logps/chosen": -67.6917953491211,
      "logps/rejected": -72.9231948852539,
      "loss": 13.6772,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04632022604346275,
      "rewards/margins": 0.023244161158800125,
      "rewards/rejected": 0.023076066747307777,
      "step": 1712
    },
    {
      "epoch": 0.9949468548527618,
      "grad_norm": 328.0347900390625,
      "learning_rate": 2.511621150493899e-06,
      "logits/chosen": -0.9256439208984375,
      "logits/rejected": -0.9502876400947571,
      "logps/chosen": -72.92536926269531,
      "logps/rejected": -72.7935562133789,
      "loss": 14.0656,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0759708434343338,
      "rewards/margins": -0.002423841506242752,
      "rewards/rejected": -0.07354700565338135,
      "step": 1713
    },
    {
      "epoch": 0.9955276761340536,
      "grad_norm": 272.72412109375,
      "learning_rate": 2.510168506682162e-06,
      "logits/chosen": -0.986652672290802,
      "logits/rejected": -0.9622478485107422,
      "logps/chosen": -66.7657470703125,
      "logps/rejected": -71.24139404296875,
      "loss": 12.6667,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0005818847566843033,
      "rewards/margins": 0.14554749429225922,
      "rewards/rejected": -0.14496560394763947,
      "step": 1714
    },
    {
      "epoch": 0.9961084974153454,
      "grad_norm": 308.6690979003906,
      "learning_rate": 2.5087158628704246e-06,
      "logits/chosen": -1.0471299886703491,
      "logits/rejected": -1.006162405014038,
      "logps/chosen": -68.63721466064453,
      "logps/rejected": -74.35594940185547,
      "loss": 12.8492,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01517643965780735,
      "rewards/margins": 0.11084318161010742,
      "rewards/rejected": -0.09566675126552582,
      "step": 1715
    },
    {
      "epoch": 0.996689318696637,
      "grad_norm": 317.9620056152344,
      "learning_rate": 2.507263219058687e-06,
      "logits/chosen": -0.8819286227226257,
      "logits/rejected": -0.8972331285476685,
      "logps/chosen": -76.63471984863281,
      "logps/rejected": -76.7853775024414,
      "loss": 13.7671,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06816191971302032,
      "rewards/margins": 0.02677130699157715,
      "rewards/rejected": -0.09493322670459747,
      "step": 1716
    },
    {
      "epoch": 0.9972701399779288,
      "grad_norm": 318.3180847167969,
      "learning_rate": 2.5058105752469496e-06,
      "logits/chosen": -0.9387394785881042,
      "logits/rejected": -1.0915039777755737,
      "logps/chosen": -70.59834289550781,
      "logps/rejected": -76.4336166381836,
      "loss": 12.6712,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.032148800790309906,
      "rewards/margins": 0.2184724062681198,
      "rewards/rejected": -0.2506211996078491,
      "step": 1717
    },
    {
      "epoch": 0.9978509612592206,
      "grad_norm": 324.6740417480469,
      "learning_rate": 2.5043579314352124e-06,
      "logits/chosen": -1.0906617641448975,
      "logits/rejected": -1.065298080444336,
      "logps/chosen": -70.38114929199219,
      "logps/rejected": -76.16145324707031,
      "loss": 12.8076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012691743671894073,
      "rewards/margins": 0.12843945622444153,
      "rewards/rejected": -0.11574769020080566,
      "step": 1718
    },
    {
      "epoch": 0.9984317825405122,
      "grad_norm": 298.0662536621094,
      "learning_rate": 2.502905287623475e-06,
      "logits/chosen": -0.8729764223098755,
      "logits/rejected": -0.90547114610672,
      "logps/chosen": -70.86561584472656,
      "logps/rejected": -70.11287689208984,
      "loss": 13.0981,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.005006933584809303,
      "rewards/margins": 0.08521091938018799,
      "rewards/rejected": -0.09021784365177155,
      "step": 1719
    },
    {
      "epoch": 0.999012603821804,
      "grad_norm": 317.7127685546875,
      "learning_rate": 2.5014526438117375e-06,
      "logits/chosen": -0.8424423933029175,
      "logits/rejected": -0.8214157223701477,
      "logps/chosen": -75.37879943847656,
      "logps/rejected": -74.88411712646484,
      "loss": 13.6922,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.091620072722435,
      "rewards/margins": 0.03128615766763687,
      "rewards/rejected": -0.12290624529123306,
      "step": 1720
    },
    {
      "epoch": 0.9995934251030958,
      "grad_norm": 286.47064208984375,
      "learning_rate": 2.5e-06,
      "logits/chosen": -0.9959999918937683,
      "logits/rejected": -1.1310381889343262,
      "logps/chosen": -71.3223648071289,
      "logps/rejected": -81.92350006103516,
      "loss": 12.1144,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.022672206163406372,
      "rewards/margins": 0.19629082083702087,
      "rewards/rejected": -0.1736186295747757,
      "step": 1721
    },
    {
      "epoch": 1.0001742463843875,
      "grad_norm": 310.3578796386719,
      "learning_rate": 2.498547356188263e-06,
      "logits/chosen": -1.0187580585479736,
      "logits/rejected": -1.0229791402816772,
      "logps/chosen": -78.88175964355469,
      "logps/rejected": -79.26593017578125,
      "loss": 13.7982,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06991003453731537,
      "rewards/margins": 0.019723568111658096,
      "rewards/rejected": -0.08963359892368317,
      "step": 1722
    },
    {
      "epoch": 1.0007550676656793,
      "grad_norm": 306.8214111328125,
      "learning_rate": 2.4970947123765253e-06,
      "logits/chosen": -1.0289098024368286,
      "logits/rejected": -0.9756290316581726,
      "logps/chosen": -76.62639617919922,
      "logps/rejected": -83.69001007080078,
      "loss": 12.4029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015295276418328285,
      "rewards/margins": 0.169394388794899,
      "rewards/rejected": -0.15409910678863525,
      "step": 1723
    },
    {
      "epoch": 1.001335888946971,
      "grad_norm": 307.2807312011719,
      "learning_rate": 2.495642068564788e-06,
      "logits/chosen": -1.0836254358291626,
      "logits/rejected": -1.143802285194397,
      "logps/chosen": -72.67015075683594,
      "logps/rejected": -77.22394561767578,
      "loss": 12.8208,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.034096889197826385,
      "rewards/margins": 0.11932823807001114,
      "rewards/rejected": -0.08523134887218475,
      "step": 1724
    },
    {
      "epoch": 1.0019167102282627,
      "grad_norm": 305.87628173828125,
      "learning_rate": 2.4941894247530508e-06,
      "logits/chosen": -0.9439589381217957,
      "logits/rejected": -0.9515148997306824,
      "logps/chosen": -87.69488525390625,
      "logps/rejected": -71.41778564453125,
      "loss": 12.3238,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0052621192298829556,
      "rewards/margins": 0.17189854383468628,
      "rewards/rejected": -0.1666364073753357,
      "step": 1725
    },
    {
      "epoch": 1.0024975315095546,
      "grad_norm": 281.7872619628906,
      "learning_rate": 2.4927367809413135e-06,
      "logits/chosen": -0.9862774610519409,
      "logits/rejected": -0.9919856786727905,
      "logps/chosen": -78.06076049804688,
      "logps/rejected": -92.29457092285156,
      "loss": 10.8852,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.04424609988927841,
      "rewards/margins": 0.3604559898376465,
      "rewards/rejected": -0.3162098526954651,
      "step": 1726
    },
    {
      "epoch": 1.0030783527908462,
      "grad_norm": 316.0426330566406,
      "learning_rate": 2.491284137129576e-06,
      "logits/chosen": -0.9686037302017212,
      "logits/rejected": -0.9354893565177917,
      "logps/chosen": -76.19658660888672,
      "logps/rejected": -77.04872131347656,
      "loss": 11.8927,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.048005081713199615,
      "rewards/margins": 0.2238675355911255,
      "rewards/rejected": -0.17586246132850647,
      "step": 1727
    },
    {
      "epoch": 1.003659174072138,
      "grad_norm": 356.8013000488281,
      "learning_rate": 2.4898314933178386e-06,
      "logits/chosen": -1.027886986732483,
      "logits/rejected": -0.9749377369880676,
      "logps/chosen": -69.18269348144531,
      "logps/rejected": -69.78416442871094,
      "loss": 13.0385,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06171412020921707,
      "rewards/margins": 0.1038701981306076,
      "rewards/rejected": -0.04215607792139053,
      "step": 1728
    },
    {
      "epoch": 1.0042399953534298,
      "grad_norm": 284.9081115722656,
      "learning_rate": 2.4883788495061013e-06,
      "logits/chosen": -1.0370898246765137,
      "logits/rejected": -1.0735399723052979,
      "logps/chosen": -74.31616973876953,
      "logps/rejected": -69.14083862304688,
      "loss": 12.2156,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.000451345753390342,
      "rewards/margins": 0.19433024525642395,
      "rewards/rejected": -0.1938788741827011,
      "step": 1729
    },
    {
      "epoch": 1.0048208166347214,
      "grad_norm": 285.5924377441406,
      "learning_rate": 2.4869262056943637e-06,
      "logits/chosen": -0.9063581228256226,
      "logits/rejected": -0.992654025554657,
      "logps/chosen": -69.09056854248047,
      "logps/rejected": -69.92359161376953,
      "loss": 11.8208,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.07115309685468674,
      "rewards/margins": 0.22874093055725098,
      "rewards/rejected": -0.15758784115314484,
      "step": 1730
    },
    {
      "epoch": 1.0054016379160133,
      "grad_norm": 290.8726501464844,
      "learning_rate": 2.4854735618826264e-06,
      "logits/chosen": -0.9257283210754395,
      "logits/rejected": -0.9341353178024292,
      "logps/chosen": -70.4848403930664,
      "logps/rejected": -71.36671447753906,
      "loss": 12.4238,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04748297110199928,
      "rewards/margins": 0.16466093063354492,
      "rewards/rejected": -0.11717796325683594,
      "step": 1731
    },
    {
      "epoch": 1.005982459197305,
      "grad_norm": 276.4698181152344,
      "learning_rate": 2.484020918070889e-06,
      "logits/chosen": -1.007843017578125,
      "logits/rejected": -0.9165021777153015,
      "logps/chosen": -66.31729125976562,
      "logps/rejected": -68.93580627441406,
      "loss": 11.8862,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.03007885254919529,
      "rewards/margins": 0.21668389439582825,
      "rewards/rejected": -0.1866050511598587,
      "step": 1732
    },
    {
      "epoch": 1.0065632804785967,
      "grad_norm": 271.5777282714844,
      "learning_rate": 2.482568274259152e-06,
      "logits/chosen": -1.1564522981643677,
      "logits/rejected": -1.1468639373779297,
      "logps/chosen": -68.81240844726562,
      "logps/rejected": -75.99501037597656,
      "loss": 11.8314,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.07771363109350204,
      "rewards/margins": 0.2282046377658844,
      "rewards/rejected": -0.15049099922180176,
      "step": 1733
    },
    {
      "epoch": 1.0071441017598886,
      "grad_norm": 296.8550720214844,
      "learning_rate": 2.4811156304474147e-06,
      "logits/chosen": -0.7842386364936829,
      "logits/rejected": -0.7807684540748596,
      "logps/chosen": -75.40328216552734,
      "logps/rejected": -73.23326110839844,
      "loss": 12.6337,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0008960515260696411,
      "rewards/margins": 0.14476574957370758,
      "rewards/rejected": -0.14386969804763794,
      "step": 1734
    },
    {
      "epoch": 1.0077249230411802,
      "grad_norm": 290.52716064453125,
      "learning_rate": 2.4796629866356774e-06,
      "logits/chosen": -1.0725663900375366,
      "logits/rejected": -0.9456678628921509,
      "logps/chosen": -70.86119842529297,
      "logps/rejected": -76.53616333007812,
      "loss": 12.5396,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.033209897577762604,
      "rewards/margins": 0.15690329670906067,
      "rewards/rejected": -0.12369339168071747,
      "step": 1735
    },
    {
      "epoch": 1.0083057443224719,
      "grad_norm": 267.59356689453125,
      "learning_rate": 2.4782103428239397e-06,
      "logits/chosen": -1.05934476852417,
      "logits/rejected": -1.0497558116912842,
      "logps/chosen": -65.6712646484375,
      "logps/rejected": -76.11734771728516,
      "loss": 12.0441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06202181428670883,
      "rewards/margins": 0.209748774766922,
      "rewards/rejected": -0.14772695302963257,
      "step": 1736
    },
    {
      "epoch": 1.0088865656037638,
      "grad_norm": 306.54168701171875,
      "learning_rate": 2.4767576990122025e-06,
      "logits/chosen": -0.8704284429550171,
      "logits/rejected": -0.8033106923103333,
      "logps/chosen": -75.1402359008789,
      "logps/rejected": -83.350830078125,
      "loss": 12.3083,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0072390856221318245,
      "rewards/margins": 0.17515526711940765,
      "rewards/rejected": -0.1823943555355072,
      "step": 1737
    },
    {
      "epoch": 1.0094673868850554,
      "grad_norm": 295.8548889160156,
      "learning_rate": 2.4753050552004652e-06,
      "logits/chosen": -1.0087684392929077,
      "logits/rejected": -1.0190235376358032,
      "logps/chosen": -81.39954376220703,
      "logps/rejected": -77.54902648925781,
      "loss": 11.6017,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.05635563284158707,
      "rewards/margins": 0.267803430557251,
      "rewards/rejected": -0.2114478051662445,
      "step": 1738
    },
    {
      "epoch": 1.010048208166347,
      "grad_norm": 297.1650695800781,
      "learning_rate": 2.4738524113887276e-06,
      "logits/chosen": -0.9970433115959167,
      "logits/rejected": -1.029784917831421,
      "logps/chosen": -69.62125396728516,
      "logps/rejected": -67.70980834960938,
      "loss": 12.5102,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.011428048834204674,
      "rewards/margins": 0.15810544788837433,
      "rewards/rejected": -0.1466774046421051,
      "step": 1739
    },
    {
      "epoch": 1.010629029447639,
      "grad_norm": 277.91485595703125,
      "learning_rate": 2.4723997675769903e-06,
      "logits/chosen": -0.9879977107048035,
      "logits/rejected": -0.938629150390625,
      "logps/chosen": -73.74698638916016,
      "logps/rejected": -79.59793090820312,
      "loss": 11.6349,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.09471438825130463,
      "rewards/margins": 0.2549354135990143,
      "rewards/rejected": -0.16022104024887085,
      "step": 1740
    },
    {
      "epoch": 1.0112098507289307,
      "grad_norm": 289.1015625,
      "learning_rate": 2.470947123765253e-06,
      "logits/chosen": -1.047290563583374,
      "logits/rejected": -1.0115858316421509,
      "logps/chosen": -71.44462585449219,
      "logps/rejected": -78.6550521850586,
      "loss": 10.8384,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.12398791313171387,
      "rewards/margins": 0.36083775758743286,
      "rewards/rejected": -0.2368498146533966,
      "step": 1741
    },
    {
      "epoch": 1.0117906720102225,
      "grad_norm": 284.77569580078125,
      "learning_rate": 2.469494479953516e-06,
      "logits/chosen": -1.0271031856536865,
      "logits/rejected": -1.0553693771362305,
      "logps/chosen": -74.97417449951172,
      "logps/rejected": -77.2381820678711,
      "loss": 12.0523,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.10335147380828857,
      "rewards/margins": 0.22173932194709778,
      "rewards/rejected": -0.1183878555893898,
      "step": 1742
    },
    {
      "epoch": 1.0123714932915142,
      "grad_norm": 279.2829895019531,
      "learning_rate": 2.468041836141778e-06,
      "logits/chosen": -1.2351869344711304,
      "logits/rejected": -1.243849515914917,
      "logps/chosen": -69.58841705322266,
      "logps/rejected": -82.63679504394531,
      "loss": 11.3664,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.042612940073013306,
      "rewards/margins": 0.2963695824146271,
      "rewards/rejected": -0.25375667214393616,
      "step": 1743
    },
    {
      "epoch": 1.0129523145728059,
      "grad_norm": 266.0357360839844,
      "learning_rate": 2.466589192330041e-06,
      "logits/chosen": -0.9723545908927917,
      "logits/rejected": -0.9949741363525391,
      "logps/chosen": -71.33142852783203,
      "logps/rejected": -83.14581298828125,
      "loss": 10.6765,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0888810083270073,
      "rewards/margins": 0.36409392952919006,
      "rewards/rejected": -0.2752128839492798,
      "step": 1744
    },
    {
      "epoch": 1.0135331358540978,
      "grad_norm": 272.3580627441406,
      "learning_rate": 2.4651365485183036e-06,
      "logits/chosen": -0.906292736530304,
      "logits/rejected": -1.1051472425460815,
      "logps/chosen": -70.36241912841797,
      "logps/rejected": -85.29285430908203,
      "loss": 11.4058,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.051963578909635544,
      "rewards/margins": 0.28600093722343445,
      "rewards/rejected": -0.2340373545885086,
      "step": 1745
    },
    {
      "epoch": 1.0141139571353894,
      "grad_norm": 290.2193298339844,
      "learning_rate": 2.463683904706566e-06,
      "logits/chosen": -1.133362054824829,
      "logits/rejected": -1.2012062072753906,
      "logps/chosen": -67.35916900634766,
      "logps/rejected": -66.3633041381836,
      "loss": 13.0282,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04202844947576523,
      "rewards/margins": 0.09714338928461075,
      "rewards/rejected": -0.05511493608355522,
      "step": 1746
    },
    {
      "epoch": 1.014694778416681,
      "grad_norm": 274.5871276855469,
      "learning_rate": 2.4622312608948287e-06,
      "logits/chosen": -1.018569827079773,
      "logits/rejected": -1.0206456184387207,
      "logps/chosen": -64.80757141113281,
      "logps/rejected": -74.65616607666016,
      "loss": 12.044,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04900861531496048,
      "rewards/margins": 0.20763413608074188,
      "rewards/rejected": -0.1586254984140396,
      "step": 1747
    },
    {
      "epoch": 1.015275599697973,
      "grad_norm": 327.37310791015625,
      "learning_rate": 2.4607786170830914e-06,
      "logits/chosen": -1.0058655738830566,
      "logits/rejected": -1.0348700284957886,
      "logps/chosen": -74.80683898925781,
      "logps/rejected": -77.69505310058594,
      "loss": 13.2394,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06611393392086029,
      "rewards/margins": 0.0843910351395607,
      "rewards/rejected": -0.1505049765110016,
      "step": 1748
    },
    {
      "epoch": 1.0158564209792647,
      "grad_norm": 278.77447509765625,
      "learning_rate": 2.4593259732713538e-06,
      "logits/chosen": -1.0118757486343384,
      "logits/rejected": -1.0431196689605713,
      "logps/chosen": -76.19218444824219,
      "logps/rejected": -76.2001724243164,
      "loss": 11.7553,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.05235186964273453,
      "rewards/margins": 0.23560091853141785,
      "rewards/rejected": -0.1832490712404251,
      "step": 1749
    },
    {
      "epoch": 1.0164372422605563,
      "grad_norm": 529.6303100585938,
      "learning_rate": 2.4578733294596165e-06,
      "logits/chosen": -1.0182701349258423,
      "logits/rejected": -1.0310004949569702,
      "logps/chosen": -79.64320373535156,
      "logps/rejected": -72.77711486816406,
      "loss": 11.9028,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.07643435895442963,
      "rewards/margins": 0.22258293628692627,
      "rewards/rejected": -0.14614860713481903,
      "step": 1750
    },
    {
      "epoch": 1.0170180635418482,
      "grad_norm": 297.3377990722656,
      "learning_rate": 2.4564206856478793e-06,
      "logits/chosen": -1.088423728942871,
      "logits/rejected": -1.1018476486206055,
      "logps/chosen": -69.29364013671875,
      "logps/rejected": -77.49354553222656,
      "loss": 11.7909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01562787964940071,
      "rewards/margins": 0.2760494351387024,
      "rewards/rejected": -0.260421484708786,
      "step": 1751
    },
    {
      "epoch": 1.0175988848231399,
      "grad_norm": 291.22344970703125,
      "learning_rate": 2.454968041836142e-06,
      "logits/chosen": -1.009781837463379,
      "logits/rejected": -1.0661695003509521,
      "logps/chosen": -70.85111999511719,
      "logps/rejected": -68.69586181640625,
      "loss": 12.9098,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.020266063511371613,
      "rewards/margins": 0.1182970255613327,
      "rewards/rejected": -0.09803096950054169,
      "step": 1752
    },
    {
      "epoch": 1.0181797061044318,
      "grad_norm": 300.30670166015625,
      "learning_rate": 2.4535153980244048e-06,
      "logits/chosen": -1.017282247543335,
      "logits/rejected": -0.9565478563308716,
      "logps/chosen": -72.61406707763672,
      "logps/rejected": -78.46427917480469,
      "loss": 12.0764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015890780836343765,
      "rewards/margins": 0.20447203516960144,
      "rewards/rejected": -0.18858130276203156,
      "step": 1753
    },
    {
      "epoch": 1.0187605273857234,
      "grad_norm": 395.31817626953125,
      "learning_rate": 2.4520627542126675e-06,
      "logits/chosen": -0.988058865070343,
      "logits/rejected": -0.7947362065315247,
      "logps/chosen": -72.217529296875,
      "logps/rejected": -71.69329071044922,
      "loss": 12.8544,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.004397256765514612,
      "rewards/margins": 0.13137534260749817,
      "rewards/rejected": -0.13577260076999664,
      "step": 1754
    },
    {
      "epoch": 1.019341348667015,
      "grad_norm": 309.5154724121094,
      "learning_rate": 2.45061011040093e-06,
      "logits/chosen": -1.1319024562835693,
      "logits/rejected": -1.2304996252059937,
      "logps/chosen": -71.77473449707031,
      "logps/rejected": -70.90110778808594,
      "loss": 12.6458,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0036634411662817,
      "rewards/margins": 0.1422930210828781,
      "rewards/rejected": -0.14595645666122437,
      "step": 1755
    },
    {
      "epoch": 1.019922169948307,
      "grad_norm": 318.6673889160156,
      "learning_rate": 2.4491574665891926e-06,
      "logits/chosen": -1.0985543727874756,
      "logits/rejected": -1.0787074565887451,
      "logps/chosen": -77.18990325927734,
      "logps/rejected": -73.70921325683594,
      "loss": 13.4627,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.058716535568237305,
      "rewards/margins": 0.06169065088033676,
      "rewards/rejected": -0.12040718644857407,
      "step": 1756
    },
    {
      "epoch": 1.0205029912295986,
      "grad_norm": 368.5150451660156,
      "learning_rate": 2.4477048227774553e-06,
      "logits/chosen": -0.8918913006782532,
      "logits/rejected": -0.9180348515510559,
      "logps/chosen": -82.34278106689453,
      "logps/rejected": -84.48460388183594,
      "loss": 13.3416,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06952496618032455,
      "rewards/margins": 0.1027899757027626,
      "rewards/rejected": -0.17231495678424835,
      "step": 1757
    },
    {
      "epoch": 1.0210838125108903,
      "grad_norm": 318.24859619140625,
      "learning_rate": 2.446252178965718e-06,
      "logits/chosen": -0.9825534820556641,
      "logits/rejected": -1.0132125616073608,
      "logps/chosen": -72.49050903320312,
      "logps/rejected": -78.62373352050781,
      "loss": 11.463,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.00828148890286684,
      "rewards/margins": 0.2658321261405945,
      "rewards/rejected": -0.2575506567955017,
      "step": 1758
    },
    {
      "epoch": 1.0216646337921822,
      "grad_norm": 468.2798156738281,
      "learning_rate": 2.4447995351539804e-06,
      "logits/chosen": -1.1734148263931274,
      "logits/rejected": -1.1108180284500122,
      "logps/chosen": -68.78428649902344,
      "logps/rejected": -71.49794006347656,
      "loss": 11.9274,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0715429037809372,
      "rewards/margins": 0.22715874016284943,
      "rewards/rejected": -0.15561582148075104,
      "step": 1759
    },
    {
      "epoch": 1.0222454550734739,
      "grad_norm": 299.5284729003906,
      "learning_rate": 2.443346891342243e-06,
      "logits/chosen": -1.0347208976745605,
      "logits/rejected": -1.1470574140548706,
      "logps/chosen": -86.3415756225586,
      "logps/rejected": -75.57364654541016,
      "loss": 12.3555,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0053415605798363686,
      "rewards/margins": 0.17316599190235138,
      "rewards/rejected": -0.17850753664970398,
      "step": 1760
    },
    {
      "epoch": 1.0228262763547655,
      "grad_norm": 299.2851867675781,
      "learning_rate": 2.441894247530506e-06,
      "logits/chosen": -1.0469765663146973,
      "logits/rejected": -1.0681182146072388,
      "logps/chosen": -70.55113220214844,
      "logps/rejected": -75.1129379272461,
      "loss": 12.0547,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.029618894681334496,
      "rewards/margins": 0.20354339480400085,
      "rewards/rejected": -0.1739244908094406,
      "step": 1761
    },
    {
      "epoch": 1.0234070976360574,
      "grad_norm": 277.2837829589844,
      "learning_rate": 2.4404416037187682e-06,
      "logits/chosen": -0.9554106593132019,
      "logits/rejected": -0.9013598561286926,
      "logps/chosen": -70.30006408691406,
      "logps/rejected": -64.8463363647461,
      "loss": 13.0213,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.008472795598208904,
      "rewards/margins": 0.09752868115901947,
      "rewards/rejected": -0.10600147396326065,
      "step": 1762
    },
    {
      "epoch": 1.023987918917349,
      "grad_norm": 296.0224609375,
      "learning_rate": 2.438988959907031e-06,
      "logits/chosen": -0.9040548205375671,
      "logits/rejected": -0.9101142883300781,
      "logps/chosen": -79.00831604003906,
      "logps/rejected": -78.0610122680664,
      "loss": 11.7458,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.026872694492340088,
      "rewards/margins": 0.24106188118457794,
      "rewards/rejected": -0.21418920159339905,
      "step": 1763
    },
    {
      "epoch": 1.024568740198641,
      "grad_norm": 312.5642395019531,
      "learning_rate": 2.4375363160952937e-06,
      "logits/chosen": -1.1683385372161865,
      "logits/rejected": -1.150472640991211,
      "logps/chosen": -81.03670501708984,
      "logps/rejected": -83.64530944824219,
      "loss": 13.3272,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04667438194155693,
      "rewards/margins": 0.0643816888332367,
      "rewards/rejected": -0.11105605214834213,
      "step": 1764
    },
    {
      "epoch": 1.0251495614799326,
      "grad_norm": 303.8731994628906,
      "learning_rate": 2.436083672283556e-06,
      "logits/chosen": -1.1134673357009888,
      "logits/rejected": -0.9937359094619751,
      "logps/chosen": -75.13594055175781,
      "logps/rejected": -73.07179260253906,
      "loss": 12.709,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.021896759048104286,
      "rewards/margins": 0.12814390659332275,
      "rewards/rejected": -0.1500406563282013,
      "step": 1765
    },
    {
      "epoch": 1.0257303827612243,
      "grad_norm": 284.25537109375,
      "learning_rate": 2.434631028471819e-06,
      "logits/chosen": -1.0828286409378052,
      "logits/rejected": -1.024355173110962,
      "logps/chosen": -75.21257019042969,
      "logps/rejected": -75.101318359375,
      "loss": 12.3471,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01720609702169895,
      "rewards/margins": 0.1741756945848465,
      "rewards/rejected": -0.1569696068763733,
      "step": 1766
    },
    {
      "epoch": 1.0263112040425162,
      "grad_norm": 304.7877502441406,
      "learning_rate": 2.4331783846600815e-06,
      "logits/chosen": -1.1682668924331665,
      "logits/rejected": -1.0933306217193604,
      "logps/chosen": -71.91011047363281,
      "logps/rejected": -81.58442687988281,
      "loss": 11.687,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0402267761528492,
      "rewards/margins": 0.2500234544277191,
      "rewards/rejected": -0.20979666709899902,
      "step": 1767
    },
    {
      "epoch": 1.0268920253238079,
      "grad_norm": 354.9096374511719,
      "learning_rate": 2.4317257408483443e-06,
      "logits/chosen": -0.9814058542251587,
      "logits/rejected": -0.8618592023849487,
      "logps/chosen": -74.69953918457031,
      "logps/rejected": -74.94476318359375,
      "loss": 13.0832,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.00495229521766305,
      "rewards/margins": 0.11072232574224472,
      "rewards/rejected": -0.11567461490631104,
      "step": 1768
    },
    {
      "epoch": 1.0274728466050995,
      "grad_norm": 279.7265319824219,
      "learning_rate": 2.4302730970366066e-06,
      "logits/chosen": -1.2584569454193115,
      "logits/rejected": -1.2268121242523193,
      "logps/chosen": -70.73481750488281,
      "logps/rejected": -81.8658676147461,
      "loss": 12.1617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.004170245490968227,
      "rewards/margins": 0.19563667476177216,
      "rewards/rejected": -0.19980691373348236,
      "step": 1769
    },
    {
      "epoch": 1.0280536678863914,
      "grad_norm": 307.786376953125,
      "learning_rate": 2.4288204532248694e-06,
      "logits/chosen": -1.0927098989486694,
      "logits/rejected": -1.2099649906158447,
      "logps/chosen": -68.34037780761719,
      "logps/rejected": -83.68338775634766,
      "loss": 11.4326,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.005060443188995123,
      "rewards/margins": 0.3119402527809143,
      "rewards/rejected": -0.31700068712234497,
      "step": 1770
    },
    {
      "epoch": 1.028634489167683,
      "grad_norm": 314.7550048828125,
      "learning_rate": 2.427367809413132e-06,
      "logits/chosen": -1.1393171548843384,
      "logits/rejected": -1.1220924854278564,
      "logps/chosen": -69.36933898925781,
      "logps/rejected": -78.43421173095703,
      "loss": 11.5055,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.08931799232959747,
      "rewards/margins": 0.26503047347068787,
      "rewards/rejected": -0.1757124960422516,
      "step": 1771
    },
    {
      "epoch": 1.0292153104489747,
      "grad_norm": 273.4578552246094,
      "learning_rate": 2.4259151656013944e-06,
      "logits/chosen": -1.0305819511413574,
      "logits/rejected": -1.1656668186187744,
      "logps/chosen": -75.16026306152344,
      "logps/rejected": -68.74202728271484,
      "loss": 12.0048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005850923713296652,
      "rewards/margins": 0.2243921011686325,
      "rewards/rejected": -0.21854114532470703,
      "step": 1772
    },
    {
      "epoch": 1.0297961317302666,
      "grad_norm": 361.3050537109375,
      "learning_rate": 2.424462521789657e-06,
      "logits/chosen": -1.0504028797149658,
      "logits/rejected": -1.123801350593567,
      "logps/chosen": -76.07279968261719,
      "logps/rejected": -81.24317932128906,
      "loss": 11.4823,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02148047275841236,
      "rewards/margins": 0.3339378237724304,
      "rewards/rejected": -0.35541829466819763,
      "step": 1773
    },
    {
      "epoch": 1.0303769530115583,
      "grad_norm": 284.7305908203125,
      "learning_rate": 2.42300987797792e-06,
      "logits/chosen": -0.8235481381416321,
      "logits/rejected": -0.7617828249931335,
      "logps/chosen": -73.07469177246094,
      "logps/rejected": -81.74129486083984,
      "loss": 11.8796,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.01117186713963747,
      "rewards/margins": 0.22988685965538025,
      "rewards/rejected": -0.24105870723724365,
      "step": 1774
    },
    {
      "epoch": 1.0309577742928502,
      "grad_norm": 287.0780029296875,
      "learning_rate": 2.4215572341661827e-06,
      "logits/chosen": -1.2456663846969604,
      "logits/rejected": -1.1421397924423218,
      "logps/chosen": -80.1615219116211,
      "logps/rejected": -88.32636260986328,
      "loss": 11.6171,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0034477233421057463,
      "rewards/margins": 0.27040886878967285,
      "rewards/rejected": -0.26696115732192993,
      "step": 1775
    },
    {
      "epoch": 1.0315385955741418,
      "grad_norm": 336.2914123535156,
      "learning_rate": 2.4201045903544454e-06,
      "logits/chosen": -1.1550507545471191,
      "logits/rejected": -1.1800119876861572,
      "logps/chosen": -75.59684753417969,
      "logps/rejected": -72.27438354492188,
      "loss": 11.7829,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.03313673287630081,
      "rewards/margins": 0.2454589605331421,
      "rewards/rejected": -0.21232223510742188,
      "step": 1776
    },
    {
      "epoch": 1.0321194168554335,
      "grad_norm": 287.0185241699219,
      "learning_rate": 2.418651946542708e-06,
      "logits/chosen": -1.1247012615203857,
      "logits/rejected": -1.074866771697998,
      "logps/chosen": -71.1903305053711,
      "logps/rejected": -80.40797424316406,
      "loss": 12.779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07777810841798782,
      "rewards/margins": 0.12616197764873505,
      "rewards/rejected": -0.20394012331962585,
      "step": 1777
    },
    {
      "epoch": 1.0327002381367254,
      "grad_norm": 342.22430419921875,
      "learning_rate": 2.4171993027309705e-06,
      "logits/chosen": -1.081476092338562,
      "logits/rejected": -1.0427172183990479,
      "logps/chosen": -72.42833709716797,
      "logps/rejected": -77.74174499511719,
      "loss": 12.7003,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02967444621026516,
      "rewards/margins": 0.13468468189239502,
      "rewards/rejected": -0.16435912251472473,
      "step": 1778
    },
    {
      "epoch": 1.033281059418017,
      "grad_norm": 279.4868469238281,
      "learning_rate": 2.4157466589192333e-06,
      "logits/chosen": -1.1080858707427979,
      "logits/rejected": -1.2145822048187256,
      "logps/chosen": -70.37612915039062,
      "logps/rejected": -92.63235473632812,
      "loss": 11.0621,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.028925588354468346,
      "rewards/margins": 0.3990841507911682,
      "rewards/rejected": -0.42800968885421753,
      "step": 1779
    },
    {
      "epoch": 1.0338618806993087,
      "grad_norm": 290.38494873046875,
      "learning_rate": 2.414294015107496e-06,
      "logits/chosen": -1.1287726163864136,
      "logits/rejected": -0.9995062947273254,
      "logps/chosen": -73.8758544921875,
      "logps/rejected": -83.56784057617188,
      "loss": 11.2852,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.001776278018951416,
      "rewards/margins": 0.2978811264038086,
      "rewards/rejected": -0.2961048185825348,
      "step": 1780
    },
    {
      "epoch": 1.0344427019806006,
      "grad_norm": 298.0079040527344,
      "learning_rate": 2.4128413712957583e-06,
      "logits/chosen": -1.022382378578186,
      "logits/rejected": -1.1161518096923828,
      "logps/chosen": -72.1476821899414,
      "logps/rejected": -71.8859634399414,
      "loss": 12.4469,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.008883072063326836,
      "rewards/margins": 0.16164202988147736,
      "rewards/rejected": -0.17052510380744934,
      "step": 1781
    },
    {
      "epoch": 1.0350235232618923,
      "grad_norm": 289.6922607421875,
      "learning_rate": 2.411388727484021e-06,
      "logits/chosen": -1.164899468421936,
      "logits/rejected": -1.135265588760376,
      "logps/chosen": -77.8196792602539,
      "logps/rejected": -77.63141632080078,
      "loss": 11.8389,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.026596810668706894,
      "rewards/margins": 0.23024630546569824,
      "rewards/rejected": -0.2568431496620178,
      "step": 1782
    },
    {
      "epoch": 1.035604344543184,
      "grad_norm": 290.9444580078125,
      "learning_rate": 2.409936083672284e-06,
      "logits/chosen": -1.0183415412902832,
      "logits/rejected": -1.0100244283676147,
      "logps/chosen": -66.28545379638672,
      "logps/rejected": -72.91636657714844,
      "loss": 12.8811,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.030678126960992813,
      "rewards/margins": 0.12311466783285141,
      "rewards/rejected": -0.15379278361797333,
      "step": 1783
    },
    {
      "epoch": 1.0361851658244758,
      "grad_norm": 280.76019287109375,
      "learning_rate": 2.4084834398605466e-06,
      "logits/chosen": -1.1244175434112549,
      "logits/rejected": -1.1172425746917725,
      "logps/chosen": -77.58967590332031,
      "logps/rejected": -74.97061157226562,
      "loss": 11.2874,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.033810317516326904,
      "rewards/margins": 0.2995678186416626,
      "rewards/rejected": -0.2657574713230133,
      "step": 1784
    },
    {
      "epoch": 1.0367659871057675,
      "grad_norm": 282.7433776855469,
      "learning_rate": 2.407030796048809e-06,
      "logits/chosen": -0.9866765141487122,
      "logits/rejected": -0.9920461773872375,
      "logps/chosen": -76.37644958496094,
      "logps/rejected": -75.4314193725586,
      "loss": 12.1634,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04232632741332054,
      "rewards/margins": 0.20775961875915527,
      "rewards/rejected": -0.16543331742286682,
      "step": 1785
    },
    {
      "epoch": 1.0373468083870594,
      "grad_norm": 265.85791015625,
      "learning_rate": 2.4055781522370716e-06,
      "logits/chosen": -0.9609881639480591,
      "logits/rejected": -1.0261566638946533,
      "logps/chosen": -68.72970581054688,
      "logps/rejected": -68.59684753417969,
      "loss": 11.9121,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.05759136751294136,
      "rewards/margins": 0.2284972220659256,
      "rewards/rejected": -0.17090585827827454,
      "step": 1786
    },
    {
      "epoch": 1.037927629668351,
      "grad_norm": 296.49517822265625,
      "learning_rate": 2.4041255084253344e-06,
      "logits/chosen": -1.133208990097046,
      "logits/rejected": -1.1493123769760132,
      "logps/chosen": -80.16184997558594,
      "logps/rejected": -82.29649353027344,
      "loss": 12.3183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.037330225110054016,
      "rewards/margins": 0.17978905141353607,
      "rewards/rejected": -0.2171192467212677,
      "step": 1787
    },
    {
      "epoch": 1.0385084509496427,
      "grad_norm": 297.3109436035156,
      "learning_rate": 2.4026728646135967e-06,
      "logits/chosen": -0.9459782838821411,
      "logits/rejected": -0.9437891840934753,
      "logps/chosen": -81.57313537597656,
      "logps/rejected": -99.79293823242188,
      "loss": 10.7098,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0311464611440897,
      "rewards/margins": 0.3756529688835144,
      "rewards/rejected": -0.34450656175613403,
      "step": 1788
    },
    {
      "epoch": 1.0390892722309346,
      "grad_norm": 280.0033264160156,
      "learning_rate": 2.4012202208018595e-06,
      "logits/chosen": -1.2231870889663696,
      "logits/rejected": -1.1726429462432861,
      "logps/chosen": -73.84477233886719,
      "logps/rejected": -76.03693389892578,
      "loss": 12.096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006459588650614023,
      "rewards/margins": 0.20097950100898743,
      "rewards/rejected": -0.19451990723609924,
      "step": 1789
    },
    {
      "epoch": 1.0396700935122263,
      "grad_norm": 290.902099609375,
      "learning_rate": 2.3997675769901222e-06,
      "logits/chosen": -1.1803739070892334,
      "logits/rejected": -1.2050305604934692,
      "logps/chosen": -81.71353149414062,
      "logps/rejected": -86.16224670410156,
      "loss": 12.4891,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0927726998925209,
      "rewards/margins": 0.18073661625385284,
      "rewards/rejected": -0.27350932359695435,
      "step": 1790
    },
    {
      "epoch": 1.040250914793518,
      "grad_norm": 286.8930969238281,
      "learning_rate": 2.398314933178385e-06,
      "logits/chosen": -1.261278510093689,
      "logits/rejected": -1.2329069375991821,
      "logps/chosen": -72.05188751220703,
      "logps/rejected": -82.3337631225586,
      "loss": 12.7563,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11942397058010101,
      "rewards/margins": 0.1308235377073288,
      "rewards/rejected": -0.2502475082874298,
      "step": 1791
    },
    {
      "epoch": 1.0408317360748098,
      "grad_norm": 308.7876281738281,
      "learning_rate": 2.3968622893666473e-06,
      "logits/chosen": -1.2346148490905762,
      "logits/rejected": -1.1794416904449463,
      "logps/chosen": -78.88829040527344,
      "logps/rejected": -76.66127014160156,
      "loss": 12.9007,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.038641270250082016,
      "rewards/margins": 0.14094164967536926,
      "rewards/rejected": -0.17958292365074158,
      "step": 1792
    },
    {
      "epoch": 1.0414125573561015,
      "grad_norm": 279.5557861328125,
      "learning_rate": 2.39540964555491e-06,
      "logits/chosen": -1.296668291091919,
      "logits/rejected": -1.2225661277770996,
      "logps/chosen": -73.86051177978516,
      "logps/rejected": -79.49795532226562,
      "loss": 12.3491,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.045357540249824524,
      "rewards/margins": 0.1776280254125595,
      "rewards/rejected": -0.13227048516273499,
      "step": 1793
    },
    {
      "epoch": 1.0419933786373932,
      "grad_norm": 290.5263671875,
      "learning_rate": 2.393957001743173e-06,
      "logits/chosen": -1.1634373664855957,
      "logits/rejected": -1.142647385597229,
      "logps/chosen": -78.15361022949219,
      "logps/rejected": -75.28469848632812,
      "loss": 12.442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.053712159395217896,
      "rewards/margins": 0.16911128163337708,
      "rewards/rejected": -0.22282347083091736,
      "step": 1794
    },
    {
      "epoch": 1.042574199918685,
      "grad_norm": 310.998291015625,
      "learning_rate": 2.3925043579314355e-06,
      "logits/chosen": -1.2551833391189575,
      "logits/rejected": -1.2467610836029053,
      "logps/chosen": -71.48469543457031,
      "logps/rejected": -74.80443572998047,
      "loss": 12.8102,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0023573674261569977,
      "rewards/margins": 0.12768803536891937,
      "rewards/rejected": -0.13004539906978607,
      "step": 1795
    },
    {
      "epoch": 1.0431550211999767,
      "grad_norm": 323.1011962890625,
      "learning_rate": 2.3910517141196983e-06,
      "logits/chosen": -1.3284083604812622,
      "logits/rejected": -1.1780775785446167,
      "logps/chosen": -73.05310821533203,
      "logps/rejected": -72.46563720703125,
      "loss": 12.9206,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.025364290922880173,
      "rewards/margins": 0.1122143492102623,
      "rewards/rejected": -0.13757865130901337,
      "step": 1796
    },
    {
      "epoch": 1.0437358424812686,
      "grad_norm": 290.84698486328125,
      "learning_rate": 2.3895990703079606e-06,
      "logits/chosen": -1.145970106124878,
      "logits/rejected": -1.1923578977584839,
      "logps/chosen": -71.29136657714844,
      "logps/rejected": -82.99765014648438,
      "loss": 12.678,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11989359557628632,
      "rewards/margins": 0.14581026136875153,
      "rewards/rejected": -0.26570385694503784,
      "step": 1797
    },
    {
      "epoch": 1.0443166637625603,
      "grad_norm": 318.05670166015625,
      "learning_rate": 2.3881464264962234e-06,
      "logits/chosen": -1.2242205142974854,
      "logits/rejected": -1.3686120510101318,
      "logps/chosen": -77.58811950683594,
      "logps/rejected": -78.39141845703125,
      "loss": 11.6007,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.006837140768766403,
      "rewards/margins": 0.26938849687576294,
      "rewards/rejected": -0.26255136728286743,
      "step": 1798
    },
    {
      "epoch": 1.044897485043852,
      "grad_norm": 300.98284912109375,
      "learning_rate": 2.386693782684486e-06,
      "logits/chosen": -1.1400254964828491,
      "logits/rejected": -1.1709778308868408,
      "logps/chosen": -79.81010437011719,
      "logps/rejected": -70.65646362304688,
      "loss": 12.8347,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05340326949954033,
      "rewards/margins": 0.13481150567531586,
      "rewards/rejected": -0.1882147639989853,
      "step": 1799
    },
    {
      "epoch": 1.0454783063251438,
      "grad_norm": 270.8587341308594,
      "learning_rate": 2.385241138872749e-06,
      "logits/chosen": -1.0681713819503784,
      "logits/rejected": -1.0120853185653687,
      "logps/chosen": -74.66976928710938,
      "logps/rejected": -74.67829132080078,
      "loss": 11.9847,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.011710884049534798,
      "rewards/margins": 0.2203114926815033,
      "rewards/rejected": -0.23202237486839294,
      "step": 1800
    },
    {
      "epoch": 1.0460591276064355,
      "grad_norm": 272.3807373046875,
      "learning_rate": 2.383788495061011e-06,
      "logits/chosen": -1.2399804592132568,
      "logits/rejected": -1.2050937414169312,
      "logps/chosen": -73.98487854003906,
      "logps/rejected": -79.21199035644531,
      "loss": 11.5269,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.019887972623109818,
      "rewards/margins": 0.27182629704475403,
      "rewards/rejected": -0.2519383132457733,
      "step": 1801
    },
    {
      "epoch": 1.0466399488877272,
      "grad_norm": 280.67901611328125,
      "learning_rate": 2.382335851249274e-06,
      "logits/chosen": -1.253383994102478,
      "logits/rejected": -1.38071608543396,
      "logps/chosen": -70.5538558959961,
      "logps/rejected": -73.8531723022461,
      "loss": 12.3127,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.016190359368920326,
      "rewards/margins": 0.18490445613861084,
      "rewards/rejected": -0.2010948210954666,
      "step": 1802
    },
    {
      "epoch": 1.047220770169019,
      "grad_norm": 298.3965148925781,
      "learning_rate": 2.3808832074375367e-06,
      "logits/chosen": -1.086688756942749,
      "logits/rejected": -1.2468153238296509,
      "logps/chosen": -76.81304931640625,
      "logps/rejected": -67.44319152832031,
      "loss": 12.6456,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04800144582986832,
      "rewards/margins": 0.1507321298122406,
      "rewards/rejected": -0.19873356819152832,
      "step": 1803
    },
    {
      "epoch": 1.0478015914503107,
      "grad_norm": 289.0165710449219,
      "learning_rate": 2.379430563625799e-06,
      "logits/chosen": -1.1440999507904053,
      "logits/rejected": -1.1437499523162842,
      "logps/chosen": -78.52664947509766,
      "logps/rejected": -75.87870025634766,
      "loss": 12.2708,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.01002211682498455,
      "rewards/margins": 0.20415373146533966,
      "rewards/rejected": -0.21417586505413055,
      "step": 1804
    },
    {
      "epoch": 1.0483824127316024,
      "grad_norm": 315.9421691894531,
      "learning_rate": 2.3779779198140618e-06,
      "logits/chosen": -1.1186802387237549,
      "logits/rejected": -1.1962964534759521,
      "logps/chosen": -85.64881896972656,
      "logps/rejected": -73.1773452758789,
      "loss": 12.4823,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06448259949684143,
      "rewards/margins": 0.2340502291917801,
      "rewards/rejected": -0.2985328137874603,
      "step": 1805
    },
    {
      "epoch": 1.0489632340128943,
      "grad_norm": 322.95550537109375,
      "learning_rate": 2.3765252760023245e-06,
      "logits/chosen": -1.0654499530792236,
      "logits/rejected": -1.119666576385498,
      "logps/chosen": -78.18614959716797,
      "logps/rejected": -83.8582534790039,
      "loss": 12.3643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06570643931627274,
      "rewards/margins": 0.17531459033489227,
      "rewards/rejected": -0.24102100729942322,
      "step": 1806
    },
    {
      "epoch": 1.049544055294186,
      "grad_norm": 314.0057067871094,
      "learning_rate": 2.375072632190587e-06,
      "logits/chosen": -1.2928205728530884,
      "logits/rejected": -1.3698197603225708,
      "logps/chosen": -82.49971008300781,
      "logps/rejected": -81.00529479980469,
      "loss": 12.4894,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09000097960233688,
      "rewards/margins": 0.16648072004318237,
      "rewards/rejected": -0.25648170709609985,
      "step": 1807
    },
    {
      "epoch": 1.0501248765754778,
      "grad_norm": 290.1881408691406,
      "learning_rate": 2.3736199883788496e-06,
      "logits/chosen": -1.0696035623550415,
      "logits/rejected": -1.0367176532745361,
      "logps/chosen": -72.80193328857422,
      "logps/rejected": -77.41194915771484,
      "loss": 11.9916,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.014666035771369934,
      "rewards/margins": 0.2182949036359787,
      "rewards/rejected": -0.20362886786460876,
      "step": 1808
    },
    {
      "epoch": 1.0507056978567695,
      "grad_norm": 296.10943603515625,
      "learning_rate": 2.3721673445671123e-06,
      "logits/chosen": -1.088547945022583,
      "logits/rejected": -1.1794053316116333,
      "logps/chosen": -71.31549072265625,
      "logps/rejected": -76.64225769042969,
      "loss": 12.055,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0024872974026948214,
      "rewards/margins": 0.20438556373119354,
      "rewards/rejected": -0.20189829170703888,
      "step": 1809
    },
    {
      "epoch": 1.0512865191380611,
      "grad_norm": 311.1213684082031,
      "learning_rate": 2.370714700755375e-06,
      "logits/chosen": -1.1315990686416626,
      "logits/rejected": -1.0816015005111694,
      "logps/chosen": -78.72032928466797,
      "logps/rejected": -80.1206283569336,
      "loss": 13.2042,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11577626317739487,
      "rewards/margins": 0.08589179813861847,
      "rewards/rejected": -0.20166806876659393,
      "step": 1810
    },
    {
      "epoch": 1.051867340419353,
      "grad_norm": 275.2270202636719,
      "learning_rate": 2.3692620569436374e-06,
      "logits/chosen": -1.2009618282318115,
      "logits/rejected": -1.1827770471572876,
      "logps/chosen": -68.66722106933594,
      "logps/rejected": -78.26915740966797,
      "loss": 11.6287,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0027723959647119045,
      "rewards/margins": 0.2555071711540222,
      "rewards/rejected": -0.25827959179878235,
      "step": 1811
    },
    {
      "epoch": 1.0524481617006447,
      "grad_norm": 287.5706787109375,
      "learning_rate": 2.3678094131319e-06,
      "logits/chosen": -1.3368051052093506,
      "logits/rejected": -1.3193013668060303,
      "logps/chosen": -70.38104248046875,
      "logps/rejected": -74.90707397460938,
      "loss": 12.1388,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.00605876836925745,
      "rewards/margins": 0.1935560256242752,
      "rewards/rejected": -0.18749725818634033,
      "step": 1812
    },
    {
      "epoch": 1.0530289829819364,
      "grad_norm": 308.96966552734375,
      "learning_rate": 2.366356769320163e-06,
      "logits/chosen": -1.2344517707824707,
      "logits/rejected": -1.2728044986724854,
      "logps/chosen": -78.21991729736328,
      "logps/rejected": -70.33300018310547,
      "loss": 12.6287,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0618162527680397,
      "rewards/margins": 0.14954325556755066,
      "rewards/rejected": -0.21135953068733215,
      "step": 1813
    },
    {
      "epoch": 1.0536098042632283,
      "grad_norm": 282.9297790527344,
      "learning_rate": 2.3649041255084252e-06,
      "logits/chosen": -1.0618566274642944,
      "logits/rejected": -1.0387117862701416,
      "logps/chosen": -73.0753402709961,
      "logps/rejected": -78.67092895507812,
      "loss": 12.0975,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01851743273437023,
      "rewards/margins": 0.2326612025499344,
      "rewards/rejected": -0.2141437530517578,
      "step": 1814
    },
    {
      "epoch": 1.05419062554452,
      "grad_norm": 305.0246276855469,
      "learning_rate": 2.363451481696688e-06,
      "logits/chosen": -1.1686697006225586,
      "logits/rejected": -1.2344142198562622,
      "logps/chosen": -67.4321060180664,
      "logps/rejected": -77.95721435546875,
      "loss": 13.0576,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06939910352230072,
      "rewards/margins": 0.1014992967247963,
      "rewards/rejected": -0.1708984076976776,
      "step": 1815
    },
    {
      "epoch": 1.0547714468258116,
      "grad_norm": 301.83245849609375,
      "learning_rate": 2.361998837884951e-06,
      "logits/chosen": -1.1127091646194458,
      "logits/rejected": -1.081343412399292,
      "logps/chosen": -78.5421371459961,
      "logps/rejected": -83.04389190673828,
      "loss": 11.6482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03340291231870651,
      "rewards/margins": 0.2592235207557678,
      "rewards/rejected": -0.29262644052505493,
      "step": 1816
    },
    {
      "epoch": 1.0553522681071035,
      "grad_norm": 292.6458740234375,
      "learning_rate": 2.3605461940732135e-06,
      "logits/chosen": -1.2257633209228516,
      "logits/rejected": -1.2264759540557861,
      "logps/chosen": -77.64219665527344,
      "logps/rejected": -74.64775085449219,
      "loss": 12.4452,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04765477776527405,
      "rewards/margins": 0.17377400398254395,
      "rewards/rejected": -0.2214287966489792,
      "step": 1817
    },
    {
      "epoch": 1.0559330893883951,
      "grad_norm": 318.1414794921875,
      "learning_rate": 2.359093550261476e-06,
      "logits/chosen": -1.1936510801315308,
      "logits/rejected": -1.1652354001998901,
      "logps/chosen": -75.5849609375,
      "logps/rejected": -84.6615219116211,
      "loss": 13.135,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10834760963916779,
      "rewards/margins": 0.09478280693292618,
      "rewards/rejected": -0.20313043892383575,
      "step": 1818
    },
    {
      "epoch": 1.056513910669687,
      "grad_norm": 301.3826904296875,
      "learning_rate": 2.357640906449739e-06,
      "logits/chosen": -1.2157015800476074,
      "logits/rejected": -1.337953805923462,
      "logps/chosen": -75.48429107666016,
      "logps/rejected": -86.51458740234375,
      "loss": 11.1245,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.033996645361185074,
      "rewards/margins": 0.3307928442955017,
      "rewards/rejected": -0.3647894859313965,
      "step": 1819
    },
    {
      "epoch": 1.0570947319509787,
      "grad_norm": 276.97589111328125,
      "learning_rate": 2.3561882626380013e-06,
      "logits/chosen": -1.149330496788025,
      "logits/rejected": -1.1030328273773193,
      "logps/chosen": -68.86640930175781,
      "logps/rejected": -71.16368103027344,
      "loss": 12.1533,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04609895870089531,
      "rewards/margins": 0.1932513266801834,
      "rewards/rejected": -0.1471523493528366,
      "step": 1820
    },
    {
      "epoch": 1.0576755532322704,
      "grad_norm": 303.4784240722656,
      "learning_rate": 2.354735618826264e-06,
      "logits/chosen": -0.9871416091918945,
      "logits/rejected": -1.0590120553970337,
      "logps/chosen": -77.62509155273438,
      "logps/rejected": -78.97621154785156,
      "loss": 13.0577,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0909157320857048,
      "rewards/margins": 0.14562661945819855,
      "rewards/rejected": -0.23654231429100037,
      "step": 1821
    },
    {
      "epoch": 1.0582563745135622,
      "grad_norm": 292.2515869140625,
      "learning_rate": 2.3532829750145268e-06,
      "logits/chosen": -1.1806539297103882,
      "logits/rejected": -1.2223962545394897,
      "logps/chosen": -83.14176940917969,
      "logps/rejected": -79.19784545898438,
      "loss": 12.1834,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03939155489206314,
      "rewards/margins": 0.21206173300743103,
      "rewards/rejected": -0.25145331025123596,
      "step": 1822
    },
    {
      "epoch": 1.058837195794854,
      "grad_norm": 279.5367431640625,
      "learning_rate": 2.351830331202789e-06,
      "logits/chosen": -1.306338906288147,
      "logits/rejected": -1.1898423433303833,
      "logps/chosen": -72.88128662109375,
      "logps/rejected": -81.03486633300781,
      "loss": 12.2514,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.013178798370063305,
      "rewards/margins": 0.1906774342060089,
      "rewards/rejected": -0.20385625958442688,
      "step": 1823
    },
    {
      "epoch": 1.0594180170761456,
      "grad_norm": 294.1966857910156,
      "learning_rate": 2.350377687391052e-06,
      "logits/chosen": -1.211908221244812,
      "logits/rejected": -1.3351070880889893,
      "logps/chosen": -67.79579162597656,
      "logps/rejected": -81.80858612060547,
      "loss": 11.7453,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05510979890823364,
      "rewards/margins": 0.24255287647247314,
      "rewards/rejected": -0.2976626455783844,
      "step": 1824
    },
    {
      "epoch": 1.0599988383574375,
      "grad_norm": 294.05548095703125,
      "learning_rate": 2.3489250435793146e-06,
      "logits/chosen": -1.277356505393982,
      "logits/rejected": -1.224443793296814,
      "logps/chosen": -64.48851013183594,
      "logps/rejected": -80.20997619628906,
      "loss": 12.5488,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05418543145060539,
      "rewards/margins": 0.15883472561836243,
      "rewards/rejected": -0.21302016079425812,
      "step": 1825
    },
    {
      "epoch": 1.0605796596387291,
      "grad_norm": 299.06317138671875,
      "learning_rate": 2.3474723997675773e-06,
      "logits/chosen": -1.2715439796447754,
      "logits/rejected": -1.2545589208602905,
      "logps/chosen": -74.84748077392578,
      "logps/rejected": -75.6917953491211,
      "loss": 12.8931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08535346388816833,
      "rewards/margins": 0.12101536989212036,
      "rewards/rejected": -0.2063688337802887,
      "step": 1826
    },
    {
      "epoch": 1.0611604809200208,
      "grad_norm": 278.2451477050781,
      "learning_rate": 2.3460197559558397e-06,
      "logits/chosen": -1.1142429113388062,
      "logits/rejected": -1.0348351001739502,
      "logps/chosen": -77.03903198242188,
      "logps/rejected": -76.2451171875,
      "loss": 12.2683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13327233493328094,
      "rewards/margins": 0.18383851647377014,
      "rewards/rejected": -0.3171108365058899,
      "step": 1827
    },
    {
      "epoch": 1.0617413022013127,
      "grad_norm": 308.69293212890625,
      "learning_rate": 2.3445671121441024e-06,
      "logits/chosen": -1.1369335651397705,
      "logits/rejected": -1.2784185409545898,
      "logps/chosen": -78.07801818847656,
      "logps/rejected": -73.11485290527344,
      "loss": 13.5133,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1476178616285324,
      "rewards/margins": 0.059361089020967484,
      "rewards/rejected": -0.2069789618253708,
      "step": 1828
    },
    {
      "epoch": 1.0623221234826044,
      "grad_norm": 273.6282653808594,
      "learning_rate": 2.343114468332365e-06,
      "logits/chosen": -1.265187382698059,
      "logits/rejected": -1.2053581476211548,
      "logps/chosen": -69.40978240966797,
      "logps/rejected": -85.99472045898438,
      "loss": 11.1173,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.021193798631429672,
      "rewards/margins": 0.3169749081134796,
      "rewards/rejected": -0.29578110575675964,
      "step": 1829
    },
    {
      "epoch": 1.0629029447638962,
      "grad_norm": 269.9952392578125,
      "learning_rate": 2.3416618245206275e-06,
      "logits/chosen": -1.1798746585845947,
      "logits/rejected": -1.2568330764770508,
      "logps/chosen": -72.36671447753906,
      "logps/rejected": -74.95005798339844,
      "loss": 10.8464,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.031924664974212646,
      "rewards/margins": 0.35574638843536377,
      "rewards/rejected": -0.32382169365882874,
      "step": 1830
    },
    {
      "epoch": 1.063483766045188,
      "grad_norm": 331.3447570800781,
      "learning_rate": 2.3402091807088902e-06,
      "logits/chosen": -1.2912403345108032,
      "logits/rejected": -1.2267673015594482,
      "logps/chosen": -78.6216812133789,
      "logps/rejected": -80.69587707519531,
      "loss": 12.4763,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05326671153306961,
      "rewards/margins": 0.16838888823986053,
      "rewards/rejected": -0.22165557742118835,
      "step": 1831
    },
    {
      "epoch": 1.0640645873264796,
      "grad_norm": 288.1050720214844,
      "learning_rate": 2.338756536897153e-06,
      "logits/chosen": -1.2442501783370972,
      "logits/rejected": -1.2563748359680176,
      "logps/chosen": -70.74190521240234,
      "logps/rejected": -77.92393493652344,
      "loss": 12.6086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05277365446090698,
      "rewards/margins": 0.16080155968666077,
      "rewards/rejected": -0.21357519924640656,
      "step": 1832
    },
    {
      "epoch": 1.0646454086077715,
      "grad_norm": 298.66094970703125,
      "learning_rate": 2.3373038930854157e-06,
      "logits/chosen": -1.3558439016342163,
      "logits/rejected": -1.3400027751922607,
      "logps/chosen": -74.9997787475586,
      "logps/rejected": -78.2625732421875,
      "loss": 12.6308,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08019869029521942,
      "rewards/margins": 0.14247417449951172,
      "rewards/rejected": -0.22267286479473114,
      "step": 1833
    },
    {
      "epoch": 1.0652262298890631,
      "grad_norm": 285.65826416015625,
      "learning_rate": 2.335851249273678e-06,
      "logits/chosen": -1.212263584136963,
      "logits/rejected": -1.2475782632827759,
      "logps/chosen": -74.08516693115234,
      "logps/rejected": -82.26819610595703,
      "loss": 11.8406,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0647260993719101,
      "rewards/margins": 0.22320258617401123,
      "rewards/rejected": -0.2879287004470825,
      "step": 1834
    },
    {
      "epoch": 1.0658070511703548,
      "grad_norm": 288.3097839355469,
      "learning_rate": 2.334398605461941e-06,
      "logits/chosen": -1.3400421142578125,
      "logits/rejected": -1.3965580463409424,
      "logps/chosen": -74.5050277709961,
      "logps/rejected": -77.81584930419922,
      "loss": 11.4902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013376163318753242,
      "rewards/margins": 0.29900267720222473,
      "rewards/rejected": -0.28562647104263306,
      "step": 1835
    },
    {
      "epoch": 1.0663878724516467,
      "grad_norm": 302.7481689453125,
      "learning_rate": 2.3329459616502036e-06,
      "logits/chosen": -1.217735767364502,
      "logits/rejected": -1.2486951351165771,
      "logps/chosen": -76.30255126953125,
      "logps/rejected": -72.14408111572266,
      "loss": 12.8934,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07950372993946075,
      "rewards/margins": 0.12100023031234741,
      "rewards/rejected": -0.20050394535064697,
      "step": 1836
    },
    {
      "epoch": 1.0669686937329383,
      "grad_norm": 310.4986877441406,
      "learning_rate": 2.3314933178384663e-06,
      "logits/chosen": -1.4266477823257446,
      "logits/rejected": -1.368774175643921,
      "logps/chosen": -82.4830093383789,
      "logps/rejected": -71.07676696777344,
      "loss": 12.8811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10153285413980484,
      "rewards/margins": 0.1268554925918579,
      "rewards/rejected": -0.22838835418224335,
      "step": 1837
    },
    {
      "epoch": 1.06754951501423,
      "grad_norm": 294.4962463378906,
      "learning_rate": 2.330040674026729e-06,
      "logits/chosen": -1.2054665088653564,
      "logits/rejected": -1.1285340785980225,
      "logps/chosen": -70.24253845214844,
      "logps/rejected": -78.65782165527344,
      "loss": 10.7279,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.03173526003956795,
      "rewards/margins": 0.36913853883743286,
      "rewards/rejected": -0.337403267621994,
      "step": 1838
    },
    {
      "epoch": 1.068130336295522,
      "grad_norm": 287.00970458984375,
      "learning_rate": 2.3285880302149914e-06,
      "logits/chosen": -1.2540881633758545,
      "logits/rejected": -1.2929776906967163,
      "logps/chosen": -77.9581069946289,
      "logps/rejected": -81.64662170410156,
      "loss": 11.8902,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06314466893672943,
      "rewards/margins": 0.239525705575943,
      "rewards/rejected": -0.30267032980918884,
      "step": 1839
    },
    {
      "epoch": 1.0687111575768136,
      "grad_norm": 266.86309814453125,
      "learning_rate": 2.327135386403254e-06,
      "logits/chosen": -1.4850796461105347,
      "logits/rejected": -1.302920937538147,
      "logps/chosen": -70.59464263916016,
      "logps/rejected": -73.81462097167969,
      "loss": 12.5114,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10395938158035278,
      "rewards/margins": 0.15829969942569733,
      "rewards/rejected": -0.2622590661048889,
      "step": 1840
    },
    {
      "epoch": 1.0692919788581055,
      "grad_norm": 274.6494140625,
      "learning_rate": 2.325682742591517e-06,
      "logits/chosen": -1.1762568950653076,
      "logits/rejected": -1.168436884880066,
      "logps/chosen": -68.6141357421875,
      "logps/rejected": -76.59086608886719,
      "loss": 11.8385,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.006103237625211477,
      "rewards/margins": 0.2304351031780243,
      "rewards/rejected": -0.22433185577392578,
      "step": 1841
    },
    {
      "epoch": 1.0698728001393971,
      "grad_norm": 307.10650634765625,
      "learning_rate": 2.3242300987797796e-06,
      "logits/chosen": -1.3367271423339844,
      "logits/rejected": -1.376089334487915,
      "logps/chosen": -85.16807556152344,
      "logps/rejected": -81.94376373291016,
      "loss": 12.9678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1122315376996994,
      "rewards/margins": 0.12183103710412979,
      "rewards/rejected": -0.23406259715557098,
      "step": 1842
    },
    {
      "epoch": 1.0704536214206888,
      "grad_norm": 284.85491943359375,
      "learning_rate": 2.322777454968042e-06,
      "logits/chosen": -1.370794415473938,
      "logits/rejected": -1.315099835395813,
      "logps/chosen": -77.26371765136719,
      "logps/rejected": -71.97315216064453,
      "loss": 12.6547,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.040176715701818466,
      "rewards/margins": 0.15831358730793,
      "rewards/rejected": -0.19849029183387756,
      "step": 1843
    },
    {
      "epoch": 1.0710344427019807,
      "grad_norm": 293.847412109375,
      "learning_rate": 2.3213248111563047e-06,
      "logits/chosen": -1.2156918048858643,
      "logits/rejected": -1.3590480089187622,
      "logps/chosen": -75.9840316772461,
      "logps/rejected": -90.80524444580078,
      "loss": 11.1998,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.00910272728651762,
      "rewards/margins": 0.3193199038505554,
      "rewards/rejected": -0.3102172017097473,
      "step": 1844
    },
    {
      "epoch": 1.0716152639832723,
      "grad_norm": 295.0451965332031,
      "learning_rate": 2.3198721673445675e-06,
      "logits/chosen": -1.2723419666290283,
      "logits/rejected": -1.1823184490203857,
      "logps/chosen": -75.76826477050781,
      "logps/rejected": -82.82530212402344,
      "loss": 12.4378,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11894576251506805,
      "rewards/margins": 0.18062159419059753,
      "rewards/rejected": -0.29956740140914917,
      "step": 1845
    },
    {
      "epoch": 1.072196085264564,
      "grad_norm": 308.6762390136719,
      "learning_rate": 2.3184195235328298e-06,
      "logits/chosen": -1.3527491092681885,
      "logits/rejected": -1.4032938480377197,
      "logps/chosen": -75.24395751953125,
      "logps/rejected": -72.67012023925781,
      "loss": 11.5219,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.013237908482551575,
      "rewards/margins": 0.2737814784049988,
      "rewards/rejected": -0.2605435848236084,
      "step": 1846
    },
    {
      "epoch": 1.072776906545856,
      "grad_norm": 285.98828125,
      "learning_rate": 2.3169668797210925e-06,
      "logits/chosen": -1.2104380130767822,
      "logits/rejected": -1.1465580463409424,
      "logps/chosen": -71.46925354003906,
      "logps/rejected": -78.69825744628906,
      "loss": 12.2847,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09712918102741241,
      "rewards/margins": 0.18867741525173187,
      "rewards/rejected": -0.2858065962791443,
      "step": 1847
    },
    {
      "epoch": 1.0733577278271476,
      "grad_norm": 280.13275146484375,
      "learning_rate": 2.3155142359093553e-06,
      "logits/chosen": -1.0124825239181519,
      "logits/rejected": -0.9435278177261353,
      "logps/chosen": -76.59584045410156,
      "logps/rejected": -83.52967834472656,
      "loss": 11.4978,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0626763179898262,
      "rewards/margins": 0.28530097007751465,
      "rewards/rejected": -0.34797731041908264,
      "step": 1848
    },
    {
      "epoch": 1.0739385491084392,
      "grad_norm": 297.3227844238281,
      "learning_rate": 2.314061592097618e-06,
      "logits/chosen": -1.2095751762390137,
      "logits/rejected": -1.2973079681396484,
      "logps/chosen": -78.3536605834961,
      "logps/rejected": -79.5671615600586,
      "loss": 11.373,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08722156286239624,
      "rewards/margins": 0.28964340686798096,
      "rewards/rejected": -0.3768649697303772,
      "step": 1849
    },
    {
      "epoch": 1.074519370389731,
      "grad_norm": 557.2191772460938,
      "learning_rate": 2.3126089482858803e-06,
      "logits/chosen": -1.2340757846832275,
      "logits/rejected": -1.1780078411102295,
      "logps/chosen": -77.78102111816406,
      "logps/rejected": -76.90254211425781,
      "loss": 12.2202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11052794754505157,
      "rewards/margins": 0.19404003024101257,
      "rewards/rejected": -0.30456796288490295,
      "step": 1850
    },
    {
      "epoch": 1.0751001916710228,
      "grad_norm": 285.372314453125,
      "learning_rate": 2.311156304474143e-06,
      "logits/chosen": -1.2881758213043213,
      "logits/rejected": -1.2939411401748657,
      "logps/chosen": -67.89484405517578,
      "logps/rejected": -84.93338012695312,
      "loss": 11.919,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09519805014133453,
      "rewards/margins": 0.23213514685630798,
      "rewards/rejected": -0.3273331820964813,
      "step": 1851
    },
    {
      "epoch": 1.0756810129523147,
      "grad_norm": 277.7596435546875,
      "learning_rate": 2.309703660662406e-06,
      "logits/chosen": -1.1061570644378662,
      "logits/rejected": -1.1383107900619507,
      "logps/chosen": -69.48309326171875,
      "logps/rejected": -77.86546325683594,
      "loss": 11.8942,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08602216094732285,
      "rewards/margins": 0.2254772186279297,
      "rewards/rejected": -0.3114994168281555,
      "step": 1852
    },
    {
      "epoch": 1.0762618342336063,
      "grad_norm": 271.91778564453125,
      "learning_rate": 2.308251016850668e-06,
      "logits/chosen": -1.4708317518234253,
      "logits/rejected": -1.43101167678833,
      "logps/chosen": -70.11153411865234,
      "logps/rejected": -75.78926849365234,
      "loss": 12.2845,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07912683486938477,
      "rewards/margins": 0.1800125688314438,
      "rewards/rejected": -0.25913938879966736,
      "step": 1853
    },
    {
      "epoch": 1.076842655514898,
      "grad_norm": 294.4907531738281,
      "learning_rate": 2.306798373038931e-06,
      "logits/chosen": -1.449562907218933,
      "logits/rejected": -1.3950929641723633,
      "logps/chosen": -75.42185974121094,
      "logps/rejected": -78.86067199707031,
      "loss": 12.1174,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07921554893255234,
      "rewards/margins": 0.2106180191040039,
      "rewards/rejected": -0.28983360528945923,
      "step": 1854
    },
    {
      "epoch": 1.0774234767961899,
      "grad_norm": 293.58984375,
      "learning_rate": 2.3053457292271937e-06,
      "logits/chosen": -1.2915606498718262,
      "logits/rejected": -1.3630459308624268,
      "logps/chosen": -72.368408203125,
      "logps/rejected": -76.18818664550781,
      "loss": 12.7297,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.058180373162031174,
      "rewards/margins": 0.15856976807117462,
      "rewards/rejected": -0.2167501449584961,
      "step": 1855
    },
    {
      "epoch": 1.0780042980774815,
      "grad_norm": 331.4824523925781,
      "learning_rate": 2.303893085415456e-06,
      "logits/chosen": -1.2043336629867554,
      "logits/rejected": -1.297723650932312,
      "logps/chosen": -72.92744445800781,
      "logps/rejected": -75.0640869140625,
      "loss": 12.5461,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10127633810043335,
      "rewards/margins": 0.1480613648891449,
      "rewards/rejected": -0.24933771789073944,
      "step": 1856
    },
    {
      "epoch": 1.0785851193587732,
      "grad_norm": 323.2904968261719,
      "learning_rate": 2.3024404416037187e-06,
      "logits/chosen": -1.4392999410629272,
      "logits/rejected": -1.5091079473495483,
      "logps/chosen": -68.75377655029297,
      "logps/rejected": -72.04249572753906,
      "loss": 12.702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12643282115459442,
      "rewards/margins": 0.1421770453453064,
      "rewards/rejected": -0.2686098515987396,
      "step": 1857
    },
    {
      "epoch": 1.079165940640065,
      "grad_norm": 311.45928955078125,
      "learning_rate": 2.300987797791982e-06,
      "logits/chosen": -1.3029788732528687,
      "logits/rejected": -1.4044101238250732,
      "logps/chosen": -77.7531509399414,
      "logps/rejected": -86.9994888305664,
      "loss": 11.1506,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1297357678413391,
      "rewards/margins": 0.37783583998680115,
      "rewards/rejected": -0.5075716376304626,
      "step": 1858
    },
    {
      "epoch": 1.0797467619213568,
      "grad_norm": 319.54095458984375,
      "learning_rate": 2.2995351539802442e-06,
      "logits/chosen": -1.355452060699463,
      "logits/rejected": -1.2840461730957031,
      "logps/chosen": -73.30493927001953,
      "logps/rejected": -77.99787902832031,
      "loss": 11.8744,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05415702611207962,
      "rewards/margins": 0.23061206936836243,
      "rewards/rejected": -0.28476908802986145,
      "step": 1859
    },
    {
      "epoch": 1.0803275832026484,
      "grad_norm": 272.79022216796875,
      "learning_rate": 2.298082510168507e-06,
      "logits/chosen": -1.256400465965271,
      "logits/rejected": -1.2169406414031982,
      "logps/chosen": -72.30365753173828,
      "logps/rejected": -86.6010971069336,
      "loss": 11.1396,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06652600318193436,
      "rewards/margins": 0.327131450176239,
      "rewards/rejected": -0.39365747570991516,
      "step": 1860
    },
    {
      "epoch": 1.0809084044839403,
      "grad_norm": 282.36474609375,
      "learning_rate": 2.2966298663567697e-06,
      "logits/chosen": -1.2665935754776,
      "logits/rejected": -1.1860650777816772,
      "logps/chosen": -71.394287109375,
      "logps/rejected": -84.48927307128906,
      "loss": 11.5218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06446046382188797,
      "rewards/margins": 0.28394967317581177,
      "rewards/rejected": -0.34841015934944153,
      "step": 1861
    },
    {
      "epoch": 1.081489225765232,
      "grad_norm": 307.82550048828125,
      "learning_rate": 2.295177222545032e-06,
      "logits/chosen": -1.3782317638397217,
      "logits/rejected": -1.3229846954345703,
      "logps/chosen": -70.89363098144531,
      "logps/rejected": -75.4673080444336,
      "loss": 12.398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06966067850589752,
      "rewards/margins": 0.1810016930103302,
      "rewards/rejected": -0.25066232681274414,
      "step": 1862
    },
    {
      "epoch": 1.0820700470465239,
      "grad_norm": 303.7426452636719,
      "learning_rate": 2.293724578733295e-06,
      "logits/chosen": -1.4068634510040283,
      "logits/rejected": -1.4247969388961792,
      "logps/chosen": -76.7747802734375,
      "logps/rejected": -72.27479553222656,
      "loss": 11.7195,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09843023121356964,
      "rewards/margins": 0.27342188358306885,
      "rewards/rejected": -0.37185215950012207,
      "step": 1863
    },
    {
      "epoch": 1.0826508683278155,
      "grad_norm": 306.2264404296875,
      "learning_rate": 2.2922719349215576e-06,
      "logits/chosen": -1.3469550609588623,
      "logits/rejected": -1.433326244354248,
      "logps/chosen": -73.90471649169922,
      "logps/rejected": -79.22645568847656,
      "loss": 10.9204,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11632704734802246,
      "rewards/margins": 0.3984544575214386,
      "rewards/rejected": -0.5147815346717834,
      "step": 1864
    },
    {
      "epoch": 1.0832316896091072,
      "grad_norm": 294.5223083496094,
      "learning_rate": 2.2908192911098203e-06,
      "logits/chosen": -1.300051212310791,
      "logits/rejected": -1.2132049798965454,
      "logps/chosen": -77.37844848632812,
      "logps/rejected": -83.87139892578125,
      "loss": 11.7377,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0917651355266571,
      "rewards/margins": 0.25181400775909424,
      "rewards/rejected": -0.34357914328575134,
      "step": 1865
    },
    {
      "epoch": 1.083812510890399,
      "grad_norm": 278.10565185546875,
      "learning_rate": 2.2893666472980826e-06,
      "logits/chosen": -1.4184238910675049,
      "logits/rejected": -1.3206241130828857,
      "logps/chosen": -76.57219696044922,
      "logps/rejected": -70.97447204589844,
      "loss": 12.6441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1752898097038269,
      "rewards/margins": 0.14112383127212524,
      "rewards/rejected": -0.31641364097595215,
      "step": 1866
    },
    {
      "epoch": 1.0843933321716908,
      "grad_norm": 332.9988708496094,
      "learning_rate": 2.2879140034863454e-06,
      "logits/chosen": -1.3018518686294556,
      "logits/rejected": -1.3394484519958496,
      "logps/chosen": -85.90144348144531,
      "logps/rejected": -82.41697692871094,
      "loss": 12.8482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18753674626350403,
      "rewards/margins": 0.1289936751127243,
      "rewards/rejected": -0.31653040647506714,
      "step": 1867
    },
    {
      "epoch": 1.0849741534529824,
      "grad_norm": 282.8656311035156,
      "learning_rate": 2.286461359674608e-06,
      "logits/chosen": -1.0735652446746826,
      "logits/rejected": -1.0225722789764404,
      "logps/chosen": -70.46998596191406,
      "logps/rejected": -87.07571411132812,
      "loss": 11.7672,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11682279407978058,
      "rewards/margins": 0.2756138741970062,
      "rewards/rejected": -0.3924367129802704,
      "step": 1868
    },
    {
      "epoch": 1.0855549747342743,
      "grad_norm": 286.0412902832031,
      "learning_rate": 2.2850087158628704e-06,
      "logits/chosen": -1.4140738248825073,
      "logits/rejected": -1.4234883785247803,
      "logps/chosen": -73.54850769042969,
      "logps/rejected": -70.362060546875,
      "loss": 12.8665,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04446287453174591,
      "rewards/margins": 0.11783261597156525,
      "rewards/rejected": -0.16229549050331116,
      "step": 1869
    },
    {
      "epoch": 1.086135796015566,
      "grad_norm": 289.5560302734375,
      "learning_rate": 2.283556072051133e-06,
      "logits/chosen": -1.4574428796768188,
      "logits/rejected": -1.366193413734436,
      "logps/chosen": -72.31874084472656,
      "logps/rejected": -91.21329498291016,
      "loss": 12.037,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08397255837917328,
      "rewards/margins": 0.2363775670528412,
      "rewards/rejected": -0.32035014033317566,
      "step": 1870
    },
    {
      "epoch": 1.0867166172968576,
      "grad_norm": 256.3517761230469,
      "learning_rate": 2.282103428239396e-06,
      "logits/chosen": -1.450507402420044,
      "logits/rejected": -1.3494465351104736,
      "logps/chosen": -70.27610778808594,
      "logps/rejected": -90.80995178222656,
      "loss": 10.6757,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0076746148988604546,
      "rewards/margins": 0.381161093711853,
      "rewards/rejected": -0.38883569836616516,
      "step": 1871
    },
    {
      "epoch": 1.0872974385781495,
      "grad_norm": 252.17544555664062,
      "learning_rate": 2.2806507844276583e-06,
      "logits/chosen": -1.337768316268921,
      "logits/rejected": -1.3080776929855347,
      "logps/chosen": -72.86946105957031,
      "logps/rejected": -86.91899108886719,
      "loss": 10.5667,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.737194336077664e-05,
      "rewards/margins": 0.3948022425174713,
      "rewards/rejected": -0.394774854183197,
      "step": 1872
    },
    {
      "epoch": 1.0878782598594412,
      "grad_norm": 273.23248291015625,
      "learning_rate": 2.279198140615921e-06,
      "logits/chosen": -1.3566663265228271,
      "logits/rejected": -1.249966025352478,
      "logps/chosen": -70.2577133178711,
      "logps/rejected": -76.09548950195312,
      "loss": 11.3925,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05293440818786621,
      "rewards/margins": 0.2877849042415619,
      "rewards/rejected": -0.3407193422317505,
      "step": 1873
    },
    {
      "epoch": 1.088459081140733,
      "grad_norm": 292.2498779296875,
      "learning_rate": 2.2777454968041838e-06,
      "logits/chosen": -1.1885011196136475,
      "logits/rejected": -1.2621798515319824,
      "logps/chosen": -67.79953002929688,
      "logps/rejected": -85.64979553222656,
      "loss": 11.491,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.04751775786280632,
      "rewards/margins": 0.2840045988559723,
      "rewards/rejected": -0.3315223753452301,
      "step": 1874
    },
    {
      "epoch": 1.0890399024220248,
      "grad_norm": 316.7376708984375,
      "learning_rate": 2.2762928529924465e-06,
      "logits/chosen": -1.4971121549606323,
      "logits/rejected": -1.4802935123443604,
      "logps/chosen": -77.48785400390625,
      "logps/rejected": -74.62857818603516,
      "loss": 13.4967,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15965226292610168,
      "rewards/margins": 0.05938272550702095,
      "rewards/rejected": -0.21903495490550995,
      "step": 1875
    },
    {
      "epoch": 1.0896207237033164,
      "grad_norm": 290.35968017578125,
      "learning_rate": 2.274840209180709e-06,
      "logits/chosen": -1.335652470588684,
      "logits/rejected": -1.3224215507507324,
      "logps/chosen": -71.61116790771484,
      "logps/rejected": -77.87721252441406,
      "loss": 12.1563,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.035627953708171844,
      "rewards/margins": 0.22073645889759064,
      "rewards/rejected": -0.2563644051551819,
      "step": 1876
    },
    {
      "epoch": 1.0902015449846083,
      "grad_norm": 333.16815185546875,
      "learning_rate": 2.2733875653689716e-06,
      "logits/chosen": -1.208413004875183,
      "logits/rejected": -1.2747244834899902,
      "logps/chosen": -81.5730209350586,
      "logps/rejected": -68.9091796875,
      "loss": 12.2125,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09135030210018158,
      "rewards/margins": 0.1913330852985382,
      "rewards/rejected": -0.282683402299881,
      "step": 1877
    },
    {
      "epoch": 1.0907823662659,
      "grad_norm": 319.01202392578125,
      "learning_rate": 2.2719349215572343e-06,
      "logits/chosen": -1.3959678411483765,
      "logits/rejected": -1.2531898021697998,
      "logps/chosen": -79.62081146240234,
      "logps/rejected": -77.94842529296875,
      "loss": 12.9557,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17629526555538177,
      "rewards/margins": 0.11534170061349869,
      "rewards/rejected": -0.29163694381713867,
      "step": 1878
    },
    {
      "epoch": 1.0913631875471916,
      "grad_norm": 263.5522766113281,
      "learning_rate": 2.270482277745497e-06,
      "logits/chosen": -1.3907314538955688,
      "logits/rejected": -1.397842526435852,
      "logps/chosen": -69.31333923339844,
      "logps/rejected": -76.0733871459961,
      "loss": 11.5471,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0770336166024208,
      "rewards/margins": 0.28445449471473694,
      "rewards/rejected": -0.36148807406425476,
      "step": 1879
    },
    {
      "epoch": 1.0919440088284835,
      "grad_norm": 305.3263854980469,
      "learning_rate": 2.26902963393376e-06,
      "logits/chosen": -1.0955054759979248,
      "logits/rejected": -1.1298199892044067,
      "logps/chosen": -83.28105163574219,
      "logps/rejected": -80.27665710449219,
      "loss": 12.5916,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1308450847864151,
      "rewards/margins": 0.16803763806819916,
      "rewards/rejected": -0.2988826632499695,
      "step": 1880
    },
    {
      "epoch": 1.0925248301097752,
      "grad_norm": 297.12945556640625,
      "learning_rate": 2.2675769901220226e-06,
      "logits/chosen": -1.3585107326507568,
      "logits/rejected": -1.4070830345153809,
      "logps/chosen": -77.00254821777344,
      "logps/rejected": -75.41547393798828,
      "loss": 12.807,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1528879851102829,
      "rewards/margins": 0.13752040266990662,
      "rewards/rejected": -0.2904083728790283,
      "step": 1881
    },
    {
      "epoch": 1.0931056513910669,
      "grad_norm": 296.3562316894531,
      "learning_rate": 2.266124346310285e-06,
      "logits/chosen": -1.399975061416626,
      "logits/rejected": -1.4085720777511597,
      "logps/chosen": -75.47244262695312,
      "logps/rejected": -83.76031494140625,
      "loss": 12.3528,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13767746090888977,
      "rewards/margins": 0.18673963844776154,
      "rewards/rejected": -0.3244171142578125,
      "step": 1882
    },
    {
      "epoch": 1.0936864726723587,
      "grad_norm": 296.68505859375,
      "learning_rate": 2.2646717024985477e-06,
      "logits/chosen": -1.278159737586975,
      "logits/rejected": -1.2933261394500732,
      "logps/chosen": -75.3333511352539,
      "logps/rejected": -78.18989562988281,
      "loss": 13.5882,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2516975998878479,
      "rewards/margins": 0.046706728637218475,
      "rewards/rejected": -0.298404335975647,
      "step": 1883
    },
    {
      "epoch": 1.0942672939536504,
      "grad_norm": 304.4270935058594,
      "learning_rate": 2.2632190586868104e-06,
      "logits/chosen": -1.2930570840835571,
      "logits/rejected": -1.2696404457092285,
      "logps/chosen": -83.64619445800781,
      "logps/rejected": -84.5613021850586,
      "loss": 11.758,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.15822724997997284,
      "rewards/margins": 0.2628728747367859,
      "rewards/rejected": -0.4211001396179199,
      "step": 1884
    },
    {
      "epoch": 1.0948481152349423,
      "grad_norm": 284.06640625,
      "learning_rate": 2.2617664148750727e-06,
      "logits/chosen": -1.5002076625823975,
      "logits/rejected": -1.457118034362793,
      "logps/chosen": -78.55818176269531,
      "logps/rejected": -86.4919204711914,
      "loss": 11.161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.051411401480436325,
      "rewards/margins": 0.3260098099708557,
      "rewards/rejected": -0.37742120027542114,
      "step": 1885
    },
    {
      "epoch": 1.095428936516234,
      "grad_norm": 310.96856689453125,
      "learning_rate": 2.2603137710633355e-06,
      "logits/chosen": -1.314337134361267,
      "logits/rejected": -1.3771755695343018,
      "logps/chosen": -74.21797180175781,
      "logps/rejected": -77.42240142822266,
      "loss": 11.3332,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.02515893243253231,
      "rewards/margins": 0.29195111989974976,
      "rewards/rejected": -0.3171100318431854,
      "step": 1886
    },
    {
      "epoch": 1.0960097577975256,
      "grad_norm": 291.7381896972656,
      "learning_rate": 2.2588611272515982e-06,
      "logits/chosen": -1.2196760177612305,
      "logits/rejected": -1.2928193807601929,
      "logps/chosen": -68.09748840332031,
      "logps/rejected": -90.88526916503906,
      "loss": 10.7496,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.018759526312351227,
      "rewards/margins": 0.3714521825313568,
      "rewards/rejected": -0.3526926636695862,
      "step": 1887
    },
    {
      "epoch": 1.0965905790788175,
      "grad_norm": 306.9683837890625,
      "learning_rate": 2.2574084834398605e-06,
      "logits/chosen": -1.256040334701538,
      "logits/rejected": -1.2561142444610596,
      "logps/chosen": -70.33595275878906,
      "logps/rejected": -77.12580871582031,
      "loss": 11.8756,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.059070032089948654,
      "rewards/margins": 0.24459059536457062,
      "rewards/rejected": -0.30366066098213196,
      "step": 1888
    },
    {
      "epoch": 1.0971714003601092,
      "grad_norm": 316.89697265625,
      "learning_rate": 2.2559558396281233e-06,
      "logits/chosen": -1.1252800226211548,
      "logits/rejected": -1.069270372390747,
      "logps/chosen": -78.318359375,
      "logps/rejected": -80.91255187988281,
      "loss": 12.9842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19383275508880615,
      "rewards/margins": 0.10379709303379059,
      "rewards/rejected": -0.29762980341911316,
      "step": 1889
    },
    {
      "epoch": 1.0977522216414008,
      "grad_norm": 304.9154357910156,
      "learning_rate": 2.254503195816386e-06,
      "logits/chosen": -1.2836596965789795,
      "logits/rejected": -1.3131887912750244,
      "logps/chosen": -75.15878295898438,
      "logps/rejected": -85.10432434082031,
      "loss": 12.4633,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15085235238075256,
      "rewards/margins": 0.1747625768184662,
      "rewards/rejected": -0.32561495900154114,
      "step": 1890
    },
    {
      "epoch": 1.0983330429226927,
      "grad_norm": 276.10308837890625,
      "learning_rate": 2.253050552004649e-06,
      "logits/chosen": -1.18367600440979,
      "logits/rejected": -1.2541491985321045,
      "logps/chosen": -75.31745910644531,
      "logps/rejected": -81.8232650756836,
      "loss": 11.0394,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.018416618928313255,
      "rewards/margins": 0.3374725878238678,
      "rewards/rejected": -0.355889230966568,
      "step": 1891
    },
    {
      "epoch": 1.0989138642039844,
      "grad_norm": 293.7543640136719,
      "learning_rate": 2.251597908192911e-06,
      "logits/chosen": -1.3513258695602417,
      "logits/rejected": -1.3945108652114868,
      "logps/chosen": -69.6405029296875,
      "logps/rejected": -81.89004516601562,
      "loss": 12.5893,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11231887340545654,
      "rewards/margins": 0.15113350749015808,
      "rewards/rejected": -0.2634523808956146,
      "step": 1892
    },
    {
      "epoch": 1.099494685485276,
      "grad_norm": 312.7142639160156,
      "learning_rate": 2.250145264381174e-06,
      "logits/chosen": -1.4540537595748901,
      "logits/rejected": -1.4543588161468506,
      "logps/chosen": -75.36080932617188,
      "logps/rejected": -75.6168441772461,
      "loss": 12.8134,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09082247316837311,
      "rewards/margins": 0.14235137403011322,
      "rewards/rejected": -0.23317387700080872,
      "step": 1893
    },
    {
      "epoch": 1.100075506766568,
      "grad_norm": 280.76019287109375,
      "learning_rate": 2.2486926205694366e-06,
      "logits/chosen": -1.1974594593048096,
      "logits/rejected": -1.262658715248108,
      "logps/chosen": -82.63468933105469,
      "logps/rejected": -84.64886474609375,
      "loss": 11.2727,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.035703353583812714,
      "rewards/margins": 0.3075450360774994,
      "rewards/rejected": -0.3432484269142151,
      "step": 1894
    },
    {
      "epoch": 1.1006563280478596,
      "grad_norm": 277.2602844238281,
      "learning_rate": 2.247239976757699e-06,
      "logits/chosen": -1.172642469406128,
      "logits/rejected": -1.1284213066101074,
      "logps/chosen": -74.31792449951172,
      "logps/rejected": -79.50527954101562,
      "loss": 11.3126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03262670710682869,
      "rewards/margins": 0.3543560206890106,
      "rewards/rejected": -0.32172930240631104,
      "step": 1895
    },
    {
      "epoch": 1.1012371493291515,
      "grad_norm": 323.99554443359375,
      "learning_rate": 2.2457873329459617e-06,
      "logits/chosen": -1.261904001235962,
      "logits/rejected": -1.2497020959854126,
      "logps/chosen": -70.64785766601562,
      "logps/rejected": -76.39424133300781,
      "loss": 13.1492,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08945821225643158,
      "rewards/margins": 0.09435537457466125,
      "rewards/rejected": -0.18381358683109283,
      "step": 1896
    },
    {
      "epoch": 1.1018179706104432,
      "grad_norm": 684.6720581054688,
      "learning_rate": 2.2443346891342244e-06,
      "logits/chosen": -1.2243711948394775,
      "logits/rejected": -1.2799022197723389,
      "logps/chosen": -68.80662536621094,
      "logps/rejected": -78.3041000366211,
      "loss": 12.0766,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0803842842578888,
      "rewards/margins": 0.21447820961475372,
      "rewards/rejected": -0.2948624789714813,
      "step": 1897
    },
    {
      "epoch": 1.1023987918917348,
      "grad_norm": 299.1340637207031,
      "learning_rate": 2.242882045322487e-06,
      "logits/chosen": -1.4365450143814087,
      "logits/rejected": -1.3485090732574463,
      "logps/chosen": -79.56819152832031,
      "logps/rejected": -75.73643493652344,
      "loss": 12.5382,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11234170198440552,
      "rewards/margins": 0.1581932008266449,
      "rewards/rejected": -0.2705349326133728,
      "step": 1898
    },
    {
      "epoch": 1.1029796131730267,
      "grad_norm": 323.996337890625,
      "learning_rate": 2.2414294015107495e-06,
      "logits/chosen": -1.2346805334091187,
      "logits/rejected": -1.465852975845337,
      "logps/chosen": -74.69519805908203,
      "logps/rejected": -83.49501037597656,
      "loss": 12.7891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1042986735701561,
      "rewards/margins": 0.17223849892616272,
      "rewards/rejected": -0.276537150144577,
      "step": 1899
    },
    {
      "epoch": 1.1035604344543184,
      "grad_norm": 452.12188720703125,
      "learning_rate": 2.2399767576990127e-06,
      "logits/chosen": -1.6904752254486084,
      "logits/rejected": -1.5498231649398804,
      "logps/chosen": -77.85609436035156,
      "logps/rejected": -81.86569213867188,
      "loss": 12.4903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10929800570011139,
      "rewards/margins": 0.17659269273281097,
      "rewards/rejected": -0.2858906686306,
      "step": 1900
    },
    {
      "epoch": 1.10414125573561,
      "grad_norm": 272.31146240234375,
      "learning_rate": 2.238524113887275e-06,
      "logits/chosen": -1.0313645601272583,
      "logits/rejected": -1.1756597757339478,
      "logps/chosen": -74.51789855957031,
      "logps/rejected": -82.53508758544922,
      "loss": 11.4648,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.052155882120132446,
      "rewards/margins": 0.2757817804813385,
      "rewards/rejected": -0.32793766260147095,
      "step": 1901
    },
    {
      "epoch": 1.104722077016902,
      "grad_norm": 284.4774169921875,
      "learning_rate": 2.2370714700755378e-06,
      "logits/chosen": -1.2619701623916626,
      "logits/rejected": -1.2839032411575317,
      "logps/chosen": -77.33555603027344,
      "logps/rejected": -75.56710052490234,
      "loss": 12.194,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07690934836864471,
      "rewards/margins": 0.20775088667869568,
      "rewards/rejected": -0.284660279750824,
      "step": 1902
    },
    {
      "epoch": 1.1053028982981936,
      "grad_norm": 272.76141357421875,
      "learning_rate": 2.2356188262638005e-06,
      "logits/chosen": -1.314558744430542,
      "logits/rejected": -1.3563075065612793,
      "logps/chosen": -66.79798126220703,
      "logps/rejected": -74.23358917236328,
      "loss": 11.2513,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.026987260207533836,
      "rewards/margins": 0.3066416084766388,
      "rewards/rejected": -0.3336288332939148,
      "step": 1903
    },
    {
      "epoch": 1.1058837195794853,
      "grad_norm": 331.40167236328125,
      "learning_rate": 2.234166182452063e-06,
      "logits/chosen": -1.303720235824585,
      "logits/rejected": -1.312684178352356,
      "logps/chosen": -77.87791442871094,
      "logps/rejected": -82.04214477539062,
      "loss": 11.9883,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09221961349248886,
      "rewards/margins": 0.22665110230445862,
      "rewards/rejected": -0.3188706934452057,
      "step": 1904
    },
    {
      "epoch": 1.1064645408607772,
      "grad_norm": 269.0350341796875,
      "learning_rate": 2.2327135386403256e-06,
      "logits/chosen": -1.2132223844528198,
      "logits/rejected": -1.1458336114883423,
      "logps/chosen": -65.7417221069336,
      "logps/rejected": -73.22904968261719,
      "loss": 11.998,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03152678906917572,
      "rewards/margins": 0.2160847932100296,
      "rewards/rejected": -0.24761156737804413,
      "step": 1905
    },
    {
      "epoch": 1.1070453621420688,
      "grad_norm": 325.47454833984375,
      "learning_rate": 2.2312608948285883e-06,
      "logits/chosen": -1.384629487991333,
      "logits/rejected": -1.3841931819915771,
      "logps/chosen": -77.91289520263672,
      "logps/rejected": -69.25160217285156,
      "loss": 13.5436,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12514977157115936,
      "rewards/margins": 0.06482227891683578,
      "rewards/rejected": -0.18997205793857574,
      "step": 1906
    },
    {
      "epoch": 1.1076261834233607,
      "grad_norm": 293.05413818359375,
      "learning_rate": 2.229808251016851e-06,
      "logits/chosen": -1.272873878479004,
      "logits/rejected": -1.3451749086380005,
      "logps/chosen": -75.60696411132812,
      "logps/rejected": -71.20547485351562,
      "loss": 12.6504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13534964621067047,
      "rewards/margins": 0.14650148153305054,
      "rewards/rejected": -0.2818511426448822,
      "step": 1907
    },
    {
      "epoch": 1.1082070047046524,
      "grad_norm": 265.19073486328125,
      "learning_rate": 2.2283556072051134e-06,
      "logits/chosen": -1.232790470123291,
      "logits/rejected": -1.3268779516220093,
      "logps/chosen": -69.74960327148438,
      "logps/rejected": -79.68042755126953,
      "loss": 11.6488,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09537284821271896,
      "rewards/margins": 0.273003488779068,
      "rewards/rejected": -0.36837631464004517,
      "step": 1908
    },
    {
      "epoch": 1.108787825985944,
      "grad_norm": 326.8232116699219,
      "learning_rate": 2.226902963393376e-06,
      "logits/chosen": -1.49457585811615,
      "logits/rejected": -1.5573073625564575,
      "logps/chosen": -78.37895202636719,
      "logps/rejected": -84.21720886230469,
      "loss": 11.1658,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08945896476507187,
      "rewards/margins": 0.31430649757385254,
      "rewards/rejected": -0.4037654995918274,
      "step": 1909
    },
    {
      "epoch": 1.109368647267236,
      "grad_norm": 301.47314453125,
      "learning_rate": 2.225450319581639e-06,
      "logits/chosen": -1.269869327545166,
      "logits/rejected": -1.2704788446426392,
      "logps/chosen": -72.6441421508789,
      "logps/rejected": -75.52769470214844,
      "loss": 13.1856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1516750007867813,
      "rewards/margins": 0.09178517758846283,
      "rewards/rejected": -0.24346022307872772,
      "step": 1910
    },
    {
      "epoch": 1.1099494685485276,
      "grad_norm": 255.66351318359375,
      "learning_rate": 2.2239976757699012e-06,
      "logits/chosen": -1.371336579322815,
      "logits/rejected": -1.288886308670044,
      "logps/chosen": -65.57683563232422,
      "logps/rejected": -76.2176513671875,
      "loss": 11.5179,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03896737098693848,
      "rewards/margins": 0.2710152566432953,
      "rewards/rejected": -0.30998262763023376,
      "step": 1911
    },
    {
      "epoch": 1.1105302898298193,
      "grad_norm": 334.7614440917969,
      "learning_rate": 2.222545031958164e-06,
      "logits/chosen": -1.2594287395477295,
      "logits/rejected": -1.4102922677993774,
      "logps/chosen": -83.85694122314453,
      "logps/rejected": -76.9856185913086,
      "loss": 13.8109,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.27195674180984497,
      "rewards/margins": 0.03679296001791954,
      "rewards/rejected": -0.3087497353553772,
      "step": 1912
    },
    {
      "epoch": 1.1111111111111112,
      "grad_norm": 288.66973876953125,
      "learning_rate": 2.2210923881464267e-06,
      "logits/chosen": -1.4630868434906006,
      "logits/rejected": -1.4939908981323242,
      "logps/chosen": -73.3730239868164,
      "logps/rejected": -82.3504409790039,
      "loss": 11.8161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03914286568760872,
      "rewards/margins": 0.2647934854030609,
      "rewards/rejected": -0.3039363920688629,
      "step": 1913
    },
    {
      "epoch": 1.1116919323924028,
      "grad_norm": 283.6938171386719,
      "learning_rate": 2.2196397443346895e-06,
      "logits/chosen": -0.9348347783088684,
      "logits/rejected": -1.117328405380249,
      "logps/chosen": -77.11993408203125,
      "logps/rejected": -82.05966186523438,
      "loss": 11.0485,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06318458169698715,
      "rewards/margins": 0.40413227677345276,
      "rewards/rejected": -0.4673168659210205,
      "step": 1914
    },
    {
      "epoch": 1.1122727536736945,
      "grad_norm": 437.3043518066406,
      "learning_rate": 2.218187100522952e-06,
      "logits/chosen": -1.3355329036712646,
      "logits/rejected": -1.3525117635726929,
      "logps/chosen": -72.26240539550781,
      "logps/rejected": -76.98352813720703,
      "loss": 13.0706,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11376510560512543,
      "rewards/margins": 0.10241514444351196,
      "rewards/rejected": -0.2161802500486374,
      "step": 1915
    },
    {
      "epoch": 1.1128535749549864,
      "grad_norm": 308.1046447753906,
      "learning_rate": 2.2167344567112145e-06,
      "logits/chosen": -1.4313459396362305,
      "logits/rejected": -1.4165375232696533,
      "logps/chosen": -76.13274383544922,
      "logps/rejected": -84.38134002685547,
      "loss": 11.7628,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1176053136587143,
      "rewards/margins": 0.2526373267173767,
      "rewards/rejected": -0.3702426850795746,
      "step": 1916
    },
    {
      "epoch": 1.113434396236278,
      "grad_norm": 514.9987182617188,
      "learning_rate": 2.2152818128994773e-06,
      "logits/chosen": -1.3288246393203735,
      "logits/rejected": -1.2554047107696533,
      "logps/chosen": -72.73957824707031,
      "logps/rejected": -82.76560974121094,
      "loss": 11.7463,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05349165201187134,
      "rewards/margins": 0.25409844517707825,
      "rewards/rejected": -0.307590126991272,
      "step": 1917
    },
    {
      "epoch": 1.11401521751757,
      "grad_norm": 297.8122863769531,
      "learning_rate": 2.2138291690877396e-06,
      "logits/chosen": -1.4135069847106934,
      "logits/rejected": -1.516442060470581,
      "logps/chosen": -78.71307373046875,
      "logps/rejected": -80.36064147949219,
      "loss": 12.0154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15694531798362732,
      "rewards/margins": 0.24924182891845703,
      "rewards/rejected": -0.40618714690208435,
      "step": 1918
    },
    {
      "epoch": 1.1145960387988616,
      "grad_norm": 316.2962951660156,
      "learning_rate": 2.2123765252760024e-06,
      "logits/chosen": -1.2217949628829956,
      "logits/rejected": -1.2300158739089966,
      "logps/chosen": -77.15487670898438,
      "logps/rejected": -73.07290649414062,
      "loss": 12.6687,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10632792860269547,
      "rewards/margins": 0.14649660885334015,
      "rewards/rejected": -0.2528245151042938,
      "step": 1919
    },
    {
      "epoch": 1.1151768600801533,
      "grad_norm": 295.5701599121094,
      "learning_rate": 2.210923881464265e-06,
      "logits/chosen": -1.1811925172805786,
      "logits/rejected": -1.2259422540664673,
      "logps/chosen": -73.10960388183594,
      "logps/rejected": -68.5401840209961,
      "loss": 13.1627,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11947636306285858,
      "rewards/margins": 0.08741600811481476,
      "rewards/rejected": -0.20689240097999573,
      "step": 1920
    },
    {
      "epoch": 1.1157576813614452,
      "grad_norm": 335.7071838378906,
      "learning_rate": 2.209471237652528e-06,
      "logits/chosen": -1.3407771587371826,
      "logits/rejected": -1.3103032112121582,
      "logps/chosen": -81.46482849121094,
      "logps/rejected": -68.2290267944336,
      "loss": 13.5742,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2187584936618805,
      "rewards/margins": 0.06662143766880035,
      "rewards/rejected": -0.28537994623184204,
      "step": 1921
    },
    {
      "epoch": 1.1163385026427368,
      "grad_norm": 280.0152587890625,
      "learning_rate": 2.2080185938407906e-06,
      "logits/chosen": -1.1563310623168945,
      "logits/rejected": -1.1213818788528442,
      "logps/chosen": -67.10273742675781,
      "logps/rejected": -70.65030670166016,
      "loss": 11.8002,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05223752185702324,
      "rewards/margins": 0.2263999730348587,
      "rewards/rejected": -0.27863746881484985,
      "step": 1922
    },
    {
      "epoch": 1.1169193239240285,
      "grad_norm": 283.3345947265625,
      "learning_rate": 2.2065659500290534e-06,
      "logits/chosen": -1.3147337436676025,
      "logits/rejected": -1.3408397436141968,
      "logps/chosen": -77.82501220703125,
      "logps/rejected": -73.78407287597656,
      "loss": 12.5355,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11272376775741577,
      "rewards/margins": 0.15296077728271484,
      "rewards/rejected": -0.2656845450401306,
      "step": 1923
    },
    {
      "epoch": 1.1175001452053204,
      "grad_norm": 315.94439697265625,
      "learning_rate": 2.2051133062173157e-06,
      "logits/chosen": -1.3167445659637451,
      "logits/rejected": -1.3747785091400146,
      "logps/chosen": -76.0057373046875,
      "logps/rejected": -84.03781127929688,
      "loss": 13.0629,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12550127506256104,
      "rewards/margins": 0.10542813688516617,
      "rewards/rejected": -0.2309294193983078,
      "step": 1924
    },
    {
      "epoch": 1.118080966486612,
      "grad_norm": 360.99993896484375,
      "learning_rate": 2.2036606624055784e-06,
      "logits/chosen": -1.1320605278015137,
      "logits/rejected": -1.1136503219604492,
      "logps/chosen": -75.85591125488281,
      "logps/rejected": -76.4011001586914,
      "loss": 12.6993,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10536464303731918,
      "rewards/margins": 0.14731159806251526,
      "rewards/rejected": -0.25267624855041504,
      "step": 1925
    },
    {
      "epoch": 1.1186617877679037,
      "grad_norm": 288.77294921875,
      "learning_rate": 2.202208018593841e-06,
      "logits/chosen": -1.3016961812973022,
      "logits/rejected": -1.3648955821990967,
      "logps/chosen": -78.11613464355469,
      "logps/rejected": -79.7267837524414,
      "loss": 11.547,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10100438445806503,
      "rewards/margins": 0.26796501874923706,
      "rewards/rejected": -0.3689693808555603,
      "step": 1926
    },
    {
      "epoch": 1.1192426090491956,
      "grad_norm": 287.3154296875,
      "learning_rate": 2.2007553747821035e-06,
      "logits/chosen": -1.3344142436981201,
      "logits/rejected": -1.3087059259414673,
      "logps/chosen": -68.77854919433594,
      "logps/rejected": -76.29188537597656,
      "loss": 12.777,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14861661195755005,
      "rewards/margins": 0.13593053817749023,
      "rewards/rejected": -0.2845471501350403,
      "step": 1927
    },
    {
      "epoch": 1.1198234303304873,
      "grad_norm": 264.4040222167969,
      "learning_rate": 2.1993027309703662e-06,
      "logits/chosen": -0.9665416479110718,
      "logits/rejected": -1.0425465106964111,
      "logps/chosen": -69.12470245361328,
      "logps/rejected": -77.84590911865234,
      "loss": 11.6269,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09439492970705032,
      "rewards/margins": 0.2536908984184265,
      "rewards/rejected": -0.34808582067489624,
      "step": 1928
    },
    {
      "epoch": 1.1204042516117791,
      "grad_norm": 294.5516662597656,
      "learning_rate": 2.197850087158629e-06,
      "logits/chosen": -1.1722339391708374,
      "logits/rejected": -1.1678173542022705,
      "logps/chosen": -74.04752349853516,
      "logps/rejected": -82.97613525390625,
      "loss": 12.0144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1017851009964943,
      "rewards/margins": 0.22043628990650177,
      "rewards/rejected": -0.32222142815589905,
      "step": 1929
    },
    {
      "epoch": 1.1209850728930708,
      "grad_norm": 279.0356750488281,
      "learning_rate": 2.1963974433468917e-06,
      "logits/chosen": -1.2174415588378906,
      "logits/rejected": -1.1340147256851196,
      "logps/chosen": -71.5632095336914,
      "logps/rejected": -74.4330062866211,
      "loss": 11.5731,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.025679226964712143,
      "rewards/margins": 0.2714490294456482,
      "rewards/rejected": -0.24576978385448456,
      "step": 1930
    },
    {
      "epoch": 1.1215658941743625,
      "grad_norm": 294.675537109375,
      "learning_rate": 2.194944799535154e-06,
      "logits/chosen": -1.1632798910140991,
      "logits/rejected": -1.2432198524475098,
      "logps/chosen": -73.2660140991211,
      "logps/rejected": -74.98796081542969,
      "loss": 12.3352,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.051136624068021774,
      "rewards/margins": 0.17500144243240356,
      "rewards/rejected": -0.22613804042339325,
      "step": 1931
    },
    {
      "epoch": 1.1221467154556544,
      "grad_norm": 296.04754638671875,
      "learning_rate": 2.193492155723417e-06,
      "logits/chosen": -1.2488974332809448,
      "logits/rejected": -1.2040361166000366,
      "logps/chosen": -77.49885559082031,
      "logps/rejected": -75.98363494873047,
      "loss": 11.6827,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06657309830188751,
      "rewards/margins": 0.24671435356140137,
      "rewards/rejected": -0.3132874369621277,
      "step": 1932
    },
    {
      "epoch": 1.122727536736946,
      "grad_norm": 277.9831237792969,
      "learning_rate": 2.1920395119116796e-06,
      "logits/chosen": -1.2203619480133057,
      "logits/rejected": -1.1925016641616821,
      "logps/chosen": -74.86399841308594,
      "logps/rejected": -82.82511138916016,
      "loss": 11.0639,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.030850926414132118,
      "rewards/margins": 0.3154781758785248,
      "rewards/rejected": -0.34632909297943115,
      "step": 1933
    },
    {
      "epoch": 1.1233083580182377,
      "grad_norm": 282.0268859863281,
      "learning_rate": 2.190586868099942e-06,
      "logits/chosen": -1.2960188388824463,
      "logits/rejected": -1.2892324924468994,
      "logps/chosen": -71.91605377197266,
      "logps/rejected": -72.52494049072266,
      "loss": 12.4001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02432938478887081,
      "rewards/margins": 0.18806426227092743,
      "rewards/rejected": -0.2123936414718628,
      "step": 1934
    },
    {
      "epoch": 1.1238891792995296,
      "grad_norm": 370.7580261230469,
      "learning_rate": 2.1891342242882046e-06,
      "logits/chosen": -1.1450526714324951,
      "logits/rejected": -1.2816616296768188,
      "logps/chosen": -76.17131042480469,
      "logps/rejected": -70.14605712890625,
      "loss": 12.4661,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0858418345451355,
      "rewards/margins": 0.18080425262451172,
      "rewards/rejected": -0.26664605736732483,
      "step": 1935
    },
    {
      "epoch": 1.1244700005808212,
      "grad_norm": 300.875732421875,
      "learning_rate": 2.1876815804764674e-06,
      "logits/chosen": -1.2304325103759766,
      "logits/rejected": -1.377967119216919,
      "logps/chosen": -78.70970916748047,
      "logps/rejected": -76.04560852050781,
      "loss": 13.1371,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2100694626569748,
      "rewards/margins": 0.09031853079795837,
      "rewards/rejected": -0.30038800835609436,
      "step": 1936
    },
    {
      "epoch": 1.125050821862113,
      "grad_norm": 310.6013488769531,
      "learning_rate": 2.1862289366647297e-06,
      "logits/chosen": -1.1975767612457275,
      "logits/rejected": -1.2848495244979858,
      "logps/chosen": -69.78667449951172,
      "logps/rejected": -82.6092300415039,
      "loss": 11.7951,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03182733803987503,
      "rewards/margins": 0.23468205332756042,
      "rewards/rejected": -0.26650941371917725,
      "step": 1937
    },
    {
      "epoch": 1.1256316431434048,
      "grad_norm": 298.565673828125,
      "learning_rate": 2.1847762928529925e-06,
      "logits/chosen": -1.3946349620819092,
      "logits/rejected": -1.3628714084625244,
      "logps/chosen": -80.00898742675781,
      "logps/rejected": -71.36045837402344,
      "loss": 13.0275,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1114078015089035,
      "rewards/margins": 0.11659424006938934,
      "rewards/rejected": -0.22800207138061523,
      "step": 1938
    },
    {
      "epoch": 1.1262124644246965,
      "grad_norm": 283.6505126953125,
      "learning_rate": 2.183323649041255e-06,
      "logits/chosen": -1.2307833433151245,
      "logits/rejected": -1.3563038110733032,
      "logps/chosen": -78.7392349243164,
      "logps/rejected": -73.3707504272461,
      "loss": 11.0833,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.017166469246149063,
      "rewards/margins": 0.3301101326942444,
      "rewards/rejected": -0.3129437267780304,
      "step": 1939
    },
    {
      "epoch": 1.1267932857059884,
      "grad_norm": 286.3055725097656,
      "learning_rate": 2.181871005229518e-06,
      "logits/chosen": -1.3248400688171387,
      "logits/rejected": -1.2848931550979614,
      "logps/chosen": -72.25897979736328,
      "logps/rejected": -78.220947265625,
      "loss": 12.1339,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02242291532456875,
      "rewards/margins": 0.2113153487443924,
      "rewards/rejected": -0.2337382733821869,
      "step": 1940
    },
    {
      "epoch": 1.12737410698728,
      "grad_norm": 289.6809997558594,
      "learning_rate": 2.1804183614177803e-06,
      "logits/chosen": -1.3012433052062988,
      "logits/rejected": -1.246675729751587,
      "logps/chosen": -75.35639953613281,
      "logps/rejected": -81.76630401611328,
      "loss": 11.7926,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03944106772542,
      "rewards/margins": 0.24572589993476868,
      "rewards/rejected": -0.28516697883605957,
      "step": 1941
    },
    {
      "epoch": 1.1279549282685717,
      "grad_norm": 283.5423889160156,
      "learning_rate": 2.1789657176060435e-06,
      "logits/chosen": -1.2307687997817993,
      "logits/rejected": -1.214462161064148,
      "logps/chosen": -73.12229919433594,
      "logps/rejected": -83.8825912475586,
      "loss": 11.9366,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06290044635534286,
      "rewards/margins": 0.22151437401771545,
      "rewards/rejected": -0.28441479802131653,
      "step": 1942
    },
    {
      "epoch": 1.1285357495498636,
      "grad_norm": 314.1970520019531,
      "learning_rate": 2.1775130737943058e-06,
      "logits/chosen": -1.2253506183624268,
      "logits/rejected": -1.1813223361968994,
      "logps/chosen": -72.55824279785156,
      "logps/rejected": -80.65178680419922,
      "loss": 12.7726,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11374680697917938,
      "rewards/margins": 0.1529560536146164,
      "rewards/rejected": -0.2667028605937958,
      "step": 1943
    },
    {
      "epoch": 1.1291165708311552,
      "grad_norm": 273.2713317871094,
      "learning_rate": 2.1760604299825685e-06,
      "logits/chosen": -1.108096718788147,
      "logits/rejected": -1.1145174503326416,
      "logps/chosen": -76.70726013183594,
      "logps/rejected": -79.79026794433594,
      "loss": 11.6639,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08386872708797455,
      "rewards/margins": 0.2588963508605957,
      "rewards/rejected": -0.34276506304740906,
      "step": 1944
    },
    {
      "epoch": 1.129697392112447,
      "grad_norm": 317.1315002441406,
      "learning_rate": 2.1746077861708313e-06,
      "logits/chosen": -1.2342846393585205,
      "logits/rejected": -1.2895079851150513,
      "logps/chosen": -82.35624694824219,
      "logps/rejected": -86.94931030273438,
      "loss": 12.0287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13268692791461945,
      "rewards/margins": 0.23872098326683044,
      "rewards/rejected": -0.3714079260826111,
      "step": 1945
    },
    {
      "epoch": 1.1302782133937388,
      "grad_norm": 273.5216979980469,
      "learning_rate": 2.1731551423590936e-06,
      "logits/chosen": -1.1475584506988525,
      "logits/rejected": -1.134659767150879,
      "logps/chosen": -67.44471740722656,
      "logps/rejected": -84.37786102294922,
      "loss": 11.8725,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.027367597445845604,
      "rewards/margins": 0.2513376772403717,
      "rewards/rejected": -0.2787052392959595,
      "step": 1946
    },
    {
      "epoch": 1.1308590346750305,
      "grad_norm": 303.4730224609375,
      "learning_rate": 2.1717024985473563e-06,
      "logits/chosen": -1.1489859819412231,
      "logits/rejected": -1.3084758520126343,
      "logps/chosen": -72.19990539550781,
      "logps/rejected": -77.89209747314453,
      "loss": 11.4524,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09930064529180527,
      "rewards/margins": 0.39551395177841187,
      "rewards/rejected": -0.49481457471847534,
      "step": 1947
    },
    {
      "epoch": 1.1314398559563221,
      "grad_norm": 298.7429504394531,
      "learning_rate": 2.170249854735619e-06,
      "logits/chosen": -1.318743348121643,
      "logits/rejected": -1.380985975265503,
      "logps/chosen": -82.39265441894531,
      "logps/rejected": -79.09247589111328,
      "loss": 11.4791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04628929868340492,
      "rewards/margins": 0.29148435592651367,
      "rewards/rejected": -0.3377736508846283,
      "step": 1948
    },
    {
      "epoch": 1.132020677237614,
      "grad_norm": 291.5268859863281,
      "learning_rate": 2.168797210923882e-06,
      "logits/chosen": -1.3119580745697021,
      "logits/rejected": -1.231465220451355,
      "logps/chosen": -72.38230895996094,
      "logps/rejected": -80.19706726074219,
      "loss": 12.7648,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13021236658096313,
      "rewards/margins": 0.14271263778209686,
      "rewards/rejected": -0.2729250192642212,
      "step": 1949
    },
    {
      "epoch": 1.1326014985189057,
      "grad_norm": 311.93792724609375,
      "learning_rate": 2.167344567112144e-06,
      "logits/chosen": -1.2097591161727905,
      "logits/rejected": -1.2426872253417969,
      "logps/chosen": -76.53303527832031,
      "logps/rejected": -76.4521713256836,
      "loss": 12.8982,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.103241465985775,
      "rewards/margins": 0.12366548925638199,
      "rewards/rejected": -0.2269069403409958,
      "step": 1950
    },
    {
      "epoch": 1.1331823198001976,
      "grad_norm": 278.9921569824219,
      "learning_rate": 2.165891923300407e-06,
      "logits/chosen": -1.337873935699463,
      "logits/rejected": -1.217064380645752,
      "logps/chosen": -73.989013671875,
      "logps/rejected": -79.61365509033203,
      "loss": 11.1842,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0386163592338562,
      "rewards/margins": 0.3131384551525116,
      "rewards/rejected": -0.3517547845840454,
      "step": 1951
    },
    {
      "epoch": 1.1337631410814892,
      "grad_norm": 273.65704345703125,
      "learning_rate": 2.1644392794886697e-06,
      "logits/chosen": -1.388195276260376,
      "logits/rejected": -1.4737672805786133,
      "logps/chosen": -69.76702880859375,
      "logps/rejected": -84.04283905029297,
      "loss": 11.7906,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07384618371725082,
      "rewards/margins": 0.2571582794189453,
      "rewards/rejected": -0.33100444078445435,
      "step": 1952
    },
    {
      "epoch": 1.134343962362781,
      "grad_norm": 259.1620178222656,
      "learning_rate": 2.162986635676932e-06,
      "logits/chosen": -1.0427820682525635,
      "logits/rejected": -1.0253342390060425,
      "logps/chosen": -67.88665008544922,
      "logps/rejected": -78.9303207397461,
      "loss": 10.8616,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07229509204626083,
      "rewards/margins": 0.3531135022640228,
      "rewards/rejected": -0.4254085421562195,
      "step": 1953
    },
    {
      "epoch": 1.1349247836440728,
      "grad_norm": 291.63311767578125,
      "learning_rate": 2.1615339918651947e-06,
      "logits/chosen": -1.2983274459838867,
      "logits/rejected": -1.3404573202133179,
      "logps/chosen": -73.8432846069336,
      "logps/rejected": -79.9900894165039,
      "loss": 11.8531,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.015718702226877213,
      "rewards/margins": 0.24740460515022278,
      "rewards/rejected": -0.2631233036518097,
      "step": 1954
    },
    {
      "epoch": 1.1355056049253645,
      "grad_norm": 865.8622436523438,
      "learning_rate": 2.1600813480534575e-06,
      "logits/chosen": -1.3869479894638062,
      "logits/rejected": -1.4180402755737305,
      "logps/chosen": -70.24688720703125,
      "logps/rejected": -73.07231903076172,
      "loss": 11.795,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.014515931718051434,
      "rewards/margins": 0.2408524453639984,
      "rewards/rejected": -0.2553683817386627,
      "step": 1955
    },
    {
      "epoch": 1.1360864262066561,
      "grad_norm": 294.4327697753906,
      "learning_rate": 2.1586287042417202e-06,
      "logits/chosen": -1.3286077976226807,
      "logits/rejected": -1.244210958480835,
      "logps/chosen": -80.20768737792969,
      "logps/rejected": -82.79126739501953,
      "loss": 11.7022,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05336623638868332,
      "rewards/margins": 0.263483464717865,
      "rewards/rejected": -0.31684964895248413,
      "step": 1956
    },
    {
      "epoch": 1.136667247487948,
      "grad_norm": 313.34765625,
      "learning_rate": 2.1571760604299826e-06,
      "logits/chosen": -1.2729710340499878,
      "logits/rejected": -1.2363452911376953,
      "logps/chosen": -75.56641387939453,
      "logps/rejected": -73.6912841796875,
      "loss": 13.222,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14025214314460754,
      "rewards/margins": 0.09303469210863113,
      "rewards/rejected": -0.2332868129014969,
      "step": 1957
    },
    {
      "epoch": 1.1372480687692397,
      "grad_norm": 343.24066162109375,
      "learning_rate": 2.1557234166182453e-06,
      "logits/chosen": -1.271246075630188,
      "logits/rejected": -1.2571808099746704,
      "logps/chosen": -83.0656509399414,
      "logps/rejected": -89.42984771728516,
      "loss": 12.2425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12075678259134293,
      "rewards/margins": 0.199143648147583,
      "rewards/rejected": -0.31990042328834534,
      "step": 1958
    },
    {
      "epoch": 1.1378288900505313,
      "grad_norm": 293.4291076660156,
      "learning_rate": 2.154270772806508e-06,
      "logits/chosen": -1.1912364959716797,
      "logits/rejected": -1.321576476097107,
      "logps/chosen": -76.13489532470703,
      "logps/rejected": -82.08197021484375,
      "loss": 11.5714,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11209726333618164,
      "rewards/margins": 0.31790828704833984,
      "rewards/rejected": -0.4300055503845215,
      "step": 1959
    },
    {
      "epoch": 1.1384097113318232,
      "grad_norm": 294.86907958984375,
      "learning_rate": 2.1528181289947704e-06,
      "logits/chosen": -1.172972559928894,
      "logits/rejected": -1.1695377826690674,
      "logps/chosen": -72.71870422363281,
      "logps/rejected": -75.96612548828125,
      "loss": 12.8486,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12673591077327728,
      "rewards/margins": 0.131285160779953,
      "rewards/rejected": -0.2580210566520691,
      "step": 1960
    },
    {
      "epoch": 1.138990532613115,
      "grad_norm": 309.75823974609375,
      "learning_rate": 2.151365485183033e-06,
      "logits/chosen": -1.2526909112930298,
      "logits/rejected": -1.1622117757797241,
      "logps/chosen": -76.56922912597656,
      "logps/rejected": -74.81319427490234,
      "loss": 11.7024,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0893649309873581,
      "rewards/margins": 0.2575554847717285,
      "rewards/rejected": -0.3469204306602478,
      "step": 1961
    },
    {
      "epoch": 1.1395713538944068,
      "grad_norm": 401.8067321777344,
      "learning_rate": 2.149912841371296e-06,
      "logits/chosen": -1.3266350030899048,
      "logits/rejected": -1.2926552295684814,
      "logps/chosen": -73.2862548828125,
      "logps/rejected": -77.86517333984375,
      "loss": 12.1218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13390740752220154,
      "rewards/margins": 0.2047162503004074,
      "rewards/rejected": -0.33862370252609253,
      "step": 1962
    },
    {
      "epoch": 1.1401521751756984,
      "grad_norm": 410.91497802734375,
      "learning_rate": 2.1484601975595586e-06,
      "logits/chosen": -1.439263105392456,
      "logits/rejected": -1.4164232015609741,
      "logps/chosen": -72.84381866455078,
      "logps/rejected": -73.59004974365234,
      "loss": 12.9645,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08399099856615067,
      "rewards/margins": 0.12516102194786072,
      "rewards/rejected": -0.20915202796459198,
      "step": 1963
    },
    {
      "epoch": 1.1407329964569901,
      "grad_norm": 290.11767578125,
      "learning_rate": 2.1470075537478214e-06,
      "logits/chosen": -1.1432440280914307,
      "logits/rejected": -1.1341665983200073,
      "logps/chosen": -68.89353942871094,
      "logps/rejected": -73.94276428222656,
      "loss": 12.4099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0591636523604393,
      "rewards/margins": 0.1692221313714981,
      "rewards/rejected": -0.2283857762813568,
      "step": 1964
    },
    {
      "epoch": 1.141313817738282,
      "grad_norm": 282.0762939453125,
      "learning_rate": 2.145554909936084e-06,
      "logits/chosen": -1.27593994140625,
      "logits/rejected": -1.2727181911468506,
      "logps/chosen": -73.45965576171875,
      "logps/rejected": -75.09355163574219,
      "loss": 12.0566,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1274360567331314,
      "rewards/margins": 0.21436676383018494,
      "rewards/rejected": -0.34180283546447754,
      "step": 1965
    },
    {
      "epoch": 1.1418946390195737,
      "grad_norm": 281.84478759765625,
      "learning_rate": 2.1441022661243465e-06,
      "logits/chosen": -1.1097770929336548,
      "logits/rejected": -1.0994617938995361,
      "logps/chosen": -78.10334777832031,
      "logps/rejected": -83.05355834960938,
      "loss": 11.6123,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04400419071316719,
      "rewards/margins": 0.2833501398563385,
      "rewards/rejected": -0.3273543119430542,
      "step": 1966
    },
    {
      "epoch": 1.1424754603008653,
      "grad_norm": 707.77685546875,
      "learning_rate": 2.142649622312609e-06,
      "logits/chosen": -1.1498565673828125,
      "logits/rejected": -1.1857938766479492,
      "logps/chosen": -74.59598541259766,
      "logps/rejected": -74.82608795166016,
      "loss": 13.1684,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1487945318222046,
      "rewards/margins": 0.08569033443927765,
      "rewards/rejected": -0.23448486626148224,
      "step": 1967
    },
    {
      "epoch": 1.1430562815821572,
      "grad_norm": 273.90826416015625,
      "learning_rate": 2.141196978500872e-06,
      "logits/chosen": -1.1750829219818115,
      "logits/rejected": -1.1335279941558838,
      "logps/chosen": -73.4001235961914,
      "logps/rejected": -76.6945571899414,
      "loss": 11.9643,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10450764745473862,
      "rewards/margins": 0.2189536988735199,
      "rewards/rejected": -0.3234613537788391,
      "step": 1968
    },
    {
      "epoch": 1.1436371028634489,
      "grad_norm": 281.28070068359375,
      "learning_rate": 2.1397443346891343e-06,
      "logits/chosen": -1.2302651405334473,
      "logits/rejected": -1.2293670177459717,
      "logps/chosen": -76.97260284423828,
      "logps/rejected": -80.07366943359375,
      "loss": 11.9157,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09830137342214584,
      "rewards/margins": 0.22925904393196106,
      "rewards/rejected": -0.3275603652000427,
      "step": 1969
    },
    {
      "epoch": 1.1442179241447406,
      "grad_norm": 302.6987609863281,
      "learning_rate": 2.138291690877397e-06,
      "logits/chosen": -1.269640326499939,
      "logits/rejected": -1.2419848442077637,
      "logps/chosen": -82.02096557617188,
      "logps/rejected": -78.39289855957031,
      "loss": 12.7465,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12325485795736313,
      "rewards/margins": 0.12945276498794556,
      "rewards/rejected": -0.2527076303958893,
      "step": 1970
    },
    {
      "epoch": 1.1447987454260324,
      "grad_norm": 272.7333679199219,
      "learning_rate": 2.1368390470656598e-06,
      "logits/chosen": -1.1014125347137451,
      "logits/rejected": -1.1449190378189087,
      "logps/chosen": -74.22889709472656,
      "logps/rejected": -69.57112121582031,
      "loss": 12.5533,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11071793735027313,
      "rewards/margins": 0.16173584759235382,
      "rewards/rejected": -0.27245378494262695,
      "step": 1971
    },
    {
      "epoch": 1.145379566707324,
      "grad_norm": 303.4043273925781,
      "learning_rate": 2.1353864032539225e-06,
      "logits/chosen": -1.3740540742874146,
      "logits/rejected": -1.3383554220199585,
      "logps/chosen": -78.8419189453125,
      "logps/rejected": -83.64044952392578,
      "loss": 12.9992,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.17922472953796387,
      "rewards/margins": 0.12119539827108383,
      "rewards/rejected": -0.3004201352596283,
      "step": 1972
    },
    {
      "epoch": 1.145960387988616,
      "grad_norm": 347.2142333984375,
      "learning_rate": 2.133933759442185e-06,
      "logits/chosen": -1.2790744304656982,
      "logits/rejected": -1.2758400440216064,
      "logps/chosen": -73.78958892822266,
      "logps/rejected": -74.19044494628906,
      "loss": 11.8279,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.033005643635988235,
      "rewards/margins": 0.24207842350006104,
      "rewards/rejected": -0.27508407831192017,
      "step": 1973
    },
    {
      "epoch": 1.1465412092699077,
      "grad_norm": 293.1376647949219,
      "learning_rate": 2.1324811156304476e-06,
      "logits/chosen": -1.143145203590393,
      "logits/rejected": -1.1884276866912842,
      "logps/chosen": -75.11312866210938,
      "logps/rejected": -76.83495330810547,
      "loss": 11.9929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.059767358005046844,
      "rewards/margins": 0.2222428321838379,
      "rewards/rejected": -0.2820102274417877,
      "step": 1974
    },
    {
      "epoch": 1.1471220305511993,
      "grad_norm": 295.08441162109375,
      "learning_rate": 2.1310284718187103e-06,
      "logits/chosen": -1.4367376565933228,
      "logits/rejected": -1.3670251369476318,
      "logps/chosen": -85.57766723632812,
      "logps/rejected": -83.10955047607422,
      "loss": 11.976,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11379885673522949,
      "rewards/margins": 0.23771452903747559,
      "rewards/rejected": -0.35151341557502747,
      "step": 1975
    },
    {
      "epoch": 1.1477028518324912,
      "grad_norm": 291.2987365722656,
      "learning_rate": 2.1295758280069727e-06,
      "logits/chosen": -1.170220971107483,
      "logits/rejected": -1.2852774858474731,
      "logps/chosen": -70.36900329589844,
      "logps/rejected": -71.58023071289062,
      "loss": 12.2647,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10121617466211319,
      "rewards/margins": 0.18490923941135406,
      "rewards/rejected": -0.28612545132637024,
      "step": 1976
    },
    {
      "epoch": 1.1482836731137829,
      "grad_norm": 297.6701354980469,
      "learning_rate": 2.1281231841952354e-06,
      "logits/chosen": -1.2434123754501343,
      "logits/rejected": -1.1429994106292725,
      "logps/chosen": -75.046630859375,
      "logps/rejected": -77.37379455566406,
      "loss": 11.8401,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07914550602436066,
      "rewards/margins": 0.2404637634754181,
      "rewards/rejected": -0.31960922479629517,
      "step": 1977
    },
    {
      "epoch": 1.1488644943950745,
      "grad_norm": 302.66400146484375,
      "learning_rate": 2.126670540383498e-06,
      "logits/chosen": -1.2931678295135498,
      "logits/rejected": -1.2007125616073608,
      "logps/chosen": -69.16878509521484,
      "logps/rejected": -80.29863739013672,
      "loss": 12.8455,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11660971492528915,
      "rewards/margins": 0.1410079300403595,
      "rewards/rejected": -0.25761765241622925,
      "step": 1978
    },
    {
      "epoch": 1.1494453156763664,
      "grad_norm": 314.7141418457031,
      "learning_rate": 2.1252178965717605e-06,
      "logits/chosen": -1.1674929857254028,
      "logits/rejected": -1.1725285053253174,
      "logps/chosen": -73.80543518066406,
      "logps/rejected": -82.11598205566406,
      "loss": 12.9425,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1631750613451004,
      "rewards/margins": 0.10442064702510834,
      "rewards/rejected": -0.26759570837020874,
      "step": 1979
    },
    {
      "epoch": 1.150026136957658,
      "grad_norm": 301.03045654296875,
      "learning_rate": 2.1237652527600232e-06,
      "logits/chosen": -1.2573392391204834,
      "logits/rejected": -1.2182655334472656,
      "logps/chosen": -77.00226593017578,
      "logps/rejected": -73.67127227783203,
      "loss": 13.3953,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16648943722248077,
      "rewards/margins": 0.07097335159778595,
      "rewards/rejected": -0.23746278882026672,
      "step": 1980
    },
    {
      "epoch": 1.1506069582389498,
      "grad_norm": 278.3601989746094,
      "learning_rate": 2.122312608948286e-06,
      "logits/chosen": -1.2742327451705933,
      "logits/rejected": -1.3312662839889526,
      "logps/chosen": -76.98381042480469,
      "logps/rejected": -73.47186279296875,
      "loss": 12.5269,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06901534646749496,
      "rewards/margins": 0.1672179102897644,
      "rewards/rejected": -0.23623323440551758,
      "step": 1981
    },
    {
      "epoch": 1.1511877795202416,
      "grad_norm": 290.89471435546875,
      "learning_rate": 2.1208599651365487e-06,
      "logits/chosen": -1.1274396181106567,
      "logits/rejected": -1.2206157445907593,
      "logps/chosen": -74.77387237548828,
      "logps/rejected": -73.85691833496094,
      "loss": 12.7598,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10330319404602051,
      "rewards/margins": 0.160480335354805,
      "rewards/rejected": -0.2637835144996643,
      "step": 1982
    },
    {
      "epoch": 1.1517686008015333,
      "grad_norm": 303.51123046875,
      "learning_rate": 2.119407321324811e-06,
      "logits/chosen": -1.3936259746551514,
      "logits/rejected": -1.4834871292114258,
      "logps/chosen": -67.70186614990234,
      "logps/rejected": -75.85845947265625,
      "loss": 13.0881,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08962702751159668,
      "rewards/margins": 0.15579208731651306,
      "rewards/rejected": -0.24541911482810974,
      "step": 1983
    },
    {
      "epoch": 1.1523494220828252,
      "grad_norm": 288.0142517089844,
      "learning_rate": 2.1179546775130742e-06,
      "logits/chosen": -1.382498025894165,
      "logits/rejected": -1.3534326553344727,
      "logps/chosen": -84.1441650390625,
      "logps/rejected": -86.47523498535156,
      "loss": 11.3705,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.040281813591718674,
      "rewards/margins": 0.28384605050086975,
      "rewards/rejected": -0.3241278827190399,
      "step": 1984
    },
    {
      "epoch": 1.1529302433641169,
      "grad_norm": 267.4831848144531,
      "learning_rate": 2.1165020337013366e-06,
      "logits/chosen": -1.1870182752609253,
      "logits/rejected": -1.3577935695648193,
      "logps/chosen": -68.93897247314453,
      "logps/rejected": -81.73818969726562,
      "loss": 10.4996,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.006344237830489874,
      "rewards/margins": 0.49926048517227173,
      "rewards/rejected": -0.5056046843528748,
      "step": 1985
    },
    {
      "epoch": 1.1535110646454085,
      "grad_norm": 279.0733947753906,
      "learning_rate": 2.1150493898895993e-06,
      "logits/chosen": -1.5182942152023315,
      "logits/rejected": -1.4258301258087158,
      "logps/chosen": -89.61849975585938,
      "logps/rejected": -105.99676513671875,
      "loss": 10.5854,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.1863698959350586,
      "rewards/margins": 0.3885978162288666,
      "rewards/rejected": -0.5749677419662476,
      "step": 1986
    },
    {
      "epoch": 1.1540918859267004,
      "grad_norm": 259.148193359375,
      "learning_rate": 2.113596746077862e-06,
      "logits/chosen": -1.4490654468536377,
      "logits/rejected": -1.3639252185821533,
      "logps/chosen": -67.47819519042969,
      "logps/rejected": -81.86549377441406,
      "loss": 11.4351,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.052554525434970856,
      "rewards/margins": 0.28138935565948486,
      "rewards/rejected": -0.33394384384155273,
      "step": 1987
    },
    {
      "epoch": 1.154672707207992,
      "grad_norm": 287.5876159667969,
      "learning_rate": 2.112144102266125e-06,
      "logits/chosen": -1.245964765548706,
      "logits/rejected": -1.2531163692474365,
      "logps/chosen": -72.80818176269531,
      "logps/rejected": -80.66802978515625,
      "loss": 11.4187,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.037334442138671875,
      "rewards/margins": 0.3032122254371643,
      "rewards/rejected": -0.26587778329849243,
      "step": 1988
    },
    {
      "epoch": 1.1552535284892838,
      "grad_norm": 273.879638671875,
      "learning_rate": 2.110691458454387e-06,
      "logits/chosen": -1.3126193284988403,
      "logits/rejected": -1.2508224248886108,
      "logps/chosen": -72.2828140258789,
      "logps/rejected": -79.15380859375,
      "loss": 10.7621,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.03407635912299156,
      "rewards/margins": 0.3783310055732727,
      "rewards/rejected": -0.4124073386192322,
      "step": 1989
    },
    {
      "epoch": 1.1558343497705756,
      "grad_norm": 283.71783447265625,
      "learning_rate": 2.10923881464265e-06,
      "logits/chosen": -1.2334492206573486,
      "logits/rejected": -1.2615303993225098,
      "logps/chosen": -80.1372299194336,
      "logps/rejected": -81.16111755371094,
      "loss": 11.5231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02960994467139244,
      "rewards/margins": 0.31688594818115234,
      "rewards/rejected": -0.3464958965778351,
      "step": 1990
    },
    {
      "epoch": 1.1564151710518673,
      "grad_norm": 270.8731994628906,
      "learning_rate": 2.1077861708309126e-06,
      "logits/chosen": -1.2452480792999268,
      "logits/rejected": -1.21407151222229,
      "logps/chosen": -72.30167388916016,
      "logps/rejected": -83.7374496459961,
      "loss": 11.0981,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06611408293247223,
      "rewards/margins": 0.3408433794975281,
      "rewards/rejected": -0.4069574773311615,
      "step": 1991
    },
    {
      "epoch": 1.156995992333159,
      "grad_norm": 321.7901306152344,
      "learning_rate": 2.106333527019175e-06,
      "logits/chosen": -1.3529338836669922,
      "logits/rejected": -1.3445909023284912,
      "logps/chosen": -82.09017944335938,
      "logps/rejected": -87.83184051513672,
      "loss": 12.6958,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2313474714756012,
      "rewards/margins": 0.17157180607318878,
      "rewards/rejected": -0.4029192328453064,
      "step": 1992
    },
    {
      "epoch": 1.1575768136144509,
      "grad_norm": 296.6117248535156,
      "learning_rate": 2.1048808832074377e-06,
      "logits/chosen": -1.241965889930725,
      "logits/rejected": -1.2898015975952148,
      "logps/chosen": -80.78897857666016,
      "logps/rejected": -83.4839859008789,
      "loss": 11.681,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15363600850105286,
      "rewards/margins": 0.26910287141799927,
      "rewards/rejected": -0.4227388799190521,
      "step": 1993
    },
    {
      "epoch": 1.1581576348957425,
      "grad_norm": 300.23199462890625,
      "learning_rate": 2.1034282393957004e-06,
      "logits/chosen": -1.2160532474517822,
      "logits/rejected": -1.2571182250976562,
      "logps/chosen": -75.12602233886719,
      "logps/rejected": -81.0251235961914,
      "loss": 11.681,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06518431752920151,
      "rewards/margins": 0.2687796950340271,
      "rewards/rejected": -0.3339639902114868,
      "step": 1994
    },
    {
      "epoch": 1.1587384561770344,
      "grad_norm": 271.4822082519531,
      "learning_rate": 2.1019755955839628e-06,
      "logits/chosen": -1.2302637100219727,
      "logits/rejected": -1.2544939517974854,
      "logps/chosen": -74.82804870605469,
      "logps/rejected": -83.90788269042969,
      "loss": 11.7709,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07818736881017685,
      "rewards/margins": 0.2426837682723999,
      "rewards/rejected": -0.32087111473083496,
      "step": 1995
    },
    {
      "epoch": 1.159319277458326,
      "grad_norm": 276.760009765625,
      "learning_rate": 2.1005229517722255e-06,
      "logits/chosen": -1.332384705543518,
      "logits/rejected": -1.4606298208236694,
      "logps/chosen": -76.43563079833984,
      "logps/rejected": -76.54421997070312,
      "loss": 11.1204,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.04564739391207695,
      "rewards/margins": 0.4047534465789795,
      "rewards/rejected": -0.45040082931518555,
      "step": 1996
    },
    {
      "epoch": 1.1599000987396177,
      "grad_norm": 289.7305603027344,
      "learning_rate": 2.0990703079604883e-06,
      "logits/chosen": -1.1699789762496948,
      "logits/rejected": -1.1911818981170654,
      "logps/chosen": -72.66226959228516,
      "logps/rejected": -77.96516418457031,
      "loss": 12.4719,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06769497692584991,
      "rewards/margins": 0.1787552833557129,
      "rewards/rejected": -0.246450275182724,
      "step": 1997
    },
    {
      "epoch": 1.1604809200209096,
      "grad_norm": 289.1410217285156,
      "learning_rate": 2.097617664148751e-06,
      "logits/chosen": -1.2657649517059326,
      "logits/rejected": -1.2483242750167847,
      "logps/chosen": -76.84370422363281,
      "logps/rejected": -80.66319274902344,
      "loss": 12.0518,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.041757579892873764,
      "rewards/margins": 0.25285854935646057,
      "rewards/rejected": -0.29461613297462463,
      "step": 1998
    },
    {
      "epoch": 1.1610617413022013,
      "grad_norm": 305.1930847167969,
      "learning_rate": 2.0961650203370133e-06,
      "logits/chosen": -1.2261898517608643,
      "logits/rejected": -1.1734330654144287,
      "logps/chosen": -84.04228210449219,
      "logps/rejected": -78.4344482421875,
      "loss": 12.1335,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0776563212275505,
      "rewards/margins": 0.19069400429725647,
      "rewards/rejected": -0.2683503329753876,
      "step": 1999
    },
    {
      "epoch": 1.161642562583493,
      "grad_norm": 268.01104736328125,
      "learning_rate": 2.094712376525276e-06,
      "logits/chosen": -1.2516624927520752,
      "logits/rejected": -1.2327663898468018,
      "logps/chosen": -70.3960189819336,
      "logps/rejected": -86.23480224609375,
      "loss": 11.1557,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08508126437664032,
      "rewards/margins": 0.33029448986053467,
      "rewards/rejected": -0.41537579894065857,
      "step": 2000
    }
  ],
  "logging_steps": 1,
  "max_steps": 3442,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}