{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 625,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.936507936507937e-08,
      "logits/chosen": 0.3703172504901886,
      "logits/rejected": 0.17183463275432587,
      "logps/chosen": -1537.315673828125,
      "logps/rejected": -1665.161865234375,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507937e-07,
      "logits/chosen": 0.2896106243133545,
      "logits/rejected": 0.18362945318222046,
      "logps/chosen": -1102.260986328125,
      "logps/rejected": -2063.15380859375,
      "loss": 0.5003,
      "rewards/accuracies": 0.3263888955116272,
      "rewards/chosen": -0.0019549066200852394,
      "rewards/margins": -0.0033436818048357964,
      "rewards/rejected": 0.0013887754175812006,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015873e-06,
      "logits/chosen": 0.31699010729789734,
      "logits/rejected": 0.2042236030101776,
      "logps/chosen": -1399.4488525390625,
      "logps/rejected": -2060.23193359375,
      "loss": 0.4997,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.016661832109093666,
      "rewards/margins": 0.002577673178166151,
      "rewards/rejected": -0.019239503890275955,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": 0.27783340215682983,
      "logits/rejected": 0.18502432107925415,
      "logps/chosen": -1305.1578369140625,
      "logps/rejected": -2624.82470703125,
      "loss": 0.4965,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.060660868883132935,
      "rewards/margins": 0.04234201833605766,
      "rewards/rejected": -0.1030028909444809,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031746e-06,
      "logits/chosen": 0.3361937701702118,
      "logits/rejected": 0.21628756821155548,
      "logps/chosen": -1122.9556884765625,
      "logps/rejected": -1928.48046875,
      "loss": 0.4885,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13624535501003265,
      "rewards/margins": 0.11144338548183441,
      "rewards/rejected": -0.24768872559070587,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-06,
      "logits/chosen": 0.19992919266223907,
      "logits/rejected": 0.05517758056521416,
      "logps/chosen": -1331.64990234375,
      "logps/rejected": -2720.01611328125,
      "loss": 0.4868,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.41040802001953125,
      "rewards/margins": 0.1472548544406891,
      "rewards/rejected": -0.557662844657898,
      "step": 50
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": 0.13223984837532043,
      "logits/rejected": -0.04643384367227554,
      "logps/chosen": -1749.2845458984375,
      "logps/rejected": -3090.0,
      "loss": 0.4836,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.7042622566223145,
      "rewards/margins": 0.42035597562789917,
      "rewards/rejected": -1.1246182918548584,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998086282661188e-06,
      "logits/chosen": 0.12587395310401917,
      "logits/rejected": 0.018394339829683304,
      "logps/chosen": -2005.3883056640625,
      "logps/rejected": -3198.51171875,
      "loss": 0.4777,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8340966105461121,
      "rewards/margins": 0.4107601046562195,
      "rewards/rejected": -1.2448567152023315,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988720025682995e-06,
      "logits/chosen": 0.10518720000982285,
      "logits/rejected": -0.007509663701057434,
      "logps/chosen": -2260.054443359375,
      "logps/rejected": -3305.970703125,
      "loss": 0.4906,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.0553181171417236,
      "rewards/margins": 0.1571974903345108,
      "rewards/rejected": -1.2125155925750732,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9715789537359126e-06,
      "logits/chosen": 0.10318700969219208,
      "logits/rejected": -0.06382742524147034,
      "logps/chosen": -1909.2587890625,
      "logps/rejected": -3141.23974609375,
      "loss": 0.4808,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.9382199048995972,
      "rewards/margins": 0.18296480178833008,
      "rewards/rejected": -1.1211848258972168,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946716615897932e-06,
      "logits/chosen": 0.09833164513111115,
      "logits/rejected": -0.012232898734509945,
      "logps/chosen": -1993.842529296875,
      "logps/rejected": -2496.283935546875,
      "loss": 0.4818,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.8898054957389832,
      "rewards/margins": 0.23708383738994598,
      "rewards/rejected": -1.1268893480300903,
      "step": 100
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9142106826480114e-06,
      "logits/chosen": 0.08790755271911621,
      "logits/rejected": -0.04514267295598984,
      "logps/chosen": -2460.919677734375,
      "logps/rejected": -3302.043701171875,
      "loss": 0.4858,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.124772310256958,
      "rewards/margins": 0.3031695783138275,
      "rewards/rejected": -1.4279417991638184,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.874162703221823e-06,
      "logits/chosen": 0.14668218791484833,
      "logits/rejected": 0.010530241765081882,
      "logps/chosen": -2258.22119140625,
      "logps/rejected": -3071.97802734375,
      "loss": 0.4816,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.0068477392196655,
      "rewards/margins": 0.29694581031799316,
      "rewards/rejected": -1.3037936687469482,
      "step": 120
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826697788369752e-06,
      "logits/chosen": 0.15852120518684387,
      "logits/rejected": -0.011427855119109154,
      "logps/chosen": -1801.1246337890625,
      "logps/rejected": -3141.499267578125,
      "loss": 0.4792,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.7917305827140808,
      "rewards/margins": 0.3859596848487854,
      "rewards/rejected": -1.1776901483535767,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7719642195082224e-06,
      "logits/chosen": 0.07224290072917938,
      "logits/rejected": -0.05159113556146622,
      "logps/chosen": -2250.46484375,
      "logps/rejected": -2875.58056640625,
      "loss": 0.4904,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.009852647781372,
      "rewards/margins": 0.02716471627354622,
      "rewards/rejected": -1.0370174646377563,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.710132985485355e-06,
      "logits/chosen": 0.1444946527481079,
      "logits/rejected": -0.08803781121969223,
      "logps/chosen": -2070.7314453125,
      "logps/rejected": -3544.486328125,
      "loss": 0.4657,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.019829511642456,
      "rewards/margins": 0.6308184862136841,
      "rewards/rejected": -1.6506481170654297,
      "step": 150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.641397248408122e-06,
      "logits/chosen": -0.055302537977695465,
      "logits/rejected": -0.22802452743053436,
      "logps/chosen": -2362.42822265625,
      "logps/rejected": -3568.122314453125,
      "loss": 0.4857,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.2188419103622437,
      "rewards/margins": 0.24281807243824005,
      "rewards/rejected": -1.4616600275039673,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5659717401997655e-06,
      "logits/chosen": -0.04191827401518822,
      "logits/rejected": -0.17326447367668152,
      "logps/chosen": -2562.07080078125,
      "logps/rejected": -3827.65673828125,
      "loss": 0.4818,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.30030357837677,
      "rewards/margins": 0.4086272120475769,
      "rewards/rejected": -1.7089307308197021,
      "step": 170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4840920917726425e-06,
      "logits/chosen": 0.05106606334447861,
      "logits/rejected": -0.09808970242738724,
      "logps/chosen": -2531.837890625,
      "logps/rejected": -3445.98681640625,
      "loss": 0.4688,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.0362106561660767,
      "rewards/margins": 0.5165188908576965,
      "rewards/rejected": -1.5527294874191284,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396014096912182e-06,
      "logits/chosen": -0.020658662542700768,
      "logits/rejected": -0.20878557860851288,
      "logps/chosen": -2704.795166015625,
      "logps/rejected": -3539.75341796875,
      "loss": 0.4847,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.280983805656433,
      "rewards/margins": 0.08954890817403793,
      "rewards/rejected": -1.370532751083374,
      "step": 190
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302012913171584e-06,
      "logits/chosen": 0.13929739594459534,
      "logits/rejected": 0.010215060785412788,
      "logps/chosen": -2425.408935546875,
      "logps/rejected": -3690.294189453125,
      "loss": 0.4679,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9245489239692688,
      "rewards/margins": 0.5686434507369995,
      "rewards/rejected": -1.493192434310913,
      "step": 200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.202382202273702e-06,
      "logits/chosen": 0.02957342565059662,
      "logits/rejected": -0.1483859121799469,
      "logps/chosen": -2492.020263671875,
      "logps/rejected": -4000.793701171875,
      "loss": 0.4764,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.1149667501449585,
      "rewards/margins": 0.5438352823257446,
      "rewards/rejected": -1.6588022708892822,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097433212705492e-06,
      "logits/chosen": -0.10198907554149628,
      "logits/rejected": -0.25369173288345337,
      "logps/chosen": -2015.391845703125,
      "logps/rejected": -3833.490966796875,
      "loss": 0.4626,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.9975125193595886,
      "rewards/margins": 0.9066341519355774,
      "rewards/rejected": -1.9041467905044556,
      "step": 220
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.987493807371033e-06,
      "logits/chosen": -0.04106748104095459,
      "logits/rejected": -0.21337568759918213,
      "logps/chosen": -2646.331787109375,
      "logps/rejected": -3494.35302734375,
      "loss": 0.4828,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4516589641571045,
      "rewards/margins": 0.18022239208221436,
      "rewards/rejected": -1.6318813562393188,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.872907439340758e-06,
      "logits/chosen": -0.13159896433353424,
      "logits/rejected": -0.2913353145122528,
      "logps/chosen": -3422.151611328125,
      "logps/rejected": -4263.8818359375,
      "loss": 0.4781,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.7441692352294922,
      "rewards/margins": 0.3363359272480011,
      "rewards/rejected": -2.080505132675171,
      "step": 240
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.75403207889666e-06,
      "logits/chosen": -0.04592425376176834,
      "logits/rejected": -0.269358366727829,
      "logps/chosen": -1998.029541015625,
      "logps/rejected": -3464.591796875,
      "loss": 0.4739,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8778263926506042,
      "rewards/margins": 0.5411949157714844,
      "rewards/rejected": -1.4190213680267334,
      "step": 250
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631239095225417e-06,
      "logits/chosen": 0.02863461896777153,
      "logits/rejected": -0.086650051176548,
      "logps/chosen": -1820.9495849609375,
      "logps/rejected": -3405.172607421875,
      "loss": 0.474,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.7010027170181274,
      "rewards/margins": 0.5200620889663696,
      "rewards/rejected": -1.2210649251937866,
      "step": 260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5049120962530608e-06,
      "logits/chosen": 0.05603921413421631,
      "logits/rejected": -0.19317272305488586,
      "logps/chosen": -1709.980224609375,
      "logps/rejected": -3665.258544921875,
      "loss": 0.4608,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.6061512231826782,
      "rewards/margins": 0.5793715715408325,
      "rewards/rejected": -1.1855227947235107,
      "step": 270
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3754457302455464e-06,
      "logits/chosen": 0.01207334827631712,
      "logits/rejected": -0.21013681590557098,
      "logps/chosen": -1814.6568603515625,
      "logps/rejected": -3202.33154296875,
      "loss": 0.4768,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.7476822137832642,
      "rewards/margins": 0.5972086191177368,
      "rewards/rejected": -1.344890832901001,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2432444529190714e-06,
      "logits/chosen": -0.016334302723407745,
      "logits/rejected": -0.20580680668354034,
      "logps/chosen": -2087.526123046875,
      "logps/rejected": -2992.07861328125,
      "loss": 0.4821,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.8422229886054993,
      "rewards/margins": 0.2464129477739334,
      "rewards/rejected": -1.0886359214782715,
      "step": 290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1087212639117057e-06,
      "logits/chosen": -0.05513416603207588,
      "logits/rejected": -0.2065565139055252,
      "logps/chosen": -1855.8892822265625,
      "logps/rejected": -3107.00634765625,
      "loss": 0.4737,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7160204648971558,
      "rewards/margins": 0.4110482335090637,
      "rewards/rejected": -1.1270686388015747,
      "step": 300
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9722964165636263e-06,
      "logits/chosen": -0.10767300426959991,
      "logits/rejected": -0.1805376410484314,
      "logps/chosen": -2242.067138671875,
      "logps/rejected": -3101.77978515625,
      "loss": 0.4736,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.8544082641601562,
      "rewards/margins": 0.3649452030658722,
      "rewards/rejected": -1.219353437423706,
      "step": 310
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8343961050366275e-06,
      "logits/chosen": -0.0001668736367719248,
      "logits/rejected": -0.181897833943367,
      "logps/chosen": -1954.7802734375,
      "logps/rejected": -3072.43994140625,
      "loss": 0.4668,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8285297155380249,
      "rewards/margins": 0.40796583890914917,
      "rewards/rejected": -1.2364954948425293,
      "step": 320
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.695451132874385e-06,
      "logits/chosen": -0.18868280947208405,
      "logits/rejected": -0.33176952600479126,
      "logps/chosen": -2518.55810546875,
      "logps/rejected": -3725.364501953125,
      "loss": 0.4799,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.2756831645965576,
      "rewards/margins": 0.33692947030067444,
      "rewards/rejected": -1.6126124858856201,
      "step": 330
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5558955671628964e-06,
      "logits/chosen": -0.18861006200313568,
      "logits/rejected": -0.3129253387451172,
      "logps/chosen": -2410.373046875,
      "logps/rejected": -3373.586669921875,
      "loss": 0.4786,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.1353819370269775,
      "rewards/margins": 0.3919072449207306,
      "rewards/rejected": -1.5272890329360962,
      "step": 340
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4161653824955654e-06,
      "logits/chosen": -0.11331228911876678,
      "logits/rejected": -0.3467509150505066,
      "logps/chosen": -2082.77734375,
      "logps/rejected": -3978.74560546875,
      "loss": 0.4579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.0137884616851807,
      "rewards/margins": 0.7807275056838989,
      "rewards/rejected": -1.79451584815979,
      "step": 350
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2766970989791697e-06,
      "logits/chosen": -0.06516402959823608,
      "logits/rejected": -0.3027499318122864,
      "logps/chosen": -1849.7210693359375,
      "logps/rejected": -3917.53369140625,
      "loss": 0.4655,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8109678030014038,
      "rewards/margins": 0.7725037336349487,
      "rewards/rejected": -1.583471655845642,
      "step": 360
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1379264185356545e-06,
      "logits/chosen": -0.0026438175700604916,
      "logits/rejected": -0.0653844028711319,
      "logps/chosen": -2357.537841796875,
      "logps/rejected": -3069.9169921875,
      "loss": 0.4782,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.9414833188056946,
      "rewards/margins": 0.37831395864486694,
      "rewards/rejected": -1.3197972774505615,
      "step": 370
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.000286863759934e-06,
      "logits/chosen": -0.06546775251626968,
      "logits/rejected": -0.2656891942024231,
      "logps/chosen": -2187.612060546875,
      "logps/rejected": -4072.555908203125,
      "loss": 0.4689,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.9218450784683228,
      "rewards/margins": 0.6684916019439697,
      "rewards/rejected": -1.5903366804122925,
      "step": 380
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8642084235859764e-06,
      "logits/chosen": 0.09177455306053162,
      "logits/rejected": -0.10396850109100342,
      "logps/chosen": -1575.6951904296875,
      "logps/rejected": -3058.1181640625,
      "loss": 0.4711,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.597963809967041,
      "rewards/margins": 0.5352895855903625,
      "rewards/rejected": -1.1332534551620483,
      "step": 390
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7301162099921013e-06,
      "logits/chosen": 0.021384917199611664,
      "logits/rejected": -0.25088971853256226,
      "logps/chosen": -2101.90771484375,
      "logps/rejected": -4023.61328125,
      "loss": 0.4703,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9023422002792358,
      "rewards/margins": 0.7357300519943237,
      "rewards/rejected": -1.6380722522735596,
      "step": 400
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5984291299420117e-06,
      "logits/chosen": -0.030091216787695885,
      "logits/rejected": -0.3482876420021057,
      "logps/chosen": -2085.20166015625,
      "logps/rejected": -3580.270751953125,
      "loss": 0.4731,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.902575671672821,
      "rewards/margins": 0.486042320728302,
      "rewards/rejected": -1.388617992401123,
      "step": 410
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4695585767104092e-06,
      "logits/chosen": -0.009017439559102058,
      "logits/rejected": -0.24702921509742737,
      "logps/chosen": -2445.38037109375,
      "logps/rejected": -3752.704345703125,
      "loss": 0.4784,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.0142749547958374,
      "rewards/margins": 0.40337714552879333,
      "rewards/rejected": -1.4176521301269531,
      "step": 420
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3439071446815452e-06,
      "logits/chosen": 0.01383951585739851,
      "logits/rejected": -0.25569045543670654,
      "logps/chosen": -1831.669189453125,
      "logps/rejected": -3227.48046875,
      "loss": 0.4616,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.6138505339622498,
      "rewards/margins": 0.5082579851150513,
      "rewards/rejected": -1.1221085786819458,
      "step": 430
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2218673716356919e-06,
      "logits/chosen": -0.13071729242801666,
      "logits/rejected": -0.2272064983844757,
      "logps/chosen": -2428.37451171875,
      "logps/rejected": -2997.77197265625,
      "loss": 0.4825,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.9212499856948853,
      "rewards/margins": 0.19097386300563812,
      "rewards/rejected": -1.1122238636016846,
      "step": 440
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.103820512452661e-06,
      "logits/chosen": -0.07564810663461685,
      "logits/rejected": -0.3487449586391449,
      "logps/chosen": -2048.64697265625,
      "logps/rejected": -3798.810546875,
      "loss": 0.4649,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8849633932113647,
      "rewards/margins": 0.6834510564804077,
      "rewards/rejected": -1.5684144496917725,
      "step": 450
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.901353480633468e-07,
      "logits/chosen": -0.10444183647632599,
      "logits/rejected": -0.23747725784778595,
      "logps/chosen": -2240.57470703125,
      "logps/rejected": -3905.287109375,
      "loss": 0.4685,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.976617157459259,
      "rewards/margins": 0.6172038912773132,
      "rewards/rejected": -1.5938211679458618,
      "step": 460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.811670333701544e-07,
      "logits/chosen": -0.13727447390556335,
      "logits/rejected": -0.23075255751609802,
      "logps/chosen": -2049.42041015625,
      "logps/rejected": -3127.796630859375,
      "loss": 0.4709,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.912371039390564,
      "rewards/margins": 0.5058678984642029,
      "rewards/rejected": -1.418238878250122,
      "step": 470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.772559877354341e-07,
      "logits/chosen": -0.17451250553131104,
      "logits/rejected": -0.405754029750824,
      "logps/chosen": -2040.5706787109375,
      "logps/rejected": -4039.03125,
      "loss": 0.4696,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.9431853294372559,
      "rewards/margins": 0.766620397567749,
      "rewards/rejected": -1.7098054885864258,
      "step": 480
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787268315040604e-07,
      "logits/chosen": -0.10103428363800049,
      "logits/rejected": -0.24396558105945587,
      "logps/chosen": -2031.975341796875,
      "logps/rejected": -3454.368408203125,
      "loss": 0.4703,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9141359329223633,
      "rewards/margins": 0.6935009956359863,
      "rewards/rejected": -1.6076370477676392,
      "step": 490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.858873718824829e-07,
      "logits/chosen": -0.23661558330059052,
      "logits/rejected": -0.30403372645378113,
      "logps/chosen": -2469.70458984375,
      "logps/rejected": -3364.81005859375,
      "loss": 0.4772,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.2715297937393188,
      "rewards/margins": 0.312258243560791,
      "rewards/rejected": -1.5837881565093994,
      "step": 500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.990276413423817e-07,
      "logits/chosen": -0.09825234115123749,
      "logits/rejected": -0.26388177275657654,
      "logps/chosen": -1970.1832275390625,
      "logps/rejected": -3387.107421875,
      "loss": 0.4706,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.8808355331420898,
      "rewards/margins": 0.6419728994369507,
      "rewards/rejected": -1.5228084325790405,
      "step": 510
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.184189915529796e-07,
      "logits/chosen": -0.019504955038428307,
      "logits/rejected": -0.22602617740631104,
      "logps/chosen": -1955.7008056640625,
      "logps/rejected": -3776.561279296875,
      "loss": 0.4598,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8692920804023743,
      "rewards/margins": 0.9374381303787231,
      "rewards/rejected": -1.8067299127578735,
      "step": 520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4431324567258176e-07,
      "logits/chosen": -0.1649870127439499,
      "logits/rejected": -0.33468708395957947,
      "logps/chosen": -2207.761474609375,
      "logps/rejected": -4173.55810546875,
      "loss": 0.4613,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.9728679656982422,
      "rewards/margins": 0.8519523739814758,
      "rewards/rejected": -1.8248205184936523,
      "step": 530
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.769419116476052e-07,
      "logits/chosen": -0.2527296841144562,
      "logits/rejected": -0.37009841203689575,
      "logps/chosen": -2005.4554443359375,
      "logps/rejected": -3320.11083984375,
      "loss": 0.4659,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.983248233795166,
      "rewards/margins": 0.48160916566848755,
      "rewards/rejected": -1.4648573398590088,
      "step": 540
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1651545897676512e-07,
      "logits/chosen": -0.1346069872379303,
      "logits/rejected": -0.30417710542678833,
      "logps/chosen": -2149.47998046875,
      "logps/rejected": -3438.53271484375,
      "loss": 0.4659,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.995121955871582,
      "rewards/margins": 0.6242730021476746,
      "rewards/rejected": -1.6193952560424805,
      "step": 550
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6322266119983222e-07,
      "logits/chosen": -0.09836331009864807,
      "logits/rejected": -0.3026123046875,
      "logps/chosen": -2651.5302734375,
      "logps/rejected": -3409.03955078125,
      "loss": 0.4739,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.2515802383422852,
      "rewards/margins": 0.16534823179244995,
      "rewards/rejected": -1.4169285297393799,
      "step": 560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1723000616502167e-07,
      "logits/chosen": -0.08475066721439362,
      "logits/rejected": -0.33103296160697937,
      "logps/chosen": -2303.97900390625,
      "logps/rejected": -3867.08251953125,
      "loss": 0.4667,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.1717913150787354,
      "rewards/margins": 0.6337391138076782,
      "rewards/rejected": -1.805530309677124,
      "step": 570
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.868117591737585e-08,
      "logits/chosen": -0.1410863846540451,
      "logits/rejected": -0.3231101930141449,
      "logps/chosen": -2047.534423828125,
      "logps/rejected": -3277.69921875,
      "loss": 0.475,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9290479421615601,
      "rewards/margins": 0.6801496744155884,
      "rewards/rejected": -1.6091976165771484,
      "step": 580
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.769659783295383e-08,
      "logits/chosen": -0.1917143017053604,
      "logits/rejected": -0.3931889832019806,
      "logps/chosen": -2393.9306640625,
      "logps/rejected": -3955.93359375,
      "loss": 0.4784,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.1227269172668457,
      "rewards/margins": 0.465735524892807,
      "rewards/rejected": -1.5884625911712646,
      "step": 590
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4373068401120358e-08,
      "logits/chosen": -0.10545693337917328,
      "logits/rejected": -0.24310024082660675,
      "logps/chosen": -2470.99560546875,
      "logps/rejected": -3619.756591796875,
      "loss": 0.4742,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.1268969774246216,
      "rewards/margins": 0.5375250577926636,
      "rewards/rejected": -1.664421796798706,
      "step": 600
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.78345083022425e-09,
      "logits/chosen": -0.136544868350029,
      "logits/rejected": -0.3129388391971588,
      "logps/chosen": -2081.79248046875,
      "logps/rejected": -3515.130126953125,
      "loss": 0.4743,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.9320171475410461,
      "rewards/margins": 0.5496521592140198,
      "rewards/rejected": -1.4816694259643555,
      "step": 610
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.764474213677654e-10,
      "logits/chosen": -0.1356605440378189,
      "logits/rejected": -0.29532909393310547,
      "logps/chosen": -2376.76953125,
      "logps/rejected": -3284.452392578125,
      "loss": 0.4779,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.0924612283706665,
      "rewards/margins": 0.41884762048721313,
      "rewards/rejected": -1.5113089084625244,
      "step": 620
    },
    {
      "epoch": 1.0,
      "step": 625,
      "total_flos": 0.0,
      "train_loss": 0.4757139442443848,
      "train_runtime": 7766.6535,
      "train_samples_per_second": 2.575,
      "train_steps_per_second": 0.08
    }
  ],
  "logging_steps": 10,
  "max_steps": 625,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 20,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}