End of training

README.md

@@ -18,18 +18,18 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.2150
-- Rewards/chosen: -0.0603
-- Rewards/rejected: -0.2557
+- Loss: 0.1307
+- Rewards/chosen: -0.0532
+- Rewards/rejected: -0.4544
 - Rewards/accuracies: 1.0
-- Rewards/margins: 0.1953
-- Logps/rejected: -2.5565
-- Logps/chosen: -0.6032
-- Logits/rejected: -0.6237
-- Logits/chosen: -0.5761
-- Nll Loss: 0.1599
-- Log Odds Ratio: -0.0986
-- Log Odds Chosen: 2.6928
+- Rewards/margins: 0.4011
+- Logps/rejected: -4.5438
+- Logps/chosen: -0.5325
+- Logits/rejected: -0.6053
+- Logits/chosen: -0.5946
+- Nll Loss: 0.1229
+- Log Odds Ratio: -0.0243
+- Log Odds Chosen: 4.9661
 
 ## Model description
 
@@ -52,11 +52,8 @@ The following hyperparameters were used during training:
 - train_batch_size: 2
 - eval_batch_size: 2
 - seed: 42
-- distributed_type: multi-GPU
-- num_devices: 2
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 16
-- total_eval_batch_size: 4
+- total_train_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - num_epochs: 10
@@ -65,26 +62,48 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
-| 3.2032        | 0.4762 | 5    | 2.3866          | -0.2104        | -0.2405          | 1.0                | 0.0301          | -2.4052        | -2.1040      | 0.3000          | 0.3159        | 2.2400   | -0.5392        | 0.3377          |
-| 2.0554        | 0.9524 | 10   | 1.7548          | -0.1555        | -0.1881          | 1.0                | 0.0326          | -1.8807        | -1.5549      | -0.2962         | -0.2709       | 1.6079   | -0.5144        | 0.3986          |
-| 1.4859        | 1.4286 | 15   | 1.2684          | -0.1328        | -0.1714          | 1.0                | 0.0387          | -1.7144        | -1.3277      | -0.6098         | -0.5813       | 1.1271   | -0.4764        | 0.4978          |
-| 1.0601        | 1.9048 | 20   | 0.9296          | -0.1168        | -0.1595          | 1.0                | 0.0427          | -1.5953        | -1.1684      | -0.5901         | -0.5618       | 0.7913   | -0.4483        | 0.5763          |
-| 0.7394        | 2.3810 | 25   | 0.6509          | -0.1056        | -0.1530          | 1.0                | 0.0473          | -1.5297        | -1.0565      | -0.5690         | -0.5380       | 0.5131   | -0.4198        | 0.6621          |
-| 0.4955        | 2.8571 | 30   | 0.4543          | -0.0950        | -0.1470          | 1.0                | 0.0520          | -1.4696        | -0.9498      | -0.4061         | -0.3748       | 0.3250   | -0.3909        | 0.7564          |
-| 0.303         | 3.3333 | 35   | 0.3624          | -0.0864        | -0.1449          | 1.0                | 0.0585          | -1.4491        | -0.8643      | -0.4525         | -0.4231       | 0.2426   | -0.3584        | 0.8748          |
-| 0.2989        | 3.8095 | 40   | 0.3240          | -0.0799        | -0.1469          | 1.0                | 0.0670          | -1.4687        | -0.7987      | -0.6479         | -0.6239       | 0.2182   | -0.3223        | 1.0204          |
-| 0.218         | 4.2857 | 45   | 0.2962          | -0.0746        | -0.1565          | 1.0                | 0.0820          | -1.5652        | -0.7457      | -0.4218         | -0.3928       | 0.1992   | -0.2722        | 1.2439          |
-| 0.2639        | 4.7619 | 50   | 0.2732          | -0.0703        | -0.1649          | 1.0                | 0.0945          | -1.6489        | -0.7035      | -0.5638         | -0.5293       | 0.1872   | -0.2354        | 1.4329          |
-| 0.1963        | 5.2381 | 55   | 0.2569          | -0.0683        | -0.1860          | 1.0                | 0.1177          | -1.8598        | -0.6826      | -0.5460         | -0.5026       | 0.1803   | -0.1898        | 1.7281          |
-| 0.1853        | 5.7143 | 60   | 0.2465          | -0.0659        | -0.1935          | 1.0                | 0.1276          | -1.9351        | -0.6594      | -0.6998         | -0.6580       | 0.1747   | -0.1714        | 1.8682          |
-| 0.1904        | 6.1905 | 65   | 0.2378          | -0.0643        | -0.2038          | 1.0                | 0.1395          | -2.0377        | -0.6429      | -0.5732         | -0.5271       | 0.1702   | -0.1529        | 2.0218          |
-| 0.1813        | 6.6667 | 70   | 0.2310          | -0.0632        | -0.2197          | 1.0                | 0.1565          | -2.1970        | -0.6319      | -0.5843         | -0.5359       | 0.1671   | -0.1332        | 2.2275          |
-| 0.1771        | 7.1429 | 75   | 0.2245          | -0.0620        | -0.2324          | 1.0                | 0.1703          | -2.3239        | -0.6204      | -0.6483         | -0.6007       | 0.1644   | -0.1192        | 2.3961          |
-| 0.167         | 7.6190 | 80   | 0.2198          | -0.0611        | -0.2431          | 1.0                | 0.1820          | -2.4313        | -0.6114      | -0.7110         | -0.6634       | 0.1621   | -0.1089        | 2.5366          |
-| 0.1668        | 8.0952 | 85   | 0.2173          | -0.0609        | -0.2520          | 1.0                | 0.1911          | -2.5201        | -0.6090      | -0.6287         | -0.5819       | 0.1614   | -0.1021        | 2.6395          |
-| 0.1545        | 8.5714 | 90   | 0.2160          | -0.0604        | -0.2536          | 1.0                | 0.1932          | -2.5363        | -0.6043      | -0.6649         | -0.6180       | 0.1604   | -0.1001        | 2.6690          |
-| 0.1723        | 9.0476 | 95   | 0.2151          | -0.0604        | -0.2545          | 1.0                | 0.1941          | -2.5451        | -0.6037      | -0.6519         | -0.6047       | 0.1600   | -0.0993        | 2.6794          |
-| 0.1583        | 9.5238 | 100  | 0.2150          | -0.0603        | -0.2557          | 1.0                | 0.1953          | -2.5565        | -0.6032      | -0.6237         | -0.5761       | 0.1599   | -0.0986        | 2.6928          |
+| 3.1838        | 0.2381 | 5    | 2.3687          | -0.2141        | -0.2396          | 0.9000             | 0.0255          | -2.3962        | -2.1413      | 0.1963          | 0.2099        | 2.2996   | -0.5609        | 0.2917          |
+| 2.0549        | 0.4762 | 10   | 1.7681          | -0.1667        | -0.1957          | 0.9000             | 0.0290          | -1.9570        | -1.6671      | -0.3938         | -0.3708       | 1.7001   | -0.5337        | 0.3558          |
+| 1.52          | 0.7143 | 15   | 1.2706          | -0.1413        | -0.1758          | 1.0                | 0.0345          | -1.7581        | -1.4131      | -0.7370         | -0.7115       | 1.2041   | -0.4984        | 0.4454          |
+| 1.0136        | 0.9524 | 20   | 0.8746          | -0.1219        | -0.1611          | 1.0                | 0.0391          | -1.6107        | -1.2194      | -0.7398         | -0.7111       | 0.8093   | -0.4665        | 0.5323          |
+| 0.6723        | 1.1905 | 25   | 0.5716          | -0.1086        | -0.1524          | 1.0                | 0.0438          | -1.5241        | -1.0859      | -0.6821         | -0.6474       | 0.5078   | -0.4362        | 0.6192          |
+| 0.4086        | 1.4286 | 30   | 0.3851          | -0.0943        | -0.1439          | 1.0                | 0.0496          | -1.4391        | -0.9432      | -0.6528         | -0.6126       | 0.3261   | -0.4003        | 0.7344          |
+| 0.2733        | 1.6667 | 35   | 0.3272          | -0.0852        | -0.1441          | 1.0                | 0.0588          | -1.4407        | -0.8524      | -0.6893         | -0.6529       | 0.2740   | -0.3564        | 0.8915          |
+| 0.2696        | 1.9048 | 40   | 0.2898          | -0.0800        | -0.1517          | 1.0                | 0.0717          | -1.5166        | -0.7996      | -0.7221         | -0.6878       | 0.2438   | -0.3088        | 1.0816          |
+| 0.2171        | 2.1429 | 45   | 0.2581          | -0.0753        | -0.1708          | 1.0                | 0.0955          | -1.7082        | -0.7534      | -0.7685         | -0.7280       | 0.2206   | -0.2432        | 1.4022          |
+| 0.2065        | 2.3810 | 50   | 0.2291          | -0.0707        | -0.2048          | 1.0                | 0.1340          | -2.0477        | -0.7073      | -0.8235         | -0.7790       | 0.2000   | -0.1733        | 1.8822          |
+| 0.1714        | 2.6190 | 55   | 0.2067          | -0.0678        | -0.2475          | 1.0                | 0.1797          | -2.4753        | -0.6785      | -0.6796         | -0.6324       | 0.1842   | -0.1243        | 2.4141          |
+| 0.1603        | 2.8571 | 60   | 0.1899          | -0.0641        | -0.2729          | 1.0                | 0.2088          | -2.7291        | -0.6411      | -0.8208         | -0.7726       | 0.1707   | -0.1012        | 2.7621          |
+| 0.1217        | 3.0952 | 65   | 0.1802          | -0.0622        | -0.3110          | 1.0                | 0.2489          | -3.1102        | -0.6216      | -0.7613         | -0.7154       | 0.1639   | -0.0782        | 3.2132          |
+| 0.1121        | 3.3333 | 70   | 0.1754          | -0.0619        | -0.3498          | 1.0                | 0.2879          | -3.4979        | -0.6193      | -0.8189         | -0.7850       | 0.1608   | -0.0624        | 3.6277          |
+| 0.1207        | 3.5714 | 75   | 0.1683          | -0.0602        | -0.3715          | 1.0                | 0.3113          | -3.7145        | -0.6020      | -0.8539         | -0.8185       | 0.1545   | -0.0534        | 3.9028          |
+| 0.1468        | 3.8095 | 80   | 0.1617          | -0.0581        | -0.3716          | 1.0                | 0.3135          | -3.7163        | -0.5811      | -0.8185         | -0.7738       | 0.1485   | -0.0498        | 3.9601          |
+| 0.1076        | 4.0476 | 85   | 0.1563          | -0.0581        | -0.4033          | 1.0                | 0.3452          | -4.0326        | -0.5810      | -0.3847         | -0.3661       | 0.1447   | -0.0410        | 4.2887          |
+| 0.0976        | 4.2857 | 90   | 0.1527          | -0.0568        | -0.3869          | 1.0                | 0.3301          | -3.8687        | -0.5678      | -0.8869         | -0.8577       | 0.1412   | -0.0428        | 4.1513          |
+| 0.1237        | 4.5238 | 95   | 0.1497          | -0.0569        | -0.4033          | 1.0                | 0.3464          | -4.0334        | -0.5690      | -0.7123         | -0.6884       | 0.1388   | -0.0385        | 4.3232          |
+| 0.1098        | 4.7619 | 100  | 0.1454          | -0.0561        | -0.4116          | 1.0                | 0.3555          | -4.1158        | -0.5611      | -0.7347         | -0.7134       | 0.1351   | -0.0360        | 4.4282          |
+| 0.0781        | 5.0    | 105  | 0.1449          | -0.0567        | -0.4296          | 1.0                | 0.3729          | -4.2958        | -0.5671      | -0.6652         | -0.6537       | 0.1355   | -0.0324        | 4.6010          |
+| 0.0986        | 5.2381 | 110  | 0.1419          | -0.0556        | -0.4165          | 1.0                | 0.3608          | -4.1646        | -0.5563      | -0.7521         | -0.7314       | 0.1325   | -0.0340        | 4.4953          |
+| 0.0853        | 5.4762 | 115  | 0.1402          | -0.0552        | -0.4216          | 1.0                | 0.3663          | -4.2159        | -0.5524      | -0.6544         | -0.6378       | 0.1309   | -0.0325        | 4.5586          |
+| 0.0867        | 5.7143 | 120  | 0.1386          | -0.0547        | -0.4309          | 1.0                | 0.3761          | -4.3088        | -0.5474      | -0.5502         | -0.5379       | 0.1294   | -0.0306        | 4.6712          |
+| 0.0829        | 5.9524 | 125  | 0.1355          | -0.0534        | -0.4306          | 1.0                | 0.3772          | -4.3056        | -0.5341      | -0.7163         | -0.6946       | 0.1265   | -0.0295        | 4.7039          |
+| 0.0921        | 6.1905 | 130  | 0.1366          | -0.0547        | -0.4466          | 1.0                | 0.3919          | -4.4659        | -0.5474      | -0.6342         | -0.6229       | 0.1282   | -0.0271        | 4.8385          |
+| 0.0874        | 6.4286 | 135  | 0.1358          | -0.0549        | -0.4507          | 1.0                | 0.3959          | -4.5075        | -0.5490      | -0.6402         | -0.6263       | 0.1277   | -0.0267        | 4.8754          |
+| 0.082         | 6.6667 | 140  | 0.1346          | -0.0541        | -0.4412          | 1.0                | 0.3872          | -4.4123        | -0.5407      | -0.6918         | -0.6770       | 0.1263   | -0.0277        | 4.7993          |
+| 0.0858        | 6.9048 | 145  | 0.1343          | -0.0542        | -0.4452          | 1.0                | 0.3910          | -4.4524        | -0.5425      | -0.5903         | -0.5802       | 0.1260   | -0.0270        | 4.8382          |
+| 0.0762        | 7.1429 | 150  | 0.1331          | -0.0539        | -0.4483          | 1.0                | 0.3943          | -4.4825        | -0.5392      | -0.6384         | -0.6278       | 0.1249   | -0.0261        | 4.8798          |
+| 0.0704        | 7.3810 | 155  | 0.1321          | -0.0536        | -0.4505          | 1.0                | 0.3970          | -4.5053        | -0.5357      | -0.6361         | -0.6260       | 0.1240   | -0.0255        | 4.9141          |
+| 0.078         | 7.6190 | 160  | 0.1317          | -0.0535        | -0.4539          | 1.0                | 0.4003          | -4.5386        | -0.5351      | -0.6485         | -0.6381       | 0.1238   | -0.0248        | 4.9509          |
+| 0.0771        | 7.8571 | 165  | 0.1315          | -0.0535        | -0.4579          | 1.0                | 0.4043          | -4.5786        | -0.5355      | -0.6204         | -0.6116       | 0.1238   | -0.0242        | 4.9924          |
+| 0.0851        | 8.0952 | 170  | 0.1313          | -0.0535        | -0.4566          | 1.0                | 0.4031          | -4.5664        | -0.5355      | -0.6164         | -0.6068       | 0.1235   | -0.0242        | 4.9829          |
+| 0.0654        | 8.3333 | 175  | 0.1305          | -0.0531        | -0.4542          | 1.0                | 0.4011          | -4.5422        | -0.5313      | -0.6323         | -0.6206       | 0.1227   | -0.0243        | 4.9685          |
+| 0.082         | 8.5714 | 180  | 0.1308          | -0.0532        | -0.4510          | 1.0                | 0.3978          | -4.5101        | -0.5323      | -0.6397         | -0.6294       | 0.1229   | -0.0247        | 4.9322          |
+| 0.0724        | 8.8095 | 185  | 0.1306          | -0.0531        | -0.4539          | 1.0                | 0.4008          | -4.5388        | -0.5307      | -0.6082         | -0.5998       | 0.1227   | -0.0243        | 4.9661          |
+| 0.0836        | 9.0476 | 190  | 0.1308          | -0.0533        | -0.4551          | 1.0                | 0.4018          | -4.5512        | -0.5333      | -0.5842         | -0.5754       | 0.1231   | -0.0241        | 4.9712          |
+| 0.0702        | 9.2857 | 195  | 0.1308          | -0.0533        | -0.4552          | 1.0                | 0.4019          | -4.5521        | -0.5335      | -0.5966         | -0.5894       | 0.1230   | -0.0244        | 4.9728          |
+| 0.0726        | 9.5238 | 200  | 0.1307          | -0.0533        | -0.4547          | 1.0                | 0.4013          | -4.5466        | -0.5335      | -0.6151         | -0.6059       | 0.1229   | -0.0243        | 4.9692          |
+| 0.0837        | 9.7619 | 205  | 0.1307          | -0.0534        | -0.4554          | 1.0                | 0.4020          | -4.5538        | -0.5338      | -0.6041         | -0.5956       | 0.1229   | -0.0241        | 4.9743          |
+| 0.0796        | 10.0   | 210  | 0.1307          | -0.0532        | -0.4544          | 1.0                | 0.4011          | -4.5438        | -0.5325      | -0.6053         | -0.5946       | 0.1229   | -0.0243        | 4.9661          |
 
 
 ### Framework versions

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0632f1eaa8db6ed972e10364695010f45c712d06400f4238b72a587728e1688a
+oid sha256:e962c51dd28f2d58454c68476f9382aa7fb374a9ac5a626e882767dae349766b
 size 4965811384

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc642da4656040d7179701b3d25cc28574a27f4acafca7f9571af2a329c534a8
+oid sha256:0188d2fa37c8060a3e33c0b4c6fb438699186a53fb8949d301a37b69e4a9a07d
 size 1459729952

trainer_state.json

@@ -0,0 +1,1596 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 10.0,
+  "eval_steps": 5,
+  "global_step": 210,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.23809523809523808,
+      "grad_norm": 28.375,
+      "learning_rate": 7.80952380952381e-06,
+      "log_odds_chosen": 0.1891055405139923,
+      "log_odds_ratio": -0.6111669540405273,
+      "logits/chosen": 0.29273518919944763,
+      "logits/rejected": 0.2814817428588867,
+      "logps/chosen": -3.5652737617492676,
+      "logps/rejected": -3.743648052215576,
+      "loss": 3.1838,
+      "nll_loss": 3.1226940155029297,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3565273880958557,
+      "rewards/margins": 0.01783742383122444,
+      "rewards/rejected": -0.37436479330062866,
+      "step": 5
+    },
+    {
+      "epoch": 0.23809523809523808,
+      "eval_log_odds_chosen": 0.29169073700904846,
+      "eval_log_odds_ratio": -0.5609198808670044,
+      "eval_logits/chosen": 0.20992258191108704,
+      "eval_logits/rejected": 0.19634230434894562,
+      "eval_logps/chosen": -2.1412553787231445,
+      "eval_logps/rejected": -2.3962299823760986,
+      "eval_loss": 2.3686962127685547,
+      "eval_nll_loss": 2.299578905105591,
+      "eval_rewards/accuracies": 0.8999999761581421,
+      "eval_rewards/chosen": -0.21412554383277893,
+      "eval_rewards/margins": 0.025497451424598694,
+      "eval_rewards/rejected": -0.23962298035621643,
+      "eval_runtime": 0.489,
+      "eval_samples_per_second": 18.406,
+      "eval_steps_per_second": 10.225,
+      "step": 5
+    },
+    {
+      "epoch": 0.47619047619047616,
+      "grad_norm": 16.75,
+      "learning_rate": 7.619047619047618e-06,
+      "log_odds_chosen": 0.3308032155036926,
+      "log_odds_ratio": -0.5464655160903931,
+      "logits/chosen": 0.07425501197576523,
+      "logits/rejected": 0.05106530338525772,
+      "logps/chosen": -1.7553539276123047,
+      "logps/rejected": -2.036280870437622,
+      "loss": 2.0549,
+      "nll_loss": 2.0002410411834717,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.1755353957414627,
+      "rewards/margins": 0.02809269167482853,
+      "rewards/rejected": -0.2036280632019043,
+      "step": 10
+    },
+    {
+      "epoch": 0.47619047619047616,
+      "eval_log_odds_chosen": 0.35579484701156616,
+      "eval_log_odds_ratio": -0.5337004661560059,
+      "eval_logits/chosen": -0.37078872323036194,
+      "eval_logits/rejected": -0.3937973380088806,
+      "eval_logps/chosen": -1.6670551300048828,
+      "eval_logps/rejected": -1.9570205211639404,
+      "eval_loss": 1.768071174621582,
+      "eval_nll_loss": 1.7001245021820068,
+      "eval_rewards/accuracies": 0.8999999761581421,
+      "eval_rewards/chosen": -0.16670551896095276,
+      "eval_rewards/margins": 0.028996536508202553,
+      "eval_rewards/rejected": -0.19570204615592957,
+      "eval_runtime": 0.4888,
+      "eval_samples_per_second": 18.413,
+      "eval_steps_per_second": 10.23,
+      "step": 10
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 14.0625,
+      "learning_rate": 7.428571428571428e-06,
+      "log_odds_chosen": 0.40405726432800293,
+      "log_odds_ratio": -0.5185824632644653,
+      "logits/chosen": -0.4588231146335602,
+      "logits/rejected": -0.4600151479244232,
+      "logps/chosen": -1.4706313610076904,
+      "logps/rejected": -1.7922747135162354,
+      "loss": 1.52,
+      "nll_loss": 1.4680984020233154,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.14706313610076904,
+      "rewards/margins": 0.032164327800273895,
+      "rewards/rejected": -0.17922747135162354,
+      "step": 15
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "eval_log_odds_chosen": 0.44535937905311584,
+      "eval_log_odds_ratio": -0.4983883798122406,
+      "eval_logits/chosen": -0.7114694714546204,
+      "eval_logits/rejected": -0.7369576692581177,
+      "eval_logps/chosen": -1.4130698442459106,
+      "eval_logps/rejected": -1.758122205734253,
+      "eval_loss": 1.2705729007720947,
+      "eval_nll_loss": 1.2041345834732056,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.14130698144435883,
+      "eval_rewards/margins": 0.03450523689389229,
+      "eval_rewards/rejected": -0.17581221461296082,
+      "eval_runtime": 0.4865,
+      "eval_samples_per_second": 18.499,
+      "eval_steps_per_second": 10.277,
+      "step": 15
+    },
+    {
+      "epoch": 0.9523809523809523,
+      "grad_norm": 29.75,
+      "learning_rate": 7.238095238095238e-06,
+      "log_odds_chosen": 0.48574334383010864,
+      "log_odds_ratio": -0.4926493763923645,
+      "logits/chosen": -0.6470807790756226,
+      "logits/rejected": -0.6642199754714966,
+      "logps/chosen": -1.27854323387146,
+      "logps/rejected": -1.6447874307632446,
+      "loss": 1.0136,
+      "nll_loss": 0.9643369913101196,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.12785431742668152,
+      "rewards/margins": 0.03662443906068802,
+      "rewards/rejected": -0.16447874903678894,
+      "step": 20
+    },
+    {
+      "epoch": 0.9523809523809523,
+      "eval_log_odds_chosen": 0.532298743724823,
+      "eval_log_odds_ratio": -0.46647781133651733,
+      "eval_logits/chosen": -0.7111243009567261,
+      "eval_logits/rejected": -0.7398008704185486,
+      "eval_logps/chosen": -1.2193677425384521,
+      "eval_logps/rejected": -1.6106961965560913,
+      "eval_loss": 0.8746375441551208,
+      "eval_nll_loss": 0.8092986941337585,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.12193679809570312,
+      "eval_rewards/margins": 0.039132826030254364,
+      "eval_rewards/rejected": -0.1610696017742157,
+      "eval_runtime": 0.488,
+      "eval_samples_per_second": 18.443,
+      "eval_steps_per_second": 10.246,
+      "step": 20
+    },
+    {
+      "epoch": 1.1904761904761905,
+      "grad_norm": 7.4375,
+      "learning_rate": 7.0476190476190475e-06,
+      "log_odds_chosen": 0.6200831532478333,
+      "log_odds_ratio": -0.44031673669815063,
+      "logits/chosen": -0.6323801279067993,
+      "logits/rejected": -0.6341850757598877,
+      "logps/chosen": -1.0979182720184326,
+      "logps/rejected": -1.5436619520187378,
+      "loss": 0.6723,
+      "nll_loss": 0.6282438039779663,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.1097918376326561,
+      "rewards/margins": 0.044574350118637085,
+      "rewards/rejected": -0.15436621010303497,
+      "step": 25
+    },
+    {
+      "epoch": 1.1904761904761905,
+      "eval_log_odds_chosen": 0.6192252039909363,
+      "eval_log_odds_ratio": -0.4361681044101715,
+      "eval_logits/chosen": -0.6473670601844788,
+      "eval_logits/rejected": -0.6820846199989319,
+      "eval_logps/chosen": -1.085878610610962,
+      "eval_logps/rejected": -1.5240614414215088,
+      "eval_loss": 0.5715838670730591,
+      "eval_nll_loss": 0.5078178644180298,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10858786106109619,
+      "eval_rewards/margins": 0.04381827265024185,
+      "eval_rewards/rejected": -0.15240614116191864,
+      "eval_runtime": 0.4872,
+      "eval_samples_per_second": 18.473,
+      "eval_steps_per_second": 10.263,
+      "step": 25
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 6.5625,
+      "learning_rate": 6.857142857142856e-06,
+      "log_odds_chosen": 0.736698567867279,
+      "log_odds_ratio": -0.40588584542274475,
+      "logits/chosen": -0.5826676487922668,
+      "logits/rejected": -0.6002309918403625,
+      "logps/chosen": -0.9341448545455933,
+      "logps/rejected": -1.4290642738342285,
+      "loss": 0.4086,
+      "nll_loss": 0.3679959177970886,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09341450035572052,
+      "rewards/margins": 0.04949193447828293,
+      "rewards/rejected": -0.14290642738342285,
+      "step": 30
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "eval_log_odds_chosen": 0.7344347834587097,
+      "eval_log_odds_ratio": -0.40031012892723083,
+      "eval_logits/chosen": -0.6125622987747192,
+      "eval_logits/rejected": -0.6528064608573914,
+      "eval_logps/chosen": -0.9431861042976379,
+      "eval_logps/rejected": -1.4390549659729004,
+      "eval_loss": 0.3850952088832855,
+      "eval_nll_loss": 0.32607388496398926,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.09431861340999603,
+      "eval_rewards/margins": 0.04958688095211983,
+      "eval_rewards/rejected": -0.14390549063682556,
+      "eval_runtime": 0.4877,
+      "eval_samples_per_second": 18.452,
+      "eval_steps_per_second": 10.251,
+      "step": 30
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "grad_norm": 7.4375,
+      "learning_rate": 6.666666666666667e-06,
+      "log_odds_chosen": 0.9226770401000977,
+      "log_odds_ratio": -0.36509159207344055,
+      "logits/chosen": -0.49513015151023865,
+      "logits/rejected": -0.5086463689804077,
+      "logps/chosen": -0.7646501064300537,
+      "logps/rejected": -1.3257030248641968,
+      "loss": 0.2733,
+      "nll_loss": 0.23684072494506836,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.07646501064300537,
+      "rewards/margins": 0.056105293333530426,
+      "rewards/rejected": -0.1325702965259552,
+      "step": 35
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "eval_log_odds_chosen": 0.8915351629257202,
+      "eval_log_odds_ratio": -0.3563780188560486,
+      "eval_logits/chosen": -0.6528802514076233,
+      "eval_logits/rejected": -0.6892634630203247,
+      "eval_logps/chosen": -0.8524182438850403,
+      "eval_logps/rejected": -1.4406554698944092,
+      "eval_loss": 0.3271670639514923,
+      "eval_nll_loss": 0.2739677429199219,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.08524181693792343,
+      "eval_rewards/margins": 0.05882372334599495,
+      "eval_rewards/rejected": -0.14406554400920868,
+      "eval_runtime": 0.4869,
+      "eval_samples_per_second": 18.484,
+      "eval_steps_per_second": 10.269,
+      "step": 35
+    },
+    {
+      "epoch": 1.9047619047619047,
+      "grad_norm": 4.0,
+      "learning_rate": 6.476190476190476e-06,
+      "log_odds_chosen": 1.0913145542144775,
+      "log_odds_ratio": -0.3268836736679077,
+      "logits/chosen": -0.7565770149230957,
+      "logits/rejected": -0.7887675762176514,
+      "logps/chosen": -0.6868108510971069,
+      "logps/rejected": -1.3315098285675049,
+      "loss": 0.2696,
+      "nll_loss": 0.23689217865467072,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.06868109107017517,
+      "rewards/margins": 0.06446989625692368,
+      "rewards/rejected": -0.13315097987651825,
+      "step": 40
+    },
+    {
+      "epoch": 1.9047619047619047,
+      "eval_log_odds_chosen": 1.0815879106521606,
+      "eval_log_odds_ratio": -0.30878978967666626,
+      "eval_logits/chosen": -0.6877866983413696,
+      "eval_logits/rejected": -0.722061812877655,
+      "eval_logps/chosen": -0.7995762825012207,
+      "eval_logps/rejected": -1.5165843963623047,
+      "eval_loss": 0.2898225784301758,
+      "eval_nll_loss": 0.24379682540893555,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.07995764166116714,
+      "eval_rewards/margins": 0.0717008113861084,
+      "eval_rewards/rejected": -0.15165844559669495,
+      "eval_runtime": 0.49,
+      "eval_samples_per_second": 18.368,
+      "eval_steps_per_second": 10.204,
+      "step": 40
+    },
+    {
+      "epoch": 2.142857142857143,
+      "grad_norm": 2.59375,
+      "learning_rate": 6.285714285714285e-06,
+      "log_odds_chosen": 1.3118517398834229,
+      "log_odds_ratio": -0.27184879779815674,
+      "logits/chosen": -0.5011266469955444,
+      "logits/rejected": -0.5411444902420044,
+      "logps/chosen": -0.6962511539459229,
+      "logps/rejected": -1.5221842527389526,
+      "loss": 0.2171,
+      "nll_loss": 0.18990826606750488,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.069625124335289,
+      "rewards/margins": 0.08259329944849014,
+      "rewards/rejected": -0.15221843123435974,
+      "step": 45
+    },
+    {
+      "epoch": 2.142857142857143,
+      "eval_log_odds_chosen": 1.4021844863891602,
+      "eval_log_odds_ratio": -0.24321937561035156,
+      "eval_logits/chosen": -0.7279993295669556,
+      "eval_logits/rejected": -0.7685092687606812,
+      "eval_logps/chosen": -0.7534304261207581,
+      "eval_logps/rejected": -1.7082147598266602,
+      "eval_loss": 0.25805217027664185,
+      "eval_nll_loss": 0.22057470679283142,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.07534303516149521,
+      "eval_rewards/margins": 0.09547843784093857,
+      "eval_rewards/rejected": -0.17082147300243378,
+      "eval_runtime": 0.4875,
+      "eval_samples_per_second": 18.46,
+      "eval_steps_per_second": 10.256,
+      "step": 45
+    },
+    {
+      "epoch": 2.380952380952381,
+      "grad_norm": 4.0,
+      "learning_rate": 6.0952380952380945e-06,
+      "log_odds_chosen": 1.6636985540390015,
+      "log_odds_ratio": -0.20683148503303528,
+      "logits/chosen": -0.6579397916793823,
+      "logits/rejected": -0.6644610166549683,
+      "logps/chosen": -0.6573775410652161,
+      "logps/rejected": -1.7458206415176392,
+      "loss": 0.2065,
+      "nll_loss": 0.18582025170326233,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.06573774665594101,
+      "rewards/margins": 0.1088443249464035,
+      "rewards/rejected": -0.1745820790529251,
+      "step": 50
+    },
+    {
+      "epoch": 2.380952380952381,
+      "eval_log_odds_chosen": 1.8822059631347656,
+      "eval_log_odds_ratio": -0.17333486676216125,
+      "eval_logits/chosen": -0.7790116667747498,
+      "eval_logits/rejected": -0.8235125541687012,
+      "eval_logps/chosen": -0.7073443531990051,
+      "eval_logps/rejected": -2.0477232933044434,
+      "eval_loss": 0.22909635305404663,
+      "eval_nll_loss": 0.20003780722618103,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.07073444128036499,
+      "eval_rewards/margins": 0.13403791189193726,
+      "eval_rewards/rejected": -0.20477232336997986,
+      "eval_runtime": 0.4894,
+      "eval_samples_per_second": 18.391,
+      "eval_steps_per_second": 10.217,
+      "step": 50
+    },
+    {
+      "epoch": 2.619047619047619,
+      "grad_norm": 3.484375,
+      "learning_rate": 5.904761904761905e-06,
+      "log_odds_chosen": 2.264883041381836,
+      "log_odds_ratio": -0.12978659570217133,
+      "logits/chosen": -0.6915649175643921,
+      "logits/rejected": -0.7381215691566467,
+      "logps/chosen": -0.5361235737800598,
+      "logps/rejected": -2.021486759185791,
+      "loss": 0.1714,
+      "nll_loss": 0.15843836963176727,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0536123625934124,
+      "rewards/margins": 0.1485363095998764,
+      "rewards/rejected": -0.2021486759185791,
+      "step": 55
+    },
+    {
+      "epoch": 2.619047619047619,
+      "eval_log_odds_chosen": 2.414100170135498,
+      "eval_log_odds_ratio": -0.12427723407745361,
+      "eval_logits/chosen": -0.6324247717857361,
+      "eval_logits/rejected": -0.6796156167984009,
+      "eval_logps/chosen": -0.6784501075744629,
+      "eval_logps/rejected": -2.475348949432373,
+      "eval_loss": 0.2066802680492401,
+      "eval_nll_loss": 0.1842283010482788,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.06784500926733017,
+      "eval_rewards/margins": 0.1796899139881134,
+      "eval_rewards/rejected": -0.24753491580486298,
+      "eval_runtime": 0.4881,
+      "eval_samples_per_second": 18.44,
+      "eval_steps_per_second": 10.245,
+      "step": 55
+    },
+    {
+      "epoch": 2.857142857142857,
+      "grad_norm": 1.84375,
+      "learning_rate": 5.7142857142857145e-06,
+      "log_odds_chosen": 2.907975673675537,
+      "log_odds_ratio": -0.08882492780685425,
+      "logits/chosen": -0.6123847365379333,
+      "logits/rejected": -0.6521289348602295,
+      "logps/chosen": -0.5171164274215698,
+      "logps/rejected": -2.5270466804504395,
+      "loss": 0.1603,
+      "nll_loss": 0.1514003425836563,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.05171164125204086,
+      "rewards/margins": 0.20099301636219025,
+      "rewards/rejected": -0.2527046501636505,
+      "step": 60
+    },
+    {
+      "epoch": 2.857142857142857,
+      "eval_log_odds_chosen": 2.7621328830718994,
+      "eval_log_odds_ratio": -0.10116946697235107,
+      "eval_logits/chosen": -0.7726386785507202,
+      "eval_logits/rejected": -0.820778489112854,
+      "eval_logps/chosen": -0.6410530805587769,
+      "eval_logps/rejected": -2.7290656566619873,
+      "eval_loss": 0.18988870084285736,
+      "eval_nll_loss": 0.17068591713905334,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.0641053095459938,
+      "eval_rewards/margins": 0.2088012397289276,
+      "eval_rewards/rejected": -0.2729065716266632,
+      "eval_runtime": 0.4876,
+      "eval_samples_per_second": 18.457,
+      "eval_steps_per_second": 10.254,
+      "step": 60
+    },
+    {
+      "epoch": 3.0952380952380953,
+      "grad_norm": 2.3125,
+      "learning_rate": 5.523809523809523e-06,
+      "log_odds_chosen": 3.2325375080108643,
+      "log_odds_ratio": -0.07393848896026611,
+      "logits/chosen": -0.5876457691192627,
+      "logits/rejected": -0.6179739236831665,
+      "logps/chosen": -0.47219792008399963,
+      "logps/rejected": -2.750746488571167,
+      "loss": 0.1217,
+      "nll_loss": 0.1142902597784996,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.047219786792993546,
+      "rewards/margins": 0.2278548777103424,
+      "rewards/rejected": -0.27507466077804565,
+      "step": 65
+    },
+    {
+      "epoch": 3.0952380952380953,
+      "eval_log_odds_chosen": 3.2131500244140625,
+      "eval_log_odds_ratio": -0.07817230373620987,
+      "eval_logits/chosen": -0.7153889536857605,
+      "eval_logits/rejected": -0.7612559795379639,
+      "eval_logps/chosen": -0.6216389536857605,
+      "eval_logps/rejected": -3.110154628753662,
+      "eval_loss": 0.18020187318325043,
+      "eval_nll_loss": 0.16394628584384918,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.06216390058398247,
+      "eval_rewards/margins": 0.24885158240795135,
+      "eval_rewards/rejected": -0.31101545691490173,
+      "eval_runtime": 0.4875,
+      "eval_samples_per_second": 18.461,
+      "eval_steps_per_second": 10.256,
+      "step": 65
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.484375,
+      "learning_rate": 5.333333333333333e-06,
+      "log_odds_chosen": 4.271537780761719,
+      "log_odds_ratio": -0.03715288266539574,
+      "logits/chosen": -0.636605441570282,
+      "logits/rejected": -0.6511901617050171,
+      "logps/chosen": -0.3875546157360077,
+      "logps/rejected": -3.485497236251831,
+      "loss": 0.1121,
+      "nll_loss": 0.10841932147741318,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03875546529889107,
+      "rewards/margins": 0.30979424715042114,
+      "rewards/rejected": -0.3485496938228607,
+      "step": 70
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "eval_log_odds_chosen": 3.6277384757995605,
+      "eval_log_odds_ratio": -0.06236565113067627,
+      "eval_logits/chosen": -0.7850332260131836,
+      "eval_logits/rejected": -0.8188579678535461,
+      "eval_logps/chosen": -0.619295597076416,
+      "eval_logps/rejected": -3.4978508949279785,
+      "eval_loss": 0.17540325224399567,
+      "eval_nll_loss": 0.16080613434314728,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.06192956119775772,
+      "eval_rewards/margins": 0.2878555357456207,
+      "eval_rewards/rejected": -0.34978508949279785,
+      "eval_runtime": 0.4885,
+      "eval_samples_per_second": 18.425,
+      "eval_steps_per_second": 10.236,
+      "step": 70
+    },
+    {
+      "epoch": 3.571428571428571,
+      "grad_norm": 1.7421875,
+      "learning_rate": 5.142857142857143e-06,
+      "log_odds_chosen": 4.159672737121582,
+      "log_odds_ratio": -0.04163466766476631,
+      "logits/chosen": -0.8136247396469116,
+      "logits/rejected": -0.8268924951553345,
+      "logps/chosen": -0.3948783278465271,
+      "logps/rejected": -3.3770499229431152,
+      "loss": 0.1207,
+      "nll_loss": 0.11651208251714706,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03948783501982689,
+      "rewards/margins": 0.29821720719337463,
+      "rewards/rejected": -0.33770498633384705,
+      "step": 75
+    },
+    {
+      "epoch": 3.571428571428571,
+      "eval_log_odds_chosen": 3.9027724266052246,
+      "eval_log_odds_ratio": -0.053419072180986404,
+      "eval_logits/chosen": -0.8184521794319153,
+      "eval_logits/rejected": -0.8538778424263,
+      "eval_logps/chosen": -0.6019997596740723,
+      "eval_logps/rejected": -3.714503526687622,
+      "eval_loss": 0.1682591438293457,
+      "eval_nll_loss": 0.15445396304130554,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.06019997596740723,
+      "eval_rewards/margins": 0.31125038862228394,
+      "eval_rewards/rejected": -0.37145036458969116,
+      "eval_runtime": 0.487,
+      "eval_samples_per_second": 18.48,
+      "eval_steps_per_second": 10.267,
+      "step": 75
+    },
+    {
+      "epoch": 3.8095238095238093,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.952380952380952e-06,
+      "log_odds_chosen": 3.8730292320251465,
+      "log_odds_ratio": -0.06712771207094193,
+      "logits/chosen": -0.7053377032279968,
+      "logits/rejected": -0.7456766963005066,
+      "logps/chosen": -0.5094538331031799,
+      "logps/rejected": -3.3846638202667236,
+      "loss": 0.1468,
+      "nll_loss": 0.14012651145458221,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.050945382565259933,
+      "rewards/margins": 0.28752097487449646,
+      "rewards/rejected": -0.3384663760662079,
+      "step": 80
+    },
+    {
+      "epoch": 3.8095238095238093,
+      "eval_log_odds_chosen": 3.960108518600464,
+      "eval_log_odds_ratio": -0.049763023853302,
+      "eval_logits/chosen": -0.7738298773765564,
+      "eval_logits/rejected": -0.8184520602226257,
+      "eval_logps/chosen": -0.5810937285423279,
+      "eval_logps/rejected": -3.7163281440734863,
+      "eval_loss": 0.1617187261581421,
+      "eval_nll_loss": 0.14851662516593933,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.058109380304813385,
+      "eval_rewards/margins": 0.31352347135543823,
+      "eval_rewards/rejected": -0.37163281440734863,
+      "eval_runtime": 0.4874,
+      "eval_samples_per_second": 18.464,
+      "eval_steps_per_second": 10.258,
+      "step": 80
+    },
+    {
+      "epoch": 4.0476190476190474,
+      "grad_norm": 1.84375,
+      "learning_rate": 4.7619047619047615e-06,
+      "log_odds_chosen": 4.677986145019531,
+      "log_odds_ratio": -0.020088233053684235,
+      "logits/chosen": -0.49949830770492554,
+      "logits/rejected": -0.5240954160690308,
+      "logps/chosen": -0.38463571667671204,
+      "logps/rejected": -3.8635241985321045,
+      "loss": 0.1076,
+      "nll_loss": 0.10564112663269043,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.038463570177555084,
+      "rewards/margins": 0.34788888692855835,
+      "rewards/rejected": -0.38635244965553284,
+      "step": 85
+    },
+    {
+      "epoch": 4.0476190476190474,
+      "eval_log_odds_chosen": 4.288655757904053,
+      "eval_log_odds_ratio": -0.041029635816812515,
+      "eval_logits/chosen": -0.36610883474349976,
+      "eval_logits/rejected": -0.3847040832042694,
+      "eval_logps/chosen": -0.5810166597366333,
+      "eval_logps/rejected": -4.0325775146484375,
+      "eval_loss": 0.15631017088890076,
+      "eval_nll_loss": 0.14473801851272583,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05810166150331497,
+      "eval_rewards/margins": 0.34515610337257385,
+      "eval_rewards/rejected": -0.4032577872276306,
+      "eval_runtime": 0.4875,
+      "eval_samples_per_second": 18.463,
+      "eval_steps_per_second": 10.257,
+      "step": 85
+    },
+    {
+      "epoch": 4.285714285714286,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.571428571428571e-06,
+      "log_odds_chosen": 5.020426273345947,
+      "log_odds_ratio": -0.018852120265364647,
+      "logits/chosen": -0.3934122920036316,
+      "logits/rejected": -0.3896563649177551,
+      "logps/chosen": -0.42108678817749023,
+      "logps/rejected": -4.143835067749023,
+      "loss": 0.0976,
+      "nll_loss": 0.09568431973457336,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.042108677327632904,
+      "rewards/margins": 0.37227481603622437,
+      "rewards/rejected": -0.41438350081443787,
+      "step": 90
+    },
+    {
+      "epoch": 4.285714285714286,
+      "eval_log_odds_chosen": 4.151299476623535,
+      "eval_log_odds_ratio": -0.042820464819669724,
+      "eval_logits/chosen": -0.8576589822769165,
+      "eval_logits/rejected": -0.8869267702102661,
+      "eval_logps/chosen": -0.5677927732467651,
+      "eval_logps/rejected": -3.8686652183532715,
+      "eval_loss": 0.15273889899253845,
+      "eval_nll_loss": 0.14122863113880157,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.056779276579618454,
+      "eval_rewards/margins": 0.33008724451065063,
+      "eval_rewards/rejected": -0.3868665397167206,
+      "eval_runtime": 0.4879,
+      "eval_samples_per_second": 18.447,
+      "eval_steps_per_second": 10.249,
+      "step": 90
+    },
+    {
+      "epoch": 4.523809523809524,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.3809523809523815e-06,
+      "log_odds_chosen": 4.556994438171387,
+      "log_odds_ratio": -0.04522204026579857,
+      "logits/chosen": -0.8442004919052124,
+      "logits/rejected": -0.8662538528442383,
+      "logps/chosen": -0.364029198884964,
+      "logps/rejected": -3.7180163860321045,
+      "loss": 0.1237,
+      "nll_loss": 0.11919407546520233,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03640292212367058,
+      "rewards/margins": 0.33539873361587524,
+      "rewards/rejected": -0.3718016445636749,
+      "step": 95
+    },
+    {
+      "epoch": 4.523809523809524,
+      "eval_log_odds_chosen": 4.323246002197266,
+      "eval_log_odds_ratio": -0.03854605555534363,
+      "eval_logits/chosen": -0.6883870363235474,
+      "eval_logits/rejected": -0.7122835516929626,
+      "eval_logps/chosen": -0.5690239667892456,
+      "eval_logps/rejected": -4.033358573913574,
+      "eval_loss": 0.14968888461589813,
+      "eval_nll_loss": 0.1387859284877777,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05690239742398262,
+      "eval_rewards/margins": 0.34643349051475525,
+      "eval_rewards/rejected": -0.4033358693122864,
+      "eval_runtime": 0.487,
+      "eval_samples_per_second": 18.48,
+      "eval_steps_per_second": 10.267,
+      "step": 95
+    },
+    {
+      "epoch": 4.761904761904762,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.19047619047619e-06,
+      "log_odds_chosen": 4.853901386260986,
+      "log_odds_ratio": -0.02733095921576023,
+      "logits/chosen": -0.6483007669448853,
+      "logits/rejected": -0.6531607508659363,
+      "logps/chosen": -0.37110140919685364,
+      "logps/rejected": -3.995234727859497,
+      "loss": 0.1098,
+      "nll_loss": 0.10703416168689728,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03711014240980148,
+      "rewards/margins": 0.36241334676742554,
+      "rewards/rejected": -0.39952346682548523,
+      "step": 100
+    },
+    {
+      "epoch": 4.761904761904762,
+      "eval_log_odds_chosen": 4.428193092346191,
+      "eval_log_odds_ratio": -0.03602977469563484,
+      "eval_logits/chosen": -0.7133967876434326,
+      "eval_logits/rejected": -0.7346823215484619,
+      "eval_logps/chosen": -0.561062753200531,
+      "eval_logps/rejected": -4.115842342376709,
+      "eval_loss": 0.14537306129932404,
+      "eval_nll_loss": 0.13508550822734833,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05610628053545952,
+      "eval_rewards/margins": 0.3554779589176178,
+      "eval_rewards/rejected": -0.4115842282772064,
+      "eval_runtime": 0.4881,
+      "eval_samples_per_second": 18.439,
+      "eval_steps_per_second": 10.244,
+      "step": 100
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 3.28125,
+      "learning_rate": 4e-06,
+      "log_odds_chosen": 5.169442176818848,
+      "log_odds_ratio": -0.01285733561962843,
+      "logits/chosen": -0.5451700091362,
+      "logits/rejected": -0.5567004680633545,
+      "logps/chosen": -0.3075657784938812,
+      "logps/rejected": -4.096129417419434,
+      "loss": 0.0781,
+      "nll_loss": 0.0768006294965744,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.030756577849388123,
+      "rewards/margins": 0.378856360912323,
+      "rewards/rejected": -0.4096129536628723,
+      "step": 105
+    },
+    {
+      "epoch": 5.0,
+      "eval_log_odds_chosen": 4.601005554199219,
+      "eval_log_odds_ratio": -0.032399993389844894,
+      "eval_logits/chosen": -0.6537272930145264,
+      "eval_logits/rejected": -0.6651639938354492,
+      "eval_logps/chosen": -0.5671016573905945,
+      "eval_logps/rejected": -4.295802116394043,
+      "eval_loss": 0.14492884278297424,
+      "eval_nll_loss": 0.13551345467567444,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05671016499400139,
+      "eval_rewards/margins": 0.3728700280189514,
+      "eval_rewards/rejected": -0.4295802116394043,
+      "eval_runtime": 0.4872,
+      "eval_samples_per_second": 18.471,
+      "eval_steps_per_second": 10.262,
+      "step": 105
+    },
+    {
+      "epoch": 5.238095238095238,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.809523809523809e-06,
+      "log_odds_chosen": 5.452701568603516,
+      "log_odds_ratio": -0.01384773850440979,
+      "logits/chosen": -0.6513525247573853,
+      "logits/rejected": -0.6465297937393188,
+      "logps/chosen": -0.3701745569705963,
+      "logps/rejected": -4.411220550537109,
+      "loss": 0.0986,
+      "nll_loss": 0.09724441170692444,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03701745718717575,
+      "rewards/margins": 0.4041045308113098,
+      "rewards/rejected": -0.44112199544906616,
+      "step": 110
+    },
+    {
+      "epoch": 5.238095238095238,
+      "eval_log_odds_chosen": 4.495316505432129,
+      "eval_log_odds_ratio": -0.034047432243824005,
+      "eval_logits/chosen": -0.731423020362854,
+      "eval_logits/rejected": -0.7520760297775269,
+      "eval_logps/chosen": -0.5562735199928284,
+      "eval_logps/rejected": -4.164623260498047,
+      "eval_loss": 0.1418984979391098,
+      "eval_nll_loss": 0.13248108327388763,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.055627357214689255,
+      "eval_rewards/margins": 0.3608350157737732,
+      "eval_rewards/rejected": -0.41646236181259155,
+      "eval_runtime": 0.4879,
+      "eval_samples_per_second": 18.447,
+      "eval_steps_per_second": 10.248,
+      "step": 110
+    },
+    {
+      "epoch": 5.476190476190476,
+      "grad_norm": 3.703125,
+      "learning_rate": 3.619047619047619e-06,
+      "log_odds_chosen": 5.597304344177246,
+      "log_odds_ratio": -0.013377921655774117,
+      "logits/chosen": -0.6798397302627563,
+      "logits/rejected": -0.7005301713943481,
+      "logps/chosen": -0.3014984726905823,
+      "logps/rejected": -4.4406232833862305,
+      "loss": 0.0853,
+      "nll_loss": 0.08393532782793045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03014984354376793,
+      "rewards/margins": 0.41391244530677795,
+      "rewards/rejected": -0.44406232237815857,
+      "step": 115
+    },
+    {
+      "epoch": 5.476190476190476,
+      "eval_log_odds_chosen": 4.5586256980896,
+      "eval_log_odds_ratio": -0.03254225105047226,
+      "eval_logits/chosen": -0.6377931833267212,
+      "eval_logits/rejected": -0.6543766260147095,
+      "eval_logps/chosen": -0.5523817539215088,
+      "eval_logps/rejected": -4.2158522605896,
+      "eval_loss": 0.14017952978610992,
+      "eval_nll_loss": 0.1309087872505188,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05523817613720894,
+      "eval_rewards/margins": 0.366347074508667,
+      "eval_rewards/rejected": -0.4215852618217468,
+      "eval_runtime": 0.4911,
+      "eval_samples_per_second": 18.328,
+      "eval_steps_per_second": 10.182,
+      "step": 115
+    },
+    {
+      "epoch": 5.714285714285714,
+      "grad_norm": 2.140625,
+      "learning_rate": 3.428571428571428e-06,
+      "log_odds_chosen": 5.031613826751709,
+      "log_odds_ratio": -0.019640957936644554,
+      "logits/chosen": -0.4365396499633789,
+      "logits/rejected": -0.4198831617832184,
+      "logps/chosen": -0.31068772077560425,
+      "logps/rejected": -3.9459738731384277,
+      "loss": 0.0867,
+      "nll_loss": 0.08475493639707565,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.031068775802850723,
+      "rewards/margins": 0.36352860927581787,
+      "rewards/rejected": -0.3945973813533783,
+      "step": 120
+    },
+    {
+      "epoch": 5.714285714285714,
+      "eval_log_odds_chosen": 4.671214580535889,
+      "eval_log_odds_ratio": -0.03062080405652523,
+      "eval_logits/chosen": -0.5378702282905579,
+      "eval_logits/rejected": -0.5502060651779175,
+      "eval_logps/chosen": -0.5473546981811523,
+      "eval_logps/rejected": -4.3087615966796875,
+      "eval_loss": 0.1385972797870636,
+      "eval_nll_loss": 0.12939123809337616,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.054735470563173294,
+      "eval_rewards/margins": 0.37614068388938904,
+      "eval_rewards/rejected": -0.4308761656284332,
+      "eval_runtime": 0.4883,
+      "eval_samples_per_second": 18.431,
+      "eval_steps_per_second": 10.24,
+      "step": 120
+    },
+    {
+      "epoch": 5.9523809523809526,
+      "grad_norm": 1.9765625,
+      "learning_rate": 3.238095238095238e-06,
+      "log_odds_chosen": 4.888375282287598,
+      "log_odds_ratio": -0.0290891882032156,
+      "logits/chosen": -0.5597335696220398,
+      "logits/rejected": -0.5532962083816528,
+      "logps/chosen": -0.32100874185562134,
+      "logps/rejected": -3.9171700477600098,
+      "loss": 0.0829,
+      "nll_loss": 0.07996629178524017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03210087865591049,
+      "rewards/margins": 0.35961610078811646,
+      "rewards/rejected": -0.39171695709228516,
+      "step": 125
+    },
+    {
+      "epoch": 5.9523809523809526,
+      "eval_log_odds_chosen": 4.70390510559082,
+      "eval_log_odds_ratio": -0.029467348009347916,
+      "eval_logits/chosen": -0.6945549249649048,
+      "eval_logits/rejected": -0.7163381576538086,
+      "eval_logps/chosen": -0.5340889096260071,
+      "eval_logps/rejected": -4.305633544921875,
+      "eval_loss": 0.13552479445934296,
+      "eval_nll_loss": 0.1264742910861969,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05340889096260071,
+      "eval_rewards/margins": 0.3771544396877289,
+      "eval_rewards/rejected": -0.430563360452652,
+      "eval_runtime": 0.4874,
+      "eval_samples_per_second": 18.464,
+      "eval_steps_per_second": 10.258,
+      "step": 125
+    },
+    {
+      "epoch": 6.190476190476191,
+      "grad_norm": 2.15625,
+      "learning_rate": 3.0476190476190473e-06,
+      "log_odds_chosen": 5.590579032897949,
+      "log_odds_ratio": -0.018792379647493362,
+      "logits/chosen": -0.6233000159263611,
+      "logits/rejected": -0.6280890107154846,
+      "logps/chosen": -0.2965574264526367,
+      "logps/rejected": -4.412256240844727,
+      "loss": 0.0921,
+      "nll_loss": 0.09021884948015213,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.02965574339032173,
+      "rewards/margins": 0.41156989336013794,
+      "rewards/rejected": -0.4412256181240082,
+      "step": 130
+    },
+    {
+      "epoch": 6.190476190476191,
+      "eval_log_odds_chosen": 4.838481903076172,
+      "eval_log_odds_ratio": -0.02710099145770073,
+      "eval_logits/chosen": -0.6229408383369446,
+      "eval_logits/rejected": -0.6341615319252014,
+      "eval_logps/chosen": -0.5473824739456177,
+      "eval_logps/rejected": -4.465917587280273,
+      "eval_loss": 0.13664411008358002,
+      "eval_nll_loss": 0.12817898392677307,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05473824217915535,
+      "eval_rewards/margins": 0.3918535113334656,
+      "eval_rewards/rejected": -0.4465917646884918,
+      "eval_runtime": 0.4867,
+      "eval_samples_per_second": 18.492,
+      "eval_steps_per_second": 10.273,
+      "step": 130
+    },
+    {
+      "epoch": 6.428571428571429,
+      "grad_norm": 3.359375,
+      "learning_rate": 2.8571428571428573e-06,
+      "log_odds_chosen": 5.753912925720215,
+      "log_odds_ratio": -0.014670786447823048,
+      "logits/chosen": -0.5793005228042603,
+      "logits/rejected": -0.5792975425720215,
+      "logps/chosen": -0.29233574867248535,
+      "logps/rejected": -4.592888355255127,
+      "loss": 0.0874,
+      "nll_loss": 0.08597152680158615,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.029233578592538834,
+      "rewards/margins": 0.4300552010536194,
+      "rewards/rejected": -0.4592887759208679,
+      "step": 135
+    },
+    {
+      "epoch": 6.428571428571429,
+      "eval_log_odds_chosen": 4.875422477722168,
+      "eval_log_odds_ratio": -0.026727139949798584,
+      "eval_logits/chosen": -0.6263194680213928,
+      "eval_logits/rejected": -0.6402261257171631,
+      "eval_logps/chosen": -0.5489826798439026,
+      "eval_logps/rejected": -4.507485866546631,
+      "eval_loss": 0.13577227294445038,
+      "eval_nll_loss": 0.1276685744524002,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05489826202392578,
+      "eval_rewards/margins": 0.3958503305912018,
+      "eval_rewards/rejected": -0.45074859261512756,
+      "eval_runtime": 0.4882,
+      "eval_samples_per_second": 18.436,
+      "eval_steps_per_second": 10.242,
+      "step": 135
+    },
+    {
+      "epoch": 6.666666666666667,
+      "grad_norm": 2.109375,
+      "learning_rate": 2.6666666666666664e-06,
+      "log_odds_chosen": 5.3087663650512695,
+      "log_odds_ratio": -0.024162959307432175,
+      "logits/chosen": -0.5501323938369751,
+      "logits/rejected": -0.543838381767273,
+      "logps/chosen": -0.33894363045692444,
+      "logps/rejected": -4.1651811599731445,
+      "loss": 0.082,
+      "nll_loss": 0.07960663735866547,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0338943675160408,
+      "rewards/margins": 0.38262373208999634,
+      "rewards/rejected": -0.41651806235313416,
+      "step": 140
+    },
+    {
+      "epoch": 6.666666666666667,
+      "eval_log_odds_chosen": 4.799313545227051,
+      "eval_log_odds_ratio": -0.027702342718839645,
+      "eval_logits/chosen": -0.677030622959137,
+      "eval_logits/rejected": -0.6918050646781921,
+      "eval_logps/chosen": -0.5406588315963745,
+      "eval_logps/rejected": -4.412304878234863,
+      "eval_loss": 0.13464586436748505,
+      "eval_nll_loss": 0.12628625333309174,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05406588315963745,
+      "eval_rewards/margins": 0.3871646523475647,
+      "eval_rewards/rejected": -0.4412304759025574,
+      "eval_runtime": 0.4867,
+      "eval_samples_per_second": 18.49,
+      "eval_steps_per_second": 10.272,
+      "step": 140
+    },
+    {
+      "epoch": 6.904761904761905,
+      "grad_norm": 3.234375,
+      "learning_rate": 2.476190476190476e-06,
+      "log_odds_chosen": 5.5894694328308105,
+      "log_odds_ratio": -0.010893247090280056,
+      "logits/chosen": -0.5942927002906799,
+      "logits/rejected": -0.5888161659240723,
+      "logps/chosen": -0.2951999306678772,
+      "logps/rejected": -4.442993640899658,
+      "loss": 0.0858,
+      "nll_loss": 0.0846717581152916,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.02951999567449093,
+      "rewards/margins": 0.4147793650627136,
+      "rewards/rejected": -0.4442993700504303,
+      "step": 145
+    },
+    {
+      "epoch": 6.904761904761905,
+      "eval_log_odds_chosen": 4.838164329528809,
+      "eval_log_odds_ratio": -0.02697155997157097,
+      "eval_logits/chosen": -0.5802319645881653,
+      "eval_logits/rejected": -0.5902702808380127,
+      "eval_logps/chosen": -0.5424883961677551,
+      "eval_logps/rejected": -4.452386856079102,
+      "eval_loss": 0.13433685898780823,
+      "eval_nll_loss": 0.12595266103744507,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05424883961677551,
+      "eval_rewards/margins": 0.39098984003067017,
+      "eval_rewards/rejected": -0.4452386796474457,
+      "eval_runtime": 0.4873,
+      "eval_samples_per_second": 18.47,
+      "eval_steps_per_second": 10.261,
+      "step": 145
+    },
+    {
+      "epoch": 7.142857142857143,
+      "grad_norm": 1.9453125,
+      "learning_rate": 2.2857142857142856e-06,
+      "log_odds_chosen": 5.92537784576416,
+      "log_odds_ratio": -0.014473943039774895,
+      "logits/chosen": -0.5114859938621521,
+      "logits/rejected": -0.5051178336143494,
+      "logps/chosen": -0.25217267870903015,
+      "logps/rejected": -4.5697736740112305,
+      "loss": 0.0762,
+      "nll_loss": 0.07474460452795029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.025217268615961075,
+      "rewards/margins": 0.4317600727081299,
+      "rewards/rejected": -0.45697736740112305,
+      "step": 150
+    },
+    {
+      "epoch": 7.142857142857143,
+      "eval_log_odds_chosen": 4.879807949066162,
+      "eval_log_odds_ratio": -0.02607133984565735,
+      "eval_logits/chosen": -0.6277867555618286,
+      "eval_logits/rejected": -0.6383845806121826,
+      "eval_logps/chosen": -0.5392376184463501,
+      "eval_logps/rejected": -4.4825334548950195,
+      "eval_loss": 0.13314732909202576,
+      "eval_nll_loss": 0.12492867559194565,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05392376333475113,
+      "eval_rewards/margins": 0.39432960748672485,
+      "eval_rewards/rejected": -0.448253333568573,
+      "eval_runtime": 0.487,
+      "eval_samples_per_second": 18.479,
+      "eval_steps_per_second": 10.266,
+      "step": 150
+    },
+    {
+      "epoch": 7.380952380952381,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.095238095238095e-06,
+      "log_odds_chosen": 5.681746959686279,
+      "log_odds_ratio": -0.008144749328494072,
+      "logits/chosen": -0.556106448173523,
+      "logits/rejected": -0.5360628366470337,
+      "logps/chosen": -0.2713228166103363,
+      "logps/rejected": -4.389334678649902,
+      "loss": 0.0704,
+      "nll_loss": 0.06960402429103851,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.02713228203356266,
+      "rewards/margins": 0.41180118918418884,
+      "rewards/rejected": -0.43893352150917053,
+      "step": 155
+    },
+    {
+      "epoch": 7.380952380952381,
+      "eval_log_odds_chosen": 4.914063930511475,
+      "eval_log_odds_ratio": -0.0254636462777853,
+      "eval_logits/chosen": -0.6259689331054688,
+      "eval_logits/rejected": -0.6361022591590881,
+      "eval_logps/chosen": -0.5356873273849487,
+      "eval_logps/rejected": -4.5052995681762695,
+      "eval_loss": 0.13209427893161774,
+      "eval_nll_loss": 0.12397398054599762,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05356873944401741,
+      "eval_rewards/margins": 0.39696118235588074,
+      "eval_rewards/rejected": -0.45052990317344666,
+      "eval_runtime": 0.4875,
+      "eval_samples_per_second": 18.463,
+      "eval_steps_per_second": 10.257,
+      "step": 155
+    },
+    {
+      "epoch": 7.619047619047619,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9047619047619045e-06,
+      "log_odds_chosen": 5.47554874420166,
+      "log_odds_ratio": -0.012598132714629173,
+      "logits/chosen": -0.5787724256515503,
+      "logits/rejected": -0.575463593006134,
+      "logps/chosen": -0.26679569482803345,
+      "logps/rejected": -4.207664489746094,
+      "loss": 0.078,
+      "nll_loss": 0.07674238830804825,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.026679569855332375,
+      "rewards/margins": 0.39408689737319946,
+      "rewards/rejected": -0.4207665026187897,
+      "step": 160
+    },
+    {
+      "epoch": 7.619047619047619,
+      "eval_log_odds_chosen": 4.950940132141113,
+      "eval_log_odds_ratio": -0.024774912744760513,
+      "eval_logits/chosen": -0.6380746960639954,
+      "eval_logits/rejected": -0.6485334038734436,
+      "eval_logps/chosen": -0.5350639820098877,
+      "eval_logps/rejected": -4.538556098937988,
+      "eval_loss": 0.13170303404331207,
+      "eval_nll_loss": 0.12376340478658676,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05350639671087265,
+      "eval_rewards/margins": 0.4003491997718811,
+      "eval_rewards/rejected": -0.45385560393333435,
+      "eval_runtime": 0.4878,
+      "eval_samples_per_second": 18.45,
+      "eval_steps_per_second": 10.25,
+      "step": 160
+    },
+    {
+      "epoch": 7.857142857142857,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.714285714285714e-06,
+      "log_odds_chosen": 5.9162702560424805,
+      "log_odds_ratio": -0.013311143033206463,
+      "logits/chosen": -0.547685980796814,
+      "logits/rejected": -0.5555287003517151,
+      "logps/chosen": -0.27488571405410767,
+      "logps/rejected": -4.701210021972656,
+      "loss": 0.0771,
+      "nll_loss": 0.07575251162052155,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.027488570660352707,
+      "rewards/margins": 0.44263243675231934,
+      "rewards/rejected": -0.4701210558414459,
+      "step": 165
+    },
+    {
+      "epoch": 7.857142857142857,
+      "eval_log_odds_chosen": 4.9923577308654785,
+      "eval_log_odds_ratio": -0.024206604808568954,
+      "eval_logits/chosen": -0.6115527749061584,
+      "eval_logits/rejected": -0.6203628778457642,
+      "eval_logps/chosen": -0.5354728102684021,
+      "eval_logps/rejected": -4.578566551208496,
+      "eval_loss": 0.13151198625564575,
+      "eval_nll_loss": 0.12378431856632233,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05354728177189827,
+      "eval_rewards/margins": 0.40430936217308044,
+      "eval_rewards/rejected": -0.4578566551208496,
+      "eval_runtime": 0.4877,
+      "eval_samples_per_second": 18.454,
+      "eval_steps_per_second": 10.252,
+      "step": 165
+    },
+    {
+      "epoch": 8.095238095238095,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.5238095238095236e-06,
+      "log_odds_chosen": 5.4759697914123535,
+      "log_odds_ratio": -0.030720766633749008,
+      "logits/chosen": -0.5306688547134399,
+      "logits/rejected": -0.5217020511627197,
+      "logps/chosen": -0.33474162220954895,
+      "logps/rejected": -4.392602920532227,
+      "loss": 0.0851,
+      "nll_loss": 0.08205311000347137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0334741584956646,
+      "rewards/margins": 0.4057861268520355,
+      "rewards/rejected": -0.439260333776474,
+      "step": 170
+    },
+    {
+      "epoch": 8.095238095238095,
+      "eval_log_odds_chosen": 4.982872486114502,
+      "eval_log_odds_ratio": -0.02418256737291813,
+      "eval_logits/chosen": -0.6068294644355774,
+      "eval_logits/rejected": -0.6163716316223145,
+      "eval_logps/chosen": -0.5354769825935364,
+      "eval_logps/rejected": -4.566414833068848,
+      "eval_loss": 0.13132263720035553,
+      "eval_nll_loss": 0.1235472559928894,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.053547702729701996,
+      "eval_rewards/margins": 0.4030938148498535,
+      "eval_rewards/rejected": -0.4566414952278137,
+      "eval_runtime": 0.4867,
+      "eval_samples_per_second": 18.491,
+      "eval_steps_per_second": 10.273,
+      "step": 170
+    },
+    {
+      "epoch": 8.333333333333334,
+      "grad_norm": 3.0,
+      "learning_rate": 1.3333333333333332e-06,
+      "log_odds_chosen": 6.1932477951049805,
+      "log_odds_ratio": -0.006689209491014481,
+      "logits/chosen": -0.4947226941585541,
+      "logits/rejected": -0.506854772567749,
+      "logps/chosen": -0.25427430868148804,
+      "logps/rejected": -4.840688705444336,
+      "loss": 0.0654,
+      "nll_loss": 0.06476090848445892,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.025427427142858505,
+      "rewards/margins": 0.4586414694786072,
+      "rewards/rejected": -0.4840688705444336,
+      "step": 175
+    },
+    {
+      "epoch": 8.333333333333334,
+      "eval_log_odds_chosen": 4.968461513519287,
+      "eval_log_odds_ratio": -0.02425793744623661,
+      "eval_logits/chosen": -0.6205912828445435,
+      "eval_logits/rejected": -0.6322892904281616,
+      "eval_logps/chosen": -0.531274139881134,
+      "eval_logps/rejected": -4.542240142822266,
+      "eval_loss": 0.13052870333194733,
+      "eval_nll_loss": 0.12270841747522354,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05312741547822952,
+      "eval_rewards/margins": 0.4010965824127197,
+      "eval_rewards/rejected": -0.45422402024269104,
+      "eval_runtime": 0.4884,
+      "eval_samples_per_second": 18.429,
+      "eval_steps_per_second": 10.238,
+      "step": 175
+    },
+    {
+      "epoch": 8.571428571428571,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.1428571428571428e-06,
+      "log_odds_chosen": 6.075676918029785,
+      "log_odds_ratio": -0.00910225696861744,
+      "logits/chosen": -0.58963543176651,
+      "logits/rejected": -0.56965172290802,
+      "logps/chosen": -0.3160817623138428,
+      "logps/rejected": -4.802862167358398,
+      "loss": 0.082,
+      "nll_loss": 0.08105450868606567,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03160817548632622,
+      "rewards/margins": 0.4486781060695648,
+      "rewards/rejected": -0.48028627038002014,
+      "step": 180
+    },
+    {
+      "epoch": 8.571428571428571,
+      "eval_log_odds_chosen": 4.932239532470703,
+      "eval_log_odds_ratio": -0.024737322703003883,
+      "eval_logits/chosen": -0.6294423937797546,
+      "eval_logits/rejected": -0.6397351026535034,
+      "eval_logps/chosen": -0.5322803258895874,
+      "eval_logps/rejected": -4.510106563568115,
+      "eval_loss": 0.13084720075130463,
+      "eval_nll_loss": 0.12291731685400009,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05322803929448128,
+      "eval_rewards/margins": 0.3977826237678528,
+      "eval_rewards/rejected": -0.45101064443588257,
+      "eval_runtime": 0.489,
+      "eval_samples_per_second": 18.406,
+      "eval_steps_per_second": 10.226,
+      "step": 180
+    },
+    {
+      "epoch": 8.80952380952381,
+      "grad_norm": 2.015625,
+      "learning_rate": 9.523809523809522e-07,
+      "log_odds_chosen": 5.602750778198242,
+      "log_odds_ratio": -0.011134130880236626,
+      "logits/chosen": -0.5277374982833862,
+      "logits/rejected": -0.5226815938949585,
+      "logps/chosen": -0.27227240800857544,
+      "logps/rejected": -4.354727745056152,
+      "loss": 0.0724,
+      "nll_loss": 0.07125753164291382,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.027227237820625305,
+      "rewards/margins": 0.4082455635070801,
+      "rewards/rejected": -0.43547287583351135,
+      "step": 185
+    },
+    {
+      "epoch": 8.80952380952381,
+      "eval_log_odds_chosen": 4.966063499450684,
+      "eval_log_odds_ratio": -0.024260815232992172,
+      "eval_logits/chosen": -0.5997673273086548,
+      "eval_logits/rejected": -0.6082264184951782,
+      "eval_logps/chosen": -0.5307241082191467,
+      "eval_logps/rejected": -4.538777828216553,
+      "eval_loss": 0.13055673241615295,
+      "eval_nll_loss": 0.1226615160703659,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05307241529226303,
+      "eval_rewards/margins": 0.40080541372299194,
+      "eval_rewards/rejected": -0.45387783646583557,
+      "eval_runtime": 0.4869,
+      "eval_samples_per_second": 18.483,
+      "eval_steps_per_second": 10.268,
+      "step": 185
+    },
+    {
+      "epoch": 9.047619047619047,
+      "grad_norm": 2.21875,
+      "learning_rate": 7.619047619047618e-07,
+      "log_odds_chosen": 5.537537574768066,
+      "log_odds_ratio": -0.018197722733020782,
+      "logits/chosen": -0.5673882365226746,
+      "logits/rejected": -0.5563878417015076,
+      "logps/chosen": -0.26666221022605896,
+      "logps/rejected": -4.266029357910156,
+      "loss": 0.0836,
+      "nll_loss": 0.08179894089698792,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.026666224002838135,
+      "rewards/margins": 0.3999367356300354,
+      "rewards/rejected": -0.42660292983055115,
+      "step": 190
+    },
+    {
+      "epoch": 9.047619047619047,
+      "eval_log_odds_chosen": 4.971249103546143,
+      "eval_log_odds_ratio": -0.024101996794342995,
+      "eval_logits/chosen": -0.575426459312439,
+      "eval_logits/rejected": -0.5841690897941589,
+      "eval_logps/chosen": -0.5333296656608582,
+      "eval_logps/rejected": -4.551214694976807,
+      "eval_loss": 0.13083776831626892,
+      "eval_nll_loss": 0.12306801974773407,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.053332965821027756,
+      "eval_rewards/margins": 0.40178847312927246,
+      "eval_rewards/rejected": -0.4551214277744293,
+      "eval_runtime": 0.4872,
+      "eval_samples_per_second": 18.474,
+      "eval_steps_per_second": 10.264,
+      "step": 190
+    },
+    {
+      "epoch": 9.285714285714286,
+      "grad_norm": 3.3125,
+      "learning_rate": 5.714285714285714e-07,
+      "log_odds_chosen": 5.82122802734375,
+      "log_odds_ratio": -0.010938870720565319,
+      "logits/chosen": -0.4943665862083435,
+      "logits/rejected": -0.47990670800209045,
+      "logps/chosen": -0.2986515164375305,
+      "logps/rejected": -4.486918926239014,
+      "loss": 0.0702,
+      "nll_loss": 0.06913810223340988,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.02986515499651432,
+      "rewards/margins": 0.41882675886154175,
+      "rewards/rejected": -0.4486919343471527,
+      "step": 195
+    },
+    {
+      "epoch": 9.285714285714286,
+      "eval_log_odds_chosen": 4.972805500030518,
+      "eval_log_odds_ratio": -0.024383049458265305,
+      "eval_logits/chosen": -0.589435338973999,
+      "eval_logits/rejected": -0.5966291427612305,
+      "eval_logps/chosen": -0.5334832668304443,
+      "eval_logps/rejected": -4.552124977111816,
+      "eval_loss": 0.13075287640094757,
+      "eval_nll_loss": 0.12296122312545776,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.053348325192928314,
+      "eval_rewards/margins": 0.4018642008304596,
+      "eval_rewards/rejected": -0.4552125036716461,
+      "eval_runtime": 0.488,
+      "eval_samples_per_second": 18.444,
+      "eval_steps_per_second": 10.247,
+      "step": 195
+    },
+    {
+      "epoch": 9.523809523809524,
+      "grad_norm": 2.390625,
+      "learning_rate": 3.809523809523809e-07,
+      "log_odds_chosen": 5.9984025955200195,
+      "log_odds_ratio": -0.008187348023056984,
+      "logits/chosen": -0.534194827079773,
+      "logits/rejected": -0.5184012055397034,
+      "logps/chosen": -0.2566554546356201,
+      "logps/rejected": -4.710351467132568,
+      "loss": 0.0726,
+      "nll_loss": 0.07180726528167725,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.02566554583609104,
+      "rewards/margins": 0.44536957144737244,
+      "rewards/rejected": -0.4710351526737213,
+      "step": 200
+    },
+    {
+      "epoch": 9.523809523809524,
+      "eval_log_odds_chosen": 4.969243049621582,
+      "eval_log_odds_ratio": -0.024293312802910805,
+      "eval_logits/chosen": -0.6059412956237793,
+      "eval_logits/rejected": -0.615134596824646,
+      "eval_logps/chosen": -0.5334844589233398,
+      "eval_logps/rejected": -4.546559810638428,
+      "eval_loss": 0.13070961833000183,
+      "eval_nll_loss": 0.12287895381450653,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.053348444402217865,
+      "eval_rewards/margins": 0.40130752325057983,
+      "eval_rewards/rejected": -0.4546559453010559,
+      "eval_runtime": 0.4875,
+      "eval_samples_per_second": 18.463,
+      "eval_steps_per_second": 10.257,
+      "step": 200
+    },
+    {
+      "epoch": 9.761904761904763,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.9047619047619045e-07,
+      "log_odds_chosen": 5.707345485687256,
+      "log_odds_ratio": -0.023422162979841232,
+      "logits/chosen": -0.5711747407913208,
+      "logits/rejected": -0.5841129422187805,
+      "logps/chosen": -0.2618250250816345,
+      "logps/rejected": -4.388245105743408,
+      "loss": 0.0837,
+      "nll_loss": 0.08134286105632782,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.02618250623345375,
+      "rewards/margins": 0.4126420021057129,
+      "rewards/rejected": -0.4388245642185211,
+      "step": 205
+    },
+    {
+      "epoch": 9.761904761904763,
+      "eval_log_odds_chosen": 4.974253177642822,
+      "eval_log_odds_ratio": -0.024108218029141426,
+      "eval_logits/chosen": -0.5955663919448853,
+      "eval_logits/rejected": -0.6041454672813416,
+      "eval_logps/chosen": -0.5337764024734497,
+      "eval_logps/rejected": -4.553771018981934,
+      "eval_loss": 0.13067038357257843,
+      "eval_nll_loss": 0.12289754301309586,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.05337763950228691,
+      "eval_rewards/margins": 0.40199947357177734,
+      "eval_rewards/rejected": -0.45537716150283813,
+      "eval_runtime": 0.487,
+      "eval_samples_per_second": 18.479,
+      "eval_steps_per_second": 10.266,
+      "step": 205
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.0,
+      "log_odds_chosen": 5.524815559387207,
+      "log_odds_ratio": -0.013400805182754993,
+      "logits/chosen": -0.5060779452323914,
+      "logits/rejected": -0.49261727929115295,
+      "logps/chosen": -0.2732623517513275,
+      "logps/rejected": -4.279727458953857,
+      "loss": 0.0796,
+      "nll_loss": 0.07821428030729294,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.027326231822371483,
+      "rewards/margins": 0.40064650774002075,
+      "rewards/rejected": -0.4279727041721344,
+      "step": 210
+    },
+    {
+      "epoch": 10.0,
+      "eval_log_odds_chosen": 4.966081619262695,
+      "eval_log_odds_ratio": -0.024325253441929817,
+      "eval_logits/chosen": -0.5945724248886108,
+      "eval_logits/rejected": -0.6053321957588196,
+      "eval_logps/chosen": -0.5324550867080688,
+      "eval_logps/rejected": -4.543835639953613,
+      "eval_loss": 0.13071957230567932,
+      "eval_nll_loss": 0.12293493747711182,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.053245507180690765,
+      "eval_rewards/margins": 0.4011380076408386,
+      "eval_rewards/rejected": -0.4543835520744324,
+      "eval_runtime": 0.4882,
+      "eval_samples_per_second": 18.434,
+      "eval_steps_per_second": 10.241,
+      "step": 210
+    },
+    {
+      "epoch": 10.0,
+      "step": 210,
+      "total_flos": 0.0,
+      "train_loss": 0.3063473221801576,
+      "train_runtime": 348.5405,
+      "train_samples_per_second": 4.82,
+      "train_steps_per_second": 0.603
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 210,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21b454619cb49890b867af3be95e9c6febda9fbfb5830c01ecbd8fdb36ae352d
+oid sha256:6423ecd3ccaa89e0be1c437ad068819ded22a1d0439e0584f2de8d13eb08a962
 size 5496