End of training

Browse files

Files changed (11) hide show

README.md +35 -57
config.json +7 -4
generation_config.json +6 -3
model-00001-of-00002.safetensors +2 -2
model-00002-of-00002.safetensors +1 -1
model.safetensors.index.json +1 -1
special_tokens_map.json +15 -19
tokenizer.json +0 -18
tokenizer_config.json +4 -24
trainer_state.json +727 -1541
training_args.bin +1 -1

README.md CHANGED Viewed

@@ -18,18 +18,18 @@ should probably proofread and complete it, then remove this comment. -->
 This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1307
-- Rewards/chosen: -0.0532
-- Rewards/rejected: -0.4544
 - Rewards/accuracies: 1.0
-- Rewards/margins: 0.4011
-- Logps/rejected: -4.5438
-- Logps/chosen: -0.5325
-- Logits/rejected: -0.6053
-- Logits/chosen: -0.5946
-- Nll Loss: 0.1229
-- Log Odds Ratio: -0.0243
-- Log Odds Chosen: 4.9661
 ## Model description
@@ -52,58 +52,36 @@ The following hyperparameters were used during training:
 - train_batch_size: 2
 - eval_batch_size: 2
 - seed: 42
-- gradient_accumulation_steps: 4
-- total_train_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - num_epochs: 10
 ### Training results
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
-| 3.1838        | 0.2381 | 5    | 2.3687          | -0.2141        | -0.2396          | 0.9000             | 0.0255          | -2.3962        | -2.1413      | 0.1963          | 0.2099        | 2.2996   | -0.5609        | 0.2917          |
-| 2.0549        | 0.4762 | 10   | 1.7681          | -0.1667        | -0.1957          | 0.9000             | 0.0290          | -1.9570        | -1.6671      | -0.3938         | -0.3708       | 1.7001   | -0.5337        | 0.3558          |
-| 1.52          | 0.7143 | 15   | 1.2706          | -0.1413        | -0.1758          | 1.0                | 0.0345          | -1.7581        | -1.4131      | -0.7370         | -0.7115       | 1.2041   | -0.4984        | 0.4454          |
-| 1.0136        | 0.9524 | 20   | 0.8746          | -0.1219        | -0.1611          | 1.0                | 0.0391          | -1.6107        | -1.2194      | -0.7398         | -0.7111       | 0.8093   | -0.4665        | 0.5323          |
-| 0.6723        | 1.1905 | 25   | 0.5716          | -0.1086        | -0.1524          | 1.0                | 0.0438          | -1.5241        | -1.0859      | -0.6821         | -0.6474       | 0.5078   | -0.4362        | 0.6192          |
-| 0.4086        | 1.4286 | 30   | 0.3851          | -0.0943        | -0.1439          | 1.0                | 0.0496          | -1.4391        | -0.9432      | -0.6528         | -0.6126       | 0.3261   | -0.4003        | 0.7344          |
-| 0.2733        | 1.6667 | 35   | 0.3272          | -0.0852        | -0.1441          | 1.0                | 0.0588          | -1.4407        | -0.8524      | -0.6893         | -0.6529       | 0.2740   | -0.3564        | 0.8915          |
-| 0.2696        | 1.9048 | 40   | 0.2898          | -0.0800        | -0.1517          | 1.0                | 0.0717          | -1.5166        | -0.7996      | -0.7221         | -0.6878       | 0.2438   | -0.3088        | 1.0816          |
-| 0.2171        | 2.1429 | 45   | 0.2581          | -0.0753        | -0.1708          | 1.0                | 0.0955          | -1.7082        | -0.7534      | -0.7685         | -0.7280       | 0.2206   | -0.2432        | 1.4022          |
-| 0.2065        | 2.3810 | 50   | 0.2291          | -0.0707        | -0.2048          | 1.0                | 0.1340          | -2.0477        | -0.7073      | -0.8235         | -0.7790       | 0.2000   | -0.1733        | 1.8822          |
-| 0.1714        | 2.6190 | 55   | 0.2067          | -0.0678        | -0.2475          | 1.0                | 0.1797          | -2.4753        | -0.6785      | -0.6796         | -0.6324       | 0.1842   | -0.1243        | 2.4141          |
-| 0.1603        | 2.8571 | 60   | 0.1899          | -0.0641        | -0.2729          | 1.0                | 0.2088          | -2.7291        | -0.6411      | -0.8208         | -0.7726       | 0.1707   | -0.1012        | 2.7621          |
-| 0.1217        | 3.0952 | 65   | 0.1802          | -0.0622        | -0.3110          | 1.0                | 0.2489          | -3.1102        | -0.6216      | -0.7613         | -0.7154       | 0.1639   | -0.0782        | 3.2132          |
-| 0.1121        | 3.3333 | 70   | 0.1754          | -0.0619        | -0.3498          | 1.0                | 0.2879          | -3.4979        | -0.6193      | -0.8189         | -0.7850       | 0.1608   | -0.0624        | 3.6277          |
-| 0.1207        | 3.5714 | 75   | 0.1683          | -0.0602        | -0.3715          | 1.0                | 0.3113          | -3.7145        | -0.6020      | -0.8539         | -0.8185       | 0.1545   | -0.0534        | 3.9028          |
-| 0.1468        | 3.8095 | 80   | 0.1617          | -0.0581        | -0.3716          | 1.0                | 0.3135          | -3.7163        | -0.5811      | -0.8185         | -0.7738       | 0.1485   | -0.0498        | 3.9601          |
-| 0.1076        | 4.0476 | 85   | 0.1563          | -0.0581        | -0.4033          | 1.0                | 0.3452          | -4.0326        | -0.5810      | -0.3847         | -0.3661       | 0.1447   | -0.0410        | 4.2887          |
-| 0.0976        | 4.2857 | 90   | 0.1527          | -0.0568        | -0.3869          | 1.0                | 0.3301          | -3.8687        | -0.5678      | -0.8869         | -0.8577       | 0.1412   | -0.0428        | 4.1513          |
-| 0.1237        | 4.5238 | 95   | 0.1497          | -0.0569        | -0.4033          | 1.0                | 0.3464          | -4.0334        | -0.5690      | -0.7123         | -0.6884       | 0.1388   | -0.0385        | 4.3232          |
-| 0.1098        | 4.7619 | 100  | 0.1454          | -0.0561        | -0.4116          | 1.0                | 0.3555          | -4.1158        | -0.5611      | -0.7347         | -0.7134       | 0.1351   | -0.0360        | 4.4282          |
-| 0.0781        | 5.0    | 105  | 0.1449          | -0.0567        | -0.4296          | 1.0                | 0.3729          | -4.2958        | -0.5671      | -0.6652         | -0.6537       | 0.1355   | -0.0324        | 4.6010          |
-| 0.0986        | 5.2381 | 110  | 0.1419          | -0.0556        | -0.4165          | 1.0                | 0.3608          | -4.1646        | -0.5563      | -0.7521         | -0.7314       | 0.1325   | -0.0340        | 4.4953          |
-| 0.0853        | 5.4762 | 115  | 0.1402          | -0.0552        | -0.4216          | 1.0                | 0.3663          | -4.2159        | -0.5524      | -0.6544         | -0.6378       | 0.1309   | -0.0325        | 4.5586          |
-| 0.0867        | 5.7143 | 120  | 0.1386          | -0.0547        | -0.4309          | 1.0                | 0.3761          | -4.3088        | -0.5474      | -0.5502         | -0.5379       | 0.1294   | -0.0306        | 4.6712          |
-| 0.0829        | 5.9524 | 125  | 0.1355          | -0.0534        | -0.4306          | 1.0                | 0.3772          | -4.3056        | -0.5341      | -0.7163         | -0.6946       | 0.1265   | -0.0295        | 4.7039          |
-| 0.0921        | 6.1905 | 130  | 0.1366          | -0.0547        | -0.4466          | 1.0                | 0.3919          | -4.4659        | -0.5474      | -0.6342         | -0.6229       | 0.1282   | -0.0271        | 4.8385          |
-| 0.0874        | 6.4286 | 135  | 0.1358          | -0.0549        | -0.4507          | 1.0                | 0.3959          | -4.5075        | -0.5490      | -0.6402         | -0.6263       | 0.1277   | -0.0267        | 4.8754          |
-| 0.082         | 6.6667 | 140  | 0.1346          | -0.0541        | -0.4412          | 1.0                | 0.3872          | -4.4123        | -0.5407      | -0.6918         | -0.6770       | 0.1263   | -0.0277        | 4.7993          |
-| 0.0858        | 6.9048 | 145  | 0.1343          | -0.0542        | -0.4452          | 1.0                | 0.3910          | -4.4524        | -0.5425      | -0.5903         | -0.5802       | 0.1260   | -0.0270        | 4.8382          |
-| 0.0762        | 7.1429 | 150  | 0.1331          | -0.0539        | -0.4483          | 1.0                | 0.3943          | -4.4825        | -0.5392      | -0.6384         | -0.6278       | 0.1249   | -0.0261        | 4.8798          |
-| 0.0704        | 7.3810 | 155  | 0.1321          | -0.0536        | -0.4505          | 1.0                | 0.3970          | -4.5053        | -0.5357      | -0.6361         | -0.6260       | 0.1240   | -0.0255        | 4.9141          |
-| 0.078         | 7.6190 | 160  | 0.1317          | -0.0535        | -0.4539          | 1.0                | 0.4003          | -4.5386        | -0.5351      | -0.6485         | -0.6381       | 0.1238   | -0.0248        | 4.9509          |
-| 0.0771        | 7.8571 | 165  | 0.1315          | -0.0535        | -0.4579          | 1.0                | 0.4043          | -4.5786        | -0.5355      | -0.6204         | -0.6116       | 0.1238   | -0.0242        | 4.9924          |
-| 0.0851        | 8.0952 | 170  | 0.1313          | -0.0535        | -0.4566          | 1.0                | 0.4031          | -4.5664        | -0.5355      | -0.6164         | -0.6068       | 0.1235   | -0.0242        | 4.9829          |
-| 0.0654        | 8.3333 | 175  | 0.1305          | -0.0531        | -0.4542          | 1.0                | 0.4011          | -4.5422        | -0.5313      | -0.6323         | -0.6206       | 0.1227   | -0.0243        | 4.9685          |
-| 0.082         | 8.5714 | 180  | 0.1308          | -0.0532        | -0.4510          | 1.0                | 0.3978          | -4.5101        | -0.5323      | -0.6397         | -0.6294       | 0.1229   | -0.0247        | 4.9322          |
-| 0.0724        | 8.8095 | 185  | 0.1306          | -0.0531        | -0.4539          | 1.0                | 0.4008          | -4.5388        | -0.5307      | -0.6082         | -0.5998       | 0.1227   | -0.0243        | 4.9661          |
-| 0.0836        | 9.0476 | 190  | 0.1308          | -0.0533        | -0.4551          | 1.0                | 0.4018          | -4.5512        | -0.5333      | -0.5842         | -0.5754       | 0.1231   | -0.0241        | 4.9712          |
-| 0.0702        | 9.2857 | 195  | 0.1308          | -0.0533        | -0.4552          | 1.0                | 0.4019          | -4.5521        | -0.5335      | -0.5966         | -0.5894       | 0.1230   | -0.0244        | 4.9728          |
-| 0.0726        | 9.5238 | 200  | 0.1307          | -0.0533        | -0.4547          | 1.0                | 0.4013          | -4.5466        | -0.5335      | -0.6151         | -0.6059       | 0.1229   | -0.0243        | 4.9692          |
-| 0.0837        | 9.7619 | 205  | 0.1307          | -0.0534        | -0.4554          | 1.0                | 0.4020          | -4.5538        | -0.5338      | -0.6041         | -0.5956       | 0.1229   | -0.0241        | 4.9743          |
-| 0.0796        | 10.0   | 210  | 0.1307          | -0.0532        | -0.4544          | 1.0                | 0.4011          | -4.5438        | -0.5325      | -0.6053         | -0.5946       | 0.1229   | -0.0243        | 4.9661          |
 ### Framework versions

 This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) on an unknown dataset.
 It achieves the following results on the evaluation set:
+- Loss: 0.8110
+- Rewards/chosen: -0.1016
+- Rewards/rejected: -0.1421
 - Rewards/accuracies: 1.0
+- Rewards/margins: 0.0405
+- Logps/rejected: -1.4214
+- Logps/chosen: -1.0163
+- Logits/rejected: -0.3132
+- Logits/chosen: -0.2798
+- Nll Loss: 0.7661
+- Log Odds Ratio: -0.4487
+- Log Odds Chosen: 0.5905
 ## Model description
 - train_batch_size: 2
 - eval_batch_size: 2
 - seed: 42
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - num_epochs: 10
 ### Training results
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 1.8976        | 0.4   | 1    | 1.6382          | -0.1720        | -0.2027          | 1.0                | 0.0307          | -2.0269        | -1.7196      | -0.1521         | -0.1194       | 1.5851   | -0.5313        | 0.3650          |
+| 1.5573        | 0.8   | 2    | 1.5334          | -0.1585        | -0.1906          | 1.0                | 0.0320          | -1.9059        | -1.5854      | -0.1364         | -0.1050       | 1.4813   | -0.5207        | 0.3907          |
+| 1.4427        | 1.2   | 3    | 1.4338          | -0.1440        | -0.1771          | 1.0                | 0.0332          | -1.7712        | -1.4397      | -0.1360         | -0.1062       | 1.3828   | -0.5097        | 0.4184          |
+| 1.3493        | 1.6   | 4    | 1.3429          | -0.1341        | -0.1681          | 1.0                | 0.0341          | -1.6811        | -1.3406      | -0.1349         | -0.1049       | 1.2929   | -0.5009        | 0.4413          |
+| 1.2683        | 2.0   | 5    | 1.2643          | -0.1271        | -0.1620          | 1.0                | 0.0349          | -1.6205        | -1.2712      | -0.1409         | -0.1101       | 1.2150   | -0.4929        | 0.4625          |
+| 1.1736        | 2.4   | 6    | 1.1920          | -0.1219        | -0.1574          | 1.0                | 0.0355          | -1.5743        | -1.2191      | -0.1502         | -0.1193       | 1.1433   | -0.4869        | 0.4787          |
+| 1.1212        | 2.8   | 7    | 1.1234          | -0.1175        | -0.1539          | 1.0                | 0.0364          | -1.5392        | -1.1754      | -0.1601         | -0.1286       | 1.0754   | -0.4801        | 0.4978          |
+| 1.0518        | 3.2   | 8    | 1.0610          | -0.1138        | -0.1509          | 1.0                | 0.0371          | -1.5093        | -1.1382      | -0.1737         | -0.1422       | 1.0136   | -0.4741        | 0.5149          |
+| 0.9805        | 3.6   | 9    | 1.0012          | -0.1105        | -0.1484          | 1.0                | 0.0379          | -1.4837        | -1.1049      | -0.1969         | -0.1645       | 0.9544   | -0.4682        | 0.5320          |
+| 0.9299        | 4.0   | 10   | 0.9496          | -0.1079        | -0.1465          | 1.0                | 0.0386          | -1.4653        | -1.0794      | -0.2201         | -0.1875       | 0.9033   | -0.4628        | 0.5477          |
+| 0.8761        | 4.4   | 11   | 0.9070          | -0.1059        | -0.1451          | 1.0                | 0.0392          | -1.4510        | -1.0591      | -0.2431         | -0.2105       | 0.8612   | -0.4584        | 0.5608          |
+| 0.8337        | 4.8   | 12   | 0.8864          | -0.1049        | -0.1444          | 1.0                | 0.0394          | -1.4436        | -1.0492      | -0.2562         | -0.2232       | 0.8407   | -0.4566        | 0.5669          |
+| 0.7975        | 5.2   | 13   | 0.8664          | -0.1041        | -0.1439          | 1.0                | 0.0398          | -1.4386        | -1.0406      | -0.2727         | -0.2397       | 0.8210   | -0.4541        | 0.5740          |
+| 0.788         | 5.6   | 14   | 0.8492          | -0.1033        | -0.1433          | 1.0                | 0.0400          | -1.4329        | -1.0326      | -0.2837         | -0.2507       | 0.8040   | -0.4524        | 0.5794          |
+| 0.78          | 6.0   | 15   | 0.8334          | -0.1026        | -0.1429          | 1.0                | 0.0402          | -1.4287        | -1.0264      | -0.2944         | -0.2614       | 0.7883   | -0.4508        | 0.5839          |
+| 0.7395        | 6.4   | 16   | 0.8211          | -0.1021        | -0.1424          | 1.0                | 0.0403          | -1.4244        | -1.0214      | -0.3054         | -0.2722       | 0.7761   | -0.4500        | 0.5865          |
+| 0.7446        | 6.8   | 17   | 0.8164          | -0.1019        | -0.1423          | 1.0                | 0.0404          | -1.4229        | -1.0187      | -0.3054         | -0.2722       | 0.7715   | -0.4492        | 0.5888          |
+| 0.7518        | 7.2   | 18   | 0.8125          | -0.1018        | -0.1423          | 1.0                | 0.0405          | -1.4226        | -1.0175      | -0.3106         | -0.2775       | 0.7677   | -0.4487        | 0.5903          |
+| 0.7431        | 7.6   | 19   | 0.8107          | -0.1016        | -0.1422          | 1.0                | 0.0405          | -1.4217        | -1.0162      | -0.3104         | -0.2768       | 0.7658   | -0.4484        | 0.5912          |
+| 0.726         | 8.0   | 20   | 0.8110          | -0.1016        | -0.1421          | 1.0                | 0.0405          | -1.4214        | -1.0163      | -0.3132         | -0.2798       | 0.7661   | -0.4487        | 0.5905          |
 ### Framework versions

config.json CHANGED Viewed

@@ -5,8 +5,12 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 128256,
-  "eos_token_id": 128257,
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 3072,
@@ -18,7 +22,6 @@
   "num_attention_heads": 24,
   "num_hidden_layers": 28,
   "num_key_value_heads": 8,
-  "pad_token_id": 128257,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
@@ -33,5 +36,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.44.2",
   "use_cache": true,
-  "vocab_size": 128258
 }

   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 3072,
   "num_attention_heads": 24,
   "num_hidden_layers": 28,
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
   "torch_dtype": "bfloat16",
   "transformers_version": "4.44.2",
   "use_cache": true,
+  "vocab_size": 128256
 }

generation_config.json CHANGED Viewed

@@ -1,8 +1,11 @@
 {
-  "bos_token_id": 128256,
   "do_sample": true,
-  "eos_token_id": 128257,
-  "pad_token_id": 128257,
   "temperature": 0.6,
   "top_p": 0.9,
   "transformers_version": "4.44.2"

 {
+  "bos_token_id": 128000,
   "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
   "temperature": 0.6,
   "top_p": 0.9,
   "transformers_version": "4.44.2"

model-00001-of-00002.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e962c51dd28f2d58454c68476f9382aa7fb374a9ac5a626e882767dae349766b
-size 4965811384

 version https://git-lfs.github.com/spec/v1
+oid sha256:f0b34b00ea7987a8b658c318060509bae4b84fdd203fc3aa9d533b26a3057ce6
+size 4965799096

model-00002-of-00002.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0188d2fa37c8060a3e33c0b4c6fb438699186a53fb8949d301a37b69e4a9a07d
 size 1459729952

 version https://git-lfs.github.com/spec/v1
+oid sha256:614ebfe53ab928785dd1447f45b3f4dbfada0b14383c69f97d6f6824534cd01b
 size 1459729952

model.safetensors.index.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 6425511936
   },
   "weight_map": {
     "model.embed_tokens.weight": "model-00001-of-00002.safetensors",

 {
   "metadata": {
+    "total_size": 6425499648
   },
   "weight_map": {
     "model.embed_tokens.weight": "model-00001-of-00002.safetensors",

special_tokens_map.json CHANGED Viewed

@@ -1,21 +1,17 @@
 {
-  "additional_special_tokens": [
-    {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
-  ],
-  "bos_token": "<|im_start|>",
-  "eos_token": "<|im_end|>",
-  "pad_token": "<|im_end|>"
 }

 {
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|finetune_right_pad_id|>"
 }

tokenizer.json CHANGED Viewed

@@ -2306,24 +2306,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 128256,
-      "content": "<|im_start|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128257,
-      "content": "<|im_end|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,

       "rstrip": false,
       "normalized": false,
       "special": true
     }
   ],
   "normalizer": null,

tokenizer_config.json CHANGED Viewed

@@ -2047,37 +2047,17 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "128256": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128257": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "bos_token": "<|im_start|>",
-  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|im_end|>",
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
-  "pad_token": "<|im_end|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
+  "pad_token": "<|finetune_right_pad_id|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

trainer_state.json CHANGED Viewed

@@ -1,1579 +1,765 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
-  "eval_steps": 5,
-  "global_step": 210,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.23809523809523808,
-      "grad_norm": 28.375,
-      "learning_rate": 7.80952380952381e-06,
-      "log_odds_chosen": 0.1891055405139923,
-      "log_odds_ratio": -0.6111669540405273,
-      "logits/chosen": 0.29273518919944763,
-      "logits/rejected": 0.2814817428588867,
-      "logps/chosen": -3.5652737617492676,
-      "logps/rejected": -3.743648052215576,
-      "loss": 3.1838,
-      "nll_loss": 3.1226940155029297,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.3565273880958557,
-      "rewards/margins": 0.01783742383122444,
-      "rewards/rejected": -0.37436479330062866,
       "step": 5
     },
     {
-      "epoch": 0.23809523809523808,
-      "eval_log_odds_chosen": 0.29169073700904846,
-      "eval_log_odds_ratio": -0.5609198808670044,
-      "eval_logits/chosen": 0.20992258191108704,
-      "eval_logits/rejected": 0.19634230434894562,
-      "eval_logps/chosen": -2.1412553787231445,
-      "eval_logps/rejected": -2.3962299823760986,
-      "eval_loss": 2.3686962127685547,
-      "eval_nll_loss": 2.299578905105591,
-      "eval_rewards/accuracies": 0.8999999761581421,
-      "eval_rewards/chosen": -0.21412554383277893,
-      "eval_rewards/margins": 0.025497451424598694,
-      "eval_rewards/rejected": -0.23962298035621643,
-      "eval_runtime": 0.489,
-      "eval_samples_per_second": 18.406,
-      "eval_steps_per_second": 10.225,
       "step": 5
     },
     {
-      "epoch": 0.47619047619047616,
-      "grad_norm": 16.75,
-      "learning_rate": 7.619047619047618e-06,
-      "log_odds_chosen": 0.3308032155036926,
-      "log_odds_ratio": -0.5464655160903931,
-      "logits/chosen": 0.07425501197576523,
-      "logits/rejected": 0.05106530338525772,
-      "logps/chosen": -1.7553539276123047,
-      "logps/rejected": -2.036280870437622,
-      "loss": 2.0549,
-      "nll_loss": 2.0002410411834717,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -0.1755353957414627,
-      "rewards/margins": 0.02809269167482853,
-      "rewards/rejected": -0.2036280632019043,
       "step": 10
     },
     {
-      "epoch": 0.47619047619047616,
-      "eval_log_odds_chosen": 0.35579484701156616,
-      "eval_log_odds_ratio": -0.5337004661560059,
-      "eval_logits/chosen": -0.37078872323036194,
-      "eval_logits/rejected": -0.3937973380088806,
-      "eval_logps/chosen": -1.6670551300048828,
-      "eval_logps/rejected": -1.9570205211639404,
-      "eval_loss": 1.768071174621582,
-      "eval_nll_loss": 1.7001245021820068,
-      "eval_rewards/accuracies": 0.8999999761581421,
-      "eval_rewards/chosen": -0.16670551896095276,
-      "eval_rewards/margins": 0.028996536508202553,
-      "eval_rewards/rejected": -0.19570204615592957,
-      "eval_runtime": 0.4888,
-      "eval_samples_per_second": 18.413,
-      "eval_steps_per_second": 10.23,
       "step": 10
     },
     {
-      "epoch": 0.7142857142857143,
-      "grad_norm": 14.0625,
-      "learning_rate": 7.428571428571428e-06,
-      "log_odds_chosen": 0.40405726432800293,
-      "log_odds_ratio": -0.5185824632644653,
-      "logits/chosen": -0.4588231146335602,
-      "logits/rejected": -0.4600151479244232,
-      "logps/chosen": -1.4706313610076904,
-      "logps/rejected": -1.7922747135162354,
-      "loss": 1.52,
-      "nll_loss": 1.4680984020233154,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -0.14706313610076904,
-      "rewards/margins": 0.032164327800273895,
-      "rewards/rejected": -0.17922747135162354,
       "step": 15
     },
     {
-      "epoch": 0.7142857142857143,
-      "eval_log_odds_chosen": 0.44535937905311584,
-      "eval_log_odds_ratio": -0.4983883798122406,
-      "eval_logits/chosen": -0.7114694714546204,
-      "eval_logits/rejected": -0.7369576692581177,
-      "eval_logps/chosen": -1.4130698442459106,
-      "eval_logps/rejected": -1.758122205734253,
-      "eval_loss": 1.2705729007720947,
-      "eval_nll_loss": 1.2041345834732056,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.14130698144435883,
-      "eval_rewards/margins": 0.03450523689389229,
-      "eval_rewards/rejected": -0.17581221461296082,
-      "eval_runtime": 0.4865,
-      "eval_samples_per_second": 18.499,
-      "eval_steps_per_second": 10.277,
       "step": 15
     },
     {
-      "epoch": 0.9523809523809523,
-      "grad_norm": 29.75,
-      "learning_rate": 7.238095238095238e-06,
-      "log_odds_chosen": 0.48574334383010864,
-      "log_odds_ratio": -0.4926493763923645,
-      "logits/chosen": -0.6470807790756226,
-      "logits/rejected": -0.6642199754714966,
-      "logps/chosen": -1.27854323387146,
-      "logps/rejected": -1.6447874307632446,
-      "loss": 1.0136,
-      "nll_loss": 0.9643369913101196,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -0.12785431742668152,
-      "rewards/margins": 0.03662443906068802,
-      "rewards/rejected": -0.16447874903678894,
       "step": 20
     },
     {
-      "epoch": 0.9523809523809523,
-      "eval_log_odds_chosen": 0.532298743724823,
-      "eval_log_odds_ratio": -0.46647781133651733,
-      "eval_logits/chosen": -0.7111243009567261,
-      "eval_logits/rejected": -0.7398008704185486,
-      "eval_logps/chosen": -1.2193677425384521,
-      "eval_logps/rejected": -1.6106961965560913,
-      "eval_loss": 0.8746375441551208,
-      "eval_nll_loss": 0.8092986941337585,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.12193679809570312,
-      "eval_rewards/margins": 0.039132826030254364,
-      "eval_rewards/rejected": -0.1610696017742157,
-      "eval_runtime": 0.488,
-      "eval_samples_per_second": 18.443,
-      "eval_steps_per_second": 10.246,
       "step": 20
     },
     {
-      "epoch": 1.1904761904761905,
-      "grad_norm": 7.4375,
-      "learning_rate": 7.0476190476190475e-06,
-      "log_odds_chosen": 0.6200831532478333,
-      "log_odds_ratio": -0.44031673669815063,
-      "logits/chosen": -0.6323801279067993,
-      "logits/rejected": -0.6341850757598877,
-      "logps/chosen": -1.0979182720184326,
-      "logps/rejected": -1.5436619520187378,
-      "loss": 0.6723,
-      "nll_loss": 0.6282438039779663,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.1097918376326561,
-      "rewards/margins": 0.044574350118637085,
-      "rewards/rejected": -0.15436621010303497,
-      "step": 25
-    },
-    {
-      "epoch": 1.1904761904761905,
-      "eval_log_odds_chosen": 0.6192252039909363,
-      "eval_log_odds_ratio": -0.4361681044101715,
-      "eval_logits/chosen": -0.6473670601844788,
-      "eval_logits/rejected": -0.6820846199989319,
-      "eval_logps/chosen": -1.085878610610962,
-      "eval_logps/rejected": -1.5240614414215088,
-      "eval_loss": 0.5715838670730591,
-      "eval_nll_loss": 0.5078178644180298,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.10858786106109619,
-      "eval_rewards/margins": 0.04381827265024185,
-      "eval_rewards/rejected": -0.15240614116191864,
-      "eval_runtime": 0.4872,
-      "eval_samples_per_second": 18.473,
-      "eval_steps_per_second": 10.263,
-      "step": 25
-    },
-    {
-      "epoch": 1.4285714285714286,
-      "grad_norm": 6.5625,
-      "learning_rate": 6.857142857142856e-06,
-      "log_odds_chosen": 0.736698567867279,
-      "log_odds_ratio": -0.40588584542274475,
-      "logits/chosen": -0.5826676487922668,
-      "logits/rejected": -0.6002309918403625,
-      "logps/chosen": -0.9341448545455933,
-      "logps/rejected": -1.4290642738342285,
-      "loss": 0.4086,
-      "nll_loss": 0.3679959177970886,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.09341450035572052,
-      "rewards/margins": 0.04949193447828293,
-      "rewards/rejected": -0.14290642738342285,
-      "step": 30
-    },
-    {
-      "epoch": 1.4285714285714286,
-      "eval_log_odds_chosen": 0.7344347834587097,
-      "eval_log_odds_ratio": -0.40031012892723083,
-      "eval_logits/chosen": -0.6125622987747192,
-      "eval_logits/rejected": -0.6528064608573914,
-      "eval_logps/chosen": -0.9431861042976379,
-      "eval_logps/rejected": -1.4390549659729004,
-      "eval_loss": 0.3850952088832855,
-      "eval_nll_loss": 0.32607388496398926,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.09431861340999603,
-      "eval_rewards/margins": 0.04958688095211983,
-      "eval_rewards/rejected": -0.14390549063682556,
-      "eval_runtime": 0.4877,
-      "eval_samples_per_second": 18.452,
-      "eval_steps_per_second": 10.251,
-      "step": 30
-    },
-    {
-      "epoch": 1.6666666666666665,
-      "grad_norm": 7.4375,
-      "learning_rate": 6.666666666666667e-06,
-      "log_odds_chosen": 0.9226770401000977,
-      "log_odds_ratio": -0.36509159207344055,
-      "logits/chosen": -0.49513015151023865,
-      "logits/rejected": -0.5086463689804077,
-      "logps/chosen": -0.7646501064300537,
-      "logps/rejected": -1.3257030248641968,
-      "loss": 0.2733,
-      "nll_loss": 0.23684072494506836,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.07646501064300537,
-      "rewards/margins": 0.056105293333530426,
-      "rewards/rejected": -0.1325702965259552,
-      "step": 35
-    },
-    {
-      "epoch": 1.6666666666666665,
-      "eval_log_odds_chosen": 0.8915351629257202,
-      "eval_log_odds_ratio": -0.3563780188560486,
-      "eval_logits/chosen": -0.6528802514076233,
-      "eval_logits/rejected": -0.6892634630203247,
-      "eval_logps/chosen": -0.8524182438850403,
-      "eval_logps/rejected": -1.4406554698944092,
-      "eval_loss": 0.3271670639514923,
-      "eval_nll_loss": 0.2739677429199219,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.08524181693792343,
-      "eval_rewards/margins": 0.05882372334599495,
-      "eval_rewards/rejected": -0.14406554400920868,
-      "eval_runtime": 0.4869,
-      "eval_samples_per_second": 18.484,
-      "eval_steps_per_second": 10.269,
-      "step": 35
-    },
-    {
-      "epoch": 1.9047619047619047,
-      "grad_norm": 4.0,
-      "learning_rate": 6.476190476190476e-06,
-      "log_odds_chosen": 1.0913145542144775,
-      "log_odds_ratio": -0.3268836736679077,
-      "logits/chosen": -0.7565770149230957,
-      "logits/rejected": -0.7887675762176514,
-      "logps/chosen": -0.6868108510971069,
-      "logps/rejected": -1.3315098285675049,
-      "loss": 0.2696,
-      "nll_loss": 0.23689217865467072,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.06868109107017517,
-      "rewards/margins": 0.06446989625692368,
-      "rewards/rejected": -0.13315097987651825,
-      "step": 40
-    },
-    {
-      "epoch": 1.9047619047619047,
-      "eval_log_odds_chosen": 1.0815879106521606,
-      "eval_log_odds_ratio": -0.30878978967666626,
-      "eval_logits/chosen": -0.6877866983413696,
-      "eval_logits/rejected": -0.722061812877655,
-      "eval_logps/chosen": -0.7995762825012207,
-      "eval_logps/rejected": -1.5165843963623047,
-      "eval_loss": 0.2898225784301758,
-      "eval_nll_loss": 0.24379682540893555,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.07995764166116714,
-      "eval_rewards/margins": 0.0717008113861084,
-      "eval_rewards/rejected": -0.15165844559669495,
-      "eval_runtime": 0.49,
-      "eval_samples_per_second": 18.368,
-      "eval_steps_per_second": 10.204,
-      "step": 40
-    },
-    {
-      "epoch": 2.142857142857143,
-      "grad_norm": 2.59375,
-      "learning_rate": 6.285714285714285e-06,
-      "log_odds_chosen": 1.3118517398834229,
-      "log_odds_ratio": -0.27184879779815674,
-      "logits/chosen": -0.5011266469955444,
-      "logits/rejected": -0.5411444902420044,
-      "logps/chosen": -0.6962511539459229,
-      "logps/rejected": -1.5221842527389526,
-      "loss": 0.2171,
-      "nll_loss": 0.18990826606750488,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.069625124335289,
-      "rewards/margins": 0.08259329944849014,
-      "rewards/rejected": -0.15221843123435974,
-      "step": 45
-    },
-    {
-      "epoch": 2.142857142857143,
-      "eval_log_odds_chosen": 1.4021844863891602,
-      "eval_log_odds_ratio": -0.24321937561035156,
-      "eval_logits/chosen": -0.7279993295669556,
-      "eval_logits/rejected": -0.7685092687606812,
-      "eval_logps/chosen": -0.7534304261207581,
-      "eval_logps/rejected": -1.7082147598266602,
-      "eval_loss": 0.25805217027664185,
-      "eval_nll_loss": 0.22057470679283142,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.07534303516149521,
-      "eval_rewards/margins": 0.09547843784093857,
-      "eval_rewards/rejected": -0.17082147300243378,
-      "eval_runtime": 0.4875,
-      "eval_samples_per_second": 18.46,
-      "eval_steps_per_second": 10.256,
-      "step": 45
-    },
-    {
-      "epoch": 2.380952380952381,
-      "grad_norm": 4.0,
-      "learning_rate": 6.0952380952380945e-06,
-      "log_odds_chosen": 1.6636985540390015,
-      "log_odds_ratio": -0.20683148503303528,
-      "logits/chosen": -0.6579397916793823,
-      "logits/rejected": -0.6644610166549683,
-      "logps/chosen": -0.6573775410652161,
-      "logps/rejected": -1.7458206415176392,
-      "loss": 0.2065,
-      "nll_loss": 0.18582025170326233,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.06573774665594101,
-      "rewards/margins": 0.1088443249464035,
-      "rewards/rejected": -0.1745820790529251,
-      "step": 50
-    },
-    {
-      "epoch": 2.380952380952381,
-      "eval_log_odds_chosen": 1.8822059631347656,
-      "eval_log_odds_ratio": -0.17333486676216125,
-      "eval_logits/chosen": -0.7790116667747498,
-      "eval_logits/rejected": -0.8235125541687012,
-      "eval_logps/chosen": -0.7073443531990051,
-      "eval_logps/rejected": -2.0477232933044434,
-      "eval_loss": 0.22909635305404663,
-      "eval_nll_loss": 0.20003780722618103,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.07073444128036499,
-      "eval_rewards/margins": 0.13403791189193726,
-      "eval_rewards/rejected": -0.20477232336997986,
-      "eval_runtime": 0.4894,
-      "eval_samples_per_second": 18.391,
-      "eval_steps_per_second": 10.217,
-      "step": 50
-    },
-    {
-      "epoch": 2.619047619047619,
-      "grad_norm": 3.484375,
-      "learning_rate": 5.904761904761905e-06,
-      "log_odds_chosen": 2.264883041381836,
-      "log_odds_ratio": -0.12978659570217133,
-      "logits/chosen": -0.6915649175643921,
-      "logits/rejected": -0.7381215691566467,
-      "logps/chosen": -0.5361235737800598,
-      "logps/rejected": -2.021486759185791,
-      "loss": 0.1714,
-      "nll_loss": 0.15843836963176727,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.0536123625934124,
-      "rewards/margins": 0.1485363095998764,
-      "rewards/rejected": -0.2021486759185791,
-      "step": 55
-    },
-    {
-      "epoch": 2.619047619047619,
-      "eval_log_odds_chosen": 2.414100170135498,
-      "eval_log_odds_ratio": -0.12427723407745361,
-      "eval_logits/chosen": -0.6324247717857361,
-      "eval_logits/rejected": -0.6796156167984009,
-      "eval_logps/chosen": -0.6784501075744629,
-      "eval_logps/rejected": -2.475348949432373,
-      "eval_loss": 0.2066802680492401,
-      "eval_nll_loss": 0.1842283010482788,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.06784500926733017,
-      "eval_rewards/margins": 0.1796899139881134,
-      "eval_rewards/rejected": -0.24753491580486298,
-      "eval_runtime": 0.4881,
-      "eval_samples_per_second": 18.44,
-      "eval_steps_per_second": 10.245,
-      "step": 55
-    },
-    {
-      "epoch": 2.857142857142857,
-      "grad_norm": 1.84375,
-      "learning_rate": 5.7142857142857145e-06,
-      "log_odds_chosen": 2.907975673675537,
-      "log_odds_ratio": -0.08882492780685425,
-      "logits/chosen": -0.6123847365379333,
-      "logits/rejected": -0.6521289348602295,
-      "logps/chosen": -0.5171164274215698,
-      "logps/rejected": -2.5270466804504395,
-      "loss": 0.1603,
-      "nll_loss": 0.1514003425836563,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.05171164125204086,
-      "rewards/margins": 0.20099301636219025,
-      "rewards/rejected": -0.2527046501636505,
-      "step": 60
-    },
-    {
-      "epoch": 2.857142857142857,
-      "eval_log_odds_chosen": 2.7621328830718994,
-      "eval_log_odds_ratio": -0.10116946697235107,
-      "eval_logits/chosen": -0.7726386785507202,
-      "eval_logits/rejected": -0.820778489112854,
-      "eval_logps/chosen": -0.6410530805587769,
-      "eval_logps/rejected": -2.7290656566619873,
-      "eval_loss": 0.18988870084285736,
-      "eval_nll_loss": 0.17068591713905334,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.0641053095459938,
-      "eval_rewards/margins": 0.2088012397289276,
-      "eval_rewards/rejected": -0.2729065716266632,
-      "eval_runtime": 0.4876,
-      "eval_samples_per_second": 18.457,
-      "eval_steps_per_second": 10.254,
-      "step": 60
-    },
-    {
-      "epoch": 3.0952380952380953,
-      "grad_norm": 2.3125,
-      "learning_rate": 5.523809523809523e-06,
-      "log_odds_chosen": 3.2325375080108643,
-      "log_odds_ratio": -0.07393848896026611,
-      "logits/chosen": -0.5876457691192627,
-      "logits/rejected": -0.6179739236831665,
-      "logps/chosen": -0.47219792008399963,
-      "logps/rejected": -2.750746488571167,
-      "loss": 0.1217,
-      "nll_loss": 0.1142902597784996,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.047219786792993546,
-      "rewards/margins": 0.2278548777103424,
-      "rewards/rejected": -0.27507466077804565,
-      "step": 65
-    },
-    {
-      "epoch": 3.0952380952380953,
-      "eval_log_odds_chosen": 3.2131500244140625,
-      "eval_log_odds_ratio": -0.07817230373620987,
-      "eval_logits/chosen": -0.7153889536857605,
-      "eval_logits/rejected": -0.7612559795379639,
-      "eval_logps/chosen": -0.6216389536857605,
-      "eval_logps/rejected": -3.110154628753662,
-      "eval_loss": 0.18020187318325043,
-      "eval_nll_loss": 0.16394628584384918,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.06216390058398247,
-      "eval_rewards/margins": 0.24885158240795135,
-      "eval_rewards/rejected": -0.31101545691490173,
-      "eval_runtime": 0.4875,
-      "eval_samples_per_second": 18.461,
-      "eval_steps_per_second": 10.256,
-      "step": 65
-    },
-    {
-      "epoch": 3.3333333333333335,
-      "grad_norm": 1.484375,
-      "learning_rate": 5.333333333333333e-06,
-      "log_odds_chosen": 4.271537780761719,
-      "log_odds_ratio": -0.03715288266539574,
-      "logits/chosen": -0.636605441570282,
-      "logits/rejected": -0.6511901617050171,
-      "logps/chosen": -0.3875546157360077,
-      "logps/rejected": -3.485497236251831,
-      "loss": 0.1121,
-      "nll_loss": 0.10841932147741318,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03875546529889107,
-      "rewards/margins": 0.30979424715042114,
-      "rewards/rejected": -0.3485496938228607,
-      "step": 70
-    },
-    {
-      "epoch": 3.3333333333333335,
-      "eval_log_odds_chosen": 3.6277384757995605,
-      "eval_log_odds_ratio": -0.06236565113067627,
-      "eval_logits/chosen": -0.7850332260131836,
-      "eval_logits/rejected": -0.8188579678535461,
-      "eval_logps/chosen": -0.619295597076416,
-      "eval_logps/rejected": -3.4978508949279785,
-      "eval_loss": 0.17540325224399567,
-      "eval_nll_loss": 0.16080613434314728,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.06192956119775772,
-      "eval_rewards/margins": 0.2878555357456207,
-      "eval_rewards/rejected": -0.34978508949279785,
-      "eval_runtime": 0.4885,
-      "eval_samples_per_second": 18.425,
-      "eval_steps_per_second": 10.236,
-      "step": 70
-    },
-    {
-      "epoch": 3.571428571428571,
-      "grad_norm": 1.7421875,
-      "learning_rate": 5.142857142857143e-06,
-      "log_odds_chosen": 4.159672737121582,
-      "log_odds_ratio": -0.04163466766476631,
-      "logits/chosen": -0.8136247396469116,
-      "logits/rejected": -0.8268924951553345,
-      "logps/chosen": -0.3948783278465271,
-      "logps/rejected": -3.3770499229431152,
-      "loss": 0.1207,
-      "nll_loss": 0.11651208251714706,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03948783501982689,
-      "rewards/margins": 0.29821720719337463,
-      "rewards/rejected": -0.33770498633384705,
-      "step": 75
-    },
-    {
-      "epoch": 3.571428571428571,
-      "eval_log_odds_chosen": 3.9027724266052246,
-      "eval_log_odds_ratio": -0.053419072180986404,
-      "eval_logits/chosen": -0.8184521794319153,
-      "eval_logits/rejected": -0.8538778424263,
-      "eval_logps/chosen": -0.6019997596740723,
-      "eval_logps/rejected": -3.714503526687622,
-      "eval_loss": 0.1682591438293457,
-      "eval_nll_loss": 0.15445396304130554,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.06019997596740723,
-      "eval_rewards/margins": 0.31125038862228394,
-      "eval_rewards/rejected": -0.37145036458969116,
-      "eval_runtime": 0.487,
-      "eval_samples_per_second": 18.48,
-      "eval_steps_per_second": 10.267,
-      "step": 75
-    },
-    {
-      "epoch": 3.8095238095238093,
-      "grad_norm": 2.34375,
-      "learning_rate": 4.952380952380952e-06,
-      "log_odds_chosen": 3.8730292320251465,
-      "log_odds_ratio": -0.06712771207094193,
-      "logits/chosen": -0.7053377032279968,
-      "logits/rejected": -0.7456766963005066,
-      "logps/chosen": -0.5094538331031799,
-      "logps/rejected": -3.3846638202667236,
-      "loss": 0.1468,
-      "nll_loss": 0.14012651145458221,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.050945382565259933,
-      "rewards/margins": 0.28752097487449646,
-      "rewards/rejected": -0.3384663760662079,
-      "step": 80
-    },
-    {
-      "epoch": 3.8095238095238093,
-      "eval_log_odds_chosen": 3.960108518600464,
-      "eval_log_odds_ratio": -0.049763023853302,
-      "eval_logits/chosen": -0.7738298773765564,
-      "eval_logits/rejected": -0.8184520602226257,
-      "eval_logps/chosen": -0.5810937285423279,
-      "eval_logps/rejected": -3.7163281440734863,
-      "eval_loss": 0.1617187261581421,
-      "eval_nll_loss": 0.14851662516593933,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.058109380304813385,
-      "eval_rewards/margins": 0.31352347135543823,
-      "eval_rewards/rejected": -0.37163281440734863,
-      "eval_runtime": 0.4874,
-      "eval_samples_per_second": 18.464,
-      "eval_steps_per_second": 10.258,
-      "step": 80
-    },
-    {
-      "epoch": 4.0476190476190474,
-      "grad_norm": 1.84375,
-      "learning_rate": 4.7619047619047615e-06,
-      "log_odds_chosen": 4.677986145019531,
-      "log_odds_ratio": -0.020088233053684235,
-      "logits/chosen": -0.49949830770492554,
-      "logits/rejected": -0.5240954160690308,
-      "logps/chosen": -0.38463571667671204,
-      "logps/rejected": -3.8635241985321045,
-      "loss": 0.1076,
-      "nll_loss": 0.10564112663269043,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.038463570177555084,
-      "rewards/margins": 0.34788888692855835,
-      "rewards/rejected": -0.38635244965553284,
-      "step": 85
-    },
-    {
-      "epoch": 4.0476190476190474,
-      "eval_log_odds_chosen": 4.288655757904053,
-      "eval_log_odds_ratio": -0.041029635816812515,
-      "eval_logits/chosen": -0.36610883474349976,
-      "eval_logits/rejected": -0.3847040832042694,
-      "eval_logps/chosen": -0.5810166597366333,
-      "eval_logps/rejected": -4.0325775146484375,
-      "eval_loss": 0.15631017088890076,
-      "eval_nll_loss": 0.14473801851272583,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05810166150331497,
-      "eval_rewards/margins": 0.34515610337257385,
-      "eval_rewards/rejected": -0.4032577872276306,
-      "eval_runtime": 0.4875,
-      "eval_samples_per_second": 18.463,
-      "eval_steps_per_second": 10.257,
-      "step": 85
-    },
-    {
-      "epoch": 4.285714285714286,
-      "grad_norm": 2.3125,
-      "learning_rate": 4.571428571428571e-06,
-      "log_odds_chosen": 5.020426273345947,
-      "log_odds_ratio": -0.018852120265364647,
-      "logits/chosen": -0.3934122920036316,
-      "logits/rejected": -0.3896563649177551,
-      "logps/chosen": -0.42108678817749023,
-      "logps/rejected": -4.143835067749023,
-      "loss": 0.0976,
-      "nll_loss": 0.09568431973457336,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.042108677327632904,
-      "rewards/margins": 0.37227481603622437,
-      "rewards/rejected": -0.41438350081443787,
-      "step": 90
-    },
-    {
-      "epoch": 4.285714285714286,
-      "eval_log_odds_chosen": 4.151299476623535,
-      "eval_log_odds_ratio": -0.042820464819669724,
-      "eval_logits/chosen": -0.8576589822769165,
-      "eval_logits/rejected": -0.8869267702102661,
-      "eval_logps/chosen": -0.5677927732467651,
-      "eval_logps/rejected": -3.8686652183532715,
-      "eval_loss": 0.15273889899253845,
-      "eval_nll_loss": 0.14122863113880157,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.056779276579618454,
-      "eval_rewards/margins": 0.33008724451065063,
-      "eval_rewards/rejected": -0.3868665397167206,
-      "eval_runtime": 0.4879,
-      "eval_samples_per_second": 18.447,
-      "eval_steps_per_second": 10.249,
-      "step": 90
-    },
-    {
-      "epoch": 4.523809523809524,
-      "grad_norm": 2.296875,
-      "learning_rate": 4.3809523809523815e-06,
-      "log_odds_chosen": 4.556994438171387,
-      "log_odds_ratio": -0.04522204026579857,
-      "logits/chosen": -0.8442004919052124,
-      "logits/rejected": -0.8662538528442383,
-      "logps/chosen": -0.364029198884964,
-      "logps/rejected": -3.7180163860321045,
-      "loss": 0.1237,
-      "nll_loss": 0.11919407546520233,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03640292212367058,
-      "rewards/margins": 0.33539873361587524,
-      "rewards/rejected": -0.3718016445636749,
-      "step": 95
-    },
-    {
-      "epoch": 4.523809523809524,
-      "eval_log_odds_chosen": 4.323246002197266,
-      "eval_log_odds_ratio": -0.03854605555534363,
-      "eval_logits/chosen": -0.6883870363235474,
-      "eval_logits/rejected": -0.7122835516929626,
-      "eval_logps/chosen": -0.5690239667892456,
-      "eval_logps/rejected": -4.033358573913574,
-      "eval_loss": 0.14968888461589813,
-      "eval_nll_loss": 0.1387859284877777,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05690239742398262,
-      "eval_rewards/margins": 0.34643349051475525,
-      "eval_rewards/rejected": -0.4033358693122864,
-      "eval_runtime": 0.487,
-      "eval_samples_per_second": 18.48,
-      "eval_steps_per_second": 10.267,
-      "step": 95
-    },
-    {
-      "epoch": 4.761904761904762,
-      "grad_norm": 2.296875,
-      "learning_rate": 4.19047619047619e-06,
-      "log_odds_chosen": 4.853901386260986,
-      "log_odds_ratio": -0.02733095921576023,
-      "logits/chosen": -0.6483007669448853,
-      "logits/rejected": -0.6531607508659363,
-      "logps/chosen": -0.37110140919685364,
-      "logps/rejected": -3.995234727859497,
-      "loss": 0.1098,
-      "nll_loss": 0.10703416168689728,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03711014240980148,
-      "rewards/margins": 0.36241334676742554,
-      "rewards/rejected": -0.39952346682548523,
-      "step": 100
-    },
-    {
-      "epoch": 4.761904761904762,
-      "eval_log_odds_chosen": 4.428193092346191,
-      "eval_log_odds_ratio": -0.03602977469563484,
-      "eval_logits/chosen": -0.7133967876434326,
-      "eval_logits/rejected": -0.7346823215484619,
-      "eval_logps/chosen": -0.561062753200531,
-      "eval_logps/rejected": -4.115842342376709,
-      "eval_loss": 0.14537306129932404,
-      "eval_nll_loss": 0.13508550822734833,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05610628053545952,
-      "eval_rewards/margins": 0.3554779589176178,
-      "eval_rewards/rejected": -0.4115842282772064,
-      "eval_runtime": 0.4881,
-      "eval_samples_per_second": 18.439,
-      "eval_steps_per_second": 10.244,
-      "step": 100
-    },
-    {
-      "epoch": 5.0,
-      "grad_norm": 3.28125,
-      "learning_rate": 4e-06,
-      "log_odds_chosen": 5.169442176818848,
-      "log_odds_ratio": -0.01285733561962843,
-      "logits/chosen": -0.5451700091362,
-      "logits/rejected": -0.5567004680633545,
-      "logps/chosen": -0.3075657784938812,
-      "logps/rejected": -4.096129417419434,
-      "loss": 0.0781,
-      "nll_loss": 0.0768006294965744,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.030756577849388123,
-      "rewards/margins": 0.378856360912323,
-      "rewards/rejected": -0.4096129536628723,
-      "step": 105
-    },
-    {
-      "epoch": 5.0,
-      "eval_log_odds_chosen": 4.601005554199219,
-      "eval_log_odds_ratio": -0.032399993389844894,
-      "eval_logits/chosen": -0.6537272930145264,
-      "eval_logits/rejected": -0.6651639938354492,
-      "eval_logps/chosen": -0.5671016573905945,
-      "eval_logps/rejected": -4.295802116394043,
-      "eval_loss": 0.14492884278297424,
-      "eval_nll_loss": 0.13551345467567444,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05671016499400139,
-      "eval_rewards/margins": 0.3728700280189514,
-      "eval_rewards/rejected": -0.4295802116394043,
-      "eval_runtime": 0.4872,
-      "eval_samples_per_second": 18.471,
-      "eval_steps_per_second": 10.262,
-      "step": 105
-    },
-    {
-      "epoch": 5.238095238095238,
-      "grad_norm": 2.09375,
-      "learning_rate": 3.809523809523809e-06,
-      "log_odds_chosen": 5.452701568603516,
-      "log_odds_ratio": -0.01384773850440979,
-      "logits/chosen": -0.6513525247573853,
-      "logits/rejected": -0.6465297937393188,
-      "logps/chosen": -0.3701745569705963,
-      "logps/rejected": -4.411220550537109,
-      "loss": 0.0986,
-      "nll_loss": 0.09724441170692444,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03701745718717575,
-      "rewards/margins": 0.4041045308113098,
-      "rewards/rejected": -0.44112199544906616,
-      "step": 110
-    },
-    {
-      "epoch": 5.238095238095238,
-      "eval_log_odds_chosen": 4.495316505432129,
-      "eval_log_odds_ratio": -0.034047432243824005,
-      "eval_logits/chosen": -0.731423020362854,
-      "eval_logits/rejected": -0.7520760297775269,
-      "eval_logps/chosen": -0.5562735199928284,
-      "eval_logps/rejected": -4.164623260498047,
-      "eval_loss": 0.1418984979391098,
-      "eval_nll_loss": 0.13248108327388763,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.055627357214689255,
-      "eval_rewards/margins": 0.3608350157737732,
-      "eval_rewards/rejected": -0.41646236181259155,
-      "eval_runtime": 0.4879,
-      "eval_samples_per_second": 18.447,
-      "eval_steps_per_second": 10.248,
-      "step": 110
-    },
-    {
-      "epoch": 5.476190476190476,
-      "grad_norm": 3.703125,
-      "learning_rate": 3.619047619047619e-06,
-      "log_odds_chosen": 5.597304344177246,
-      "log_odds_ratio": -0.013377921655774117,
-      "logits/chosen": -0.6798397302627563,
-      "logits/rejected": -0.7005301713943481,
-      "logps/chosen": -0.3014984726905823,
-      "logps/rejected": -4.4406232833862305,
-      "loss": 0.0853,
-      "nll_loss": 0.08393532782793045,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03014984354376793,
-      "rewards/margins": 0.41391244530677795,
-      "rewards/rejected": -0.44406232237815857,
-      "step": 115
-    },
-    {
-      "epoch": 5.476190476190476,
-      "eval_log_odds_chosen": 4.5586256980896,
-      "eval_log_odds_ratio": -0.03254225105047226,
-      "eval_logits/chosen": -0.6377931833267212,
-      "eval_logits/rejected": -0.6543766260147095,
-      "eval_logps/chosen": -0.5523817539215088,
-      "eval_logps/rejected": -4.2158522605896,
-      "eval_loss": 0.14017952978610992,
-      "eval_nll_loss": 0.1309087872505188,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05523817613720894,
-      "eval_rewards/margins": 0.366347074508667,
-      "eval_rewards/rejected": -0.4215852618217468,
-      "eval_runtime": 0.4911,
-      "eval_samples_per_second": 18.328,
-      "eval_steps_per_second": 10.182,
-      "step": 115
-    },
-    {
-      "epoch": 5.714285714285714,
-      "grad_norm": 2.140625,
-      "learning_rate": 3.428571428571428e-06,
-      "log_odds_chosen": 5.031613826751709,
-      "log_odds_ratio": -0.019640957936644554,
-      "logits/chosen": -0.4365396499633789,
-      "logits/rejected": -0.4198831617832184,
-      "logps/chosen": -0.31068772077560425,
-      "logps/rejected": -3.9459738731384277,
-      "loss": 0.0867,
-      "nll_loss": 0.08475493639707565,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.031068775802850723,
-      "rewards/margins": 0.36352860927581787,
-      "rewards/rejected": -0.3945973813533783,
-      "step": 120
-    },
-    {
-      "epoch": 5.714285714285714,
-      "eval_log_odds_chosen": 4.671214580535889,
-      "eval_log_odds_ratio": -0.03062080405652523,
-      "eval_logits/chosen": -0.5378702282905579,
-      "eval_logits/rejected": -0.5502060651779175,
-      "eval_logps/chosen": -0.5473546981811523,
-      "eval_logps/rejected": -4.3087615966796875,
-      "eval_loss": 0.1385972797870636,
-      "eval_nll_loss": 0.12939123809337616,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.054735470563173294,
-      "eval_rewards/margins": 0.37614068388938904,
-      "eval_rewards/rejected": -0.4308761656284332,
-      "eval_runtime": 0.4883,
-      "eval_samples_per_second": 18.431,
-      "eval_steps_per_second": 10.24,
-      "step": 120
-    },
-    {
-      "epoch": 5.9523809523809526,
-      "grad_norm": 1.9765625,
-      "learning_rate": 3.238095238095238e-06,
-      "log_odds_chosen": 4.888375282287598,
-      "log_odds_ratio": -0.0290891882032156,
-      "logits/chosen": -0.5597335696220398,
-      "logits/rejected": -0.5532962083816528,
-      "logps/chosen": -0.32100874185562134,
-      "logps/rejected": -3.9171700477600098,
-      "loss": 0.0829,
-      "nll_loss": 0.07996629178524017,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03210087865591049,
-      "rewards/margins": 0.35961610078811646,
-      "rewards/rejected": -0.39171695709228516,
-      "step": 125
-    },
-    {
-      "epoch": 5.9523809523809526,
-      "eval_log_odds_chosen": 4.70390510559082,
-      "eval_log_odds_ratio": -0.029467348009347916,
-      "eval_logits/chosen": -0.6945549249649048,
-      "eval_logits/rejected": -0.7163381576538086,
-      "eval_logps/chosen": -0.5340889096260071,
-      "eval_logps/rejected": -4.305633544921875,
-      "eval_loss": 0.13552479445934296,
-      "eval_nll_loss": 0.1264742910861969,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05340889096260071,
-      "eval_rewards/margins": 0.3771544396877289,
-      "eval_rewards/rejected": -0.430563360452652,
-      "eval_runtime": 0.4874,
-      "eval_samples_per_second": 18.464,
-      "eval_steps_per_second": 10.258,
-      "step": 125
-    },
-    {
-      "epoch": 6.190476190476191,
-      "grad_norm": 2.15625,
-      "learning_rate": 3.0476190476190473e-06,
-      "log_odds_chosen": 5.590579032897949,
-      "log_odds_ratio": -0.018792379647493362,
-      "logits/chosen": -0.6233000159263611,
-      "logits/rejected": -0.6280890107154846,
-      "logps/chosen": -0.2965574264526367,
-      "logps/rejected": -4.412256240844727,
-      "loss": 0.0921,
-      "nll_loss": 0.09021884948015213,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02965574339032173,
-      "rewards/margins": 0.41156989336013794,
-      "rewards/rejected": -0.4412256181240082,
-      "step": 130
-    },
-    {
-      "epoch": 6.190476190476191,
-      "eval_log_odds_chosen": 4.838481903076172,
-      "eval_log_odds_ratio": -0.02710099145770073,
-      "eval_logits/chosen": -0.6229408383369446,
-      "eval_logits/rejected": -0.6341615319252014,
-      "eval_logps/chosen": -0.5473824739456177,
-      "eval_logps/rejected": -4.465917587280273,
-      "eval_loss": 0.13664411008358002,
-      "eval_nll_loss": 0.12817898392677307,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05473824217915535,
-      "eval_rewards/margins": 0.3918535113334656,
-      "eval_rewards/rejected": -0.4465917646884918,
-      "eval_runtime": 0.4867,
-      "eval_samples_per_second": 18.492,
-      "eval_steps_per_second": 10.273,
-      "step": 130
-    },
-    {
-      "epoch": 6.428571428571429,
-      "grad_norm": 3.359375,
-      "learning_rate": 2.8571428571428573e-06,
-      "log_odds_chosen": 5.753912925720215,
-      "log_odds_ratio": -0.014670786447823048,
-      "logits/chosen": -0.5793005228042603,
-      "logits/rejected": -0.5792975425720215,
-      "logps/chosen": -0.29233574867248535,
-      "logps/rejected": -4.592888355255127,
-      "loss": 0.0874,
-      "nll_loss": 0.08597152680158615,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.029233578592538834,
-      "rewards/margins": 0.4300552010536194,
-      "rewards/rejected": -0.4592887759208679,
-      "step": 135
-    },
-    {
-      "epoch": 6.428571428571429,
-      "eval_log_odds_chosen": 4.875422477722168,
-      "eval_log_odds_ratio": -0.026727139949798584,
-      "eval_logits/chosen": -0.6263194680213928,
-      "eval_logits/rejected": -0.6402261257171631,
-      "eval_logps/chosen": -0.5489826798439026,
-      "eval_logps/rejected": -4.507485866546631,
-      "eval_loss": 0.13577227294445038,
-      "eval_nll_loss": 0.1276685744524002,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05489826202392578,
-      "eval_rewards/margins": 0.3958503305912018,
-      "eval_rewards/rejected": -0.45074859261512756,
-      "eval_runtime": 0.4882,
-      "eval_samples_per_second": 18.436,
-      "eval_steps_per_second": 10.242,
-      "step": 135
-    },
-    {
-      "epoch": 6.666666666666667,
-      "grad_norm": 2.109375,
-      "learning_rate": 2.6666666666666664e-06,
-      "log_odds_chosen": 5.3087663650512695,
-      "log_odds_ratio": -0.024162959307432175,
-      "logits/chosen": -0.5501323938369751,
-      "logits/rejected": -0.543838381767273,
-      "logps/chosen": -0.33894363045692444,
-      "logps/rejected": -4.1651811599731445,
-      "loss": 0.082,
-      "nll_loss": 0.07960663735866547,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.0338943675160408,
-      "rewards/margins": 0.38262373208999634,
-      "rewards/rejected": -0.41651806235313416,
-      "step": 140
-    },
-    {
-      "epoch": 6.666666666666667,
-      "eval_log_odds_chosen": 4.799313545227051,
-      "eval_log_odds_ratio": -0.027702342718839645,
-      "eval_logits/chosen": -0.677030622959137,
-      "eval_logits/rejected": -0.6918050646781921,
-      "eval_logps/chosen": -0.5406588315963745,
-      "eval_logps/rejected": -4.412304878234863,
-      "eval_loss": 0.13464586436748505,
-      "eval_nll_loss": 0.12628625333309174,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05406588315963745,
-      "eval_rewards/margins": 0.3871646523475647,
-      "eval_rewards/rejected": -0.4412304759025574,
-      "eval_runtime": 0.4867,
-      "eval_samples_per_second": 18.49,
-      "eval_steps_per_second": 10.272,
-      "step": 140
-    },
-    {
-      "epoch": 6.904761904761905,
-      "grad_norm": 3.234375,
-      "learning_rate": 2.476190476190476e-06,
-      "log_odds_chosen": 5.5894694328308105,
-      "log_odds_ratio": -0.010893247090280056,
-      "logits/chosen": -0.5942927002906799,
-      "logits/rejected": -0.5888161659240723,
-      "logps/chosen": -0.2951999306678772,
-      "logps/rejected": -4.442993640899658,
-      "loss": 0.0858,
-      "nll_loss": 0.0846717581152916,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02951999567449093,
-      "rewards/margins": 0.4147793650627136,
-      "rewards/rejected": -0.4442993700504303,
-      "step": 145
-    },
-    {
-      "epoch": 6.904761904761905,
-      "eval_log_odds_chosen": 4.838164329528809,
-      "eval_log_odds_ratio": -0.02697155997157097,
-      "eval_logits/chosen": -0.5802319645881653,
-      "eval_logits/rejected": -0.5902702808380127,
-      "eval_logps/chosen": -0.5424883961677551,
-      "eval_logps/rejected": -4.452386856079102,
-      "eval_loss": 0.13433685898780823,
-      "eval_nll_loss": 0.12595266103744507,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05424883961677551,
-      "eval_rewards/margins": 0.39098984003067017,
-      "eval_rewards/rejected": -0.4452386796474457,
-      "eval_runtime": 0.4873,
-      "eval_samples_per_second": 18.47,
-      "eval_steps_per_second": 10.261,
-      "step": 145
-    },
-    {
-      "epoch": 7.142857142857143,
-      "grad_norm": 1.9453125,
-      "learning_rate": 2.2857142857142856e-06,
-      "log_odds_chosen": 5.92537784576416,
-      "log_odds_ratio": -0.014473943039774895,
-      "logits/chosen": -0.5114859938621521,
-      "logits/rejected": -0.5051178336143494,
-      "logps/chosen": -0.25217267870903015,
-      "logps/rejected": -4.5697736740112305,
-      "loss": 0.0762,
-      "nll_loss": 0.07474460452795029,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.025217268615961075,
-      "rewards/margins": 0.4317600727081299,
-      "rewards/rejected": -0.45697736740112305,
-      "step": 150
-    },
-    {
-      "epoch": 7.142857142857143,
-      "eval_log_odds_chosen": 4.879807949066162,
-      "eval_log_odds_ratio": -0.02607133984565735,
-      "eval_logits/chosen": -0.6277867555618286,
-      "eval_logits/rejected": -0.6383845806121826,
-      "eval_logps/chosen": -0.5392376184463501,
-      "eval_logps/rejected": -4.4825334548950195,
-      "eval_loss": 0.13314732909202576,
-      "eval_nll_loss": 0.12492867559194565,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05392376333475113,
-      "eval_rewards/margins": 0.39432960748672485,
-      "eval_rewards/rejected": -0.448253333568573,
-      "eval_runtime": 0.487,
-      "eval_samples_per_second": 18.479,
-      "eval_steps_per_second": 10.266,
-      "step": 150
-    },
-    {
-      "epoch": 7.380952380952381,
-      "grad_norm": 2.234375,
-      "learning_rate": 2.095238095238095e-06,
-      "log_odds_chosen": 5.681746959686279,
-      "log_odds_ratio": -0.008144749328494072,
-      "logits/chosen": -0.556106448173523,
-      "logits/rejected": -0.5360628366470337,
-      "logps/chosen": -0.2713228166103363,
-      "logps/rejected": -4.389334678649902,
-      "loss": 0.0704,
-      "nll_loss": 0.06960402429103851,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02713228203356266,
-      "rewards/margins": 0.41180118918418884,
-      "rewards/rejected": -0.43893352150917053,
-      "step": 155
-    },
-    {
-      "epoch": 7.380952380952381,
-      "eval_log_odds_chosen": 4.914063930511475,
-      "eval_log_odds_ratio": -0.0254636462777853,
-      "eval_logits/chosen": -0.6259689331054688,
-      "eval_logits/rejected": -0.6361022591590881,
-      "eval_logps/chosen": -0.5356873273849487,
-      "eval_logps/rejected": -4.5052995681762695,
-      "eval_loss": 0.13209427893161774,
-      "eval_nll_loss": 0.12397398054599762,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05356873944401741,
-      "eval_rewards/margins": 0.39696118235588074,
-      "eval_rewards/rejected": -0.45052990317344666,
-      "eval_runtime": 0.4875,
-      "eval_samples_per_second": 18.463,
-      "eval_steps_per_second": 10.257,
-      "step": 155
-    },
-    {
-      "epoch": 7.619047619047619,
-      "grad_norm": 2.34375,
-      "learning_rate": 1.9047619047619045e-06,
-      "log_odds_chosen": 5.47554874420166,
-      "log_odds_ratio": -0.012598132714629173,
-      "logits/chosen": -0.5787724256515503,
-      "logits/rejected": -0.575463593006134,
-      "logps/chosen": -0.26679569482803345,
-      "logps/rejected": -4.207664489746094,
-      "loss": 0.078,
-      "nll_loss": 0.07674238830804825,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.026679569855332375,
-      "rewards/margins": 0.39408689737319946,
-      "rewards/rejected": -0.4207665026187897,
-      "step": 160
-    },
-    {
-      "epoch": 7.619047619047619,
-      "eval_log_odds_chosen": 4.950940132141113,
-      "eval_log_odds_ratio": -0.024774912744760513,
-      "eval_logits/chosen": -0.6380746960639954,
-      "eval_logits/rejected": -0.6485334038734436,
-      "eval_logps/chosen": -0.5350639820098877,
-      "eval_logps/rejected": -4.538556098937988,
-      "eval_loss": 0.13170303404331207,
-      "eval_nll_loss": 0.12376340478658676,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05350639671087265,
-      "eval_rewards/margins": 0.4003491997718811,
-      "eval_rewards/rejected": -0.45385560393333435,
-      "eval_runtime": 0.4878,
-      "eval_samples_per_second": 18.45,
-      "eval_steps_per_second": 10.25,
-      "step": 160
-    },
-    {
-      "epoch": 7.857142857142857,
-      "grad_norm": 2.390625,
-      "learning_rate": 1.714285714285714e-06,
-      "log_odds_chosen": 5.9162702560424805,
-      "log_odds_ratio": -0.013311143033206463,
-      "logits/chosen": -0.547685980796814,
-      "logits/rejected": -0.5555287003517151,
-      "logps/chosen": -0.27488571405410767,
-      "logps/rejected": -4.701210021972656,
-      "loss": 0.0771,
-      "nll_loss": 0.07575251162052155,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.027488570660352707,
-      "rewards/margins": 0.44263243675231934,
-      "rewards/rejected": -0.4701210558414459,
-      "step": 165
-    },
-    {
-      "epoch": 7.857142857142857,
-      "eval_log_odds_chosen": 4.9923577308654785,
-      "eval_log_odds_ratio": -0.024206604808568954,
-      "eval_logits/chosen": -0.6115527749061584,
-      "eval_logits/rejected": -0.6203628778457642,
-      "eval_logps/chosen": -0.5354728102684021,
-      "eval_logps/rejected": -4.578566551208496,
-      "eval_loss": 0.13151198625564575,
-      "eval_nll_loss": 0.12378431856632233,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05354728177189827,
-      "eval_rewards/margins": 0.40430936217308044,
-      "eval_rewards/rejected": -0.4578566551208496,
-      "eval_runtime": 0.4877,
-      "eval_samples_per_second": 18.454,
-      "eval_steps_per_second": 10.252,
-      "step": 165
-    },
-    {
-      "epoch": 8.095238095238095,
-      "grad_norm": 1.8359375,
-      "learning_rate": 1.5238095238095236e-06,
-      "log_odds_chosen": 5.4759697914123535,
-      "log_odds_ratio": -0.030720766633749008,
-      "logits/chosen": -0.5306688547134399,
-      "logits/rejected": -0.5217020511627197,
-      "logps/chosen": -0.33474162220954895,
-      "logps/rejected": -4.392602920532227,
-      "loss": 0.0851,
-      "nll_loss": 0.08205311000347137,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.0334741584956646,
-      "rewards/margins": 0.4057861268520355,
-      "rewards/rejected": -0.439260333776474,
-      "step": 170
-    },
-    {
-      "epoch": 8.095238095238095,
-      "eval_log_odds_chosen": 4.982872486114502,
-      "eval_log_odds_ratio": -0.02418256737291813,
-      "eval_logits/chosen": -0.6068294644355774,
-      "eval_logits/rejected": -0.6163716316223145,
-      "eval_logps/chosen": -0.5354769825935364,
-      "eval_logps/rejected": -4.566414833068848,
-      "eval_loss": 0.13132263720035553,
-      "eval_nll_loss": 0.1235472559928894,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.053547702729701996,
-      "eval_rewards/margins": 0.4030938148498535,
-      "eval_rewards/rejected": -0.4566414952278137,
-      "eval_runtime": 0.4867,
-      "eval_samples_per_second": 18.491,
-      "eval_steps_per_second": 10.273,
-      "step": 170
-    },
-    {
-      "epoch": 8.333333333333334,
-      "grad_norm": 3.0,
-      "learning_rate": 1.3333333333333332e-06,
-      "log_odds_chosen": 6.1932477951049805,
-      "log_odds_ratio": -0.006689209491014481,
-      "logits/chosen": -0.4947226941585541,
-      "logits/rejected": -0.506854772567749,
-      "logps/chosen": -0.25427430868148804,
-      "logps/rejected": -4.840688705444336,
-      "loss": 0.0654,
-      "nll_loss": 0.06476090848445892,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.025427427142858505,
-      "rewards/margins": 0.4586414694786072,
-      "rewards/rejected": -0.4840688705444336,
-      "step": 175
-    },
-    {
-      "epoch": 8.333333333333334,
-      "eval_log_odds_chosen": 4.968461513519287,
-      "eval_log_odds_ratio": -0.02425793744623661,
-      "eval_logits/chosen": -0.6205912828445435,
-      "eval_logits/rejected": -0.6322892904281616,
-      "eval_logps/chosen": -0.531274139881134,
-      "eval_logps/rejected": -4.542240142822266,
-      "eval_loss": 0.13052870333194733,
-      "eval_nll_loss": 0.12270841747522354,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05312741547822952,
-      "eval_rewards/margins": 0.4010965824127197,
-      "eval_rewards/rejected": -0.45422402024269104,
-      "eval_runtime": 0.4884,
-      "eval_samples_per_second": 18.429,
-      "eval_steps_per_second": 10.238,
-      "step": 175
-    },
-    {
-      "epoch": 8.571428571428571,
-      "grad_norm": 2.140625,
-      "learning_rate": 1.1428571428571428e-06,
-      "log_odds_chosen": 6.075676918029785,
-      "log_odds_ratio": -0.00910225696861744,
-      "logits/chosen": -0.58963543176651,
-      "logits/rejected": -0.56965172290802,
-      "logps/chosen": -0.3160817623138428,
-      "logps/rejected": -4.802862167358398,
-      "loss": 0.082,
-      "nll_loss": 0.08105450868606567,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03160817548632622,
-      "rewards/margins": 0.4486781060695648,
-      "rewards/rejected": -0.48028627038002014,
-      "step": 180
-    },
-    {
-      "epoch": 8.571428571428571,
-      "eval_log_odds_chosen": 4.932239532470703,
-      "eval_log_odds_ratio": -0.024737322703003883,
-      "eval_logits/chosen": -0.6294423937797546,
-      "eval_logits/rejected": -0.6397351026535034,
-      "eval_logps/chosen": -0.5322803258895874,
-      "eval_logps/rejected": -4.510106563568115,
-      "eval_loss": 0.13084720075130463,
-      "eval_nll_loss": 0.12291731685400009,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05322803929448128,
-      "eval_rewards/margins": 0.3977826237678528,
-      "eval_rewards/rejected": -0.45101064443588257,
-      "eval_runtime": 0.489,
-      "eval_samples_per_second": 18.406,
-      "eval_steps_per_second": 10.226,
-      "step": 180
-    },
-    {
-      "epoch": 8.80952380952381,
-      "grad_norm": 2.015625,
-      "learning_rate": 9.523809523809522e-07,
-      "log_odds_chosen": 5.602750778198242,
-      "log_odds_ratio": -0.011134130880236626,
-      "logits/chosen": -0.5277374982833862,
-      "logits/rejected": -0.5226815938949585,
-      "logps/chosen": -0.27227240800857544,
-      "logps/rejected": -4.354727745056152,
-      "loss": 0.0724,
-      "nll_loss": 0.07125753164291382,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.027227237820625305,
-      "rewards/margins": 0.4082455635070801,
-      "rewards/rejected": -0.43547287583351135,
-      "step": 185
-    },
-    {
-      "epoch": 8.80952380952381,
-      "eval_log_odds_chosen": 4.966063499450684,
-      "eval_log_odds_ratio": -0.024260815232992172,
-      "eval_logits/chosen": -0.5997673273086548,
-      "eval_logits/rejected": -0.6082264184951782,
-      "eval_logps/chosen": -0.5307241082191467,
-      "eval_logps/rejected": -4.538777828216553,
-      "eval_loss": 0.13055673241615295,
-      "eval_nll_loss": 0.1226615160703659,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05307241529226303,
-      "eval_rewards/margins": 0.40080541372299194,
-      "eval_rewards/rejected": -0.45387783646583557,
-      "eval_runtime": 0.4869,
-      "eval_samples_per_second": 18.483,
-      "eval_steps_per_second": 10.268,
-      "step": 185
-    },
-    {
-      "epoch": 9.047619047619047,
-      "grad_norm": 2.21875,
-      "learning_rate": 7.619047619047618e-07,
-      "log_odds_chosen": 5.537537574768066,
-      "log_odds_ratio": -0.018197722733020782,
-      "logits/chosen": -0.5673882365226746,
-      "logits/rejected": -0.5563878417015076,
-      "logps/chosen": -0.26666221022605896,
-      "logps/rejected": -4.266029357910156,
-      "loss": 0.0836,
-      "nll_loss": 0.08179894089698792,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.026666224002838135,
-      "rewards/margins": 0.3999367356300354,
-      "rewards/rejected": -0.42660292983055115,
-      "step": 190
-    },
-    {
-      "epoch": 9.047619047619047,
-      "eval_log_odds_chosen": 4.971249103546143,
-      "eval_log_odds_ratio": -0.024101996794342995,
-      "eval_logits/chosen": -0.575426459312439,
-      "eval_logits/rejected": -0.5841690897941589,
-      "eval_logps/chosen": -0.5333296656608582,
-      "eval_logps/rejected": -4.551214694976807,
-      "eval_loss": 0.13083776831626892,
-      "eval_nll_loss": 0.12306801974773407,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.053332965821027756,
-      "eval_rewards/margins": 0.40178847312927246,
-      "eval_rewards/rejected": -0.4551214277744293,
-      "eval_runtime": 0.4872,
-      "eval_samples_per_second": 18.474,
-      "eval_steps_per_second": 10.264,
-      "step": 190
-    },
-    {
-      "epoch": 9.285714285714286,
-      "grad_norm": 3.3125,
-      "learning_rate": 5.714285714285714e-07,
-      "log_odds_chosen": 5.82122802734375,
-      "log_odds_ratio": -0.010938870720565319,
-      "logits/chosen": -0.4943665862083435,
-      "logits/rejected": -0.47990670800209045,
-      "logps/chosen": -0.2986515164375305,
-      "logps/rejected": -4.486918926239014,
-      "loss": 0.0702,
-      "nll_loss": 0.06913810223340988,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02986515499651432,
-      "rewards/margins": 0.41882675886154175,
-      "rewards/rejected": -0.4486919343471527,
-      "step": 195
-    },
-    {
-      "epoch": 9.285714285714286,
-      "eval_log_odds_chosen": 4.972805500030518,
-      "eval_log_odds_ratio": -0.024383049458265305,
-      "eval_logits/chosen": -0.589435338973999,
-      "eval_logits/rejected": -0.5966291427612305,
-      "eval_logps/chosen": -0.5334832668304443,
-      "eval_logps/rejected": -4.552124977111816,
-      "eval_loss": 0.13075287640094757,
-      "eval_nll_loss": 0.12296122312545776,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.053348325192928314,
-      "eval_rewards/margins": 0.4018642008304596,
-      "eval_rewards/rejected": -0.4552125036716461,
-      "eval_runtime": 0.488,
-      "eval_samples_per_second": 18.444,
-      "eval_steps_per_second": 10.247,
-      "step": 195
-    },
-    {
-      "epoch": 9.523809523809524,
-      "grad_norm": 2.390625,
-      "learning_rate": 3.809523809523809e-07,
-      "log_odds_chosen": 5.9984025955200195,
-      "log_odds_ratio": -0.008187348023056984,
-      "logits/chosen": -0.534194827079773,
-      "logits/rejected": -0.5184012055397034,
-      "logps/chosen": -0.2566554546356201,
-      "logps/rejected": -4.710351467132568,
-      "loss": 0.0726,
-      "nll_loss": 0.07180726528167725,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02566554583609104,
-      "rewards/margins": 0.44536957144737244,
-      "rewards/rejected": -0.4710351526737213,
-      "step": 200
-    },
-    {
-      "epoch": 9.523809523809524,
-      "eval_log_odds_chosen": 4.969243049621582,
-      "eval_log_odds_ratio": -0.024293312802910805,
-      "eval_logits/chosen": -0.6059412956237793,
-      "eval_logits/rejected": -0.615134596824646,
-      "eval_logps/chosen": -0.5334844589233398,
-      "eval_logps/rejected": -4.546559810638428,
-      "eval_loss": 0.13070961833000183,
-      "eval_nll_loss": 0.12287895381450653,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.053348444402217865,
-      "eval_rewards/margins": 0.40130752325057983,
-      "eval_rewards/rejected": -0.4546559453010559,
-      "eval_runtime": 0.4875,
-      "eval_samples_per_second": 18.463,
-      "eval_steps_per_second": 10.257,
-      "step": 200
-    },
-    {
-      "epoch": 9.761904761904763,
-      "grad_norm": 2.640625,
-      "learning_rate": 1.9047619047619045e-07,
-      "log_odds_chosen": 5.707345485687256,
-      "log_odds_ratio": -0.023422162979841232,
-      "logits/chosen": -0.5711747407913208,
-      "logits/rejected": -0.5841129422187805,
-      "logps/chosen": -0.2618250250816345,
-      "logps/rejected": -4.388245105743408,
-      "loss": 0.0837,
-      "nll_loss": 0.08134286105632782,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02618250623345375,
-      "rewards/margins": 0.4126420021057129,
-      "rewards/rejected": -0.4388245642185211,
-      "step": 205
-    },
-    {
-      "epoch": 9.761904761904763,
-      "eval_log_odds_chosen": 4.974253177642822,
-      "eval_log_odds_ratio": -0.024108218029141426,
-      "eval_logits/chosen": -0.5955663919448853,
-      "eval_logits/rejected": -0.6041454672813416,
-      "eval_logps/chosen": -0.5337764024734497,
-      "eval_logps/rejected": -4.553771018981934,
-      "eval_loss": 0.13067038357257843,
-      "eval_nll_loss": 0.12289754301309586,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.05337763950228691,
-      "eval_rewards/margins": 0.40199947357177734,
-      "eval_rewards/rejected": -0.45537716150283813,
-      "eval_runtime": 0.487,
-      "eval_samples_per_second": 18.479,
-      "eval_steps_per_second": 10.266,
-      "step": 205
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 2.234375,
-      "learning_rate": 0.0,
-      "log_odds_chosen": 5.524815559387207,
-      "log_odds_ratio": -0.013400805182754993,
-      "logits/chosen": -0.5060779452323914,
-      "logits/rejected": -0.49261727929115295,
-      "logps/chosen": -0.2732623517513275,
-      "logps/rejected": -4.279727458953857,
-      "loss": 0.0796,
-      "nll_loss": 0.07821428030729294,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.027326231822371483,
-      "rewards/margins": 0.40064650774002075,
-      "rewards/rejected": -0.4279727041721344,
-      "step": 210
-    },
-    {
-      "epoch": 10.0,
-      "eval_log_odds_chosen": 4.966081619262695,
-      "eval_log_odds_ratio": -0.024325253441929817,
-      "eval_logits/chosen": -0.5945724248886108,
-      "eval_logits/rejected": -0.6053321957588196,
-      "eval_logps/chosen": -0.5324550867080688,
-      "eval_logps/rejected": -4.543835639953613,
-      "eval_loss": 0.13071957230567932,
-      "eval_nll_loss": 0.12293493747711182,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -0.053245507180690765,
-      "eval_rewards/margins": 0.4011380076408386,
-      "eval_rewards/rejected": -0.4543835520744324,
-      "eval_runtime": 0.4882,
-      "eval_samples_per_second": 18.434,
-      "eval_steps_per_second": 10.241,
-      "step": 210
-    },
-    {
-      "epoch": 10.0,
-      "step": 210,
       "total_flos": 0.0,
-      "train_loss": 0.3063473221801576,
-      "train_runtime": 348.5405,
-      "train_samples_per_second": 4.82,
-      "train_steps_per_second": 0.603
     }
   ],
-  "logging_steps": 5,
-  "max_steps": 210,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 500,

 {
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 8.0,
+  "eval_steps": 1,
+  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
+      "epoch": 0.4,
+      "grad_norm": 18.625,
+      "learning_rate": 7.599999999999999e-06,
+      "log_odds_chosen": 0.3293280005455017,
+      "log_odds_ratio": -0.545608401298523,
+      "logits/chosen": -0.22181883454322815,
+      "logits/rejected": -0.2948111891746521,
+      "logps/chosen": -1.9494528770446777,
+      "logps/rejected": -2.2380290031433105,
+      "loss": 1.8976,
+      "nll_loss": 1.8430625200271606,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.1949452906847,
+      "rewards/margins": 0.028857626020908356,
+      "rewards/rejected": -0.22380293905735016,
+      "step": 1
+    },
+    {
+      "epoch": 0.4,
+      "eval_log_odds_chosen": 0.3650469183921814,
+      "eval_log_odds_ratio": -0.5312943458557129,
+      "eval_logits/chosen": -0.11938808858394623,
+      "eval_logits/rejected": -0.15210816264152527,
+      "eval_logps/chosen": -1.7196087837219238,
+      "eval_logps/rejected": -2.026923179626465,
+      "eval_loss": 1.6382209062576294,
+      "eval_nll_loss": 1.585091471672058,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.17196084558963776,
+      "eval_rewards/margins": 0.030731473118066788,
+      "eval_rewards/rejected": -0.20269232988357544,
+      "eval_runtime": 0.9093,
+      "eval_samples_per_second": 19.795,
+      "eval_steps_per_second": 9.898,
+      "step": 1
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 8.125,
+      "learning_rate": 7.2e-06,
+      "log_odds_chosen": 0.3380122482776642,
+      "log_odds_ratio": -0.5430496335029602,
+      "logits/chosen": -0.07896450906991959,
+      "logits/rejected": -0.11844252794981003,
+      "logps/chosen": -1.65217125415802,
+      "logps/rejected": -1.9326075315475464,
+      "loss": 1.5573,
+      "nll_loss": 1.5030204057693481,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.16521713137626648,
+      "rewards/margins": 0.02804364264011383,
+      "rewards/rejected": -0.19326075911521912,
+      "step": 2
+    },
+    {
+      "epoch": 0.8,
+      "eval_log_odds_chosen": 0.39069631695747375,
+      "eval_log_odds_ratio": -0.5206953883171082,
+      "eval_logits/chosen": -0.10498537868261337,
+      "eval_logits/rejected": -0.136393204331398,
+      "eval_logps/chosen": -1.5854017734527588,
+      "eval_logps/rejected": -1.9058791399002075,
+      "eval_loss": 1.5333527326583862,
+      "eval_nll_loss": 1.4812833070755005,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.15854017436504364,
+      "eval_rewards/margins": 0.032047729939222336,
+      "eval_rewards/rejected": -0.19058789312839508,
+      "eval_runtime": 0.9117,
+      "eval_samples_per_second": 19.744,
+      "eval_steps_per_second": 9.872,
+      "step": 2
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 7.875,
+      "learning_rate": 6.799999999999999e-06,
+      "log_odds_chosen": 0.36618566513061523,
+      "log_odds_ratio": -0.5316208004951477,
+      "logits/chosen": -0.045309849083423615,
+      "logits/rejected": -0.09037788212299347,
+      "logps/chosen": -1.4718542098999023,
+      "logps/rejected": -1.7647374868392944,
+      "loss": 1.4427,
+      "nll_loss": 1.3894941806793213,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.14718542993068695,
+      "rewards/margins": 0.029288342222571373,
+      "rewards/rejected": -0.17647376656532288,
+      "step": 3
+    },
+    {
+      "epoch": 1.2,
+      "eval_log_odds_chosen": 0.41840454936027527,
+      "eval_log_odds_ratio": -0.5097466111183167,
+      "eval_logits/chosen": -0.10623180121183395,
+      "eval_logits/rejected": -0.13603489100933075,
+      "eval_logps/chosen": -1.4396543502807617,
+      "eval_logps/rejected": -1.7711676359176636,
+      "eval_loss": 1.4337514638900757,
+      "eval_nll_loss": 1.3827767372131348,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.1439654380083084,
+      "eval_rewards/margins": 0.03315134346485138,
+      "eval_rewards/rejected": -0.1771167814731598,
+      "eval_runtime": 0.9101,
+      "eval_samples_per_second": 19.778,
+      "eval_steps_per_second": 9.889,
+      "step": 3
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 7.78125,
+      "learning_rate": 6.4e-06,
+      "log_odds_chosen": 0.4042999744415283,
+      "log_odds_ratio": -0.5167055726051331,
+      "logits/chosen": -0.0431833378970623,
+      "logits/rejected": -0.07951641082763672,
+      "logps/chosen": -1.3549946546554565,
+      "logps/rejected": -1.6684811115264893,
+      "loss": 1.3493,
+      "nll_loss": 1.2975877523422241,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.13549946248531342,
+      "rewards/margins": 0.03134865313768387,
+      "rewards/rejected": -0.1668480932712555,
+      "step": 4
+    },
+    {
+      "epoch": 1.6,
+      "eval_log_odds_chosen": 0.44134366512298584,
+      "eval_log_odds_ratio": -0.5008935332298279,
+      "eval_logits/chosen": -0.104909747838974,
+      "eval_logits/rejected": -0.13491111993789673,
+      "eval_logps/chosen": -1.340553641319275,
+      "eval_logps/rejected": -1.6810719966888428,
+      "eval_loss": 1.3429497480392456,
+      "eval_nll_loss": 1.292860507965088,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.13405534625053406,
+      "eval_rewards/margins": 0.03405185043811798,
+      "eval_rewards/rejected": -0.16810721158981323,
+      "eval_runtime": 0.9119,
+      "eval_samples_per_second": 19.738,
+      "eval_steps_per_second": 9.869,
+      "step": 4
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 6.96875,
+      "learning_rate": 6e-06,
+      "log_odds_chosen": 0.4279482960700989,
+      "log_odds_ratio": -0.5070147514343262,
+      "logits/chosen": -0.034176260232925415,
+      "logits/rejected": -0.07206660509109497,
+      "logps/chosen": -1.2959173917770386,
+      "logps/rejected": -1.6209981441497803,
+      "loss": 1.2683,
+      "nll_loss": 1.2175886631011963,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.12959173321723938,
+      "rewards/margins": 0.03250807896256447,
+      "rewards/rejected": -0.16209980845451355,
       "step": 5
     },
     {
+      "epoch": 2.0,
+      "eval_log_odds_chosen": 0.46249422430992126,
+      "eval_log_odds_ratio": -0.49292659759521484,
+      "eval_logits/chosen": -0.11007735878229141,
+      "eval_logits/rejected": -0.14086602628231049,
+      "eval_logps/chosen": -1.2711644172668457,
+      "eval_logps/rejected": -1.6204769611358643,
+      "eval_loss": 1.2642947435379028,
+      "eval_nll_loss": 1.215002179145813,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.12711645662784576,
+      "eval_rewards/margins": 0.034931257367134094,
+      "eval_rewards/rejected": -0.16204769909381866,
+      "eval_runtime": 0.9131,
+      "eval_samples_per_second": 19.712,
+      "eval_steps_per_second": 9.856,
       "step": 5
     },
     {
+      "epoch": 2.4,
+      "grad_norm": 6.65625,
+      "learning_rate": 5.6e-06,
+      "log_odds_chosen": 0.4413740336894989,
+      "log_odds_ratio": -0.5025829672813416,
+      "logits/chosen": -0.026908639818429947,
+      "logits/rejected": -0.06323603540658951,
+      "logps/chosen": -1.2035048007965088,
+      "logps/rejected": -1.5302585363388062,
+      "loss": 1.1736,
+      "nll_loss": 1.1233787536621094,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.12035048753023148,
+      "rewards/margins": 0.03267538174986839,
+      "rewards/rejected": -0.15302586555480957,
+      "step": 6
+    },
+    {
+      "epoch": 2.4,
+      "eval_log_odds_chosen": 0.4787115454673767,
+      "eval_log_odds_ratio": -0.4869447946548462,
+      "eval_logits/chosen": -0.11934076249599457,
+      "eval_logits/rejected": -0.15017718076705933,
+      "eval_logps/chosen": -1.2190965414047241,
+      "eval_logps/rejected": -1.5743210315704346,
+      "eval_loss": 1.1919617652893066,
+      "eval_nll_loss": 1.1432671546936035,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.12190967053174973,
+      "eval_rewards/margins": 0.03552243858575821,
+      "eval_rewards/rejected": -0.15743210911750793,
+      "eval_runtime": 0.9096,
+      "eval_samples_per_second": 19.79,
+      "eval_steps_per_second": 9.895,
+      "step": 6
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 6.96875,
+      "learning_rate": 5.2e-06,
+      "log_odds_chosen": 0.46652132272720337,
+      "log_odds_ratio": -0.4945680797100067,
+      "logits/chosen": -0.0492391437292099,
+      "logits/rejected": -0.0838259607553482,
+      "logps/chosen": -1.1840012073516846,
+      "logps/rejected": -1.525024652481079,
+      "loss": 1.1212,
+      "nll_loss": 1.0717414617538452,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.11840011179447174,
+      "rewards/margins": 0.03410235792398453,
+      "rewards/rejected": -0.15250247716903687,
+      "step": 7
+    },
+    {
+      "epoch": 2.8,
+      "eval_log_odds_chosen": 0.4977823495864868,
+      "eval_log_odds_ratio": -0.4801054000854492,
+      "eval_logits/chosen": -0.12862297892570496,
+      "eval_logits/rejected": -0.16006678342819214,
+      "eval_logps/chosen": -1.1753649711608887,
+      "eval_logps/rejected": -1.5391558408737183,
+      "eval_loss": 1.1233677864074707,
+      "eval_nll_loss": 1.0753573179244995,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.1175365075469017,
+      "eval_rewards/margins": 0.03637908399105072,
+      "eval_rewards/rejected": -0.15391558408737183,
+      "eval_runtime": 0.9109,
+      "eval_samples_per_second": 19.76,
+      "eval_steps_per_second": 9.88,
+      "step": 7
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 6.84375,
+      "learning_rate": 4.8e-06,
+      "log_odds_chosen": 0.4919642210006714,
+      "log_odds_ratio": -0.48277872800827026,
+      "logits/chosen": -0.05812246352434158,
+      "logits/rejected": -0.0962948203086853,
+      "logps/chosen": -1.1183116436004639,
+      "logps/rejected": -1.4710171222686768,
+      "loss": 1.0518,
+      "nll_loss": 1.0035254955291748,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.11183115839958191,
+      "rewards/margins": 0.03527054935693741,
+      "rewards/rejected": -0.14710170030593872,
+      "step": 8
+    },
+    {
+      "epoch": 3.2,
+      "eval_log_odds_chosen": 0.5149489045143127,
+      "eval_log_odds_ratio": -0.47412246465682983,
+      "eval_logits/chosen": -0.14218762516975403,
+      "eval_logits/rejected": -0.17368356883525848,
+      "eval_logps/chosen": -1.1381663084030151,
+      "eval_logps/rejected": -1.5092874765396118,
+      "eval_loss": 1.0610299110412598,
+      "eval_nll_loss": 1.013617753982544,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.11381663382053375,
+      "eval_rewards/margins": 0.03711211308836937,
+      "eval_rewards/rejected": -0.15092875063419342,
+      "eval_runtime": 0.9136,
+      "eval_samples_per_second": 19.703,
+      "eval_steps_per_second": 9.852,
+      "step": 8
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 6.71875,
+      "learning_rate": 4.4e-06,
+      "log_odds_chosen": 0.5135414004325867,
+      "log_odds_ratio": -0.47593823075294495,
+      "logits/chosen": -0.06834974884986877,
+      "logits/rejected": -0.11046632379293442,
+      "logps/chosen": -1.0532824993133545,
+      "logps/rejected": -1.4117248058319092,
+      "loss": 0.9805,
+      "nll_loss": 0.9329336285591125,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.1053282618522644,
+      "rewards/margins": 0.035844214260578156,
+      "rewards/rejected": -0.14117246866226196,
+      "step": 9
+    },
+    {
+      "epoch": 3.6,
+      "eval_log_odds_chosen": 0.5320238471031189,
+      "eval_log_odds_ratio": -0.4681590497493744,
+      "eval_logits/chosen": -0.1644686907529831,
+      "eval_logits/rejected": -0.19686466455459595,
+      "eval_logps/chosen": -1.104932427406311,
+      "eval_logps/rejected": -1.4836636781692505,
+      "eval_loss": 1.0012433528900146,
+      "eval_nll_loss": 0.9544275999069214,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.1104932650923729,
+      "eval_rewards/margins": 0.03787311539053917,
+      "eval_rewards/rejected": -0.14836637675762177,
+      "eval_runtime": 0.9099,
+      "eval_samples_per_second": 19.783,
+      "eval_steps_per_second": 9.892,
+      "step": 9
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 6.75,
+      "learning_rate": 4e-06,
+      "log_odds_chosen": 0.5470705032348633,
+      "log_odds_ratio": -0.4658868908882141,
+      "logits/chosen": -0.09963471442461014,
+      "logits/rejected": -0.13952209055423737,
+      "logps/chosen": -1.0679322481155396,
+      "logps/rejected": -1.4557496309280396,
+      "loss": 0.9299,
+      "nll_loss": 0.8832955956459045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.10679321736097336,
+      "rewards/margins": 0.038781747221946716,
+      "rewards/rejected": -0.14557495713233948,
       "step": 10
     },
     {
+      "epoch": 4.0,
+      "eval_log_odds_chosen": 0.547681450843811,
+      "eval_log_odds_ratio": -0.46276018023490906,
+      "eval_logits/chosen": -0.1875133067369461,
+      "eval_logits/rejected": -0.22008682787418365,
+      "eval_logps/chosen": -1.07937490940094,
+      "eval_logps/rejected": -1.4652737379074097,
+      "eval_loss": 0.9495540261268616,
+      "eval_nll_loss": 0.9032779335975647,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10793750733137131,
+      "eval_rewards/margins": 0.03858988359570503,
+      "eval_rewards/rejected": -0.14652739465236664,
+      "eval_runtime": 0.9118,
+      "eval_samples_per_second": 19.742,
+      "eval_steps_per_second": 9.871,
       "step": 10
     },
     {
+      "epoch": 4.4,
+      "grad_norm": 6.5625,
+      "learning_rate": 3.6e-06,
+      "log_odds_chosen": 0.5115205645561218,
+      "log_odds_ratio": -0.4781632423400879,
+      "logits/chosen": -0.11988667398691177,
+      "logits/rejected": -0.15675179660320282,
+      "logps/chosen": -1.0269582271575928,
+      "logps/rejected": -1.3790360689163208,
+      "loss": 0.8761,
+      "nll_loss": 0.8282526135444641,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.10269583761692047,
+      "rewards/margins": 0.03520777449011803,
+      "rewards/rejected": -0.1379036009311676,
+      "step": 11
+    },
+    {
+      "epoch": 4.4,
+      "eval_log_odds_chosen": 0.560771107673645,
+      "eval_log_odds_ratio": -0.45837968587875366,
+      "eval_logits/chosen": -0.21046772599220276,
+      "eval_logits/rejected": -0.2431277632713318,
+      "eval_logps/chosen": -1.0591222047805786,
+      "eval_logps/rejected": -1.4509668350219727,
+      "eval_loss": 0.9070050120353699,
+      "eval_nll_loss": 0.8611669540405273,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10591220110654831,
+      "eval_rewards/margins": 0.03918447345495224,
+      "eval_rewards/rejected": -0.14509668946266174,
+      "eval_runtime": 0.9119,
+      "eval_samples_per_second": 19.739,
+      "eval_steps_per_second": 9.869,
+      "step": 11
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 6.5625,
+      "learning_rate": 3.2e-06,
+      "log_odds_chosen": 0.6133227944374084,
+      "log_odds_ratio": -0.4412252902984619,
+      "logits/chosen": -0.14904728531837463,
+      "logits/rejected": -0.1963028907775879,
+      "logps/chosen": -0.9898001551628113,
+      "logps/rejected": -1.4105660915374756,
+      "loss": 0.8337,
+      "nll_loss": 0.7895629405975342,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09898000955581665,
+      "rewards/margins": 0.04207659140229225,
+      "rewards/rejected": -0.1410566121339798,
+      "step": 12
+    },
+    {
+      "epoch": 4.8,
+      "eval_log_odds_chosen": 0.566861093044281,
+      "eval_log_odds_ratio": -0.45656245946884155,
+      "eval_logits/chosen": -0.22315236926078796,
+      "eval_logits/rejected": -0.2561546862125397,
+      "eval_logps/chosen": -1.0492280721664429,
+      "eval_logps/rejected": -1.443569302558899,
+      "eval_loss": 0.8863641023635864,
+      "eval_nll_loss": 0.8407078981399536,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.1049228087067604,
+      "eval_rewards/margins": 0.03943413123488426,
+      "eval_rewards/rejected": -0.14435693621635437,
+      "eval_runtime": 0.9164,
+      "eval_samples_per_second": 19.642,
+      "eval_steps_per_second": 9.821,
+      "step": 12
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 6.5,
+      "learning_rate": 2.8e-06,
+      "log_odds_chosen": 0.49919161200523376,
+      "log_odds_ratio": -0.4824044108390808,
+      "logits/chosen": -0.15154853463172913,
+      "logits/rejected": -0.18894340097904205,
+      "logps/chosen": -1.0274688005447388,
+      "logps/rejected": -1.3725156784057617,
+      "loss": 0.7975,
+      "nll_loss": 0.7492961883544922,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.1027468740940094,
+      "rewards/margins": 0.034504685550928116,
+      "rewards/rejected": -0.13725155591964722,
+      "step": 13
+    },
+    {
+      "epoch": 5.2,
+      "eval_log_odds_chosen": 0.57403564453125,
+      "eval_log_odds_ratio": -0.45407184958457947,
+      "eval_logits/chosen": -0.23968791961669922,
+      "eval_logits/rejected": -0.2726818025112152,
+      "eval_logps/chosen": -1.0405869483947754,
+      "eval_logps/rejected": -1.4385521411895752,
+      "eval_loss": 0.8664105534553528,
+      "eval_nll_loss": 0.8210033774375916,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10405868291854858,
+      "eval_rewards/margins": 0.03979651629924774,
+      "eval_rewards/rejected": -0.14385519921779633,
+      "eval_runtime": 0.9125,
+      "eval_samples_per_second": 19.726,
+      "eval_steps_per_second": 9.863,
+      "step": 13
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 6.40625,
+      "learning_rate": 2.4e-06,
+      "log_odds_chosen": 0.6314184665679932,
+      "log_odds_ratio": -0.4354270100593567,
+      "logits/chosen": -0.1857105791568756,
+      "logits/rejected": -0.22656874358654022,
+      "logps/chosen": -0.9448862075805664,
+      "logps/rejected": -1.3670051097869873,
+      "loss": 0.788,
+      "nll_loss": 0.7444556951522827,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09448862820863724,
+      "rewards/margins": 0.04221189767122269,
+      "rewards/rejected": -0.13670052587985992,
+      "step": 14
+    },
+    {
+      "epoch": 5.6,
+      "eval_log_odds_chosen": 0.5794126987457275,
+      "eval_log_odds_ratio": -0.4523561894893646,
+      "eval_logits/chosen": -0.2507224678993225,
+      "eval_logits/rejected": -0.2837482690811157,
+      "eval_logps/chosen": -1.0325850248336792,
+      "eval_logps/rejected": -1.4329302310943604,
+      "eval_loss": 0.8492475152015686,
+      "eval_nll_loss": 0.8040118217468262,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10325851291418076,
+      "eval_rewards/margins": 0.04003452509641647,
+      "eval_rewards/rejected": -0.14329302310943604,
+      "eval_runtime": 0.9113,
+      "eval_samples_per_second": 19.752,
+      "eval_steps_per_second": 9.876,
+      "step": 14
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 6.25,
+      "learning_rate": 2e-06,
+      "log_odds_chosen": 0.5837588906288147,
+      "log_odds_ratio": -0.45432496070861816,
+      "logits/chosen": -0.18676218390464783,
+      "logits/rejected": -0.23436766862869263,
+      "logps/chosen": -1.0190256834030151,
+      "logps/rejected": -1.4252986907958984,
+      "loss": 0.78,
+      "nll_loss": 0.7345477342605591,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.1019025593996048,
+      "rewards/margins": 0.040627315640449524,
+      "rewards/rejected": -0.14252987504005432,
       "step": 15
     },
     {
+      "epoch": 6.0,
+      "eval_log_odds_chosen": 0.5839331150054932,
+      "eval_log_odds_ratio": -0.45080792903900146,
+      "eval_logits/chosen": -0.26143890619277954,
+      "eval_logits/rejected": -0.29441285133361816,
+      "eval_logps/chosen": -1.0263959169387817,
+      "eval_logps/rejected": -1.4286550283432007,
+      "eval_loss": 0.8333800435066223,
+      "eval_nll_loss": 0.7882992625236511,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10263960063457489,
+      "eval_rewards/margins": 0.040225885808467865,
+      "eval_rewards/rejected": -0.14286547899246216,
+      "eval_runtime": 0.9128,
+      "eval_samples_per_second": 19.72,
+      "eval_steps_per_second": 9.86,
       "step": 15
     },
     {
+      "epoch": 6.4,
+      "grad_norm": 6.25,
+      "learning_rate": 1.6e-06,
+      "log_odds_chosen": 0.6216727495193481,
+      "log_odds_ratio": -0.4399999976158142,
+      "logits/chosen": -0.19661211967468262,
+      "logits/rejected": -0.24193710088729858,
+      "logps/chosen": -0.949080228805542,
+      "logps/rejected": -1.3682817220687866,
+      "loss": 0.7395,
+      "nll_loss": 0.6955283880233765,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09490802884101868,
+      "rewards/margins": 0.04192016273736954,
+      "rewards/rejected": -0.13682818412780762,
+      "step": 16
+    },
+    {
+      "epoch": 6.4,
+      "eval_log_odds_chosen": 0.5865243673324585,
+      "eval_log_odds_ratio": -0.44995343685150146,
+      "eval_logits/chosen": -0.2722480595111847,
+      "eval_logits/rejected": -0.30540305376052856,
+      "eval_logps/chosen": -1.0213567018508911,
+      "eval_logps/rejected": -1.4244111776351929,
+      "eval_loss": 0.821089506149292,
+      "eval_nll_loss": 0.7760941386222839,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10213566571474075,
+      "eval_rewards/margins": 0.040305450558662415,
+      "eval_rewards/rejected": -0.14244110882282257,
+      "eval_runtime": 0.9109,
+      "eval_samples_per_second": 19.76,
+      "eval_steps_per_second": 9.88,
+      "step": 16
+    },
+    {
+      "epoch": 6.8,
+      "grad_norm": 6.15625,
+      "learning_rate": 1.2e-06,
+      "log_odds_chosen": 0.5441011786460876,
+      "log_odds_ratio": -0.4661071002483368,
+      "logits/chosen": -0.20644214749336243,
+      "logits/rejected": -0.2489599585533142,
+      "logps/chosen": -1.002626657485962,
+      "logps/rejected": -1.3740018606185913,
+      "loss": 0.7446,
+      "nll_loss": 0.6980130076408386,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.10026266425848007,
+      "rewards/margins": 0.037137530744075775,
+      "rewards/rejected": -0.13740019500255585,
+      "step": 17
+    },
+    {
+      "epoch": 6.8,
+      "eval_log_odds_chosen": 0.5888115763664246,
+      "eval_log_odds_ratio": -0.44922754168510437,
+      "eval_logits/chosen": -0.2722025513648987,
+      "eval_logits/rejected": -0.30542224645614624,
+      "eval_logps/chosen": -1.0186693668365479,
+      "eval_logps/rejected": -1.4229329824447632,
+      "eval_loss": 0.8164036870002747,
+      "eval_nll_loss": 0.7714808583259583,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10186693072319031,
+      "eval_rewards/margins": 0.040426358580589294,
+      "eval_rewards/rejected": -0.1422932893037796,
+      "eval_runtime": 0.9099,
+      "eval_samples_per_second": 19.782,
+      "eval_steps_per_second": 9.891,
+      "step": 17
+    },
+    {
+      "epoch": 7.2,
+      "grad_norm": 6.125,
+      "learning_rate": 8e-07,
+      "log_odds_chosen": 0.603852391242981,
+      "log_odds_ratio": -0.44559258222579956,
+      "logits/chosen": -0.21281519532203674,
+      "logits/rejected": -0.2561994791030884,
+      "logps/chosen": -0.9362643957138062,
+      "logps/rejected": -1.3377454280853271,
+      "loss": 0.7518,
+      "nll_loss": 0.7072104811668396,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09362644702196121,
+      "rewards/margins": 0.040148116648197174,
+      "rewards/rejected": -0.1337745636701584,
+      "step": 18
+    },
+    {
+      "epoch": 7.2,
+      "eval_log_odds_chosen": 0.5903448462486267,
+      "eval_log_odds_ratio": -0.4487246870994568,
+      "eval_logits/chosen": -0.2774750292301178,
+      "eval_logits/rejected": -0.3105829358100891,
+      "eval_logps/chosen": -1.0175344944000244,
+      "eval_logps/rejected": -1.4226274490356445,
+      "eval_loss": 0.812524139881134,
+      "eval_nll_loss": 0.7676517963409424,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10175344347953796,
+      "eval_rewards/margins": 0.040509287267923355,
+      "eval_rewards/rejected": -0.1422627568244934,
+      "eval_runtime": 0.9145,
+      "eval_samples_per_second": 19.683,
+      "eval_steps_per_second": 9.841,
+      "step": 18
+    },
+    {
+      "epoch": 7.6,
+      "grad_norm": 6.21875,
+      "learning_rate": 4e-07,
+      "log_odds_chosen": 0.6179953217506409,
+      "log_odds_ratio": -0.4413047730922699,
+      "logits/chosen": -0.22949087619781494,
+      "logits/rejected": -0.27671945095062256,
+      "logps/chosen": -0.9877333045005798,
+      "logps/rejected": -1.4103630781173706,
+      "loss": 0.7431,
+      "nll_loss": 0.6990159749984741,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.09877333790063858,
+      "rewards/margins": 0.0422629676759243,
+      "rewards/rejected": -0.14103631675243378,
+      "step": 19
+    },
+    {
+      "epoch": 7.6,
+      "eval_log_odds_chosen": 0.5911502838134766,
+      "eval_log_odds_ratio": -0.44839149713516235,
+      "eval_logits/chosen": -0.276792049407959,
+      "eval_logits/rejected": -0.31038832664489746,
+      "eval_logps/chosen": -1.0161813497543335,
+      "eval_logps/rejected": -1.4216675758361816,
+      "eval_loss": 0.8106683492660522,
+      "eval_nll_loss": 0.7658291459083557,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10161812603473663,
+      "eval_rewards/margins": 0.04054862633347511,
+      "eval_rewards/rejected": -0.14216677844524384,
+      "eval_runtime": 0.9137,
+      "eval_samples_per_second": 19.701,
+      "eval_steps_per_second": 9.85,
+      "step": 19
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 6.15625,
+      "learning_rate": 0.0,
+      "log_odds_chosen": 0.578275740146637,
+      "log_odds_ratio": -0.45558279752731323,
+      "logits/chosen": -0.210488423705101,
+      "logits/rejected": -0.24888469278812408,
+      "logps/chosen": -0.9616943001747131,
+      "logps/rejected": -1.3508220911026,
+      "loss": 0.726,
+      "nll_loss": 0.6804530620574951,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.09616944193840027,
+      "rewards/margins": 0.038912780582904816,
+      "rewards/rejected": -0.1350822150707245,
       "step": 20
     },
     {
+      "epoch": 8.0,
+      "eval_log_odds_chosen": 0.5904660224914551,
+      "eval_log_odds_ratio": -0.44868627190589905,
+      "eval_logits/chosen": -0.27975308895111084,
+      "eval_logits/rejected": -0.3132215440273285,
+      "eval_logps/chosen": -1.0163097381591797,
+      "eval_logps/rejected": -1.4213618040084839,
+      "eval_loss": 0.8110137581825256,
+      "eval_nll_loss": 0.7661450505256653,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.10163097083568573,
+      "eval_rewards/margins": 0.04050520807504654,
+      "eval_rewards/rejected": -0.14213618636131287,
+      "eval_runtime": 0.9114,
+      "eval_samples_per_second": 19.75,
+      "eval_steps_per_second": 9.875,
       "step": 20
     },
     {
+      "epoch": 8.0,
+      "step": 20,
       "total_flos": 0.0,
+      "train_loss": 1.0276277005672454,
+      "train_runtime": 261.791,
+      "train_samples_per_second": 6.074,
+      "train_steps_per_second": 0.076
     }
   ],
+  "logging_steps": 1,
+  "max_steps": 20,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 500,

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6423ecd3ccaa89e0be1c437ad068819ded22a1d0439e0584f2de8d13eb08a962
 size 5496

 version https://git-lfs.github.com/spec/v1
+oid sha256:68a289fefa0539d7c070c4455fa974fae25943053b0f7d6f0a6b03a9832618f3
 size 5496