WSX
/

Qwen2.5-1.5B-Open-R1-GRPO-FC

@@ -26,7 +26,7 @@ print(output["generated_text"])
 ## Training procedure
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/wsx/grpo/runs/tjif0i4g)
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
@@ -34,9 +34,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 - TRL: 0.16.0.dev0
-- Transformers: 4.49.0.dev0
 - Pytorch: 2.5.1
-- Datasets: 3.3.0
 - Tokenizers: 0.21.0
 ## Citations

 ## Training procedure
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/wsx/grpo/runs/qjognkqg)
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
 ### Framework versions
 - TRL: 0.16.0.dev0
+- Transformers: 4.50.0.dev0
 - Pytorch: 2.5.1
+- Datasets: 3.3.1
 - Tokenizers: 0.21.0
 ## Citations

all_results.json CHANGED Viewed

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 20.984000422099324,
-    "train_runtime": 3511.8094,
-    "train_samples": 5316,
-    "train_samples_per_second": 1.514,
-    "train_steps_per_second": 0.006
 }

 {
     "total_flos": 0.0,
+    "train_loss": 1.7066374281421304,
+    "train_runtime": 5498.7014,
+    "train_samples": 11300,
+    "train_samples_per_second": 2.055,
+    "train_steps_per_second": 0.004
 }

generation_config.json CHANGED Viewed

@@ -1,6 +1,14 @@
 {
   "bos_token_id": 151643,
-  "eos_token_id": 151643,
-  "max_new_tokens": 2048,
-  "transformers_version": "4.49.0.dev0"
 }

 {
   "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.50.0.dev0"
 }

train_results.json CHANGED Viewed

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 20.984000422099324,
-    "train_runtime": 3511.8094,
-    "train_samples": 5316,
-    "train_samples_per_second": 1.514,
-    "train_steps_per_second": 0.006
 }

 {
     "total_flos": 0.0,
+    "train_loss": 1.7066374281421304,
+    "train_runtime": 5498.7014,
+    "train_samples": 11300,
+    "train_samples_per_second": 2.055,
+    "train_steps_per_second": 0.004
 }

trainer_state.json CHANGED Viewed

@@ -1,322 +1,328 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.960960960960961,
-  "eval_steps": 100,
-  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 199.19922256469727,
-      "epoch": 0.04804804804804805,
-      "grad_norm": 23.81310616225955,
       "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": 0.0,
-      "reward": 1.0142708644270897,
-      "reward_std": 0.47196217253804207,
-      "rewards/fc_accuracy_reward": 0.42833332903683186,
-      "rewards/format_reward": 0.5846354328095913,
-      "rewards/reasoning_steps_reward": 0.0013020833721384406,
       "step": 1
     },
     {
-      "completion_length": 196.983078956604,
-      "epoch": 0.0960960960960961,
-      "grad_norm": 17.400559836416594,
       "kl": 0.0,
-      "learning_rate": 2e-05,
-      "loss": 0.0,
-      "reward": 1.0837717466056347,
-      "reward_std": 0.46565326303243637,
-      "rewards/fc_accuracy_reward": 0.49175781942903996,
-      "rewards/format_reward": 0.5898437723517418,
-      "rewards/reasoning_steps_reward": 0.002170138992369175,
       "step": 2
     },
     {
-      "completion_length": 219.9648494720459,
-      "epoch": 0.14414414414414414,
-      "grad_norm": 295118.37380778877,
-      "kl": 9113.812822580338,
-      "learning_rate": 1.9848077530122083e-05,
-      "loss": 365.0355,
-      "reward": 1.238880269229412,
-      "reward_std": 0.40484684705734253,
-      "rewards/fc_accuracy_reward": 0.5162239633500576,
-      "rewards/format_reward": 0.7226562686264515,
-      "rewards/reasoning_steps_reward": 0.0,
       "step": 3
     },
     {
-      "completion_length": 274.19922733306885,
-      "epoch": 0.1921921921921922,
-      "grad_norm": 7394.899853833754,
-      "kl": 308.7130870819092,
-      "learning_rate": 1.9396926207859085e-05,
-      "loss": 12.3462,
-      "reward": 1.36618060618639,
-      "reward_std": 0.33165138494223356,
-      "rewards/fc_accuracy_reward": 0.49421875923871994,
-      "rewards/format_reward": 0.8710937723517418,
-      "rewards/reasoning_steps_reward": 0.0008680556202307343,
       "step": 4
     },
     {
-      "completion_length": 282.16277027130127,
-      "epoch": 0.24024024024024024,
-      "grad_norm": 1580.0757931975875,
-      "kl": 40.46273612976074,
-      "learning_rate": 1.866025403784439e-05,
-      "loss": 1.6191,
-      "reward": 1.4065451845526695,
-      "reward_std": 0.27760336082428694,
-      "rewards/fc_accuracy_reward": 0.46296875178813934,
-      "rewards/format_reward": 0.9388020969927311,
-      "rewards/reasoning_steps_reward": 0.004774305736646056,
       "step": 5
     },
     {
-      "completion_length": 287.5546989440918,
-      "epoch": 0.2882882882882883,
-      "grad_norm": 66.23782695389403,
-      "kl": 1.2862091064453125,
-      "learning_rate": 1.766044443118978e-05,
-      "loss": 0.0516,
-      "reward": 1.366289108991623,
-      "reward_std": 0.26060857344418764,
-      "rewards/fc_accuracy_reward": 0.42748698592185974,
-      "rewards/format_reward": 0.9375000186264515,
-      "rewards/reasoning_steps_reward": 0.0013020833721384406,
       "step": 6
     },
     {
-      "completion_length": 276.5859441757202,
-      "epoch": 0.33633633633633636,
-      "grad_norm": 10.29947581325369,
-      "kl": 0.10057258605957031,
-      "learning_rate": 1.6427876096865394e-05,
-      "loss": 0.0041,
-      "reward": 1.4556901454925537,
-      "reward_std": 0.269397790543735,
-      "rewards/fc_accuracy_reward": 0.5129817798733711,
-      "rewards/format_reward": 0.9414062723517418,
-      "rewards/reasoning_steps_reward": 0.0013020833721384406,
       "step": 7
     },
     {
-      "completion_length": 274.68620681762695,
-      "epoch": 0.3843843843843844,
-      "grad_norm": 135.31823127931858,
-      "kl": 2.202852249145508,
-      "learning_rate": 1.5000000000000002e-05,
-      "loss": 0.0884,
-      "reward": 1.454049527645111,
-      "reward_std": 0.25068292673677206,
-      "rewards/fc_accuracy_reward": 0.5048307403922081,
-      "rewards/format_reward": 0.9479166902601719,
-      "rewards/reasoning_steps_reward": 0.0013020833721384406,
       "step": 8
     },
     {
-      "completion_length": 274.01433277130127,
-      "epoch": 0.43243243243243246,
-      "grad_norm": 50.596020384988215,
-      "kl": 1.0600624084472656,
-      "learning_rate": 1.342020143325669e-05,
-      "loss": 0.0426,
-      "reward": 1.4849176108837128,
-      "reward_std": 0.23833946604281664,
-      "rewards/fc_accuracy_reward": 0.5400390792638063,
-      "rewards/format_reward": 0.9414062760770321,
-      "rewards/reasoning_steps_reward": 0.0034722223645076156,
       "step": 9
     },
     {
-      "completion_length": 270.1119842529297,
-      "epoch": 0.4804804804804805,
-      "grad_norm": 3.4647264465831866,
-      "kl": 0.18325424194335938,
-      "learning_rate": 1.1736481776669307e-05,
-      "loss": 0.0075,
-      "reward": 1.4754557833075523,
-      "reward_std": 0.24551822617650032,
-      "rewards/fc_accuracy_reward": 0.534049479290843,
-      "rewards/format_reward": 0.9414062686264515,
-      "rewards/reasoning_steps_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 268.4127674102783,
-      "epoch": 0.5285285285285285,
-      "grad_norm": 21954.461363262388,
-      "kl": 1002.1737289428711,
-      "learning_rate": 1e-05,
-      "loss": 40.019,
-      "reward": 1.4405295476317406,
-      "reward_std": 0.26797763630747795,
-      "rewards/fc_accuracy_reward": 0.4969531334936619,
-      "rewards/format_reward": 0.9401041865348816,
-      "rewards/reasoning_steps_reward": 0.0034722223645076156,
       "step": 11
     },
     {
-      "completion_length": 267.64323711395264,
-      "epoch": 0.5765765765765766,
-      "grad_norm": 122.50226092016571,
-      "kl": 7.038032531738281,
-      "learning_rate": 8.263518223330698e-06,
-      "loss": 0.2818,
-      "reward": 1.4381510838866234,
-      "reward_std": 0.27108312491327524,
-      "rewards/fc_accuracy_reward": 0.4954427257180214,
-      "rewards/format_reward": 0.9427083507180214,
-      "rewards/reasoning_steps_reward": 0.0,
       "step": 12
     },
     {
-      "completion_length": 268.85547828674316,
-      "epoch": 0.6246246246246246,
-      "grad_norm": 7.964152420466192,
-      "kl": 0.13129806518554688,
-      "learning_rate": 6.579798566743314e-06,
-      "loss": 0.0053,
-      "reward": 1.400338590145111,
-      "reward_std": 0.24977970868349075,
-      "rewards/fc_accuracy_reward": 0.47716146148741245,
-      "rewards/format_reward": 0.9231771007180214,
-      "rewards/reasoning_steps_reward": 0.0,
       "step": 13
     },
     {
-      "completion_length": 265.68360328674316,
-      "epoch": 0.6726726726726727,
-      "grad_norm": 8.142268019517086,
-      "kl": 0.7337493896484375,
-      "learning_rate": 5.000000000000003e-06,
-      "loss": 0.0295,
-      "reward": 1.4463759139180183,
-      "reward_std": 0.26006509829312563,
-      "rewards/fc_accuracy_reward": 0.509309895336628,
-      "rewards/format_reward": 0.9361979402601719,
-      "rewards/reasoning_steps_reward": 0.0008680556202307343,
       "step": 14
     },
     {
-      "completion_length": 251.79297924041748,
-      "epoch": 0.7207207207207207,
-      "grad_norm": 5.0488043029036005,
-      "kl": 0.1615753173828125,
-      "learning_rate": 3.5721239031346067e-06,
-      "loss": 0.0065,
-      "reward": 1.46674482524395,
-      "reward_std": 0.24269275972619653,
-      "rewards/fc_accuracy_reward": 0.5214322991669178,
-      "rewards/format_reward": 0.9440104365348816,
-      "rewards/reasoning_steps_reward": 0.0013020833721384406,
       "step": 15
     },
     {
-      "completion_length": 252.04167461395264,
-      "epoch": 0.7687687687687688,
-      "grad_norm": 67.89594390549263,
-      "kl": 2.355632781982422,
-      "learning_rate": 2.339555568810221e-06,
-      "loss": 0.0944,
-      "reward": 1.4977865368127823,
-      "reward_std": 0.2157565113157034,
-      "rewards/fc_accuracy_reward": 0.5446614678949118,
-      "rewards/format_reward": 0.9505208544433117,
-      "rewards/reasoning_steps_reward": 0.0026041667442768812,
       "step": 16
     },
     {
-      "completion_length": 253.686203956604,
-      "epoch": 0.8168168168168168,
-      "grad_norm": 4.6776561505879375,
-      "kl": 0.4775199890136719,
-      "learning_rate": 1.339745962155613e-06,
-      "loss": 0.0192,
-      "reward": 1.4627083614468575,
-      "reward_std": 0.22646328434348106,
-      "rewards/fc_accuracy_reward": 0.5004687625914812,
-      "rewards/format_reward": 0.9596354402601719,
-      "rewards/reasoning_steps_reward": 0.0026041667442768812,
       "step": 17
     },
     {
-      "completion_length": 259.276047706604,
-      "epoch": 0.8648648648648649,
-      "grad_norm": 3.3536158930168773,
-      "kl": 0.314849853515625,
-      "learning_rate": 6.030737921409169e-07,
-      "loss": 0.0127,
-      "reward": 1.3979297056794167,
-      "reward_std": 0.23455267632380128,
-      "rewards/fc_accuracy_reward": 0.46694011986255646,
-      "rewards/format_reward": 0.9309896007180214,
-      "rewards/reasoning_steps_reward": 0.0,
       "step": 18
     },
     {
-      "completion_length": 256.66146755218506,
-      "epoch": 0.9129129129129129,
-      "grad_norm": 1.5380141665723024,
-      "kl": 0.17271041870117188,
-      "learning_rate": 1.519224698779198e-07,
-      "loss": 0.007,
-      "reward": 1.479730948805809,
-      "reward_std": 0.22270044265314937,
-      "rewards/fc_accuracy_reward": 0.5261718854308128,
-      "rewards/format_reward": 0.9531250186264515,
-      "rewards/reasoning_steps_reward": 0.0004340278101153672,
       "step": 19
     },
     {
-      "completion_length": 259.4140691757202,
-      "epoch": 0.960960960960961,
-      "grad_norm": 0.8561283187855229,
-      "kl": 0.24119949340820312,
-      "learning_rate": 0.0,
-      "loss": 0.0097,
-      "reward": 1.453385479748249,
-      "reward_std": 0.2447610031813383,
-      "rewards/fc_accuracy_reward": 0.5054687578231096,
-      "rewards/format_reward": 0.9453125149011612,
-      "rewards/reasoning_steps_reward": 0.0026041667442768812,
       "step": 20
     },
     {
-      "epoch": 0.960960960960961,
-      "step": 20,
       "total_flos": 0.0,
-      "train_loss": 20.984000422099324,
-      "train_runtime": 3511.8094,
-      "train_samples_per_second": 1.514,
-      "train_steps_per_second": 0.006
     }
   ],
   "logging_steps": 1,
-  "max_steps": 20,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
         "should_evaluate": false,
         "should_log": false,
-        "should_save": false,
-        "should_training_stop": false
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

 {
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 0.9943502824858758,
+  "eval_steps": 500,
+  "global_step": 22,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
+      "completion_length": 276.02735233306885,
+      "epoch": 0.04519774011299435,
+      "grad_norm": 0.13002647486414434,
       "kl": 0.0,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.0303,
+      "reward": 1.3704427555203438,
+      "reward_std": 0.1837239097803831,
+      "rewards/fc_accuracy_reward": 0.4309895895421505,
+      "rewards/format_reward": 0.9394531473517418,
       "step": 1
     },
     {
+      "completion_length": 292.13477420806885,
+      "epoch": 0.0903954802259887,
+      "grad_norm": 0.14619501253296655,
       "kl": 0.0,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 0.0469,
+      "reward": 1.3990885764360428,
+      "reward_std": 0.21469345316290855,
+      "rewards/fc_accuracy_reward": 0.44856772013008595,
+      "rewards/format_reward": 0.9505208507180214,
       "step": 2
     },
     {
+      "completion_length": 268.9563903808594,
+      "epoch": 0.13559322033898305,
+      "grad_norm": 0.13036409926543296,
+      "kl": 0.0002338886260986328,
+      "learning_rate": 2e-05,
+      "loss": 0.0319,
+      "reward": 1.3704427406191826,
+      "reward_std": 0.18289842084050179,
+      "rewards/fc_accuracy_reward": 0.41992188431322575,
+      "rewards/format_reward": 0.9505208656191826,
       "step": 3
     },
     {
+      "completion_length": 237.50716876983643,
+      "epoch": 0.1807909604519774,
+      "grad_norm": 0.12085621262181591,
+      "kl": 0.00720977783203125,
+      "learning_rate": 1.9863613034027224e-05,
+      "loss": 0.0065,
+      "reward": 1.427734412252903,
+      "reward_std": 0.1744816219434142,
+      "rewards/fc_accuracy_reward": 0.46614584885537624,
+      "rewards/format_reward": 0.9615885503590107,
       "step": 4
     },
     {
+      "completion_length": 198.7910213470459,
+      "epoch": 0.22598870056497175,
+      "grad_norm": 1.1449772190729142,
+      "kl": 0.0814666748046875,
+      "learning_rate": 1.9458172417006347e-05,
+      "loss": -0.007,
+      "reward": 1.2617187947034836,
+      "reward_std": 0.17365613672882318,
+      "rewards/fc_accuracy_reward": 0.3222656352445483,
+      "rewards/format_reward": 0.939453125,
       "step": 5
     },
     {
+      "completion_length": 231.09896564483643,
+      "epoch": 0.2711864406779661,
+      "grad_norm": 0.1913660515760745,
+      "kl": 0.045501708984375,
+      "learning_rate": 1.879473751206489e-05,
+      "loss": 0.0077,
+      "reward": 1.3893229588866234,
+      "reward_std": 0.14095465373247862,
+      "rewards/fc_accuracy_reward": 0.43489584513008595,
+      "rewards/format_reward": 0.9544270932674408,
       "step": 6
     },
     {
+      "completion_length": 247.44727039337158,
+      "epoch": 0.3163841807909605,
+      "grad_norm": 0.13567082283951828,
+      "kl": 0.045623779296875,
+      "learning_rate": 1.789140509396394e-05,
+      "loss": 0.0065,
+      "reward": 1.3470052555203438,
+      "reward_std": 0.14320992957800627,
+      "rewards/fc_accuracy_reward": 0.3938802182674408,
+      "rewards/format_reward": 0.9531250074505806,
       "step": 7
     },
     {
+      "completion_length": 269.38086795806885,
+      "epoch": 0.3615819209039548,
+      "grad_norm": 0.16971700336774798,
+      "kl": 0.060699462890625,
+      "learning_rate": 1.6772815716257414e-05,
+      "loss": 0.0085,
+      "reward": 1.3372396156191826,
+      "reward_std": 0.1406525035854429,
+      "rewards/fc_accuracy_reward": 0.39257813431322575,
+      "rewards/format_reward": 0.9446614682674408,
       "step": 8
     },
     {
+      "completion_length": 295.05665016174316,
+      "epoch": 0.4067796610169492,
+      "grad_norm": 1.3287845887377334,
+      "kl": 0.070892333984375,
+      "learning_rate": 1.5469481581224274e-05,
+      "loss": 0.0192,
+      "reward": 1.4036458730697632,
+      "reward_std": 0.18207293096929789,
+      "rewards/fc_accuracy_reward": 0.43750000931322575,
+      "rewards/format_reward": 0.9661458507180214,
       "step": 9
     },
     {
+      "completion_length": 307.4420690536499,
+      "epoch": 0.4519774011299435,
+      "grad_norm": 54738.02031388349,
+      "kl": 394.0651397705078,
+      "learning_rate": 1.4016954246529697e-05,
+      "loss": 37.1931,
+      "reward": 1.3352864980697632,
+      "reward_std": 0.2137870010919869,
+      "rewards/fc_accuracy_reward": 0.4199218861758709,
+      "rewards/format_reward": 0.915364608168602,
       "step": 10
     },
     {
+      "completion_length": 314.0898551940918,
+      "epoch": 0.4971751412429379,
+      "grad_norm": 2.107091829265173,
+      "kl": 0.095855712890625,
+      "learning_rate": 1.2454854871407993e-05,
+      "loss": 0.0338,
+      "reward": 1.3177083730697632,
+      "reward_std": 0.2345268540084362,
+      "rewards/fc_accuracy_reward": 0.4108072966337204,
+      "rewards/format_reward": 0.9069010689854622,
       "step": 11
     },
     {
+      "completion_length": 318.7447986602783,
+      "epoch": 0.5423728813559322,
+      "grad_norm": 0.20351280734446026,
+      "kl": 0.081268310546875,
+      "learning_rate": 1.0825793454723325e-05,
+      "loss": 0.0176,
+      "reward": 1.2675781697034836,
+      "reward_std": 0.3210527803748846,
+      "rewards/fc_accuracy_reward": 0.397135429084301,
+      "rewards/format_reward": 0.8704427294433117,
       "step": 12
     },
     {
+      "completion_length": 311.57813262939453,
+      "epoch": 0.5875706214689266,
+      "grad_norm": 0.1737011976596144,
+      "kl": 0.07958984375,
+      "learning_rate": 9.174206545276678e-06,
+      "loss": 0.0088,
+      "reward": 1.3684896156191826,
+      "reward_std": 0.23807168938219547,
+      "rewards/fc_accuracy_reward": 0.4479166828095913,
+      "rewards/format_reward": 0.9205729365348816,
       "step": 13
     },
     {
+      "completion_length": 301.56706619262695,
+      "epoch": 0.632768361581921,
+      "grad_norm": 18.70037949885207,
+      "kl": 0.586456298828125,
+      "learning_rate": 7.545145128592009e-06,
+      "loss": 0.0599,
+      "reward": 1.3203125298023224,
+      "reward_std": 0.20432353112846613,
+      "rewards/fc_accuracy_reward": 0.40820313803851604,
+      "rewards/format_reward": 0.9121094010770321,
       "step": 14
     },
     {
+      "completion_length": 289.6764450073242,
+      "epoch": 0.6779661016949152,
+      "grad_norm": 0.15577324164046125,
+      "kl": 0.077789306640625,
+      "learning_rate": 5.983045753470308e-06,
+      "loss": 0.011,
+      "reward": 1.3496094048023224,
+      "reward_std": 0.21356581384316087,
+      "rewards/fc_accuracy_reward": 0.4186198003590107,
+      "rewards/format_reward": 0.9309896044433117,
       "step": 15
     },
     {
+      "completion_length": 286.3886785507202,
+      "epoch": 0.7231638418079096,
+      "grad_norm": 5.326707764035434,
+      "kl": 0.0885009765625,
+      "learning_rate": 4.530518418775734e-06,
+      "loss": 0.0102,
+      "reward": 1.3658854514360428,
+      "reward_std": 0.23490996472537518,
+      "rewards/fc_accuracy_reward": 0.44401043094694614,
+      "rewards/format_reward": 0.9218750260770321,
       "step": 16
     },
     {
+      "completion_length": 272.7597780227661,
+      "epoch": 0.768361581920904,
+      "grad_norm": 0.33239438768575263,
+      "kl": 0.075653076171875,
+      "learning_rate": 3.2271842837425917e-06,
+      "loss": 0.0083,
+      "reward": 1.3222656697034836,
+      "reward_std": 0.23243350349366665,
+      "rewards/fc_accuracy_reward": 0.4173177182674408,
+      "rewards/format_reward": 0.9049479365348816,
       "step": 17
     },
     {
+      "completion_length": 267.5846462249756,
+      "epoch": 0.8135593220338984,
+      "grad_norm": 0.16069383676764726,
+      "kl": 0.07232666015625,
+      "learning_rate": 2.1085949060360654e-06,
+      "loss": 0.0114,
+      "reward": 1.3854166939854622,
+      "reward_std": 0.242501275613904,
+      "rewards/fc_accuracy_reward": 0.47070313803851604,
+      "rewards/format_reward": 0.9147135652601719,
       "step": 18
     },
     {
+      "completion_length": 266.1367254257202,
+      "epoch": 0.8587570621468926,
+      "grad_norm": 0.14326034602692705,
+      "kl": 0.0689239501953125,
+      "learning_rate": 1.2052624879351105e-06,
+      "loss": 0.0075,
+      "reward": 1.3854166939854622,
+      "reward_std": 0.20935742277652025,
+      "rewards/fc_accuracy_reward": 0.44531251676380634,
+      "rewards/format_reward": 0.940104179084301,
       "step": 19
     },
     {
+      "completion_length": 260.7949285507202,
+      "epoch": 0.903954802259887,
+      "grad_norm": 0.14485988944724762,
+      "kl": 0.06939697265625,
+      "learning_rate": 5.418275829936537e-07,
+      "loss": 0.0098,
+      "reward": 1.395182341337204,
+      "reward_std": 0.2115317303687334,
+      "rewards/fc_accuracy_reward": 0.47330730222165585,
+      "rewards/format_reward": 0.9218750260770321,
       "step": 20
     },
     {
+      "completion_length": 262.3704528808594,
+      "epoch": 0.9491525423728814,
+      "grad_norm": 0.19459843380674965,
+      "kl": 0.0714874267578125,
+      "learning_rate": 1.3638696597277678e-07,
+      "loss": 0.0072,
+      "reward": 1.3489583656191826,
+      "reward_std": 0.20432352973148227,
+      "rewards/fc_accuracy_reward": 0.4348958469927311,
+      "rewards/format_reward": 0.9140625186264515,
+      "step": 21
+    },
+    {
+      "completion_length": 263.5794382095337,
+      "epoch": 0.9943502824858758,
+      "grad_norm": 0.16933108100699076,
+      "kl": 0.067138671875,
+      "learning_rate": 0.0,
+      "loss": 0.0169,
+      "reward": 1.354817733168602,
+      "reward_std": 0.242501275613904,
+      "rewards/fc_accuracy_reward": 0.4348958469927311,
+      "rewards/format_reward": 0.9199219010770321,
+      "step": 22
+    },
+    {
+      "epoch": 0.9943502824858758,
+      "step": 22,
       "total_flos": 0.0,
+      "train_loss": 1.7066374281421304,
+      "train_runtime": 5498.7014,
+      "train_samples_per_second": 2.055,
+      "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 1,
+  "max_steps": 22,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
+  "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
         "should_evaluate": false,
         "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
+  "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null
 }