Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,68 @@
+---
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+library_name: transformers
+model_name: Qwen2.5-1.5B-Open-R1-GRPO
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Open-R1-GRPO
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="byteXWJ/Qwen2.5-1.5B-Open-R1-GRPO", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.3506326697535499,
+    "train_runtime": 41982.506,
+    "train_samples": 72441,
+    "train_samples_per_second": 1.726,
+    "train_steps_per_second": 0.015
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.49.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2fc9097478fce2d549d4e27e3927581af17d34e0351d8d0497abc8dd0b9dc13
+size 3087467144

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.3506326697535499,
+    "train_runtime": 41982.506,
+    "train_samples": 72441,
+    "train_samples_per_second": 1.726,
+    "train_steps_per_second": 0.015
+}

trainer_state.json

@@ -0,0 +1,2081 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987438399845395,
+  "eval_steps": 100,
+  "global_step": 646,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 392.36966094970705,
+      "epoch": 0.007730215479756498,
+      "grad_norm": 0.9390482577141096,
+      "kl": 0.00012568235397338867,
+      "learning_rate": 1.5384615384615387e-06,
+      "loss": 0.0,
+      "reward": 0.7186287339776755,
+      "reward_std": 0.5490757117047906,
+      "rewards/accuracy_reward": 0.16696429448202252,
+      "rewards/cosine_scaled_reward": -0.0902998779551126,
+      "rewards/format_reward": 0.46250001853331923,
+      "rewards/reasoning_steps_reward": 0.17946429955773055,
+      "step": 5
+    },
+    {
+      "completion_length": 365.6544815063477,
+      "epoch": 0.015460430959512996,
+      "grad_norm": 8.833524212599093,
+      "kl": 0.0298675537109375,
+      "learning_rate": 3.0769230769230774e-06,
+      "loss": 0.0012,
+      "reward": 0.7676249794661999,
+      "reward_std": 0.47266554515808823,
+      "rewards/accuracy_reward": 0.14375000735744833,
+      "rewards/cosine_scaled_reward": -0.0978512549365405,
+      "rewards/format_reward": 0.5580357387661934,
+      "rewards/reasoning_steps_reward": 0.16369048785418272,
+      "step": 10
+    },
+    {
+      "completion_length": 180.57322311401367,
+      "epoch": 0.023190646439269495,
+      "grad_norm": 1.7344816570791748,
+      "kl": 0.171295166015625,
+      "learning_rate": 4.615384615384616e-06,
+      "loss": 0.0069,
+      "reward": 0.8723336957395077,
+      "reward_std": 0.3432145787868649,
+      "rewards/accuracy_reward": 0.06339286053553224,
+      "rewards/cosine_scaled_reward": -0.08332110820629168,
+      "rewards/format_reward": 0.8366071864962578,
+      "rewards/reasoning_steps_reward": 0.055654766084626314,
+      "step": 15
+    },
+    {
+      "completion_length": 166.2580440044403,
+      "epoch": 0.03092086191902599,
+      "grad_norm": 1.3202255516946677,
+      "kl": 0.08023223876953126,
+      "learning_rate": 6.153846153846155e-06,
+      "loss": 0.0032,
+      "reward": 0.882199002802372,
+      "reward_std": 0.3599051775876433,
+      "rewards/accuracy_reward": 0.07500000409781933,
+      "rewards/cosine_scaled_reward": -0.06333675118803513,
+      "rewards/format_reward": 0.8276786141097545,
+      "rewards/reasoning_steps_reward": 0.042857146076858046,
+      "step": 20
+    },
+    {
+      "completion_length": 101.15804007053376,
+      "epoch": 0.03865107739878249,
+      "grad_norm": 1.6249005624090773,
+      "kl": 0.1916595458984375,
+      "learning_rate": 7.692307692307694e-06,
+      "loss": 0.0077,
+      "reward": 0.9729942634701729,
+      "reward_std": 0.22987019338761455,
+      "rewards/accuracy_reward": 0.05000000260770321,
+      "rewards/cosine_scaled_reward": -0.02730340599810006,
+      "rewards/format_reward": 0.9223214581608772,
+      "rewards/reasoning_steps_reward": 0.027976192627102138,
+      "step": 25
+    },
+    {
+      "completion_length": 94.2044686794281,
+      "epoch": 0.04638129287853899,
+      "grad_norm": 1.7538543203867525,
+      "kl": 0.36331787109375,
+      "learning_rate": 9.230769230769232e-06,
+      "loss": 0.0145,
+      "reward": 1.0303254418075085,
+      "reward_std": 0.22137819393537939,
+      "rewards/accuracy_reward": 0.055357145611196756,
+      "rewards/cosine_scaled_reward": -0.01074603625565942,
+      "rewards/format_reward": 0.923214315623045,
+      "rewards/reasoning_steps_reward": 0.06250000461004675,
+      "step": 30
+    },
+    {
+      "completion_length": 74.22143168449402,
+      "epoch": 0.054111508358295486,
+      "grad_norm": 2.008556705979058,
+      "kl": 0.74228515625,
+      "learning_rate": 1.076923076923077e-05,
+      "loss": 0.0297,
+      "reward": 1.0773570157587529,
+      "reward_std": 0.20800948874093592,
+      "rewards/accuracy_reward": 0.03125000139698386,
+      "rewards/cosine_scaled_reward": -0.017881136175128633,
+      "rewards/format_reward": 0.9267857424914837,
+      "rewards/reasoning_steps_reward": 0.13720238991081715,
+      "step": 35
+    },
+    {
+      "completion_length": 44.866966414451596,
+      "epoch": 0.06184172383805198,
+      "grad_norm": 1.5958148574832594,
+      "kl": 0.7339111328125,
+      "learning_rate": 1.230769230769231e-05,
+      "loss": 0.0294,
+      "reward": 1.301904872059822,
+      "reward_std": 0.08922360279975691,
+      "rewards/accuracy_reward": 0.012500000558793545,
+      "rewards/cosine_scaled_reward": -0.01178566825692542,
+      "rewards/format_reward": 0.9660714447498322,
+      "rewards/reasoning_steps_reward": 0.3351190730929375,
+      "step": 40
+    },
+    {
+      "completion_length": 32.241965675354,
+      "epoch": 0.06957193931780849,
+      "grad_norm": 1.7018249159830048,
+      "kl": 1.53359375,
+      "learning_rate": 1.3846153846153847e-05,
+      "loss": 0.0613,
+      "reward": 1.27698794901371,
+      "reward_std": 0.08601129058224614,
+      "rewards/accuracy_reward": 0.0026785715483129023,
+      "rewards/cosine_scaled_reward": -0.006940663917339407,
+      "rewards/format_reward": 0.9419643089175225,
+      "rewards/reasoning_steps_reward": 0.3392857421189547,
+      "step": 45
+    },
+    {
+      "completion_length": 16.62410787343979,
+      "epoch": 0.07730215479756498,
+      "grad_norm": 2.0636678600862,
+      "kl": 1.9171875,
+      "learning_rate": 1.5384615384615387e-05,
+      "loss": 0.0767,
+      "reward": 1.327180388569832,
+      "reward_std": 0.01873247450253075,
+      "rewards/accuracy_reward": 0.00357142873108387,
+      "rewards/cosine_scaled_reward": 0.00039462426211684943,
+      "rewards/format_reward": 0.9919642888009548,
+      "rewards/reasoning_steps_reward": 0.33125003390014174,
+      "step": 50
+    },
+    {
+      "completion_length": 15.018750631809235,
+      "epoch": 0.08503237027732148,
+      "grad_norm": 0.4101138858085747,
+      "kl": 2.75224609375,
+      "learning_rate": 1.6923076923076924e-05,
+      "loss": 0.11,
+      "reward": 1.3266640812158585,
+      "reward_std": 0.008146386814166818,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0022050292027415708,
+      "rewards/format_reward": 0.9955357164144516,
+      "rewards/reasoning_steps_reward": 0.3333333697170019,
+      "step": 55
+    },
+    {
+      "completion_length": 14.050893533229829,
+      "epoch": 0.09276258575707798,
+      "grad_norm": 0.6632975119048585,
+      "kl": 3.4400390625,
+      "learning_rate": 1.8461538461538465e-05,
+      "loss": 0.1376,
+      "reward": 1.319350515305996,
+      "reward_std": 0.016851214813584647,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0020780946069862694,
+      "rewards/format_reward": 0.9964285731315613,
+      "rewards/reasoning_steps_reward": 0.3250000324100256,
+      "step": 60
+    },
+    {
+      "completion_length": 90.5026834487915,
+      "epoch": 0.10049280123683448,
+      "grad_norm": 10.020830564657501,
+      "kl": 3.48125,
+      "learning_rate": 2e-05,
+      "loss": 0.1393,
+      "reward": 1.207764456421137,
+      "reward_std": 0.13331688460179975,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.033902262800256724,
+      "rewards/format_reward": 0.9187500149011611,
+      "rewards/reasoning_steps_reward": 0.32291669528931377,
+      "step": 65
+    },
+    {
+      "completion_length": 51.040181362628935,
+      "epoch": 0.10822301671659097,
+      "grad_norm": 1.2114764591084524,
+      "kl": 3.489453125,
+      "learning_rate": 1.999634547413886e-05,
+      "loss": 0.1396,
+      "reward": 1.1045873086899518,
+      "reward_std": 0.10469552398151336,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.015650834681582636,
+      "rewards/format_reward": 0.784821440745145,
+      "rewards/reasoning_steps_reward": 0.3354166992008686,
+      "step": 70
+    },
+    {
+      "completion_length": 386.1705480456352,
+      "epoch": 0.11595323219634747,
+      "grad_norm": 1.081250544412763,
+      "kl": 2.19903564453125,
+      "learning_rate": 1.9985384567667278e-05,
+      "loss": 0.088,
+      "reward": 1.3853926189243793,
+      "reward_std": 0.2650093133095652,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.09764315947249998,
+      "rewards/format_reward": 0.6366071619093419,
+      "rewards/reasoning_steps_reward": 0.8455357570201159,
+      "step": 75
+    },
+    {
+      "completion_length": 995.5151908874511,
+      "epoch": 0.12368344767610397,
+      "grad_norm": 0.8658636576368606,
+      "kl": 0.270458984375,
+      "learning_rate": 1.9967125291968495e-05,
+      "loss": 0.0108,
+      "reward": 0.7942588612437248,
+      "reward_std": 0.11519759800285101,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.2646697614341974,
+      "rewards/format_reward": 0.05892857387661934,
+      "rewards/reasoning_steps_reward": 1.0,
+      "step": 80
+    },
+    {
+      "completion_length": 399.2928756713867,
+      "epoch": 0.13141366315586048,
+      "grad_norm": 1.1675243662227284,
+      "kl": 1.223876953125,
+      "learning_rate": 1.9941580992841562e-05,
+      "loss": 0.0489,
+      "reward": 1.7452217385172843,
+      "reward_std": 0.18216142036835664,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.12293311327230186,
+      "rewards/format_reward": 0.8696428865194321,
+      "rewards/reasoning_steps_reward": 0.9985119082033634,
+      "step": 85
+    },
+    {
+      "completion_length": 26.440179657936095,
+      "epoch": 0.13914387863561697,
+      "grad_norm": 0.8658089821111639,
+      "kl": 3.4791015625,
+      "learning_rate": 1.990877034074683e-05,
+      "loss": 0.1391,
+      "reward": 1.8058076962828635,
+      "reward_std": 0.1962852551096148,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.007287628838093951,
+      "rewards/format_reward": 0.8803571738302708,
+      "rewards/reasoning_steps_reward": 0.9327381618320942,
+      "step": 90
+    },
+    {
+      "completion_length": 56.44821660518646,
+      "epoch": 0.14687409411537347,
+      "grad_norm": 0.6032597111944057,
+      "kl": 3.191796875,
+      "learning_rate": 1.9868717317159617e-05,
+      "loss": 0.1277,
+      "reward": 1.866142961382866,
+      "reward_std": 0.15099413888340224,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.019869021500926464,
+      "rewards/format_reward": 0.9276785932481288,
+      "rewards/reasoning_steps_reward": 0.9583333849906921,
+      "step": 95
+    },
+    {
+      "completion_length": 190.62947311401368,
+      "epoch": 0.15460430959512997,
+      "grad_norm": 0.278335659297726,
+      "kl": 3.030078125,
+      "learning_rate": 1.9821451197042028e-05,
+      "loss": 0.1212,
+      "reward": 1.7083741463720798,
+      "reward_std": 0.13575071440020564,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.06960212688427418,
+      "rewards/format_reward": 0.8107142995111645,
+      "rewards/reasoning_steps_reward": 0.96726194024086,
+      "step": 100
+    },
+    {
+      "epoch": 0.15460430959512997,
+      "eval_completion_length": 42.085418462753296,
+      "eval_kl": 3.35546875,
+      "eval_loss": 0.13430055975914001,
+      "eval_reward": 1.88418547809124,
+      "eval_reward_std": 0.14376993896439672,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.007183671579696238,
+      "eval_rewards/format_reward": 0.9464285969734192,
+      "eval_rewards/reasoning_steps_reward": 0.944940522313118,
+      "eval_runtime": 19.4962,
+      "eval_samples_per_second": 5.078,
+      "eval_steps_per_second": 0.205,
+      "step": 100
+    },
+    {
+      "completion_length": 129.1232194662094,
+      "epoch": 0.16233452507488647,
+      "grad_norm": 1.4087679882993498,
+      "kl": 3.0447265625,
+      "learning_rate": 1.9767006527445728e-05,
+      "loss": 0.1218,
+      "reward": 1.803493769466877,
+      "reward_std": 0.21765968978722866,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.017339627695037052,
+      "rewards/format_reward": 0.8535714633762836,
+      "rewards/reasoning_steps_reward": 0.9672619417309761,
+      "step": 105
+    },
+    {
+      "completion_length": 61.853573369979856,
+      "epoch": 0.17006474055464296,
+      "grad_norm": 0.3073740874519991,
+      "kl": 3.32109375,
+      "learning_rate": 1.9705423102261324e-05,
+      "loss": 0.1329,
+      "reward": 1.9153445169329644,
+      "reward_std": 0.10697445697765033,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.016203196794958786,
+      "rewards/format_reward": 0.941964304447174,
+      "rewards/reasoning_steps_reward": 0.9895833551883697,
+      "step": 110
+    },
+    {
+      "completion_length": 32.900001358985904,
+      "epoch": 0.17779495603439946,
+      "grad_norm": 1.2224402940084464,
+      "kl": 3.3640625,
+      "learning_rate": 1.9636745933132807e-05,
+      "loss": 0.1346,
+      "reward": 1.8428309723734855,
+      "reward_std": 0.18514500717083138,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.022050061996560545,
+      "rewards/format_reward": 0.9008928887546063,
+      "rewards/reasoning_steps_reward": 0.9639881268143654,
+      "step": 115
+    },
+    {
+      "completion_length": 58.67053807973862,
+      "epoch": 0.18552517151415596,
+      "grad_norm": 1.5030016268180966,
+      "kl": 3.31328125,
+      "learning_rate": 1.956102521655831e-05,
+      "loss": 0.1325,
+      "reward": 1.5131114095449447,
+      "reward_std": 0.2015471445057983,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03182920304534491,
+      "rewards/format_reward": 0.8651786059141159,
+      "rewards/reasoning_steps_reward": 0.6797619640827179,
+      "step": 120
+    },
+    {
+      "completion_length": 49.43928836584091,
+      "epoch": 0.19325538699391245,
+      "grad_norm": 9.605593016742555,
+      "kl": 3.896484375,
+      "learning_rate": 1.9478316297201218e-05,
+      "loss": 0.1559,
+      "reward": 1.6346509769558906,
+      "reward_std": 0.12935965392530308,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.018325358303263782,
+      "rewards/format_reward": 0.9294643089175224,
+      "rewards/reasoning_steps_reward": 0.7235119700431824,
+      "step": 125
+    },
+    {
+      "completion_length": 164.66250653266906,
+      "epoch": 0.20098560247366895,
+      "grad_norm": 41.45434131312338,
+      "kl": 4.0005859375,
+      "learning_rate": 1.9388679627438486e-05,
+      "loss": 0.16,
+      "reward": 1.6668862670660018,
+      "reward_std": 0.3199146263883449,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.06525669391849079,
+      "rewards/format_reward": 0.7357143245637416,
+      "rewards/reasoning_steps_reward": 0.9964285805821419,
+      "step": 130
+    },
+    {
+      "completion_length": 239.10536794662477,
+      "epoch": 0.20871581795342545,
+      "grad_norm": 20.949149212624942,
+      "kl": 4.421484375,
+      "learning_rate": 1.9292180723175656e-05,
+      "loss": 0.1769,
+      "reward": 1.6670863956212998,
+      "reward_std": 0.20173981295956764,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.05225894197938032,
+      "rewards/format_reward": 0.7223214648663998,
+      "rewards/reasoning_steps_reward": 0.9970238178968429,
+      "step": 135
+    },
+    {
+      "completion_length": 305.64019243717195,
+      "epoch": 0.21644603343318194,
+      "grad_norm": 1.211567877000997,
+      "kl": 3.112890625,
+      "learning_rate": 1.9188890115960967e-05,
+      "loss": 0.1245,
+      "reward": 1.6698275178670883,
+      "reward_std": 0.12347855939142391,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030172577398479915,
+      "rewards/format_reward": 0.7000000348314643,
+      "rewards/reasoning_steps_reward": 1.0,
+      "step": 140
+    },
+    {
+      "completion_length": 153.34643466472625,
+      "epoch": 0.22417624891293844,
+      "grad_norm": 2.332988253148305,
+      "kl": 3.383984375,
+      "learning_rate": 1.9078883301433488e-05,
+      "loss": 0.1354,
+      "reward": 1.8375755593180656,
+      "reward_std": 0.13659321218001425,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.022841239851550198,
+      "rewards/format_reward": 0.8607143275439739,
+      "rewards/reasoning_steps_reward": 0.9997023820877076,
+      "step": 145
+    },
+    {
+      "completion_length": 461.5205446720123,
+      "epoch": 0.23190646439269494,
+      "grad_norm": 17.964768544049164,
+      "kl": 3.4033203125,
+      "learning_rate": 1.8962240684142923e-05,
+      "loss": 0.1361,
+      "reward": 1.5285535104572774,
+      "reward_std": 0.0944752536539454,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.021148963196901606,
+      "rewards/format_reward": 0.5544643050990998,
+      "rewards/reasoning_steps_reward": 0.9952381037175655,
+      "step": 150
+    },
+    {
+      "completion_length": 1015.0125053405761,
+      "epoch": 0.23963667987245144,
+      "grad_norm": 0.6199515139941986,
+      "kl": 0.829443359375,
+      "learning_rate": 1.883904751878156e-05,
+      "loss": 0.0332,
+      "reward": 0.952112901210785,
+      "reward_std": 0.06256022888337612,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02258952285628766,
+      "rewards/format_reward": 0.00357142873108387,
+      "rewards/reasoning_steps_reward": 0.9711309850215912,
+      "step": 155
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.24736689535220793,
+      "grad_norm": 0.503878803731676,
+      "kl": 0.19283447265625,
+      "learning_rate": 1.8709393847871146e-05,
+      "loss": 0.0077,
+      "reward": 0.9781682834029197,
+      "reward_std": 0.03131380465492839,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.005760333975194954,
+      "rewards/format_reward": 0.001785714365541935,
+      "rewards/reasoning_steps_reward": 0.9821428745985031,
+      "step": 160
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.25509711083196446,
+      "grad_norm": 0.4904560952590957,
+      "kl": 0.30223388671875,
+      "learning_rate": 1.857337443595034e-05,
+      "loss": 0.0121,
+      "reward": 0.9860361784696579,
+      "reward_std": 0.02012283246149309,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.010987673864292447,
+      "rewards/format_reward": 0.001785714365541935,
+      "rewards/reasoning_steps_reward": 0.9952380999922752,
+      "step": 165
+    },
+    {
+      "completion_length": 1023.7392860412598,
+      "epoch": 0.26282732631172095,
+      "grad_norm": 4.56651185484627,
+      "kl": 3.3035888671875,
+      "learning_rate": 1.8431088700310846e-05,
+      "loss": 0.1323,
+      "reward": 0.8368007011711598,
+      "reward_std": 0.07920540247578174,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.16498505512718112,
+      "rewards/format_reward": 0.001785714365541935,
+      "rewards/reasoning_steps_reward": 1.0,
+      "step": 170
+    },
+    {
+      "completion_length": 1020.1089332580566,
+      "epoch": 0.27055754179147745,
+      "grad_norm": 51.0734022575601,
+      "kl": 2.13564453125,
+      "learning_rate": 1.8282640638332773e-05,
+      "loss": 0.0855,
+      "reward": 0.9397083148360252,
+      "reward_std": 0.07600208498770371,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0760655484162271,
+      "rewards/format_reward": 0.016071429289877415,
+      "rewards/reasoning_steps_reward": 0.9997023820877076,
+      "step": 175
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.27828775727123395,
+      "grad_norm": 9.026347870282267,
+      "kl": 0.5303955078125,
+      "learning_rate": 1.8128138751472432e-05,
+      "loss": 0.0212,
+      "reward": 1.032665991783142,
+      "reward_std": 0.12255189336137846,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.04769120002165437,
+      "rewards/format_reward": 0.08035714738070965,
+      "rewards/reasoning_steps_reward": 1.0,
+      "step": 180
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.28601797275099045,
+      "grad_norm": 0.4517769901102366,
+      "kl": 0.4077880859375,
+      "learning_rate": 1.7967695965958044e-05,
+      "loss": 0.0163,
+      "reward": 1.5157621785998345,
+      "reward_std": 0.3208800745662302,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02798790571396239,
+      "rewards/format_reward": 0.5767857381142676,
+      "rewards/reasoning_steps_reward": 0.9669643253087997,
+      "step": 185
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.29374818823074694,
+      "grad_norm": 0.4242780022201252,
+      "kl": 0.410595703125,
+      "learning_rate": 1.780142955025139e-05,
+      "loss": 0.0164,
+      "reward": 1.9678164646029472,
+      "reward_std": 0.04180803272465709,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.009564569016220048,
+      "rewards/format_reward": 0.9785714380443096,
+      "rewards/reasoning_steps_reward": 0.9979166723787785,
+      "step": 190
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.30147840371050344,
+      "grad_norm": 0.3332720853311019,
+      "kl": 0.345654296875,
+      "learning_rate": 1.7629461029335683e-05,
+      "loss": 0.0138,
+      "reward": 1.9703096732497216,
+      "reward_std": 0.04092628928920021,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.007964215916581452,
+      "rewards/format_reward": 0.9848214358091354,
+      "rewards/reasoning_steps_reward": 0.9925595372915268,
+      "step": 195
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.30920861919025994,
+      "grad_norm": 0.2795217499990793,
+      "kl": 0.3870361328125,
+      "learning_rate": 1.745191609589231e-05,
+      "loss": 0.0155,
+      "reward": 1.9680001959204674,
+      "reward_std": 0.041035071920487096,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.01592847361753229,
+      "rewards/format_reward": 0.9919642895460129,
+      "rewards/reasoning_steps_reward": 0.9910714499652385,
+      "step": 200
+    },
+    {
+      "epoch": 0.30920861919025994,
+      "eval_completion_length": 1024.0,
+      "eval_kl": 0.381103515625,
+      "eval_loss": 0.015190178528428078,
+      "eval_reward": 1.927598088979721,
+      "eval_reward_std": 0.07101984624750912,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.030735314125195146,
+      "eval_rewards/format_reward": 0.9598214477300644,
+      "eval_rewards/reasoning_steps_reward": 0.9985119178891182,
+      "eval_runtime": 61.5768,
+      "eval_samples_per_second": 1.608,
+      "eval_steps_per_second": 0.065,
+      "step": 200
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.31693883467001643,
+      "grad_norm": 0.39741821896953167,
+      "kl": 771.5985595703125,
+      "learning_rate": 1.7268924518431437e-05,
+      "loss": 30.9403,
+      "reward": 1.8754769906401634,
+      "reward_std": 0.143933101760922,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.031368333660066125,
+      "rewards/format_reward": 0.9098214559257031,
+      "rewards/reasoning_steps_reward": 0.996130958944559,
+      "step": 205
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.32466905014977293,
+      "grad_norm": 0.24983284857106747,
+      "kl": 0.30125732421875,
+      "learning_rate": 1.7080620046443503e-05,
+      "loss": 0.0121,
+      "reward": 1.8183680206537247,
+      "reward_std": 0.2094764916575514,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.03490588075947017,
+      "rewards/format_reward": 0.8544643260538578,
+      "rewards/reasoning_steps_reward": 0.9979166716337204,
+      "step": 210
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.33239926562952943,
+      "grad_norm": 0.2796584952581468,
+      "kl": 0.2487060546875,
+      "learning_rate": 1.6887140312641036e-05,
+      "loss": 0.0099,
+      "reward": 1.688377882540226,
+      "reward_std": 0.30055564286885783,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.024122193665243687,
+      "rewards/format_reward": 0.7116071753203869,
+      "rewards/reasoning_steps_reward": 1.0,
+      "step": 215
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3401294811092859,
+      "grad_norm": 0.3394156120918996,
+      "kl": 0.26431884765625,
+      "learning_rate": 1.6688626732362192e-05,
+      "loss": 0.0106,
+      "reward": 1.622536237537861,
+      "reward_std": 0.3328998180571944,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.0631781310075894,
+      "rewards/format_reward": 0.6919643215835094,
+      "rewards/reasoning_steps_reward": 0.9928571574389935,
+      "step": 220
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3478596965890424,
+      "grad_norm": 0.12323180634844798,
+      "kl": 0.163543701171875,
+      "learning_rate": 1.6485224400209557e-05,
+      "loss": 0.0065,
+      "reward": 1.4264108300209046,
+      "reward_std": 0.36164557933807373,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.045315440162085,
+      "rewards/format_reward": 0.4794643077999353,
+      "rewards/reasoning_steps_reward": 0.9922619216144085,
+      "step": 225
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3555899120687989,
+      "grad_norm": 1.1461722123281664,
+      "kl": 0.20413818359375,
+      "learning_rate": 1.6277081983999742e-05,
+      "loss": 0.0082,
+      "reward": 1.5457165658473968,
+      "reward_std": 0.2846676936373115,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.16886685080826283,
+      "rewards/format_reward": 0.7214285962283611,
+      "rewards/reasoning_steps_reward": 0.9931547753512859,
+      "step": 230
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3633201275485554,
+      "grad_norm": 0.28315172743754835,
+      "kl": 0.27060546875,
+      "learning_rate": 1.6064351616101318e-05,
+      "loss": 0.0108,
+      "reward": 1.8082718402147293,
+      "reward_std": 0.11225779644446447,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.13309730420587584,
+      "rewards/format_reward": 0.9812500089406967,
+      "rewards/reasoning_steps_reward": 0.96011907979846,
+      "step": 235
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3710503430283119,
+      "grad_norm": 0.23337110623082213,
+      "kl": 0.2306396484375,
+      "learning_rate": 1.5847188782240473e-05,
+      "loss": 0.0092,
+      "reward": 1.8728422731161118,
+      "reward_std": 0.1430520822061226,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.028943523665657268,
+      "rewards/format_reward": 0.9071428924798965,
+      "rewards/reasoning_steps_reward": 0.9946428649127483,
+      "step": 240
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3787805585080684,
+      "grad_norm": 0.18427188675046358,
+      "kl": 0.2426513671875,
+      "learning_rate": 1.562575220785569e-05,
+      "loss": 0.0097,
+      "reward": 1.8290013462305068,
+      "reward_std": 0.19572628067107872,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.018320175528060645,
+      "rewards/format_reward": 0.8848214596509933,
+      "rewards/reasoning_steps_reward": 0.9616071864962578,
+      "step": 245
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.3865107739878249,
+      "grad_norm": 0.21464170252675563,
+      "kl": 0.24188232421875,
+      "learning_rate": 1.5400203742084508e-05,
+      "loss": 0.0097,
+      "reward": 1.9442964002490044,
+      "reward_std": 0.07978458781144582,
+      "rewards/accuracy_reward": 0.00357142873108387,
+      "rewards/cosine_scaled_reward": -0.016715598275186493,
+      "rewards/format_reward": 0.9883928626775742,
+      "rewards/reasoning_steps_reward": 0.9690476559102535,
+      "step": 250
+    },
+    {
+      "completion_length": 1023.6294647216797,
+      "epoch": 0.3942409894675814,
+      "grad_norm": 0.2407491270261808,
+      "kl": 0.26353759765625,
+      "learning_rate": 1.5170708239467143e-05,
+      "loss": 0.0105,
+      "reward": 1.9458773970603942,
+      "reward_std": 0.07613754472695292,
+      "rewards/accuracy_reward": 0.00357142873108387,
+      "rewards/cosine_scaled_reward": -0.03537268368527293,
+      "rewards/format_reward": 0.9883928626775742,
+      "rewards/reasoning_steps_reward": 0.989285732805729,
+      "step": 255
+    },
+    {
+      "completion_length": 1014.4705436706543,
+      "epoch": 0.4019712049473379,
+      "grad_norm": 0.3467057009002833,
+      "kl": 0.3231689453125,
+      "learning_rate": 1.4937433439453465e-05,
+      "loss": 0.0129,
+      "reward": 1.8792840793728829,
+      "reward_std": 0.1289262903505005,
+      "rewards/accuracy_reward": 0.0026785715483129023,
+      "rewards/cosine_scaled_reward": -0.06268029478378594,
+      "rewards/format_reward": 0.9517857365310192,
+      "rewards/reasoning_steps_reward": 0.9875000208616257,
+      "step": 260
+    },
+    {
+      "completion_length": 198.87232966423034,
+      "epoch": 0.4097014204270944,
+      "grad_norm": 1.889219820728968,
+      "kl": 1.1971435546875,
+      "learning_rate": 1.4700549843801359e-05,
+      "loss": 0.0479,
+      "reward": 1.617885561287403,
+      "reward_std": 0.18369729482219554,
+      "rewards/accuracy_reward": 0.001785714365541935,
+      "rewards/cosine_scaled_reward": -0.09044785196310841,
+      "rewards/format_reward": 0.9500000230967999,
+      "rewards/reasoning_steps_reward": 0.7565476797521115,
+      "step": 265
+    },
+    {
+      "completion_length": 61.36785981655121,
+      "epoch": 0.4174316359068509,
+      "grad_norm": 1.5439650810408614,
+      "kl": 2.06083984375,
+      "learning_rate": 1.4460230591956097e-05,
+      "loss": 0.0824,
+      "reward": 1.7324005201458932,
+      "reward_std": 0.1723117540634121,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.07265909540001303,
+      "rewards/format_reward": 0.8964285887777805,
+      "rewards/reasoning_steps_reward": 0.9086310066282749,
+      "step": 270
+    },
+    {
+      "completion_length": 30.82410855293274,
+      "epoch": 0.4251618513866074,
+      "grad_norm": 1.5711994372600742,
+      "kl": 2.94609375,
+      "learning_rate": 1.421665133450184e-05,
+      "loss": 0.1178,
+      "reward": 1.8442469641566277,
+      "reward_std": 0.172434140794212,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.020038836018647997,
+      "rewards/format_reward": 0.9116071671247482,
+      "rewards/reasoning_steps_reward": 0.9526786282658577,
+      "step": 275
+    },
+    {
+      "completion_length": 31.485715675354005,
+      "epoch": 0.4328920668663639,
+      "grad_norm": 1.254143062627419,
+      "kl": 3.3177734375,
+      "learning_rate": 1.3969990104777712e-05,
+      "loss": 0.1327,
+      "reward": 1.9396235466003418,
+      "reward_std": 0.07859349314680912,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.019305108429398386,
+      "rewards/format_reward": 0.9723214410245419,
+      "rewards/reasoning_steps_reward": 0.985714317113161,
+      "step": 280
+    },
+    {
+      "completion_length": 112.89821949005128,
+      "epoch": 0.4406222823461204,
+      "grad_norm": 1.7486323762646898,
+      "kl": 3.2439453125,
+      "learning_rate": 1.3720427188752306e-05,
+      "loss": 0.1297,
+      "reward": 1.8264620706439019,
+      "reward_std": 0.17633500225992976,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02085943255224265,
+      "rewards/format_reward": 0.8508928880095482,
+      "rewards/reasoning_steps_reward": 0.9964285835623741,
+      "step": 285
+    },
+    {
+      "completion_length": 85.93482549190522,
+      "epoch": 0.4483524978258769,
+      "grad_norm": 0.18585642120389043,
+      "kl": 3.4283203125,
+      "learning_rate": 1.3468144993251735e-05,
+      "loss": 0.1372,
+      "reward": 1.9039041504263878,
+      "reward_std": 0.11914442183588107,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.008595946489367633,
+      "rewards/format_reward": 0.9250000327825546,
+      "rewards/reasoning_steps_reward": 0.9875000223517418,
+      "step": 290
+    },
+    {
+      "completion_length": 68.39286019802094,
+      "epoch": 0.4560827133056334,
+      "grad_norm": 0.6138249675103672,
+      "kl": 3.4900390625,
+      "learning_rate": 1.3213327912637563e-05,
+      "loss": 0.1396,
+      "reward": 1.9247175946831703,
+      "reward_std": 0.09705542341289401,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.005044402932981029,
+      "rewards/format_reward": 0.9473214514553547,
+      "rewards/reasoning_steps_reward": 0.9824405036866665,
+      "step": 295
+    },
+    {
+      "completion_length": 92.07054011821747,
+      "epoch": 0.4638129287853899,
+      "grad_norm": 17.64447481437311,
+      "kl": 6.3109375,
+      "learning_rate": 1.295616219403197e-05,
+      "loss": 0.2532,
+      "reward": 1.8822270065546036,
+      "reward_std": 0.15449890223480905,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.007951658876845614,
+      "rewards/format_reward": 0.9205357454717159,
+      "rewards/reasoning_steps_reward": 0.9696429014205933,
+      "step": 300
+    },
+    {
+      "epoch": 0.4638129287853899,
+      "eval_completion_length": 53.17351460456848,
+      "eval_kl": 3.720703125,
+      "eval_loss": 0.14936980605125427,
+      "eval_reward": 1.9487695544958115,
+      "eval_reward_std": 0.06659787986427546,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.006587653420865536,
+      "eval_rewards/format_reward": 0.9642857313156128,
+      "eval_rewards/reasoning_steps_reward": 0.9910714626312256,
+      "eval_runtime": 27.7064,
+      "eval_samples_per_second": 3.573,
+      "eval_steps_per_second": 0.144,
+      "step": 300
+    },
+    {
+      "completion_length": 48.73571631908417,
+      "epoch": 0.4715431442651464,
+      "grad_norm": 0.5068156353598838,
+      "kl": 3.772265625,
+      "learning_rate": 1.2696835801188816e-05,
+      "loss": 0.1509,
+      "reward": 1.9468622133135796,
+      "reward_std": 0.06508203058242543,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.005816462484654039,
+      "rewards/format_reward": 0.9642857305705548,
+      "rewards/reasoning_steps_reward": 0.9883928790688514,
+      "step": 305
+    },
+    {
+      "completion_length": 43.2633944272995,
+      "epoch": 0.47927335974490287,
+      "grad_norm": 0.3176925953315301,
+      "kl": 3.947265625,
+      "learning_rate": 1.2435538277109919e-05,
+      "loss": 0.1578,
+      "reward": 1.9513282284140587,
+      "reward_std": 0.059372010298375244,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.010576637217309327,
+      "rewards/format_reward": 0.9714285850524902,
+      "rewards/reasoning_steps_reward": 0.9904762081801891,
+      "step": 310
+    },
+    {
+      "completion_length": 39.936609101295474,
+      "epoch": 0.48700357522465937,
+      "grad_norm": 0.6443106715438625,
+      "kl": 3.9490234375,
+      "learning_rate": 1.2172460605507126e-05,
+      "loss": 0.1579,
+      "reward": 1.9332506194710732,
+      "reward_std": 0.08854502255671833,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.014070913463365286,
+      "rewards/format_reward": 0.963392873108387,
+      "rewards/reasoning_steps_reward": 0.9839286029338836,
+      "step": 315
+    },
+    {
+      "completion_length": 52.19107358455658,
+      "epoch": 0.49473379070441587,
+      "grad_norm": 0.7546037355193578,
+      "kl": 4.55859375,
+      "learning_rate": 1.19077950712113e-05,
+      "loss": 0.1824,
+      "reward": 1.8329863145947456,
+      "reward_std": 0.2112982230493799,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0262399717932567,
+      "rewards/format_reward": 0.9071428939700127,
+      "rewards/reasoning_steps_reward": 0.9520833820104599,
+      "step": 320
+    },
+    {
+      "completion_length": 43.95535898208618,
+      "epoch": 0.5024640061841724,
+      "grad_norm": 1.7212994430025008,
+      "kl": 4.2908203125,
+      "learning_rate": 1.1641735119630373e-05,
+      "loss": 0.1716,
+      "reward": 1.8960628926753997,
+      "reward_std": 0.13772980722906142,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.016139589733211324,
+      "rewards/format_reward": 0.9419643118977546,
+      "rewards/reasoning_steps_reward": 0.9702381290495395,
+      "step": 325
+    },
+    {
+      "completion_length": 88.43839681148529,
+      "epoch": 0.5101942216639289,
+      "grad_norm": 2.833261374391085,
+      "kl": 4.2220703125,
+      "learning_rate": 1.137447521535908e-05,
+      "loss": 0.1689,
+      "reward": 1.8282963901758194,
+      "reward_std": 0.19430164659138427,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.025275125188636595,
+      "rewards/format_reward": 0.8919643208384513,
+      "rewards/reasoning_steps_reward": 0.9616071946918965,
+      "step": 330
+    },
+    {
+      "completion_length": 66.55982444286346,
+      "epoch": 0.5179244371436854,
+      "grad_norm": 0.6221989550919217,
+      "kl": 4.0482421875,
+      "learning_rate": 1.110621070004378e-05,
+      "loss": 0.1619,
+      "reward": 1.8733744263648986,
+      "reward_std": 0.16404558259237093,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.021268530780798756,
+      "rewards/format_reward": 0.9267857410013676,
+      "rewards/reasoning_steps_reward": 0.9678571939468383,
+      "step": 335
+    },
+    {
+      "completion_length": 70.68839602470398,
+      "epoch": 0.5256546526234419,
+      "grad_norm": 1.497593841474171,
+      "kl": 3.9134765625,
+      "learning_rate": 1.0837137649606241e-05,
+      "loss": 0.1565,
+      "reward": 1.8916518941521645,
+      "reward_std": 0.14140696468043643,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02441962165758014,
+      "rewards/format_reward": 0.9410714536905289,
+      "rewards/reasoning_steps_reward": 0.9750000379979611,
+      "step": 340
+    },
+    {
+      "completion_length": 89.36161134243011,
+      "epoch": 0.5333848681031984,
+      "grad_norm": 0.35372687668193753,
+      "kl": 3.51796875,
+      "learning_rate": 1.0567452730930743e-05,
+      "loss": 0.1407,
+      "reward": 1.8679632544517517,
+      "reward_std": 0.1587062255845922,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03739399563637562,
+      "rewards/format_reward": 0.9312500260770321,
+      "rewards/reasoning_steps_reward": 0.9741071820259094,
+      "step": 345
+    },
+    {
+      "completion_length": 71.34107451438904,
+      "epoch": 0.5411150835829549,
+      "grad_norm": 0.5702206180346077,
+      "kl": 3.4873046875,
+      "learning_rate": 1.0297353058119209e-05,
+      "loss": 0.1395,
+      "reward": 1.8956190168857574,
+      "reward_std": 0.13545800935062288,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030273928574752064,
+      "rewards/format_reward": 0.951785734295845,
+      "rewards/reasoning_steps_reward": 0.9741071708500385,
+      "step": 350
+    },
+    {
+      "completion_length": 72.94018166065216,
+      "epoch": 0.5488452990627114,
+      "grad_norm": 0.196755343328829,
+      "kl": 3.54375,
+      "learning_rate": 1.0027036048419514e-05,
+      "loss": 0.1417,
+      "reward": 1.8875077337026596,
+      "reward_std": 0.1440115618094751,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030944747471949086,
+      "rewards/format_reward": 0.9455357365310192,
+      "rewards/reasoning_steps_reward": 0.9729167051613331,
+      "step": 355
+    },
+    {
+      "completion_length": 54.92678787708282,
+      "epoch": 0.5565755145424679,
+      "grad_norm": 0.2567810162757019,
+      "kl": 3.3958984375,
+      "learning_rate": 9.756699277932196e-06,
+      "loss": 0.1358,
+      "reward": 1.9269503444433211,
+      "reward_std": 0.0966148825835262,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0221569019544404,
+      "rewards/format_reward": 0.963392873108387,
+      "rewards/reasoning_steps_reward": 0.9857143104076386,
+      "step": 360
+    },
+    {
+      "completion_length": 72.01518182754516,
+      "epoch": 0.5643057300222244,
+      "grad_norm": 0.1829959012401895,
+      "kl": 3.1505859375,
+      "learning_rate": 9.486540337201046e-06,
+      "loss": 0.126,
+      "reward": 1.8959941297769547,
+      "reward_std": 0.13998180008102282,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030494074343005197,
+      "rewards/format_reward": 0.9482143081724643,
+      "rewards/reasoning_steps_reward": 0.9782738350331783,
+      "step": 365
+    },
+    {
+      "completion_length": 64.86786003112793,
+      "epoch": 0.5720359455019809,
+      "grad_norm": 0.2587368698858213,
+      "kl": 3.344140625,
+      "learning_rate": 9.216756686793163e-06,
+      "loss": 0.1338,
+      "reward": 1.9080506026744843,
+      "reward_std": 0.12041498598309772,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.027068544877693057,
+      "rewards/format_reward": 0.9571428768336773,
+      "rewards/reasoning_steps_reward": 0.9779762201011181,
+      "step": 370
+    },
+    {
+      "completion_length": 79.13750350475311,
+      "epoch": 0.5797661609817374,
+      "grad_norm": 0.6472814862064507,
+      "kl": 3.48828125,
+      "learning_rate": 8.94754551297402e-06,
+      "loss": 0.1395,
+      "reward": 1.87237848341465,
+      "reward_std": 0.16542400652033393,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.033871618512785064,
+      "rewards/format_reward": 0.9375000268220901,
+      "rewards/reasoning_steps_reward": 0.9687500350177288,
+      "step": 375
+    },
+    {
+      "completion_length": 73.87143204212188,
+      "epoch": 0.5874963764614939,
+      "grad_norm": 0.2300340441130014,
+      "kl": 3.5958984375,
+      "learning_rate": 8.67910358358298e-06,
+      "loss": 0.1438,
+      "reward": 1.8885457441210747,
+      "reward_std": 0.13497921331181714,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.03079959342139773,
+      "rewards/format_reward": 0.945535734295845,
+      "rewards/reasoning_steps_reward": 0.972916702926159,
+      "step": 380
+    },
+    {
+      "completion_length": 73.79464621543885,
+      "epoch": 0.5952265919412504,
+      "grad_norm": 0.18795751532079963,
+      "kl": 3.540234375,
+      "learning_rate": 8.411627104214675e-06,
+      "loss": 0.1416,
+      "reward": 1.8874747961759568,
+      "reward_std": 0.14899413716896107,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03127530172932893,
+      "rewards/format_reward": 0.9446428820490838,
+      "rewards/reasoning_steps_reward": 0.9741071827709675,
+      "step": 385
+    },
+    {
+      "completion_length": 76.45803918838502,
+      "epoch": 0.6029568074210069,
+      "grad_norm": 0.2560833861847512,
+      "kl": 3.5017578125,
+      "learning_rate": 8.145311574811325e-06,
+      "loss": 0.14,
+      "reward": 1.8978156805038453,
+      "reward_std": 0.1357477414895129,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.032541560102254154,
+      "rewards/format_reward": 0.9526785925030709,
+      "rewards/reasoning_steps_reward": 0.9776786133646965,
+      "step": 390
+    },
+    {
+      "completion_length": 61.17500262260437,
+      "epoch": 0.6106870229007634,
+      "grad_norm": 0.14156843477971917,
+      "kl": 3.583984375,
+      "learning_rate": 7.880351646770824e-06,
+      "loss": 0.1433,
+      "reward": 1.9174254342913628,
+      "reward_std": 0.11136556981223293,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.024241326967603526,
+      "rewards/format_reward": 0.958035734295845,
+      "rewards/reasoning_steps_reward": 0.9827381305396556,
+      "step": 395
+    },
+    {
+      "completion_length": 69.2750031709671,
+      "epoch": 0.6184172383805199,
+      "grad_norm": 0.5154686075500935,
+      "kl": 3.551953125,
+      "learning_rate": 7.616940980675004e-06,
+      "loss": 0.1421,
+      "reward": 1.891478382050991,
+      "reward_std": 0.13758877764275895,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.029057439247844742,
+      "rewards/format_reward": 0.9482143096625805,
+      "rewards/reasoning_steps_reward": 0.9723214752972126,
+      "step": 400
+    },
+    {
+      "epoch": 0.6184172383805199,
+      "eval_completion_length": 107.8214340209961,
+      "eval_kl": 3.58203125,
+      "eval_loss": 0.14280247688293457,
+      "eval_reward": 1.8317049890756607,
+      "eval_reward_std": 0.19036237383261323,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.047759354987647384,
+      "eval_rewards/format_reward": 0.9107143208384514,
+      "eval_rewards/reasoning_steps_reward": 0.9687500298023224,
+      "eval_runtime": 35.747,
+      "eval_samples_per_second": 2.769,
+      "eval_steps_per_second": 0.112,
+      "step": 400
+    },
+    {
+      "completion_length": 63.0035742521286,
+      "epoch": 0.6261474538602764,
+      "grad_norm": 0.04709338457327443,
+      "kl": 3.5291015625,
+      "learning_rate": 7.355272104742132e-06,
+      "loss": 0.1412,
+      "reward": 1.9105920106172563,
+      "reward_std": 0.12058914793014992,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.026015246647875755,
+      "rewards/format_reward": 0.9544643066823483,
+      "rewards/reasoning_steps_reward": 0.9821428917348385,
+      "step": 405
+    },
+    {
+      "completion_length": 58.422323846817015,
+      "epoch": 0.6338776693400329,
+      "grad_norm": 0.18715535959210597,
+      "kl": 3.5099609375,
+      "learning_rate": 7.095536274107046e-06,
+      "loss": 0.1404,
+      "reward": 1.917353543639183,
+      "reward_std": 0.11102688768487497,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.023717991745797917,
+      "rewards/format_reward": 0.9589285902678967,
+      "rewards/reasoning_steps_reward": 0.9821428820490837,
+      "step": 410
+    },
+    {
+      "completion_length": 72.81786034107208,
+      "epoch": 0.6416078848197894,
+      "grad_norm": 0.1071854451049798,
+      "kl": 3.4794921875,
+      "learning_rate": 6.837923331031761e-06,
+      "loss": 0.1392,
+      "reward": 1.8966768980026245,
+      "reward_std": 0.13567318589484784,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03070416395785287,
+      "rewards/format_reward": 0.9473214507102966,
+      "rewards/reasoning_steps_reward": 0.9800595581531525,
+      "step": 415
+    },
+    {
+      "completion_length": 97.09911193847657,
+      "epoch": 0.6493381002995459,
+      "grad_norm": 0.2244006344733829,
+      "kl": 3.446484375,
+      "learning_rate": 6.58262156614881e-06,
+      "loss": 0.1378,
+      "reward": 1.85521010607481,
+      "reward_std": 0.187637352827187,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.042409035906894134,
+      "rewards/format_reward": 0.9232143186032772,
+      "rewards/reasoning_steps_reward": 0.9744048126041889,
+      "step": 420
+    },
+    {
+      "completion_length": 89.34196858406067,
+      "epoch": 0.6570683157793024,
+      "grad_norm": 0.12727317452906473,
+      "kl": 3.47890625,
+      "learning_rate": 6.3298175808386284e-06,
+      "loss": 0.1391,
+      "reward": 1.8599596157670022,
+      "reward_std": 0.1839674000402738,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03765952198300511,
+      "rewards/format_reward": 0.9285714574158191,
+      "rewards/reasoning_steps_reward": 0.96904766112566,
+      "step": 425
+    },
+    {
+      "completion_length": 77.31250357627869,
+      "epoch": 0.6647985312590589,
+      "grad_norm": 0.1370553058236246,
+      "kl": 3.48828125,
+      "learning_rate": 6.079696150841634e-06,
+      "loss": 0.1395,
+      "reward": 1.889148586988449,
+      "reward_std": 0.14615388929378242,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030196763610001655,
+      "rewards/format_reward": 0.9419643133878708,
+      "rewards/reasoning_steps_reward": 0.9773809894919395,
+      "step": 430
+    },
+    {
+      "completion_length": 79.07946805953979,
+      "epoch": 0.6725287467388154,
+      "grad_norm": 0.19615244247307984,
+      "kl": 3.5166015625,
+      "learning_rate": 5.832440091204698e-06,
+      "loss": 0.1406,
+      "reward": 1.8841874316334724,
+      "reward_std": 0.15793084264860227,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03128885845653713,
+      "rewards/format_reward": 0.9419643118977546,
+      "rewards/reasoning_steps_reward": 0.9735119409859181,
+      "step": 435
+    },
+    {
+      "completion_length": 88.07857568264008,
+      "epoch": 0.6802589622185718,
+      "grad_norm": 0.4724963752927991,
+      "kl": 3.5033203125,
+      "learning_rate": 5.588230122660672e-06,
+      "loss": 0.1401,
+      "reward": 1.8664761379361152,
+      "reward_std": 0.164639704185538,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.036500148748746145,
+      "rewards/format_reward": 0.932142885029316,
+      "rewards/reasoning_steps_reward": 0.9708333738148213,
+      "step": 440
+    },
+    {
+      "completion_length": 113.22857713699341,
+      "epoch": 0.6879891776983283,
+      "grad_norm": 0.3097996813031107,
+      "kl": 3.45,
+      "learning_rate": 5.347244739538677e-06,
+      "loss": 0.138,
+      "reward": 1.8203087821602821,
+      "reward_std": 0.24133571856264097,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.046655593352625147,
+      "rewards/format_reward": 0.9080357499420643,
+      "rewards/reasoning_steps_reward": 0.9589286193251609,
+      "step": 445
+    },
+    {
+      "completion_length": 111.4133985042572,
+      "epoch": 0.6957193931780848,
+      "grad_norm": 0.2012585894133353,
+      "kl": 3.41640625,
+      "learning_rate": 5.109660079301668e-06,
+      "loss": 0.1366,
+      "reward": 1.8180661857128144,
+      "reward_std": 0.23534413103952828,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0471124786825385,
+      "rewards/format_reward": 0.9098214633762837,
+      "rewards/reasoning_steps_reward": 0.9553571894764901,
+      "step": 450
+    },
+    {
+      "completion_length": 77.29286074638367,
+      "epoch": 0.7034496086578413,
+      "grad_norm": 0.17652208862596358,
+      "kl": 3.4974609375,
+      "learning_rate": 4.875649793806655e-06,
+      "loss": 0.1399,
+      "reward": 1.8818938612937928,
+      "reward_std": 0.15572975755203516,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03179671107791364,
+      "rewards/format_reward": 0.9419643104076385,
+      "rewards/reasoning_steps_reward": 0.9717262268066407,
+      "step": 455
+    },
+    {
+      "completion_length": 62.00982408523559,
+      "epoch": 0.7111798241375978,
+      "grad_norm": 0.04887797991180135,
+      "kl": 3.5248046875,
+      "learning_rate": 4.64538492238166e-06,
+      "loss": 0.141,
+      "reward": 1.9128967747092247,
+      "reward_std": 0.11732972208019418,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.024603341292822732,
+      "rewards/format_reward": 0.9562500201165676,
+      "rewards/reasoning_steps_reward": 0.9812500312924385,
+      "step": 460
+    },
+    {
+      "completion_length": 62.917859840393064,
+      "epoch": 0.7189100396173543,
+      "grad_norm": 0.08187645718348353,
+      "kl": 3.5171875,
+      "learning_rate": 4.4190337668121964e-06,
+      "loss": 0.1407,
+      "reward": 1.9146563693881036,
+      "reward_std": 0.1148412693213686,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.024927071237470955,
+      "rewards/format_reward": 0.9562500201165676,
+      "rewards/reasoning_steps_reward": 0.9833333626389503,
+      "step": 465
+    },
+    {
+      "completion_length": 63.83928849697113,
+      "epoch": 0.7266402550971108,
+      "grad_norm": 0.07879219054017059,
+      "kl": 3.510546875,
+      "learning_rate": 4.196761768328599e-06,
+      "loss": 0.1404,
+      "reward": 1.9097336187958718,
+      "reward_std": 0.114427309948951,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.025385533989174293,
+      "rewards/format_reward": 0.9535714477300644,
+      "rewards/reasoning_steps_reward": 0.9815476454794407,
+      "step": 470
+    },
+    {
+      "completion_length": 73.70536048412323,
+      "epoch": 0.7343704705768673,
+      "grad_norm": 0.027892131042639323,
+      "kl": 3.4958984375,
+      "learning_rate": 3.978731386684206e-06,
+      "loss": 0.1398,
+      "reward": 1.8938029453158378,
+      "reward_std": 0.14057790259539615,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030006676557241006,
+      "rewards/format_reward": 0.9446428827941418,
+      "rewards/reasoning_steps_reward": 0.9791667051613331,
+      "step": 475
+    },
+    {
+      "completion_length": 84.47589683532715,
+      "epoch": 0.7421006860566238,
+      "grad_norm": 0.08677532303093845,
+      "kl": 3.4443359375,
+      "learning_rate": 3.7651019814126656e-06,
+      "loss": 0.1378,
+      "reward": 1.8761127531528472,
+      "reward_std": 0.1564179871153101,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03549448415869847,
+      "rewards/format_reward": 0.9357143126428127,
+      "rewards/reasoning_steps_reward": 0.9758928962051868,
+      "step": 480
+    },
+    {
+      "completion_length": 91.65446877479553,
+      "epoch": 0.7498309015363803,
+      "grad_norm": 0.24200171800321144,
+      "kl": 3.43359375,
+      "learning_rate": 3.5560296953512296e-06,
+      "loss": 0.1373,
+      "reward": 1.8591931834816933,
+      "reward_std": 0.16599050024733514,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03842596691683866,
+      "rewards/format_reward": 0.9276785954833031,
+      "rewards/reasoning_steps_reward": 0.9699405111372471,
+      "step": 485
+    },
+    {
+      "completion_length": 85.36607551574707,
+      "epoch": 0.7575611170161368,
+      "grad_norm": 0.056901915490173295,
+      "kl": 3.43671875,
+      "learning_rate": 3.3516673405151546e-06,
+      "loss": 0.1375,
+      "reward": 1.8713713854551315,
+      "reward_std": 0.17605545576270742,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.035771561722503974,
+      "rewards/format_reward": 0.9339285984635353,
+      "rewards/reasoning_steps_reward": 0.97321432903409,
+      "step": 490
+    },
+    {
+      "completion_length": 72.79286041259766,
+      "epoch": 0.7652913324958933,
+      "grad_norm": 0.21511832859969,
+      "kl": 3.4658203125,
+      "learning_rate": 3.1521642864065905e-06,
+      "loss": 0.1387,
+      "reward": 1.8944186642765999,
+      "reward_std": 0.12519371660437173,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.029688578064087777,
+      "rewards/format_reward": 0.9464285932481289,
+      "rewards/reasoning_steps_reward": 0.9776786051690578,
+      "step": 495
+    },
+    {
+      "completion_length": 72.79107475280762,
+      "epoch": 0.7730215479756498,
+      "grad_norm": 0.04223723247860192,
+      "kl": 3.4537109375,
+      "learning_rate": 2.957666350839663e-06,
+      "loss": 0.1381,
+      "reward": 1.8936428263783456,
+      "reward_std": 0.13871936192344947,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.029869176616193725,
+      "rewards/format_reward": 0.9464285969734192,
+      "rewards/reasoning_steps_reward": 0.9770833663642406,
+      "step": 500
+    },
+    {
+      "epoch": 0.7730215479756498,
+      "eval_completion_length": 98.31190967559814,
+      "eval_kl": 3.498046875,
+      "eval_loss": 0.13821543753147125,
+      "eval_reward": 1.8459261506795883,
+      "eval_reward_std": 0.1891471924027428,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.03949062270112336,
+      "eval_rewards/format_reward": 0.9241071790456772,
+      "eval_rewards/reasoning_steps_reward": 0.961309552192688,
+      "eval_runtime": 45.7093,
+      "eval_samples_per_second": 2.166,
+      "eval_steps_per_second": 0.088,
+      "step": 500
+    },
+    {
+      "completion_length": 66.50357437133789,
+      "epoch": 0.7807517634554063,
+      "grad_norm": 0.0248301767642567,
+      "kl": 3.4912109375,
+      "learning_rate": 2.768315693361474e-06,
+      "loss": 0.1397,
+      "reward": 1.9070332109928132,
+      "reward_std": 0.11746567972004414,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.026597837242297827,
+      "rewards/format_reward": 0.9526785925030709,
+      "rewards/reasoning_steps_reward": 0.9809524178504944,
+      "step": 505
+    },
+    {
+      "completion_length": 72.8000033378601,
+      "epoch": 0.7884819789351628,
+      "grad_norm": 0.06143871493916858,
+      "kl": 3.4701171875,
+      "learning_rate": 2.5842507113469307e-06,
+      "loss": 0.1388,
+      "reward": 1.8958711713552474,
+      "reward_std": 0.1414075436458006,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.030021787446457893,
+      "rewards/format_reward": 0.9464285939931869,
+      "rewards/reasoning_steps_reward": 0.9794643186032772,
+      "step": 510
+    },
+    {
+      "completion_length": 75.4857177734375,
+      "epoch": 0.7962121944149193,
+      "grad_norm": 0.028586002230415825,
+      "kl": 3.4546875,
+      "learning_rate": 2.405605938843416e-06,
+      "loss": 0.1382,
+      "reward": 1.8888832941651343,
+      "reward_std": 0.1412749015726149,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03135490003041923,
+      "rewards/format_reward": 0.9446428820490838,
+      "rewards/reasoning_steps_reward": 0.9755952760577202,
+      "step": 515
+    },
+    {
+      "completion_length": 95.24732608795166,
+      "epoch": 0.8039424098946758,
+      "grad_norm": 0.08861098505851854,
+      "kl": 3.4146484375,
+      "learning_rate": 2.2325119482391466e-06,
+      "loss": 0.1366,
+      "reward": 1.852172775566578,
+      "reward_std": 0.18426264682784677,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.040684470417909326,
+      "rewards/format_reward": 0.9241071723401546,
+      "rewards/reasoning_steps_reward": 0.9687500439584256,
+      "step": 520
+    },
+    {
+      "completion_length": 81.48393249511719,
+      "epoch": 0.8116726253744323,
+      "grad_norm": 710.8421619250436,
+      "kl": 11.9974609375,
+      "learning_rate": 2.065095254827133e-06,
+      "loss": 0.4807,
+      "reward": 1.8779245063662529,
+      "reward_std": 0.15769884596811606,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.033980353159131484,
+      "rewards/format_reward": 0.9375000268220901,
+      "rewards/reasoning_steps_reward": 0.9744048058986664,
+      "step": 525
+    },
+    {
+      "completion_length": 64.70714569091797,
+      "epoch": 0.8194028408541888,
+      "grad_norm": 0.04884046335826561,
+      "kl": 3.493359375,
+      "learning_rate": 1.9034782243345074e-06,
+      "loss": 0.1397,
+      "reward": 1.9050584733486176,
+      "reward_std": 0.11774966189013866,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02619163042982109,
+      "rewards/format_reward": 0.9535714477300644,
+      "rewards/reasoning_steps_reward": 0.9776785969734192,
+      "step": 530
+    },
+    {
+      "completion_length": 66.50357437133789,
+      "epoch": 0.8271330563339453,
+      "grad_norm": 0.06425799238919987,
+      "kl": 3.5025390625,
+      "learning_rate": 1.7477789834847835e-06,
+      "loss": 0.1401,
+      "reward": 1.9041022166609765,
+      "reward_std": 0.12976709343101903,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.026850270089926197,
+      "rewards/format_reward": 0.9526785917580127,
+      "rewards/reasoning_steps_reward": 0.9782738409936428,
+      "step": 535
+    },
+    {
+      "completion_length": 69.19821739196777,
+      "epoch": 0.8348632718137018,
+      "grad_norm": 0.0533265407861989,
+      "kl": 3.4892578125,
+      "learning_rate": 1.5981113336584041e-06,
+      "loss": 0.1395,
+      "reward": 1.9006277784705161,
+      "reward_std": 0.125570208276622,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.027943752938881516,
+      "rewards/format_reward": 0.949107164144516,
+      "rewards/reasoning_steps_reward": 0.9794643156230449,
+      "step": 540
+    },
+    {
+      "completion_length": 67.40625302791595,
+      "epoch": 0.8425934872934583,
+      "grad_norm": 0.06527419814300027,
+      "kl": 3.5046875,
+      "learning_rate": 1.4545846677147446e-06,
+      "loss": 0.1402,
+      "reward": 1.9023914471268655,
+      "reward_std": 0.13218648402971667,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.027370562584837898,
+      "rewards/format_reward": 0.9508928805589676,
+      "rewards/reasoning_steps_reward": 0.9788690812885761,
+      "step": 545
+    },
+    {
+      "completion_length": 70.99911036491395,
+      "epoch": 0.8503237027732148,
+      "grad_norm": 0.14692504997903305,
+      "kl": 3.49453125,
+      "learning_rate": 1.3173038900362977e-06,
+      "loss": 0.1398,
+      "reward": 1.8956558883190155,
+      "reward_std": 0.12889199228147846,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02904659230262041,
+      "rewards/format_reward": 0.9473214522004128,
+      "rewards/reasoning_steps_reward": 0.977380982786417,
+      "step": 550
+    },
+    {
+      "completion_length": 67.55535998344422,
+      "epoch": 0.8580539182529713,
+      "grad_norm": 0.1394448990351517,
+      "kl": 3.48125,
+      "learning_rate": 1.1863693398535115e-06,
+      "loss": 0.1392,
+      "reward": 1.9022439941763878,
+      "reward_std": 0.13239500699564816,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02811324886861257,
+      "rewards/format_reward": 0.9517857365310192,
+      "rewards/reasoning_steps_reward": 0.9785714581608772,
+      "step": 555
+    },
+    {
+      "completion_length": 62.0455384016037,
+      "epoch": 0.8657841337327278,
+      "grad_norm": 0.10614107580517884,
+      "kl": 3.5326171875,
+      "learning_rate": 1.0618767179063416e-06,
+      "loss": 0.1413,
+      "reward": 1.9079707980155944,
+      "reward_std": 0.12429608543943686,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02476739838020876,
+      "rewards/format_reward": 0.9553571626543998,
+      "rewards/reasoning_steps_reward": 0.9773809798061848,
+      "step": 560
+    },
+    {
+      "completion_length": 65.60714576244354,
+      "epoch": 0.8735143492124843,
+      "grad_norm": 0.08347131383243112,
+      "kl": 3.4798828125,
+      "learning_rate": 9.439170164960765e-07,
+      "loss": 0.1392,
+      "reward": 1.907164917886257,
+      "reward_std": 0.12543576840839704,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.026763751916587353,
+      "rewards/format_reward": 0.9535714492201806,
+      "rewards/reasoning_steps_reward": 0.9803571715950966,
+      "step": 565
+    },
+    {
+      "completion_length": 68.30803880691528,
+      "epoch": 0.8812445646922408,
+      "grad_norm": 0.27661869101085307,
+      "kl": 3.4837890625,
+      "learning_rate": 8.325764529785851e-07,
+      "loss": 0.1394,
+      "reward": 1.8964042320847512,
+      "reward_std": 0.13058434696868063,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.028000628220615907,
+      "rewards/format_reward": 0.9500000230967999,
+      "rewards/reasoning_steps_reward": 0.9744047962129117,
+      "step": 570
+    },
+    {
+      "completion_length": 88.06964702606201,
+      "epoch": 0.8889747801719973,
+      "grad_norm": 0.1964548865756102,
+      "kl": 3.4263671875,
+      "learning_rate": 7.279364067476247e-07,
+      "loss": 0.137,
+      "reward": 1.861181303858757,
+      "reward_std": 0.18066303946543485,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03733069502632134,
+      "rewards/format_reward": 0.9312500290572643,
+      "rewards/reasoning_steps_reward": 0.9672619484364986,
+      "step": 575
+    },
+    {
+      "completion_length": 78.18661077022553,
+      "epoch": 0.8967049956517538,
+      "grad_norm": 0.04068328201949373,
+      "kl": 3.4509765625,
+      "learning_rate": 6.300733597542086e-07,
+      "loss": 0.138,
+      "reward": 1.8831033661961556,
+      "reward_std": 0.15025150256306005,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03237292571575381,
+      "rewards/format_reward": 0.9410714529454708,
+      "rewards/reasoning_steps_reward": 0.9744047962129117,
+      "step": 580
+    },
+    {
+      "completion_length": 70.1169674873352,
+      "epoch": 0.9044352111315103,
+      "grad_norm": 0.2203913242004148,
+      "kl": 3.4599609375,
+      "learning_rate": 5.390588406055497e-07,
+      "loss": 0.1384,
+      "reward": 1.895041187107563,
+      "reward_std": 0.1379903966987513,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.029066056082956492,
+      "rewards/format_reward": 0.949107164144516,
+      "rewards/reasoning_steps_reward": 0.9750000342726708,
+      "step": 585
+    },
+    {
+      "completion_length": 77.28482503890991,
+      "epoch": 0.9121654266112668,
+      "grad_norm": 0.16584654551452574,
+      "kl": 3.4349609375,
+      "learning_rate": 4.549593722844492e-07,
+      "loss": 0.1374,
+      "reward": 1.8838764503598213,
+      "reward_std": 0.15387297290526475,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.032195083069382235,
+      "rewards/format_reward": 0.9410714536905289,
+      "rewards/reasoning_steps_reward": 0.9750000394880771,
+      "step": 590
+    },
+    {
+      "completion_length": 74.61607468128204,
+      "epoch": 0.9198956420910233,
+      "grad_norm": 0.19216245987467911,
+      "kl": 3.475390625,
+      "learning_rate": 3.77836423527278e-07,
+      "loss": 0.139,
+      "reward": 1.8887845054268837,
+      "reward_std": 0.14236255070909465,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03085845318273641,
+      "rewards/format_reward": 0.9437500238418579,
+      "rewards/reasoning_steps_reward": 0.975892896950245,
+      "step": 595
+    },
+    {
+      "completion_length": 76.38393211364746,
+      "epoch": 0.9276258575707798,
+      "grad_norm": 0.060252180136433846,
+      "kl": 3.4697265625,
+      "learning_rate": 3.0774636389618196e-07,
+      "loss": 0.1388,
+      "reward": 1.884899152815342,
+      "reward_std": 0.147100730240345,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03147000586614013,
+      "rewards/format_reward": 0.9419643118977546,
+      "rewards/reasoning_steps_reward": 0.9744047917425632,
+      "step": 600
+    },
+    {
+      "epoch": 0.9276258575707798,
+      "eval_completion_length": 89.25833797454834,
+      "eval_kl": 3.470703125,
+      "eval_loss": 0.13820315897464752,
+      "eval_reward": 1.8693622648715973,
+      "eval_reward_std": 0.15594222582876682,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.03539974894374609,
+      "eval_rewards/format_reward": 0.933035746216774,
+      "eval_rewards/reasoning_steps_reward": 0.9717262163758278,
+      "eval_runtime": 33.1947,
+      "eval_samples_per_second": 2.982,
+      "eval_steps_per_second": 0.121,
+      "step": 600
+    },
+    {
+      "completion_length": 80.05089657306671,
+      "epoch": 0.9353560730505363,
+      "grad_norm": 0.057380352644946994,
+      "kl": 3.4642578125,
+      "learning_rate": 2.44740422578269e-07,
+      "loss": 0.1386,
+      "reward": 1.876345480978489,
+      "reward_std": 0.15905879642814397,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03347604034934193,
+      "rewards/format_reward": 0.9375000275671482,
+      "rewards/reasoning_steps_reward": 0.9723214760422707,
+      "step": 605
+    },
+    {
+      "completion_length": 85.38750400543213,
+      "epoch": 0.9430862885302927,
+      "grad_norm": 0.12308221099914948,
+      "kl": 3.4466796875,
+      "learning_rate": 1.8886465094192895e-07,
+      "loss": 0.1378,
+      "reward": 1.8649245023727417,
+      "reward_std": 0.17699857261031865,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03596844737767242,
+      "rewards/format_reward": 0.9330357424914837,
+      "rewards/reasoning_steps_reward": 0.9678571827709674,
+      "step": 610
+    },
+    {
+      "completion_length": 75.48661060333252,
+      "epoch": 0.9508165040100492,
+      "grad_norm": 0.08698913640999234,
+      "kl": 3.4462890625,
+      "learning_rate": 1.401598888776523e-07,
+      "loss": 0.1378,
+      "reward": 1.8878486067056657,
+      "reward_std": 0.14815408168360591,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.031496733688982204,
+      "rewards/format_reward": 0.946428595483303,
+      "rewards/reasoning_steps_reward": 0.9729166962206364,
+      "step": 615
+    },
+    {
+      "completion_length": 80.87678947448731,
+      "epoch": 0.9585467194898057,
+      "grad_norm": 0.0751696581199154,
+      "kl": 3.4388671875,
+      "learning_rate": 9.866173494794462e-08,
+      "loss": 0.1375,
+      "reward": 1.8755054756999017,
+      "reward_std": 0.14723004253473845,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.034018434659810734,
+      "rewards/format_reward": 0.9383928820490837,
+      "rewards/reasoning_steps_reward": 0.9711309857666492,
+      "step": 620
+    },
+    {
+      "completion_length": 83.59286117553711,
+      "epoch": 0.9662769349695622,
+      "grad_norm": 0.10937622032847172,
+      "kl": 3.457421875,
+      "learning_rate": 6.440052036815081e-08,
+      "loss": 0.1383,
+      "reward": 1.8713971391320228,
+      "reward_std": 0.16230505469911805,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03515056184260175,
+      "rewards/format_reward": 0.9357143118977547,
+      "rewards/reasoning_steps_reward": 0.970833370089531,
+      "step": 625
+    },
+    {
+      "completion_length": 83.54196829795838,
+      "epoch": 0.9740071504493187,
+      "grad_norm": 0.11908490334435194,
+      "kl": 3.4828125,
+      "learning_rate": 3.7401286837214224e-08,
+      "loss": 0.1393,
+      "reward": 1.8702244937419892,
+      "reward_std": 0.15762649106984555,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03513275724835694,
+      "rewards/format_reward": 0.9357143118977547,
+      "rewards/reasoning_steps_reward": 0.9696428954601288,
+      "step": 630
+    },
+    {
+      "completion_length": 78.18839647769929,
+      "epoch": 0.9817373659290752,
+      "grad_norm": 0.11683355049170727,
+      "kl": 3.4583984375,
+      "learning_rate": 1.7683768234568745e-08,
+      "loss": 0.1383,
+      "reward": 1.8763473451137542,
+      "reward_std": 0.15354155295281088,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03258133551571518,
+      "rewards/format_reward": 0.9401785962283611,
+      "rewards/reasoning_steps_reward": 0.9687500402331353,
+      "step": 635
+    },
+    {
+      "completion_length": 81.77321815490723,
+      "epoch": 0.9894675814088317,
+      "grad_norm": 0.5345026896039206,
+      "kl": 3.4708984375,
+      "learning_rate": 5.262376196544239e-09,
+      "loss": 0.1388,
+      "reward": 1.8747070506215096,
+      "reward_std": 0.1622747814282775,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03362638182006776,
+      "rewards/format_reward": 0.9366071708500385,
+      "rewards/reasoning_steps_reward": 0.9717262290418148,
+      "step": 640
+    },
+    {
+      "completion_length": 84.52768263816833,
+      "epoch": 0.9971977968885882,
+      "grad_norm": 0.13671175640957767,
+      "kl": 3.4451171875,
+      "learning_rate": 1.461895828280824e-10,
+      "loss": 0.1378,
+      "reward": 1.8673253536224366,
+      "reward_std": 0.16173910862949015,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03565093849319965,
+      "rewards/format_reward": 0.934821455180645,
+      "rewards/reasoning_steps_reward": 0.9681547932326794,
+      "step": 645
+    },
+    {
+      "completion_length": 71.89286041259766,
+      "epoch": 0.9987438399845395,
+      "kl": 3.4921875,
+      "reward": 1.8919485434889793,
+      "reward_std": 0.14695495925843716,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02918250160291791,
+      "rewards/format_reward": 0.9464285969734192,
+      "rewards/reasoning_steps_reward": 0.9747024178504944,
+      "step": 646,
+      "total_flos": 0.0,
+      "train_loss": 0.3506326697535499,
+      "train_runtime": 41982.506,
+      "train_samples_per_second": 1.726,
+      "train_steps_per_second": 0.015
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 646,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b94c655a8fafa729f0608210f9fbfa68c66c03da60c773a7ba53c622752f9af
+size 7544