Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-Math-7B
-datasets: DigitalLearningGmbH/MATH-lighteval
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B) on the [DigitalLearningGmbH/MATH-lighteval](https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/3gmilpkh) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/eybgfkwc) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 3.3746,
-    "train_samples": 7500,
-    "train_samples_per_second": 2222.49,
-    "train_steps_per_second": 17.187
+    "train_loss": 6.178523520551001e-05,
+    "train_runtime": 3794.8453,
+    "train_samples": 1000,
+    "train_samples_per_second": 0.264,
+    "train_steps_per_second": 0.004
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 3.3746,
-    "train_samples": 7500,
-    "train_samples_per_second": 2222.49,
-    "train_steps_per_second": 17.187
+    "train_loss": 6.178523520551001e-05,
+    "train_runtime": 3794.8453,
+    "train_samples": 1000,
+    "train_samples_per_second": 0.264,
+    "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -1,184 +1,80 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9893390191897654,
-  "eval_steps": 100,
-  "global_step": 58,
+  "epoch": 0.96,
+  "eval_steps": 500,
+  "global_step": 15,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 598.6428833007812,
-      "epoch": 0.017057569296375266,
-      "grad_norm": 1.5689994621266787,
+      "completion_length": 752.8906402587891,
+      "epoch": 0.064,
+      "grad_norm": 0.8492731776446195,
       "kl": 0.0,
-      "learning_rate": 5e-07,
+      "learning_rate": 1.5e-06,
       "loss": 0.0,
-      "reward": 0.6093750223517418,
-      "reward_std": 0.35872430354356766,
-      "rewards/accuracy_reward": 0.6093750223517418,
+      "reward": 0.5052083432674408,
+      "reward_std": 0.0497216647490859,
+      "rewards/accuracy_reward": 0.5052083432674408,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
-      "completion_length": 598.8225679397583,
-      "epoch": 0.08528784648187633,
-      "grad_norm": 7.756384521859563,
-      "kl": 0.00030493736267089844,
-      "learning_rate": 2.5e-06,
+      "completion_length": 777.4062652587891,
+      "epoch": 0.32,
+      "grad_norm": 0.10438261848562716,
+      "kl": 0.000284731388092041,
+      "learning_rate": 2.6227661222566517e-06,
       "loss": 0.0,
-      "reward": 0.6102120848372579,
-      "reward_std": 0.3614693288691342,
-      "rewards/accuracy_reward": 0.6102120848372579,
+      "reward": 0.5052083432674408,
+      "reward_std": 0.03829827485606074,
+      "rewards/accuracy_reward": 0.5052083432674408,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
-      "completion_length": 635.0709037780762,
-      "epoch": 0.17057569296375266,
-      "grad_norm": 0.24554990369771146,
-      "kl": 0.0016298294067382812,
-      "learning_rate": 2.956412726139078e-06,
+      "completion_length": 765.1005416870117,
+      "epoch": 0.64,
+      "grad_norm": 0.9738634307537658,
+      "kl": 0.0014707565307617188,
+      "learning_rate": 9.680926694361964e-07,
       "loss": 0.0001,
-      "reward": 0.7444196678698063,
-      "reward_std": 0.25730893574655056,
-      "rewards/accuracy_reward": 0.7444196678698063,
+      "reward": 0.47135417237877847,
+      "reward_std": 0.03448774488642812,
+      "rewards/accuracy_reward": 0.47135417237877847,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 613.8451156616211,
-      "epoch": 0.255863539445629,
-      "grad_norm": 0.2932648725811773,
-      "kl": 0.004305648803710938,
-      "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0002,
-      "reward": 0.7533482506871223,
-      "reward_std": 0.22977835088968276,
-      "rewards/accuracy_reward": 0.7533482506871223,
-      "rewards/format_reward": 0.0,
-      "step": 15
-    },
-    {
-      "completion_length": 604.8868591308594,
-      "epoch": 0.3411513859275053,
-      "grad_norm": 0.16889654761639042,
-      "kl": 0.0032588958740234373,
-      "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0001,
-      "reward": 0.7676339626312256,
-      "reward_std": 0.20254891095682978,
-      "rewards/accuracy_reward": 0.7676339626312256,
-      "rewards/format_reward": 0.0,
-      "step": 20
-    },
-    {
-      "completion_length": 599.3297142028808,
-      "epoch": 0.42643923240938164,
-      "grad_norm": 0.14357212406142408,
-      "kl": 9.20305938720703,
-      "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.3681,
-      "reward": 0.7622768104076385,
-      "reward_std": 0.19452331885695456,
-      "rewards/accuracy_reward": 0.7622768104076385,
-      "rewards/format_reward": 0.0,
-      "step": 25
-    },
-    {
-      "completion_length": 596.0473510742188,
-      "epoch": 0.511727078891258,
-      "grad_norm": 0.20095541353253643,
-      "kl": 0.002848052978515625,
-      "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0001,
-      "reward": 0.7611607506871223,
-      "reward_std": 0.16808430003002287,
-      "rewards/accuracy_reward": 0.7611607506871223,
-      "rewards/format_reward": 0.0,
-      "step": 30
-    },
-    {
-      "completion_length": 584.6814987182618,
-      "epoch": 0.5970149253731343,
-      "grad_norm": 0.0743109219899615,
-      "kl": 0.003064537048339844,
-      "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0001,
-      "reward": 0.7562500342726708,
-      "reward_std": 0.16550147179514169,
-      "rewards/accuracy_reward": 0.7562500342726708,
-      "rewards/format_reward": 0.0,
-      "step": 35
-    },
-    {
-      "completion_length": 574.5002449035644,
-      "epoch": 0.6823027718550106,
-      "grad_norm": 0.06794240162987708,
-      "kl": 0.003296661376953125,
-      "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0001,
-      "reward": 0.7834821805357933,
-      "reward_std": 0.15637031104415655,
-      "rewards/accuracy_reward": 0.7834821805357933,
-      "rewards/format_reward": 0.0,
-      "step": 40
-    },
-    {
-      "completion_length": 589.9893081665039,
-      "epoch": 0.767590618336887,
-      "grad_norm": 0.08693057149783366,
-      "kl": 0.0029300689697265626,
-      "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0001,
-      "reward": 0.7680803880095481,
-      "reward_std": 0.17124480060301722,
-      "rewards/accuracy_reward": 0.7680803880095481,
-      "rewards/format_reward": 0.0,
-      "step": 45
-    },
-    {
-      "completion_length": 587.243330001831,
-      "epoch": 0.8528784648187633,
-      "grad_norm": 0.1426044961551274,
-      "kl": 0.0030112266540527344,
-      "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0001,
-      "reward": 0.7603236939758062,
-      "reward_std": 0.1613232684903778,
-      "rewards/accuracy_reward": 0.7603236939758062,
-      "rewards/format_reward": 0.0,
-      "step": 50
-    },
-    {
-      "completion_length": 583.5995719909668,
-      "epoch": 0.9381663113006397,
-      "grad_norm": 0.11866102279196994,
-      "kl": 0.003066253662109375,
-      "learning_rate": 2.4570139579284723e-08,
+      "completion_length": 820.3849151611328,
+      "epoch": 0.96,
+      "grad_norm": 0.16045206350580687,
+      "kl": 0.0029309749603271484,
+      "learning_rate": 0.0,
       "loss": 0.0001,
-      "reward": 0.7906250342726707,
-      "reward_std": 0.17242762465029954,
-      "rewards/accuracy_reward": 0.7906250342726707,
+      "reward": 0.5114583395421505,
+      "reward_std": 0.04463785570114851,
+      "rewards/accuracy_reward": 0.5114583395421505,
       "rewards/format_reward": 0.0,
-      "step": 55
+      "step": 15
     },
     {
-      "epoch": 0.9893390191897654,
-      "step": 58,
+      "epoch": 0.96,
+      "step": 15,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 3.3746,
-      "train_samples_per_second": 2222.49,
-      "train_steps_per_second": 17.187
+      "train_loss": 6.178523520551001e-05,
+      "train_runtime": 3794.8453,
+      "train_samples_per_second": 0.264,
+      "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 5,
-  "max_steps": 58,
+  "max_steps": 15,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 1,
+  "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {