Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/cosineai/deepseek-reproduce/runs/mr9qnll8) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/cosineai/deepseek-reproduce/runs/a2zpk9j5) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.1328258249329856e-06,
-    "train_runtime": 38.497,
-    "train_samples": 6725,
-    "train_samples_per_second": 311.713,
-    "train_steps_per_second": 6.494
+    "train_loss": 0.08981622397834212,
+    "train_runtime": 6983.3902,
+    "train_samples": 45000,
+    "train_samples_per_second": 3.093,
+    "train_steps_per_second": 0.064
 }

config.json

@@ -36,6 +36,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.48.1",
-  "use_cache": false,
+  "use_cache": true,
   "vocab_size": 128256
 }

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.48.1"
+}

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.1328258249329856e-06,
-    "train_runtime": 38.497,
-    "train_samples": 6725,
-    "train_samples_per_second": 311.713,
-    "train_steps_per_second": 6.494
+    "train_loss": 0.08981622397834212,
+    "train_runtime": 6983.3902,
+    "train_samples": 45000,
+    "train_samples_per_second": 3.093,
+    "train_steps_per_second": 0.064
 }