Training in progress, step 632, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2b4cc1cef93d033abfc2f87d3b5b9c5dbacd72a5b0dc730c6659bff3c4ed2cb
+oid sha256:139b19928fa97b7c1693b19298b775f6c1f3de936d5316bec307c782a04054c1
 size 1001465824

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d3ed47ca30bb1233df822d49b40e5d03f9ce09e404501ed6c579fd67b6e7255
+oid sha256:12adce1c2d0ed7d9b98485bf23a9ec1cfa71c5d5145fef3624db221bf4212989
 size 509177556

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40a9574e57f7a253b64d50c5f5629f324b4a1a5ac841f256ec2dca1bec61ee62
+oid sha256:f945a76a27af6da77da26bea2b0d33efbecc3a8bfb8cdd31c6f06af07cbed2fe
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1feb2c4e4f0ebf5517a21670a0fdcf3d94d1b879845629079677b365bcee24f4
+oid sha256:a709aa6a7d0d804b893952095b83e581a0b61a8d7425622873aa0f43ed10f0a0
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 1.073432445526123,
   "best_model_checkpoint": "miner_id_24/checkpoint-600",
-  "epoch": 0.11036512462061988,
+  "epoch": 0.11625126460038628,
   "eval_steps": 100,
-  "global_step": 600,
+  "global_step": 632,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4263,6 +4263,230 @@
       "eval_samples_per_second": 3.909,
       "eval_steps_per_second": 0.977,
       "step": 600
+    },
+    {
+      "epoch": 0.11054906649498758,
+      "grad_norm": 1.0617674589157104,
+      "learning_rate": 1.2232773595138415e-06,
+      "loss": 7.3825,
+      "step": 601
+    },
+    {
+      "epoch": 0.11073300836935529,
+      "grad_norm": 1.2446156740188599,
+      "learning_rate": 1.145777792614189e-06,
+      "loss": 8.8698,
+      "step": 602
+    },
+    {
+      "epoch": 0.11091695024372299,
+      "grad_norm": 1.2019675970077515,
+      "learning_rate": 1.070800041019504e-06,
+      "loss": 8.9471,
+      "step": 603
+    },
+    {
+      "epoch": 0.11110089211809068,
+      "grad_norm": 1.4310685396194458,
+      "learning_rate": 9.98346017445706e-07,
+      "loss": 8.777,
+      "step": 604
+    },
+    {
+      "epoch": 0.11128483399245838,
+      "grad_norm": 1.1273508071899414,
+      "learning_rate": 9.284175702272246e-07,
+      "loss": 7.8448,
+      "step": 605
+    },
+    {
+      "epoch": 0.11146877586682609,
+      "grad_norm": 1.1338694095611572,
+      "learning_rate": 8.610164832699608e-07,
+      "loss": 7.9259,
+      "step": 606
+    },
+    {
+      "epoch": 0.11165271774119379,
+      "grad_norm": 1.441076397895813,
+      "learning_rate": 7.961444760056891e-07,
+      "loss": 9.1154,
+      "step": 607
+    },
+    {
+      "epoch": 0.11183665961556148,
+      "grad_norm": 1.3927642107009888,
+      "learning_rate": 7.338032033482711e-07,
+      "loss": 7.2621,
+      "step": 608
+    },
+    {
+      "epoch": 0.11202060148992918,
+      "grad_norm": 1.31500244140625,
+      "learning_rate": 6.739942556513889e-07,
+      "loss": 7.2625,
+      "step": 609
+    },
+    {
+      "epoch": 0.11220454336429689,
+      "grad_norm": 1.1202362775802612,
+      "learning_rate": 6.167191586679555e-07,
+      "loss": 9.2775,
+      "step": 610
+    },
+    {
+      "epoch": 0.11238848523866458,
+      "grad_norm": 1.0659925937652588,
+      "learning_rate": 5.619793735112566e-07,
+      "loss": 9.2408,
+      "step": 611
+    },
+    {
+      "epoch": 0.11257242711303228,
+      "grad_norm": 1.259950041770935,
+      "learning_rate": 5.097762966176256e-07,
+      "loss": 7.6575,
+      "step": 612
+    },
+    {
+      "epoch": 0.11275636898739998,
+      "grad_norm": 1.474528193473816,
+      "learning_rate": 4.6011125971084924e-07,
+      "loss": 8.3618,
+      "step": 613
+    },
+    {
+      "epoch": 0.11294031086176769,
+      "grad_norm": 1.1022167205810547,
+      "learning_rate": 4.129855297681617e-07,
+      "loss": 7.7316,
+      "step": 614
+    },
+    {
+      "epoch": 0.11312425273613538,
+      "grad_norm": 1.2318350076675415,
+      "learning_rate": 3.684003089879484e-07,
+      "loss": 9.4046,
+      "step": 615
+    },
+    {
+      "epoch": 0.11330819461050308,
+      "grad_norm": 1.2324371337890625,
+      "learning_rate": 3.2635673475910344e-07,
+      "loss": 8.4448,
+      "step": 616
+    },
+    {
+      "epoch": 0.11349213648487078,
+      "grad_norm": 1.224507212638855,
+      "learning_rate": 2.8685587963194206e-07,
+      "loss": 10.1946,
+      "step": 617
+    },
+    {
+      "epoch": 0.11367607835923849,
+      "grad_norm": 1.0900365114212036,
+      "learning_rate": 2.4989875129091125e-07,
+      "loss": 7.4567,
+      "step": 618
+    },
+    {
+      "epoch": 0.11386002023360618,
+      "grad_norm": 1.2144701480865479,
+      "learning_rate": 2.1548629252883256e-07,
+      "loss": 9.9855,
+      "step": 619
+    },
+    {
+      "epoch": 0.11404396210797388,
+      "grad_norm": 1.4588361978530884,
+      "learning_rate": 1.8361938122287703e-07,
+      "loss": 9.7914,
+      "step": 620
+    },
+    {
+      "epoch": 0.11422790398234157,
+      "grad_norm": 1.5850588083267212,
+      "learning_rate": 1.5429883031217173e-07,
+      "loss": 9.6143,
+      "step": 621
+    },
+    {
+      "epoch": 0.11441184585670928,
+      "grad_norm": 1.0361146926879883,
+      "learning_rate": 1.2752538777704992e-07,
+      "loss": 6.8237,
+      "step": 622
+    },
+    {
+      "epoch": 0.11459578773107698,
+      "grad_norm": 1.4321608543395996,
+      "learning_rate": 1.0329973661996617e-07,
+      "loss": 9.9172,
+      "step": 623
+    },
+    {
+      "epoch": 0.11477972960544468,
+      "grad_norm": 1.2243539094924927,
+      "learning_rate": 8.162249484809925e-08,
+      "loss": 10.1801,
+      "step": 624
+    },
+    {
+      "epoch": 0.11496367147981237,
+      "grad_norm": 0.9993793368339539,
+      "learning_rate": 6.249421545755364e-08,
+      "loss": 10.4931,
+      "step": 625
+    },
+    {
+      "epoch": 0.11514761335418008,
+      "grad_norm": 1.1969102621078491,
+      "learning_rate": 4.5915386419270736e-08,
+      "loss": 9.4625,
+      "step": 626
+    },
+    {
+      "epoch": 0.11533155522854778,
+      "grad_norm": 1.2495107650756836,
+      "learning_rate": 3.188643066656116e-08,
+      "loss": 8.6992,
+      "step": 627
+    },
+    {
+      "epoch": 0.11551549710291548,
+      "grad_norm": 0.9973385334014893,
+      "learning_rate": 2.0407706084368815e-08,
+      "loss": 8.9503,
+      "step": 628
+    },
+    {
+      "epoch": 0.11569943897728317,
+      "grad_norm": 1.3621132373809814,
+      "learning_rate": 1.1479505500044951e-08,
+      "loss": 8.2804,
+      "step": 629
+    },
+    {
+      "epoch": 0.11588338085165088,
+      "grad_norm": 1.1957571506500244,
+      "learning_rate": 5.102056675998501e-09,
+      "loss": 9.484,
+      "step": 630
+    },
+    {
+      "epoch": 0.11606732272601858,
+      "grad_norm": 1.095003604888916,
+      "learning_rate": 1.2755223037896892e-09,
+      "loss": 8.6529,
+      "step": 631
+    },
+    {
+      "epoch": 0.11625126460038628,
+      "grad_norm": 1.1216403245925903,
+      "learning_rate": 0.0,
+      "loss": 10.7205,
+      "step": 632
     }
   ],
   "logging_steps": 1,
@@ -4286,12 +4510,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.5600972702652826e+18,
+  "total_flos": 3.7507247651998925e+18,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null