Training in progress, step 49, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58a044faebcf0660c059236976c7ae26375203aae123b5c283a651f822c78c15
+oid sha256:c794e01bc794c9528f8ad671916ce397054fcd194c1f065ca9c5615e5dc68b7c
 size 671466706

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cdbab2448fd6961b2a07487f43f27dd07f956e01ad3d64befcb4995549b9f475
+oid sha256:26efe027ff8b9d9fda780767367d83fae932c472105c44e39d341f01e69fa5bc
 size 14960

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d623bf1cec420c2e0ddd2d8ef8b5d8e65a6d21bbb96c3dee2ca6499f13523bcd
+oid sha256:a480a33dbd5ad263e0db59cfb3e4d0db1cfa3454acdf4d3a2d9c2bf0ec390392
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbde44c60bc9c7eec7e5ab61af2d30b00d1be7cd88e2f0664181ecc4741c93a8
+oid sha256:7178d7b836b8b592ea63a2cb12cc55c5c8a278ef622ff1a8ea032ca8b9d82b4c
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddb999e8ee92bef6acab3ea76e3fb11d92f352c1c46cf9ad9abc50ae3a9c8b0a
+oid sha256:1f576b9677afe005965981ac89d77c8b7ea5ee29985b8e69b4cafd051445eded
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:840657c72d1644d32902ed3baed4169f8fa8749780cabcde307874a07544bcbd
+oid sha256:25704ee5478cd4521ee5db3e9e14050e9c72edebe1088acf05e49ba2bb8ed7fb
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 1.556420233463035,
+  "epoch": 3.0505836575875485,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 49,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,174 @@
       "eval_samples_per_second": 36.17,
       "eval_steps_per_second": 4.689,
       "step": 25
+    },
+    {
+      "epoch": 1.6186770428015564,
+      "grad_norm": NaN,
+      "learning_rate": 4.832925114961629e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 1.6809338521400778,
+      "grad_norm": NaN,
+      "learning_rate": 4.4995215418795085e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 1.7431906614785992,
+      "grad_norm": NaN,
+      "learning_rate": 4.1683532270843504e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 1.8054474708171206,
+      "grad_norm": NaN,
+      "learning_rate": 3.840899248662358e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 1.867704280155642,
+      "grad_norm": NaN,
+      "learning_rate": 3.5186220955718306e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 1.9299610894941635,
+      "grad_norm": NaN,
+      "learning_rate": 3.202961135812437e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 1.9922178988326849,
+      "grad_norm": NaN,
+      "learning_rate": 2.895326187858326e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 2.054474708171206,
+      "grad_norm": NaN,
+      "learning_rate": 2.5970912240665813e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 2.1167315175097277,
+      "grad_norm": NaN,
+      "learning_rate": 2.3095882341831372e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 2.178988326848249,
+      "grad_norm": NaN,
+      "learning_rate": 2.0341012763532243e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 2.2412451361867705,
+      "grad_norm": NaN,
+      "learning_rate": 1.771860742205988e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 2.3035019455252916,
+      "grad_norm": NaN,
+      "learning_rate": 1.5240378616267886e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 2.3657587548638133,
+      "grad_norm": NaN,
+      "learning_rate": 1.2917394717602121e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 2.4280155642023344,
+      "grad_norm": NaN,
+      "learning_rate": 1.0760030736066951e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 2.490272373540856,
+      "grad_norm": NaN,
+      "learning_rate": 8.777921982911996e-06,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 2.5525291828793772,
+      "grad_norm": NaN,
+      "learning_rate": 6.979921036993042e-06,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 2.614785992217899,
+      "grad_norm": NaN,
+      "learning_rate": 5.374058207005944e-06,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 2.6770428015564205,
+      "grad_norm": NaN,
+      "learning_rate": 3.967505666178556e-06,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 2.7392996108949417,
+      "grad_norm": NaN,
+      "learning_rate": 2.7665454196040664e-06,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 2.801556420233463,
+      "grad_norm": NaN,
+      "learning_rate": 1.7765412472811771e-06,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 2.8638132295719845,
+      "grad_norm": NaN,
+      "learning_rate": 1.0019147481706625e-06,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 2.926070038910506,
+      "grad_norm": NaN,
+      "learning_rate": 4.461255922609986e-07,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 2.9883268482490273,
+      "grad_norm": NaN,
+      "learning_rate": 1.1165606884234181e-07,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 3.0505836575875485,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 49
     }
   ],
   "logging_steps": 1,
@@ -221,12 +389,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.829013890367488e+17,
+  "total_flos": 5.5448672251202765e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null