Training in progress, epoch 0

all_results.json

@@ -1,13 +1,8 @@
 {
-    "epoch": 9.090909090909092,
-    "eval_accuracy": 1.0,
-    "eval_loss": 0.8444482088088989,
-    "eval_runtime": 0.2618,
-    "eval_samples_per_second": 294.076,
-    "eval_steps_per_second": 11.458,
-    "total_flos": 6.343101899766374e+16,
-    "train_loss": 0.9457284736633301,
-    "train_runtime": 45.6784,
-    "train_samples_per_second": 151.275,
-    "train_steps_per_second": 1.095
+    "epoch": 8.88888888888889,
+    "total_flos": 7.753465317556224e+16,
+    "train_loss": 1.240843003892173e+28,
+    "train_runtime": 60.0746,
+    "train_samples_per_second": 143.821,
+    "train_steps_per_second": 0.999
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d03ad4c7e680e46ead29aebee4dcbc57f12b52872a831b7c4bd5b7a7f71c8414
+oid sha256:fb05f24a6e7a0c70dc713da46761a67e4c05783466a8590f6f771cfabaf2c154
 size 44766388

runs/Jun06_03-51-27_b446df81307f/events.out.tfevents.1717645898.b446df81307f.157.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e62595a4bb050022ca460d9e6ebca31ca3381260957e61c9844b27eb2f47647c
+size 5233

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 9.090909090909092,
-    "total_flos": 6.343101899766374e+16,
-    "train_loss": 0.9457284736633301,
-    "train_runtime": 45.6784,
-    "train_samples_per_second": 151.275,
-    "train_steps_per_second": 1.095
+    "epoch": 8.88888888888889,
+    "total_flos": 7.753465317556224e+16,
+    "train_loss": 1.240843003892173e+28,
+    "train_runtime": 60.0746,
+    "train_samples_per_second": 143.821,
+    "train_steps_per_second": 0.999
 }

trainer_state.json

@@ -1,150 +1,148 @@
 {
-  "best_metric": 1.0,
-  "best_model_checkpoint": "resnet-18/checkpoint-22",
-  "epoch": 9.090909090909092,
+  "best_metric": 0.3645833333333333,
+  "best_model_checkpoint": "resnet-18/checkpoint-6",
+  "epoch": 8.88888888888889,
   "eval_steps": 500,
-  "global_step": 50,
+  "global_step": 60,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.9090909090909091,
-      "eval_accuracy": 0.4155844155844156,
-      "eval_loss": 1.0317904949188232,
-      "eval_runtime": 0.3039,
-      "eval_samples_per_second": 253.336,
-      "eval_steps_per_second": 9.87,
-      "step": 5
+      "epoch": 0.8888888888888888,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.3463,
+      "eval_samples_per_second": 277.207,
+      "eval_steps_per_second": 8.663,
+      "step": 6
     },
     {
-      "epoch": 1.8181818181818183,
-      "grad_norm": 2.841965913772583,
-      "learning_rate": 4.4444444444444447e-05,
-      "loss": 1.0893,
+      "epoch": 1.4814814814814814,
+      "grad_norm": 24.085277557373047,
+      "learning_rate": 4.62962962962963e-05,
+      "loss": 1.22806547652237e+28,
       "step": 10
     },
     {
-      "epoch": 2.0,
-      "eval_accuracy": 0.6363636363636364,
-      "eval_loss": 0.9519639611244202,
-      "eval_runtime": 0.2748,
-      "eval_samples_per_second": 280.217,
-      "eval_steps_per_second": 10.918,
-      "step": 11
+      "epoch": 1.925925925925926,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.334,
+      "eval_samples_per_second": 287.396,
+      "eval_steps_per_second": 8.981,
+      "step": 13
     },
     {
-      "epoch": 2.909090909090909,
-      "eval_accuracy": 0.8441558441558441,
-      "eval_loss": 0.9016602039337158,
-      "eval_runtime": 0.3011,
-      "eval_samples_per_second": 255.709,
-      "eval_steps_per_second": 9.963,
-      "step": 16
+      "epoch": 2.962962962962963,
+      "grad_norm": 27.859230041503906,
+      "learning_rate": 3.7037037037037037e-05,
+      "loss": 1.246868900936531e+28,
+      "step": 20
     },
     {
-      "epoch": 3.6363636363636362,
-      "grad_norm": 3.1325621604919434,
-      "learning_rate": 3.3333333333333335e-05,
-      "loss": 0.9912,
+      "epoch": 2.962962962962963,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.3614,
+      "eval_samples_per_second": 265.642,
+      "eval_steps_per_second": 8.301,
       "step": 20
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.8444482088088989,
-      "eval_runtime": 0.2042,
-      "eval_samples_per_second": 377.122,
-      "eval_steps_per_second": 14.693,
-      "step": 22
-    },
-    {
-      "epoch": 4.909090909090909,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.802738606929779,
-      "eval_runtime": 0.221,
-      "eval_samples_per_second": 348.461,
-      "eval_steps_per_second": 13.576,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.3509,
+      "eval_samples_per_second": 273.57,
+      "eval_steps_per_second": 8.549,
       "step": 27
     },
     {
-      "epoch": 5.454545454545454,
-      "grad_norm": 4.148845195770264,
-      "learning_rate": 2.2222222222222223e-05,
-      "loss": 0.9248,
+      "epoch": 4.444444444444445,
+      "grad_norm": 29.561874389648438,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 1.2353039198673127e+28,
       "step": 30
     },
     {
-      "epoch": 6.0,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.763073205947876,
-      "eval_runtime": 0.2945,
-      "eval_samples_per_second": 261.469,
-      "eval_steps_per_second": 10.187,
+      "epoch": 4.888888888888889,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.337,
+      "eval_samples_per_second": 284.861,
+      "eval_steps_per_second": 8.902,
       "step": 33
     },
     {
-      "epoch": 6.909090909090909,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.736884593963623,
-      "eval_runtime": 0.2212,
-      "eval_samples_per_second": 348.098,
-      "eval_steps_per_second": 13.562,
-      "step": 38
+      "epoch": 5.925925925925926,
+      "grad_norm": 30.260478973388672,
+      "learning_rate": 1.8518518518518518e-05,
+      "loss": 1.2598814762274078e+28,
+      "step": 40
     },
     {
-      "epoch": 7.2727272727272725,
-      "grad_norm": 3.343644142150879,
-      "learning_rate": 1.1111111111111112e-05,
-      "loss": 0.8716,
+      "epoch": 5.925925925925926,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.3324,
+      "eval_samples_per_second": 288.789,
+      "eval_steps_per_second": 9.025,
       "step": 40
     },
     {
-      "epoch": 8.0,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.7155924439430237,
-      "eval_runtime": 0.261,
-      "eval_samples_per_second": 295.058,
-      "eval_steps_per_second": 11.496,
-      "step": 44
+      "epoch": 6.962962962962963,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.5708,
+      "eval_samples_per_second": 168.188,
+      "eval_steps_per_second": 5.256,
+      "step": 47
     },
     {
-      "epoch": 8.909090909090908,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.7137082815170288,
-      "eval_runtime": 0.2787,
-      "eval_samples_per_second": 276.309,
-      "eval_steps_per_second": 10.765,
-      "step": 49
+      "epoch": 7.407407407407407,
+      "grad_norm": 30.463253021240234,
+      "learning_rate": 9.259259259259259e-06,
+      "loss": 1.2352969307649181e+28,
+      "step": 50
     },
     {
-      "epoch": 9.090909090909092,
-      "grad_norm": 3.096590757369995,
+      "epoch": 8.0,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.3468,
+      "eval_samples_per_second": 276.801,
+      "eval_steps_per_second": 8.65,
+      "step": 54
+    },
+    {
+      "epoch": 8.88888888888889,
+      "grad_norm": 30.54263687133789,
       "learning_rate": 0.0,
-      "loss": 0.8517,
-      "step": 50
+      "loss": 1.239641319034499e+28,
+      "step": 60
     },
     {
-      "epoch": 9.090909090909092,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.7116669416427612,
-      "eval_runtime": 0.4991,
-      "eval_samples_per_second": 154.284,
-      "eval_steps_per_second": 6.011,
-      "step": 50
+      "epoch": 8.88888888888889,
+      "eval_accuracy": 0.3645833333333333,
+      "eval_loss": 1.1764246045329271e+28,
+      "eval_runtime": 0.3434,
+      "eval_samples_per_second": 279.567,
+      "eval_steps_per_second": 8.736,
+      "step": 60
     },
     {
-      "epoch": 9.090909090909092,
-      "step": 50,
-      "total_flos": 6.343101899766374e+16,
-      "train_loss": 0.9457284736633301,
-      "train_runtime": 45.6784,
-      "train_samples_per_second": 151.275,
-      "train_steps_per_second": 1.095
+      "epoch": 8.88888888888889,
+      "step": 60,
+      "total_flos": 7.753465317556224e+16,
+      "train_loss": 1.240843003892173e+28,
+      "train_runtime": 60.0746,
+      "train_samples_per_second": 143.821,
+      "train_steps_per_second": 0.999
     }
   ],
   "logging_steps": 10,
-  "max_steps": 50,
+  "max_steps": 60,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 500,
@@ -160,7 +158,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.343101899766374e+16,
+  "total_flos": 7.753465317556224e+16,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6215e1709c54d82fb3bf303e3b05b5289d964c8bc3be64bfb0069a5737e020b4
+oid sha256:3a9ed8ec4d077c6e80c227c21450a586f7309e4efa184b66f3a4e8d15447b033
 size 5112