Training in progress, step 80

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:961d8dd736214c061f78791eb2be5af9a93b35a6a64f7b4e351a8f092b707b3b
+oid sha256:c7a1f20bc83390e7ce46330ad59e22144f38538f7f9633312f01b659bf5ba6a6
 size 261131840

runs/Mar13_11-36-50_7acbe5a74b63/events.out.tfevents.1710329812.7acbe5a74b63.337.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e665f030cc79f4cf63238d4b2773a58f7318f6298038fdae3e2ad95033172a8
-size 6999
+oid sha256:ca7fd3f4141cc93c775dcc7fe52054e06791b8659ed484c1e8e7b766f394343d
+size 7206

wandb/debug-internal.log

@@ -862,3 +862,46 @@
 2024-03-13 11:47:12,056 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
 2024-03-13 11:47:15,311 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:47:20,312 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:22,333 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:47:25,334 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:26,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:47:26,083 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:47:26,083 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:47:31,230 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:36,230 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:41,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:47:41,082 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:47:41,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:47:42,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:47,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:52,161 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:52,334 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:47:56,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:47:56,082 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:47:56,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:47:57,347 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:02,349 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:07,350 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:11,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:11,083 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:11,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:48:13,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:18,161 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:22,335 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:48:23,335 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:26,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:26,083 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:26,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:48:29,165 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:34,166 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:39,167 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:39,870 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:48:39,871 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:48:39,871 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:48:39,871 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:48:40,085 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:48:41,114 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:41,114 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:41,191 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:48:44,236 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:49,237 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.2382, "train/grad_norm": 0.7199187874794006, "train/learning_rate": 7.546600811643816e-05, "train/epoch": 5.6, "train/global_step": 70, "_timestamp": 1710330431.3066437, "_runtime": 1607.924634695053, "_step": 10, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}
+{"train/loss": 0.2297, "train/grad_norm": 0.6079784035682678, "train/learning_rate": 6.819524684817438e-05, "train/epoch": 6.4, "train/global_step": 80, "_timestamp": 1710330519.8694649, "_runtime": 1696.4874558448792, "_step": 11, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}

wandb/run-20240313_112023-t8o614bo/logs/debug-internal.log

@@ -862,3 +862,46 @@
 2024-03-13 11:47:12,056 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
 2024-03-13 11:47:15,311 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:47:20,312 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:22,333 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:47:25,334 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:26,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:47:26,083 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:47:26,083 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:47:31,230 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:36,230 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:41,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:47:41,082 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:47:41,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:47:42,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:47,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:52,161 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:47:52,334 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:47:56,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:47:56,082 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:47:56,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:47:57,347 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:02,349 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:07,350 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:11,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:11,083 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:11,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:48:13,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:18,161 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:22,335 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:48:23,335 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:26,082 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:26,083 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:26,123 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:48:29,165 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:34,166 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:39,167 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:39,870 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:48:39,871 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:48:39,871 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:48:39,871 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:48:40,085 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:48:41,114 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:41,114 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:41,191 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:48:44,236 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:49,237 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report