Training in progress, step 90

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7a1f20bc83390e7ce46330ad59e22144f38538f7f9633312f01b659bf5ba6a6
+oid sha256:d5b94916d954962c5a0ae07658de35b98e2e53f254ded5ba3de5ac1a4af467ec
 size 261131840

runs/Mar13_11-36-50_7acbe5a74b63/events.out.tfevents.1710329812.7acbe5a74b63.337.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca7fd3f4141cc93c775dcc7fe52054e06791b8659ed484c1e8e7b766f394343d
-size 7206
+oid sha256:dc8e6aed1719e3bab9eb684a5c63618dac708e9954ed1e92bff1fcc9e5091d65
+size 7413

wandb/debug-internal.log

@@ -905,3 +905,46 @@
 2024-03-13 11:48:41,191 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
 2024-03-13 11:48:44,236 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:48:49,237 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:52,335 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:48:54,336 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:56,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:56,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:56,136 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:00,243 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:05,244 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:10,245 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:11,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:11,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:11,135 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:16,168 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:21,169 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:22,336 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:49:26,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:26,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:26,136 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:26,213 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:31,214 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:36,214 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:41,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:41,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:41,135 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:42,203 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:47,204 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:52,205 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:52,337 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:49:56,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:56,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:56,135 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:57,226 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:02,226 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:07,227 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:10,262 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:50:10,263 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:50:10,264 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:50:10,264 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:50:11,114 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:50:11,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:50:11,178 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:50:11,179 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:50:12,350 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:17,351 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.2297, "train/grad_norm": 0.6079784035682678, "train/learning_rate": 6.819524684817438e-05, "train/epoch": 6.4, "train/global_step": 80, "_timestamp": 1710330519.8694649, "_runtime": 1696.4874558448792, "_step": 11, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}
+{"train/loss": 0.1823, "train/grad_norm": 0.6431258320808411, "train/learning_rate": 6.044837815156377e-05, "train/epoch": 7.2, "train/global_step": 90, "_timestamp": 1710330610.2621236, "_runtime": 1786.880114555359, "_step": 12, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}

wandb/run-20240313_112023-t8o614bo/logs/debug-internal.log

@@ -905,3 +905,46 @@
 2024-03-13 11:48:41,191 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
 2024-03-13 11:48:44,236 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:48:49,237 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:52,335 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:48:54,336 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:48:56,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:48:56,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:48:56,136 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:00,243 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:05,244 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:10,245 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:11,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:11,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:11,135 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:16,168 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:21,169 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:22,336 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:49:26,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:26,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:26,136 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:26,213 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:31,214 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:36,214 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:41,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:41,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:41,135 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:42,203 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:47,204 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:52,205 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:49:52,337 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:49:56,095 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:49:56,095 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:49:56,135 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:49:57,226 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:02,226 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:07,227 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:10,262 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:50:10,263 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:50:10,264 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:50:10,264 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:50:11,114 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:50:11,160 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:50:11,178 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:50:11,179 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:50:12,350 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:50:17,351 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report