Training in progress, step 130

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13a78ee11a6a85e17a32e58b02cf1884dbc7bb521e1b404621979b034ad21058
+oid sha256:0585ce58894546e59cc5cee54b41f60e92d6a743bb9059cb6f6767b6784753ce
 size 261131840

runs/Mar13_11-36-50_7acbe5a74b63/events.out.tfevents.1710329812.7acbe5a74b63.337.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:561ea9511b11c1dae9411f14dd81d941f360640119474776a9179bca75eb6c8d
-size 8034
+oid sha256:38df48e9c8c6a4b11e6b04c465327e5ed0433d8cf51e28fb3e3b59f339c1a101
+size 8245

wandb/debug-internal.log

@@ -1078,3 +1078,46 @@
 2024-03-13 11:54:44,201 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
 2024-03-13 11:54:47,655 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:54:52,344 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:54:53,344 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:54:56,116 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:54:56,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:54:56,157 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:54:59,190 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:04,190 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:09,191 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:11,116 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:11,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:11,157 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:14,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:19,277 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:22,344 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:55:24,345 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:26,116 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:26,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:26,157 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:30,239 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:35,240 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:40,241 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:41,117 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:41,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:41,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:46,234 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:51,234 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:52,345 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:55:56,117 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:56,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:56,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:56,262 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:01,263 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:06,264 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:11,117 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:56:11,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:56:11,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:56:12,194 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:13,134 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:56:13,135 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:56:13,136 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:56:13,136 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:56:13,230 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:56:18,137 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:22,346 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats

wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.1722, "train/grad_norm": 0.7571136951446533, "train/learning_rate": 3.640849638818286e-05, "train/epoch": 9.6, "train/global_step": 120, "_timestamp": 1710330883.652137, "_runtime": 2060.2701280117035, "_step": 15, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}
+{"train/loss": 0.1519, "train/grad_norm": 0.6272870898246765, "train/learning_rate": 2.882832728712551e-05, "train/epoch": 10.4, "train/global_step": 130, "_timestamp": 1710330973.1336446, "_runtime": 2149.7516355514526, "_step": 16, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}

wandb/run-20240313_112023-t8o614bo/logs/debug-internal.log

@@ -1078,3 +1078,46 @@
 2024-03-13 11:54:44,201 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
 2024-03-13 11:54:47,655 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:54:52,344 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:54:53,344 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:54:56,116 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:54:56,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:54:56,157 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:54:59,190 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:04,190 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:09,191 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:11,116 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:11,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:11,157 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:14,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:19,277 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:22,344 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:55:24,345 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:26,116 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:26,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:26,157 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:30,239 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:35,240 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:40,241 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:41,117 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:41,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:41,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:46,234 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:51,234 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:55:52,345 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:55:56,117 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:55:56,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:55:56,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:55:56,262 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:01,263 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:06,264 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:11,117 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:56:11,117 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:56:11,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:56:12,194 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:13,134 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:56:13,135 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:56:13,136 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:56:13,136 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:56:13,230 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:56:18,137 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:56:22,346 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats