Training in progress, step 150

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f2467336ae8ee4f6be6b2a0e57089221c721ad4ad7ba28ebce910094377268a
+oid sha256:a1fb795e0cc6af12382d10993f6c8e6fbb16004274d82cfcf420af1a6c313484
 size 261131840

runs/Mar13_11-36-50_7acbe5a74b63/events.out.tfevents.1710329812.7acbe5a74b63.337.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b64714caa06bda6673ad4e9255136d369b14df1cba96b7e6efc1717fff914f35
-size 8456
+oid sha256:23b132b6e14eff6961b5db5025caa65183f6b022d567ee1be41851a375cb7922
+size 8667

wandb/debug-internal.log

@@ -1165,3 +1165,46 @@
 2024-03-13 11:57:47,996 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:57:52,348 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
 2024-03-13 11:57:53,348 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:57:56,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:57:56,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:57:56,159 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:57:59,193 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:04,195 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:09,196 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:11,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:11,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:11,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:58:14,245 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:19,246 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:22,348 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:58:24,349 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:26,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:26,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:26,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:58:30,227 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:35,228 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:40,229 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:41,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:41,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:41,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:58:45,268 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:50,268 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:52,349 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:58:55,350 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:56,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:56,119 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:56,159 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:59:00,881 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:05,881 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:10,882 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:11,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:59:11,119 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:59:11,159 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:59:14,130 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:59:14,132 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:59:14,132 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:59:14,132 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:59:14,287 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:59:16,133 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:21,134 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:22,349 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats

wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.1453, "train/grad_norm": 0.5252134203910828, "train/learning_rate": 2.180214850745467e-05, "train/epoch": 11.2, "train/global_step": 140, "_timestamp": 1710331064.9926612, "_runtime": 2241.6106522083282, "_step": 17, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}
+{"train/loss": 0.1461, "train/grad_norm": 0.6350714564323425, "train/learning_rate": 1.5513811136094787e-05, "train/epoch": 12.0, "train/global_step": 150, "_timestamp": 1710331154.1302264, "_runtime": 2330.748217344284, "_step": 18, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}

wandb/run-20240313_112023-t8o614bo/logs/debug-internal.log

@@ -1165,3 +1165,46 @@
 2024-03-13 11:57:47,996 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
 2024-03-13 11:57:52,348 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
 2024-03-13 11:57:53,348 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:57:56,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:57:56,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:57:56,159 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:57:59,193 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:04,195 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:09,196 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:11,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:11,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:11,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:58:14,245 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:19,246 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:22,348 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:58:24,349 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:26,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:26,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:26,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:58:30,227 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:35,228 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:40,229 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:41,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:41,118 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:41,158 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:58:45,268 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:50,268 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:52,349 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 11:58:55,350 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:58:56,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:58:56,119 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:58:56,159 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:59:00,881 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:05,881 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:10,882 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:11,118 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 11:59:11,119 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 11:59:11,159 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 11:59:14,130 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 11:59:14,132 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 11:59:14,132 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 11:59:14,132 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 11:59:14,287 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 11:59:16,133 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:21,134 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 11:59:22,349 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats