diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,914 +1,106 @@ -2024-03-08 13:02:00,787 INFO StreamThr :25394 [internal.py:wandb_internal():86] W&B internal server running at pid: 25394, started at: 2024-03-08 13:02:00.786482 -2024-03-08 13:02:00,788 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status -2024-03-08 13:02:05,791 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:05,792 INFO WriterThread:25394 [datastore.py:open_for_write():87] open: /content/wandb/run-20240308_130200-qomx2zsg/run-qomx2zsg.wandb -2024-03-08 13:02:05,817 DEBUG SenderThread:25394 [sender.py:send():378] send: header -2024-03-08 13:02:05,820 DEBUG SenderThread:25394 [sender.py:send():378] send: run -2024-03-08 13:02:06,337 INFO SenderThread:25394 [dir_watcher.py:__init__():211] watching files in: /content/wandb/run-20240308_130200-qomx2zsg/files -2024-03-08 13:02:06,337 INFO SenderThread:25394 [sender.py:_start_run_threads():1099] run started: qomx2zsg with start time 1709902920.786651 -2024-03-08 13:02:06,339 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: check_version -2024-03-08 13:02:06,339 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: check_version -2024-03-08 13:02:06,567 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: run_start -2024-03-08 13:02:06,578 DEBUG HandlerThread:25394 [system_info.py:__init__():26] System info init -2024-03-08 13:02:06,578 DEBUG HandlerThread:25394 [system_info.py:__init__():41] System info init done -2024-03-08 13:02:06,579 INFO HandlerThread:25394 [system_monitor.py:start():194] Starting system monitor -2024-03-08 13:02:06,579 INFO SystemMonitor:25394 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-08 13:02:06,579 INFO HandlerThread:25394 [system_monitor.py:probe():214] Collecting system info -2024-03-08 13:02:06,580 INFO SystemMonitor:25394 [interfaces.py:start():190] Started cpu monitoring -2024-03-08 13:02:06,580 INFO SystemMonitor:25394 [interfaces.py:start():190] Started disk monitoring -2024-03-08 13:02:06,581 INFO SystemMonitor:25394 [interfaces.py:start():190] Started gpu monitoring -2024-03-08 13:02:06,582 INFO SystemMonitor:25394 [interfaces.py:start():190] Started memory monitoring -2024-03-08 13:02:06,583 INFO SystemMonitor:25394 [interfaces.py:start():190] Started network monitoring -2024-03-08 13:02:06,589 DEBUG HandlerThread:25394 [system_info.py:probe():150] Probing system -2024-03-08 13:02:06,590 DEBUG HandlerThread:25394 [gitlib.py:_init_repo():56] git repository is invalid -2024-03-08 13:02:06,590 DEBUG HandlerThread:25394 [system_info.py:probe():198] Probing system done -2024-03-08 13:02:06,590 DEBUG HandlerThread:25394 [system_monitor.py:probe():223] {'os': 'Linux-6.1.58+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-03-08T13:02:06.589080', 'startedAt': '2024-03-08T13:02:00.780539', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'Smart_Agriculture_Falcon.ipynb', 'codePathLocal': None, 'colab': 'https://colab.research.google.com/notebook#fileId=1kv2VO7q0c8xX_DQt5NERLeJJZtHW6zHO', 'host': '4997a2709b0f', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 4, 'cpu_count_logical': 8, 'cpu_freq': {'current': 2199.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 201.22513961791992, 'used': 58.329315185546875}}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 50.993690490722656}} -2024-03-08 13:02:06,590 INFO HandlerThread:25394 [system_monitor.py:probe():224] Finished collecting system info -2024-03-08 13:02:06,590 INFO HandlerThread:25394 [system_monitor.py:probe():227] Publishing system info -2024-03-08 13:02:06,591 INFO HandlerThread:25394 [system_monitor.py:probe():229] Finished publishing system info -2024-03-08 13:02:06,597 DEBUG SenderThread:25394 [sender.py:send():378] send: files -2024-03-08 13:02:06,597 INFO SenderThread:25394 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now -2024-03-08 13:02:07,304 INFO wandb-upload_0:25394 [upload_job.py:push():131] Uploaded file /tmp/tmpnr4g6anewandb/kyjo8nn9-wandb-metadata.json -2024-03-08 13:02:07,339 INFO Thread-12 :25394 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_130200-qomx2zsg/files/wandb-metadata.json -2024-03-08 13:02:07,428 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: python_packages -2024-03-08 13:02:07,428 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: python_packages -2024-03-08 13:02:07,468 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:02:07,468 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:02:07,469 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:02:07,708 DEBUG SenderThread:25394 [sender.py:send():378] send: telemetry -2024-03-08 13:02:07,708 DEBUG SenderThread:25394 [sender.py:send():378] send: config -2024-03-08 13:02:07,709 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:02:07,709 DEBUG SenderThread:25394 [sender.py:send():378] send: telemetry -2024-03-08 13:02:07,709 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:02:07,709 WARNING SenderThread:25394 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) -2024-03-08 13:02:07,709 DEBUG SenderThread:25394 [sender.py:send():378] send: telemetry -2024-03-08 13:02:08,338 INFO Thread-12 :25394 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_130200-qomx2zsg/files/requirements.txt -2024-03-08 13:02:11,710 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:16,712 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:21,713 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:22,429 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:02:22,429 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:02:22,469 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:02:26,798 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:31,803 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:32,345 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/config.yaml -2024-03-08 13:02:37,102 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:37,429 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:02:37,429 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:02:37,470 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:02:42,616 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:47,617 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:52,429 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:02:52,430 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:02:52,470 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:02:52,659 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:02:57,660 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:02,661 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:06,583 DEBUG SystemMonitor:25394 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-08 13:03:06,585 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:03:07,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:03:07,430 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:03:07,470 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:03:07,718 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:12,719 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:17,720 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:22,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:03:22,430 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:03:22,470 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:03:23,640 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:28,641 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:33,642 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:36,586 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:03:37,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:03:37,430 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:03:37,470 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:03:38,727 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:43,728 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:47,466 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: partial_history -2024-03-08 13:03:47,468 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:03:47,468 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:03:47,468 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:03:47,468 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:03:47,469 DEBUG SenderThread:25394 [sender.py:send():378] send: history -2024-03-08 13:03:47,469 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: summary_record -2024-03-08 13:03:47,471 INFO SenderThread:25394 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-08 13:03:48,371 INFO Thread-12 :25394 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_130200-qomx2zsg/files/output.log -2024-03-08 13:03:48,372 INFO Thread-12 :25394 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_130200-qomx2zsg/files/wandb-summary.json -2024-03-08 13:03:49,507 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:50,372 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/output.log -2024-03-08 13:03:52,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:03:52,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:03:52,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:03:54,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:03:59,677 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:04,683 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:05,378 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/config.yaml -2024-03-08 13:04:06,586 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:04:07,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:04:07,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:04:07,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:04:10,687 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:15,688 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:20,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:22,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:04:22,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:04:22,471 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:04:25,695 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:30,696 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:35,697 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:36,587 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:04:37,430 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:04:37,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:04:37,471 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:04:40,733 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:45,735 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:50,735 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:04:52,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:04:52,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:04:52,471 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:04:56,638 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:01,639 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:06,588 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:05:07,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:05:07,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:05:07,471 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:05:07,720 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:12,721 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:17,722 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:22,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:05:22,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:05:22,472 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:05:23,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:28,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:33,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:36,589 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:05:37,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:05:37,431 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:05:37,472 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:05:38,252 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: partial_history -2024-03-08 13:05:38,253 DEBUG SenderThread:25394 [sender.py:send():378] send: history -2024-03-08 13:05:38,253 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: summary_record -2024-03-08 13:05:38,254 INFO SenderThread:25394 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-08 13:05:38,411 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/wandb-summary.json -2024-03-08 13:05:39,294 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:40,412 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/output.log -2024-03-08 13:05:44,295 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:49,296 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:52,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:05:52,432 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:05:52,432 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:05:54,647 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:05:59,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:04,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:06,590 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:06:07,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:06:07,432 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:06:07,472 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:06:10,626 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:15,627 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:20,628 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:22,431 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:06:22,432 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:06:22,472 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:06:25,728 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:30,729 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:35,730 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:36,590 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:06:37,432 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:06:37,432 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:06:37,472 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:06:41,647 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:46,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:51,649 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:06:52,432 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:06:52,432 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:06:52,473 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:06:57,635 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:02,636 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:06,591 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:07:07,432 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:07:07,432 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:07:07,473 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:07:07,682 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:12,683 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:17,684 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:22,432 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:07:22,433 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:07:22,473 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:07:23,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:28,649 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:33,635 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: partial_history -2024-03-08 13:07:33,637 DEBUG SenderThread:25394 [sender.py:send():378] send: history -2024-03-08 13:07:33,637 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: summary_record -2024-03-08 13:07:33,638 INFO SenderThread:25394 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-08 13:07:33,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:34,458 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/wandb-summary.json -2024-03-08 13:07:36,459 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/output.log -2024-03-08 13:07:36,592 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:07:37,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:07:37,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:07:37,434 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:07:38,284 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: partial_history -2024-03-08 13:07:38,286 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:07:38,286 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:07:38,286 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:07:38,286 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:07:38,286 DEBUG SenderThread:25394 [sender.py:send():378] send: metric -2024-03-08 13:07:38,287 DEBUG SenderThread:25394 [sender.py:send():378] send: history -2024-03-08 13:07:38,287 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: summary_record -2024-03-08 13:07:38,287 INFO SenderThread:25394 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-08 13:07:38,460 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/wandb-summary.json -2024-03-08 13:07:39,288 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:44,293 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:45,462 INFO Thread-12 :25394 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_130200-qomx2zsg/files/config.yaml -2024-03-08 13:07:49,581 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:52,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:07:52,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:07:52,434 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:07:54,708 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:07:59,710 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:04,711 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:06,593 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:08:07,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:08:07,433 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:08:07,474 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:08:10,623 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:15,624 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:17,140 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: pause -2024-03-08 13:08:17,140 INFO HandlerThread:25394 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-03-08 13:08:17,140 INFO HandlerThread:25394 [system_monitor.py:finish():203] Stopping system monitor -2024-03-08 13:08:17,140 DEBUG SystemMonitor:25394 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-03-08 13:08:17,141 DEBUG SystemMonitor:25394 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-03-08 13:08:17,142 INFO HandlerThread:25394 [interfaces.py:finish():202] Joined cpu monitor -2024-03-08 13:08:17,142 INFO HandlerThread:25394 [interfaces.py:finish():202] Joined disk monitor -2024-03-08 13:08:17,148 INFO HandlerThread:25394 [interfaces.py:finish():202] Joined gpu monitor -2024-03-08 13:08:17,148 INFO HandlerThread:25394 [interfaces.py:finish():202] Joined memory monitor -2024-03-08 13:08:17,149 INFO HandlerThread:25394 [interfaces.py:finish():202] Joined network monitor -2024-03-08 13:08:17,149 DEBUG SenderThread:25394 [sender.py:send():378] send: stats -2024-03-08 13:08:21,151 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:22,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:08:22,434 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:08:22,434 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:08:26,284 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:31,285 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:36,286 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:37,433 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:08:37,434 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:08:37,474 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:08:41,713 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:46,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:51,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:08:52,434 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:08:52,434 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:08:52,474 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:08:57,665 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:02,666 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:07,434 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:09:07,434 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:09:07,475 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:09:08,611 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:13,613 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:18,614 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:22,434 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:09:22,435 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:09:22,475 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:09:23,688 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:28,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:33,690 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:37,435 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:09:37,435 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:09:37,475 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:09:39,682 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:44,684 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:49,685 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:09:52,435 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:09:52,435 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:09:52,475 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:09:55,644 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:00,646 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:05,647 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:07,435 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:10:07,435 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:10:07,476 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:10:11,635 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:16,636 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:21,638 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:22,435 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:10:22,436 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:10:22,476 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:10:26,680 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:31,681 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:36,683 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:37,435 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:10:37,436 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:10:37,476 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:10:42,665 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:47,666 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:52,436 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:10:52,436 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:10:52,476 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:10:52,722 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:10:57,723 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:02,724 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:07,436 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:11:07,436 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:11:07,476 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:11:08,645 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:13,646 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:18,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:22,436 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:11:22,436 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:11:22,477 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:11:24,636 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:29,637 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:34,639 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:37,436 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:11:37,437 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:11:37,477 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:11:39,706 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:44,707 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:49,708 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:11:52,436 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:11:52,437 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:11:52,477 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:11:55,673 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:00,674 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:05,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:07,437 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:12:07,437 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:12:07,477 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:12:11,668 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:16,669 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:21,671 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:22,437 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:12:22,437 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:12:22,477 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:12:26,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:31,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:36,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:37,437 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:12:37,438 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:12:37,478 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:12:42,635 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:47,636 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:52,437 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:12:52,438 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:12:52,478 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:12:52,675 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:12:57,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:02,677 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:07,437 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:13:07,438 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:13:07,478 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:13:07,692 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:12,694 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:17,695 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:22,438 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:13:22,438 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:13:22,478 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:13:24,155 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:29,156 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:34,157 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:37,438 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:13:37,438 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:13:37,479 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:13:39,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:44,718 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:49,719 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:13:52,438 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:13:52,438 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:13:52,479 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:13:55,643 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:00,644 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:05,645 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:07,438 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:14:07,439 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:14:07,479 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:14:10,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:15,690 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:20,691 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:22,438 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:14:22,439 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:14:22,479 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:14:26,687 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:31,688 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:36,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:37,439 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:14:37,439 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:14:37,479 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:14:41,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:46,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:51,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:14:52,439 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:14:52,440 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:14:52,480 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:14:56,718 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:01,719 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:06,720 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:07,439 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:15:07,440 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:15:07,480 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:15:11,723 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:16,725 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:21,726 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:22,439 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:15:22,440 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:15:22,480 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:15:27,626 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:32,628 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:37,440 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:15:37,440 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:15:37,481 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:15:37,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:42,677 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:47,678 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:52,440 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:15:52,440 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:15:52,481 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:15:54,330 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:15:59,331 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:04,332 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:07,440 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:16:07,441 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:16:07,481 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:16:09,630 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:14,631 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:19,633 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:22,440 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:16:22,441 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:16:22,481 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:16:24,691 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:29,692 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:34,693 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:37,441 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:16:37,441 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:16:37,481 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:16:40,685 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:45,686 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:50,687 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:16:52,441 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:16:52,441 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:16:52,481 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:16:55,810 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:00,812 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:05,813 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:07,441 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:17:07,441 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:17:07,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:17:11,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:16,678 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:21,679 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:22,441 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:17:22,442 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:17:22,482 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:17:27,593 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:32,594 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:37,441 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:17:37,442 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:17:37,482 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:17:37,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:42,718 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:47,720 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:52,442 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:17:52,442 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:17:52,482 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:17:54,802 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:17:59,804 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:04,805 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:07,442 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:18:07,442 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:18:07,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:18:10,700 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:15,701 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:20,702 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:22,442 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:18:22,443 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:18:22,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:18:25,721 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:30,722 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:35,723 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:37,442 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:18:37,443 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:18:37,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:18:41,707 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:46,708 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:51,709 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:18:52,442 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:18:52,443 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:18:52,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:18:56,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:01,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:06,718 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:07,443 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:19:07,443 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:19:07,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:19:12,701 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:17,702 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:22,443 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:19:22,443 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:19:22,483 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:19:22,719 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:27,720 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:32,721 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:37,443 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:19:37,443 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:19:37,484 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:19:38,657 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:43,658 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:48,659 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:52,443 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:19:52,444 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:19:52,484 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:19:53,698 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:19:58,699 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:03,700 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:07,443 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:20:07,444 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:20:07,484 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:20:08,714 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:13,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:18,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:22,443 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:20:22,444 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:20:22,484 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:20:24,705 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:29,706 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:34,708 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:37,444 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:20:37,444 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:20:37,485 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:20:39,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:44,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:49,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:20:52,444 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:20:52,445 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:20:52,485 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:20:55,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:00,690 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:05,691 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:07,444 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:21:07,445 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:21:07,485 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:21:11,657 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:16,659 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:21,660 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:22,444 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:21:22,445 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:21:22,485 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:21:26,677 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:31,678 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:36,679 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:37,445 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:21:37,445 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:21:37,485 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:21:42,660 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:47,661 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:52,445 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:21:52,446 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:21:52,485 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:21:54,095 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:21:59,096 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:04,097 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:07,445 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:22:07,445 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:22:07,486 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:22:09,673 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:14,675 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:19,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:22,445 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:22:22,446 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:22:22,486 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:22:25,199 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:30,200 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:35,201 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:37,445 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:22:37,446 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:22:37,486 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:22:40,735 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:45,737 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:50,738 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:22:52,445 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:22:52,446 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:22:52,486 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:22:56,726 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:01,728 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:06,729 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:07,446 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:23:07,446 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:23:07,486 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:23:12,723 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:17,724 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:22,446 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:23:22,446 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:23:22,487 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:23:23,722 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:28,723 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:33,724 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:37,446 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:23:37,447 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:23:37,487 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:23:39,634 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:44,635 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:49,636 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:52,446 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:23:52,447 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:23:52,487 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:23:54,918 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:23:59,919 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:04,921 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:07,447 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:24:07,447 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:24:07,487 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:24:10,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:15,691 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:20,692 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:22,447 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:24:22,447 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:24:22,487 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:24:26,088 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:31,089 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:36,091 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:37,447 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:24:37,448 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:24:37,488 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:24:41,724 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:46,725 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:51,726 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:24:52,447 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:24:52,448 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:24:52,488 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:24:56,735 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:01,736 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:06,738 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:07,448 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:25:07,448 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:25:07,488 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:25:12,647 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:17,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:22,447 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:25:22,448 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:25:22,488 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:25:22,658 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:27,660 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:32,661 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:37,448 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:25:37,448 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:25:37,489 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:25:37,720 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:42,721 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:47,722 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:52,448 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:25:52,449 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:25:52,489 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:25:53,704 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:25:58,706 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:03,707 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:07,448 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:26:07,449 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:26:07,490 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:26:08,736 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:13,737 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:18,739 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:22,448 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:26:22,449 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:26:22,489 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:26:24,665 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:29,666 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:34,667 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:37,449 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:26:37,449 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:26:37,489 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:26:39,677 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:44,678 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:49,679 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:52,449 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:26:52,449 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:26:52,490 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:26:54,736 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:26:59,737 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:04,739 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:07,449 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:27:07,450 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:27:07,490 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:27:10,709 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:15,710 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:20,712 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:22,449 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:27:22,450 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:27:22,490 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:27:25,713 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:30,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:35,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:37,450 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:27:37,450 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:27:37,490 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:27:41,709 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:46,710 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:51,711 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:27:52,450 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:27:52,450 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:27:52,490 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:27:56,727 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:01,728 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:06,728 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:07,450 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:28:07,450 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:28:07,491 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:28:12,699 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:17,700 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:22,450 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:28:22,450 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:28:22,491 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:28:22,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:27,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:32,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:37,450 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:28:37,451 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:28:37,491 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:28:38,703 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:43,705 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:48,706 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:52,451 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:28:52,451 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:28:52,491 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:28:54,704 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:28:59,705 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:04,706 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:07,451 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:29:07,451 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:29:07,491 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:29:10,645 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:15,647 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:20,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:22,451 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:29:22,451 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:29:22,492 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:29:25,748 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:30,749 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:35,750 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:37,451 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:29:37,452 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:29:37,492 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:29:41,642 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:46,643 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:51,645 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:29:52,452 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:29:52,452 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:29:52,492 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:29:57,639 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:02,641 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:07,451 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:30:07,452 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:30:07,492 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:30:07,747 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:12,748 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:17,749 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:22,452 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:30:22,452 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:30:22,492 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:30:25,128 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:30,129 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:35,130 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:37,452 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:30:37,453 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:30:37,493 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:30:40,689 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:45,690 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:50,692 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:30:52,452 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:30:52,453 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:30:52,493 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:30:55,718 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:00,719 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:05,721 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:07,452 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:31:07,453 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:31:07,493 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:31:11,714 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:16,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:21,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:22,452 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:31:22,453 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:31:22,493 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:31:27,647 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:32,648 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:37,453 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:31:37,453 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:31:37,493 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:31:37,730 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:42,731 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:47,732 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:52,453 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:31:52,453 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:31:52,494 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:31:53,804 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:31:58,805 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:03,806 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:07,453 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:32:07,454 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:32:07,494 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:32:09,732 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:14,733 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:19,735 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:22,453 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:32:22,454 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:32:22,494 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:32:25,502 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:30,503 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:35,505 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:37,453 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:32:37,454 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:32:37,494 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:32:40,674 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:45,675 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:50,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:32:52,454 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:32:52,454 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:32:52,494 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:32:55,926 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:00,927 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:05,928 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:07,454 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:33:07,454 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:33:07,495 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:33:11,764 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:16,765 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:21,767 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:22,454 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:33:22,454 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:33:22,495 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:33:27,629 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:32,630 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:37,454 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:33:37,455 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:33:37,495 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:33:37,715 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:42,716 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:47,717 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:52,454 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:33:52,455 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:33:52,495 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:33:52,740 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:33:57,741 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:02,742 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:07,455 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:34:07,455 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:34:07,495 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:34:08,688 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:13,690 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:18,691 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:22,455 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:34:22,455 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:34:22,496 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:34:24,678 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:29,679 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:34,680 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:37,455 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:34:37,456 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:34:37,496 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:34:39,721 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:44,722 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:49,723 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:52,456 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:34:52,456 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:34:52,496 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:34:54,727 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:34:59,729 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:04,730 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:07,455 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:35:07,456 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:35:07,496 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:35:10,685 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:15,687 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:20,688 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:22,456 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:35:22,456 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:35:22,496 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:35:26,658 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:31,659 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:36,661 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:37,456 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:35:37,456 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:35:37,497 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:35:41,676 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:46,677 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:51,678 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:35:52,456 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:35:52,457 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:35:52,497 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:35:56,731 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:36:01,733 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:36:06,734 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report -2024-03-08 13:36:07,456 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: stop_status -2024-03-08 13:36:07,457 DEBUG SenderThread:25394 [sender.py:send_request():405] send_request: stop_status -2024-03-08 13:36:07,497 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-08 13:36:10,759 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: resume -2024-03-08 13:36:10,759 INFO HandlerThread:25394 [handler.py:handle_request_resume():699] starting system metrics thread -2024-03-08 13:36:10,759 INFO HandlerThread:25394 [system_monitor.py:start():194] Starting system monitor -2024-03-08 13:36:10,760 INFO SystemMonitor:25394 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-08 13:36:10,760 INFO SystemMonitor:25394 [interfaces.py:start():190] Started cpu monitoring -2024-03-08 13:36:10,761 INFO SystemMonitor:25394 [interfaces.py:start():190] Started disk monitoring -2024-03-08 13:36:10,761 INFO SystemMonitor:25394 [interfaces.py:start():190] Started gpu monitoring -2024-03-08 13:36:10,762 INFO SystemMonitor:25394 [interfaces.py:start():190] Started memory monitoring -2024-03-08 13:36:10,763 INFO SystemMonitor:25394 [interfaces.py:start():190] Started network monitoring -2024-03-08 13:36:12,667 DEBUG HandlerThread:25394 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:01,420 INFO StreamThr :37167 [internal.py:wandb_internal():86] W&B internal server running at pid: 37167, started at: 2024-03-08 13:49:01.420105 +2024-03-08 13:49:01,422 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status +2024-03-08 13:49:06,425 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:06,425 INFO WriterThread:37167 [datastore.py:open_for_write():87] open: /content/wandb/run-20240308_134901-hlg2f18u/run-hlg2f18u.wandb +2024-03-08 13:49:06,451 DEBUG SenderThread:37167 [sender.py:send():378] send: header +2024-03-08 13:49:06,454 DEBUG SenderThread:37167 [sender.py:send():378] send: run +2024-03-08 13:49:06,939 INFO SenderThread:37167 [dir_watcher.py:__init__():211] watching files in: /content/wandb/run-20240308_134901-hlg2f18u/files +2024-03-08 13:49:06,939 INFO SenderThread:37167 [sender.py:_start_run_threads():1099] run started: hlg2f18u with start time 1709905741.420297 +2024-03-08 13:49:06,942 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: check_version +2024-03-08 13:49:06,942 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: check_version +2024-03-08 13:49:07,153 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: run_start +2024-03-08 13:49:07,163 DEBUG HandlerThread:37167 [system_info.py:__init__():26] System info init +2024-03-08 13:49:07,163 DEBUG HandlerThread:37167 [system_info.py:__init__():41] System info init done +2024-03-08 13:49:07,163 INFO HandlerThread:37167 [system_monitor.py:start():194] Starting system monitor +2024-03-08 13:49:07,164 INFO SystemMonitor:37167 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 13:49:07,164 INFO HandlerThread:37167 [system_monitor.py:probe():214] Collecting system info +2024-03-08 13:49:07,164 INFO SystemMonitor:37167 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 13:49:07,165 INFO SystemMonitor:37167 [interfaces.py:start():190] Started disk monitoring +2024-03-08 13:49:07,166 INFO SystemMonitor:37167 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 13:49:07,167 INFO SystemMonitor:37167 [interfaces.py:start():190] Started memory monitoring +2024-03-08 13:49:07,168 INFO SystemMonitor:37167 [interfaces.py:start():190] Started network monitoring +2024-03-08 13:49:07,172 DEBUG HandlerThread:37167 [system_info.py:probe():150] Probing system +2024-03-08 13:49:07,174 DEBUG HandlerThread:37167 [gitlib.py:_init_repo():56] git repository is invalid +2024-03-08 13:49:07,174 DEBUG HandlerThread:37167 [system_info.py:probe():198] Probing system done +2024-03-08 13:49:07,174 DEBUG HandlerThread:37167 [system_monitor.py:probe():223] {'os': 'Linux-6.1.58+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-03-08T13:49:07.172649', 'startedAt': '2024-03-08T13:49:01.413484', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'Smart_Agriculture_Falcon.ipynb', 'codePathLocal': None, 'colab': 'https://colab.research.google.com/notebook#fileId=1kv2VO7q0c8xX_DQt5NERLeJJZtHW6zHO', 'host': '4997a2709b0f', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 4, 'cpu_count_logical': 8, 'cpu_freq': {'current': 2199.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}, {'current': 2199.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 201.22513961791992, 'used': 58.57290267944336}}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 50.993690490722656}} +2024-03-08 13:49:07,174 INFO HandlerThread:37167 [system_monitor.py:probe():224] Finished collecting system info +2024-03-08 13:49:07,174 INFO HandlerThread:37167 [system_monitor.py:probe():227] Publishing system info +2024-03-08 13:49:07,174 INFO HandlerThread:37167 [system_monitor.py:probe():229] Finished publishing system info +2024-03-08 13:49:07,186 DEBUG SenderThread:37167 [sender.py:send():378] send: files +2024-03-08 13:49:07,186 INFO SenderThread:37167 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now +2024-03-08 13:49:07,646 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: python_packages +2024-03-08 13:49:07,647 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: python_packages +2024-03-08 13:49:07,686 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:49:07,687 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:49:07,688 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:49:07,940 INFO Thread-12 :37167 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_134901-hlg2f18u/files/wandb-metadata.json +2024-03-08 13:49:07,941 INFO Thread-12 :37167 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_134901-hlg2f18u/files/requirements.txt +2024-03-08 13:49:08,065 INFO wandb-upload_0:37167 [upload_job.py:push():131] Uploaded file /tmp/tmpgtstjuiqwandb/o7wr7mlg-wandb-metadata.json +2024-03-08 13:49:08,292 DEBUG SenderThread:37167 [sender.py:send():378] send: telemetry +2024-03-08 13:49:08,292 DEBUG SenderThread:37167 [sender.py:send():378] send: config +2024-03-08 13:49:08,293 DEBUG SenderThread:37167 [sender.py:send():378] send: metric +2024-03-08 13:49:08,293 DEBUG SenderThread:37167 [sender.py:send():378] send: telemetry +2024-03-08 13:49:08,293 DEBUG SenderThread:37167 [sender.py:send():378] send: metric +2024-03-08 13:49:08,293 WARNING SenderThread:37167 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) +2024-03-08 13:49:08,293 DEBUG SenderThread:37167 [sender.py:send():378] send: telemetry +2024-03-08 13:49:12,294 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:17,295 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:22,296 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:22,647 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:49:22,647 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:49:22,688 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:49:27,889 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:32,894 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:33,949 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/config.yaml +2024-03-08 13:49:37,647 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:49:37,648 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:49:37,688 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:49:38,897 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:43,898 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:48,899 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:52,647 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:49:52,648 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:49:52,688 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:49:54,851 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:49:59,852 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:04,853 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:07,168 DEBUG SystemMonitor:37167 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 13:50:07,169 DEBUG SenderThread:37167 [sender.py:send():378] send: stats +2024-03-08 13:50:07,647 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:50:07,648 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:50:07,688 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:50:09,898 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:14,899 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:19,900 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:22,648 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:50:22,648 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:50:22,688 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:50:25,835 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:30,836 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:30,851 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: partial_history +2024-03-08 13:50:30,853 DEBUG SenderThread:37167 [sender.py:send():378] send: metric +2024-03-08 13:50:30,854 DEBUG SenderThread:37167 [sender.py:send():378] send: metric +2024-03-08 13:50:30,854 DEBUG SenderThread:37167 [sender.py:send():378] send: metric +2024-03-08 13:50:30,854 DEBUG SenderThread:37167 [sender.py:send():378] send: metric +2024-03-08 13:50:30,854 DEBUG SenderThread:37167 [sender.py:send():378] send: history +2024-03-08 13:50:30,854 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: summary_record +2024-03-08 13:50:30,856 INFO SenderThread:37167 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-08 13:50:30,969 INFO Thread-12 :37167 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_134901-hlg2f18u/files/output.log +2024-03-08 13:50:30,970 INFO Thread-12 :37167 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_134901-hlg2f18u/files/wandb-summary.json +2024-03-08 13:50:32,971 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/output.log +2024-03-08 13:50:35,899 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:36,972 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/config.yaml +2024-03-08 13:50:37,170 DEBUG SenderThread:37167 [sender.py:send():378] send: stats +2024-03-08 13:50:37,648 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:50:37,648 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:50:37,649 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:50:41,848 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:46,849 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:51,850 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:50:52,648 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 13:50:52,648 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status +2024-03-08 13:50:52,689 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 13:50:56,936 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:51:01,937 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:51:06,939 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 13:51:07,171 DEBUG SenderThread:37167 [sender.py:send():378] send: stats