diff --git "a/run.log" "b/run.log"
--- "a/run.log"
+++ "b/run.log"
@@ -36044,3 +36044,1344 @@ Time to load utils op: 0.00040650367736816406 seconds
 [2022-12-20 17:20:08,330] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-2000/global_step2024/zero_pp_rank_0_mp_rank_00_optim_states.pt
 [2022-12-20 17:20:08,330] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2024 is ready now!
 [INFO|feature_extraction_utils.py:368] 2022-12-20 17:20:13,382 >> Feature extractor saved in ./preprocessor_config.json
+[2022-12-20 17:22:16,998] [INFO] [timer.py:197:stop] 0/4050, RunningAvgSamplesPerSec=5.869386070314113, CurrSamplesPerSec=5.068967296185505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2001/5000 [6:59:52<366:28:31, 439.92s/it][2022-12-20 17:22:28,005] [INFO] [timer.py:197:stop] 0/4052, RunningAvgSamplesPerSec=5.869391927530963, CurrSamplesPerSec=5.300500373019847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2002/5000 [7:00:03<259:11:50, 311.24s/it][2022-12-20 17:22:39,002] [INFO] [timer.py:197:stop] 0/4054, RunningAvgSamplesPerSec=5.869400390538758, CurrSamplesPerSec=5.325949626901429, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2003/5000 [7:00:14<184:07:26, 221.17s/it][2022-12-20 17:22:49,982] [INFO] [timer.py:197:stop] 0/4056, RunningAvgSamplesPerSec=5.869413120329157, CurrSamplesPerSec=5.35738468026286, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2004/5000 [7:00:25<131:35:06, 158.11s/it][2022-12-20 17:23:01,364] [INFO] [timer.py:197:stop] 0/4058, RunningAvgSamplesPerSec=5.869319147153795, CurrSamplesPerSec=5.026528964955302, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2005/5000 [7:00:36<94:55:11, 114.09s/it] [2022-12-20 17:23:12,303] [INFO] [logging.py:68:log_dist] [Rank 0] step=2030, skipped=3, lr=[6.608888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:23:12,305] [INFO] [timer.py:197:stop] 0/4060, RunningAvgSamplesPerSec=5.8693435398836895, CurrSamplesPerSec=5.35441881870314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2006/5000 [7:00:47<69:09:04, 83.15s/it] [2022-12-20 17:23:23,344] [INFO] [timer.py:197:stop] 0/4062, RunningAvgSamplesPerSec=5.869340453271744, CurrSamplesPerSec=5.3007804664473515, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2007/5000 [7:00:58<51:08:34, 61.52s/it][2022-12-20 17:23:34,500] [INFO] [timer.py:197:stop] 0/4064, RunningAvgSamplesPerSec=5.869305965318879, CurrSamplesPerSec=5.1783726478217575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2008/5000 [7:01:10<38:34:11, 46.41s/it][2022-12-20 17:23:45,484] [INFO] [timer.py:197:stop] 0/4066, RunningAvgSamplesPerSec=5.869317420587939, CurrSamplesPerSec=5.322756063620821, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2009/5000 [7:01:21<29:43:39, 35.78s/it][2022-12-20 17:23:55,607] [INFO] [timer.py:197:stop] 0/4068, RunningAvgSamplesPerSec=5.869554713583558, CurrSamplesPerSec=5.306858742725285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2010/5000 [7:01:36<24:32:51, 29.56s/it][2022-12-20 17:24:06,588] [INFO] [timer.py:197:stop] 0/4070, RunningAvgSamplesPerSec=5.869566827895841, CurrSamplesPerSec=5.325485350699582, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2011/5000 [7:01:47<19:54:55, 23.99s/it][2022-12-20 17:24:17,566] [INFO] [timer.py:197:stop] 0/4072, RunningAvgSamplesPerSec=5.869579598936879, CurrSamplesPerSec=5.33692707881441, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2012/5000 [7:01:58<16:40:17, 20.09s/it][2022-12-20 17:24:28,607] [INFO] [timer.py:197:stop] 0/4074, RunningAvgSamplesPerSec=5.869575694735896, CurrSamplesPerSec=5.287936732481226, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2013/5000 [7:02:09<14:25:10, 17.38s/it][2022-12-20 17:24:39,601] [INFO] [timer.py:197:stop] 0/4076, RunningAvgSamplesPerSec=5.869584134818115, CurrSamplesPerSec=5.347553982542901, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2014/5000 [7:02:20<12:49:27, 15.46s/it][2022-12-20 17:24:50,620] [INFO] [timer.py:197:stop] 0/4078, RunningAvgSamplesPerSec=5.869585991184868, CurrSamplesPerSec=5.319675120965998, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2015/5000 [7:02:31<11:43:01, 14.13s/it][2022-12-20 17:25:01,601] [INFO] [logging.py:68:log_dist] [Rank 0] step=2040, skipped=3, lr=[6.5866666666666666e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:25:01,603] [INFO] [timer.py:197:stop] 0/4080, RunningAvgSamplesPerSec=5.869597521017075, CurrSamplesPerSec=5.358913882848374, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2016/5000 [7:02:42<10:55:10, 13.17s/it][2022-12-20 17:25:12,669] [INFO] [timer.py:197:stop] 0/4082, RunningAvgSamplesPerSec=5.869587089116151, CurrSamplesPerSec=5.248799867131394, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2017/5000 [7:02:53<10:23:58, 12.55s/it][2022-12-20 17:25:23,670] [INFO] [timer.py:197:stop] 0/4084, RunningAvgSamplesPerSec=5.8695936001170415, CurrSamplesPerSec=5.332328985556303, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2018/5000 [7:03:04<10:00:53, 12.09s/it][2022-12-20 17:25:34,708] [INFO] [timer.py:197:stop] 0/4086, RunningAvgSamplesPerSec=5.869590693513245, CurrSamplesPerSec=5.313192001828257, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2019/5000 [7:03:15<9:44:35, 11.77s/it] [2022-12-20 17:25:45,732] [INFO] [timer.py:197:stop] 0/4088, RunningAvgSamplesPerSec=5.8695919667493515, CurrSamplesPerSec=5.3033937948372945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2020/5000 [7:03:26<9:33:43, 11.55s/it][2022-12-20 17:25:56,763] [INFO] [timer.py:197:stop] 0/4090, RunningAvgSamplesPerSec=5.869590714367239, CurrSamplesPerSec=5.318152423079144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2021/5000 [7:03:37<9:26:09, 11.40s/it][2022-12-20 17:26:07,777] [INFO] [timer.py:197:stop] 0/4092, RunningAvgSamplesPerSec=5.869594352964098, CurrSamplesPerSec=5.357536941066056, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2022/5000 [7:03:48<9:19:48, 11.28s/it][2022-12-20 17:26:18,833] [INFO] [timer.py:197:stop] 0/4094, RunningAvgSamplesPerSec=5.86958656563254, CurrSamplesPerSec=5.297363382387859, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2023/5000 [7:03:59<9:15:52, 11.20s/it][2022-12-20 17:26:29,842] [INFO] [timer.py:197:stop] 0/4096, RunningAvgSamplesPerSec=5.86959094824509, CurrSamplesPerSec=5.312335466008394, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2024/5000 [7:04:10<9:13:08, 11.15s/it][2022-12-20 17:26:40,823] [INFO] [timer.py:197:stop] 0/4098, RunningAvgSamplesPerSec=5.86960310088504, CurrSamplesPerSec=5.3579978393593795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 40%|████      | 2025/5000 [7:04:21<9:11:07, 11.12s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.566666666666667e-06, 'epoch': 49.39}
+
 40%|████      | 2025/5000 [7:04:21<9:11:07, 11.12s/it][2022-12-20 17:26:51,964] [INFO] [logging.py:68:log_dist] [Rank 0] step=2050, skipped=3, lr=[6.564444444444446e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:26:51,966] [INFO] [timer.py:197:stop] 0/4100, RunningAvgSamplesPerSec=5.869589859741664, CurrSamplesPerSec=5.314459542150491, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2026/5000 [7:04:32<9:10:45, 11.11s/it][2022-12-20 17:27:02,990] [INFO] [timer.py:197:stop] 0/4102, RunningAvgSamplesPerSec=5.869590172591566, CurrSamplesPerSec=5.3264220164408345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2027/5000 [7:04:43<9:09:34, 11.09s/it][2022-12-20 17:27:14,016] [INFO] [timer.py:197:stop] 0/4104, RunningAvgSamplesPerSec=5.869593641112581, CurrSamplesPerSec=5.3428252687032405, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2028/5000 [7:04:54<9:07:58, 11.06s/it][2022-12-20 17:27:25,099] [INFO] [timer.py:197:stop] 0/4106, RunningAvgSamplesPerSec=5.869592780393165, CurrSamplesPerSec=5.312320537437671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2029/5000 [7:05:05<9:07:46, 11.06s/it][2022-12-20 17:27:36,072] [INFO] [timer.py:197:stop] 0/4108, RunningAvgSamplesPerSec=5.869606700689769, CurrSamplesPerSec=5.345410613300867, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2030/5000 [7:05:16<9:07:16, 11.06s/it][2022-12-20 17:27:47,148] [INFO] [timer.py:197:stop] 0/4110, RunningAvgSamplesPerSec=5.869595055009693, CurrSamplesPerSec=5.314151279604911, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2031/5000 [7:05:27<9:07:03, 11.06s/it][2022-12-20 17:27:58,184] [INFO] [timer.py:197:stop] 0/4112, RunningAvgSamplesPerSec=5.869594091246176, CurrSamplesPerSec=5.3348206322016525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2032/5000 [7:05:38<9:06:16, 11.04s/it][2022-12-20 17:28:09,233] [INFO] [timer.py:197:stop] 0/4114, RunningAvgSamplesPerSec=5.869588120567921, CurrSamplesPerSec=5.300064591434987, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2033/5000 [7:05:49<9:06:36, 11.05s/it][2022-12-20 17:28:20,310] [INFO] [timer.py:197:stop] 0/4116, RunningAvgSamplesPerSec=5.869578971512619, CurrSamplesPerSec=5.300665117960555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2034/5000 [7:06:00<9:06:23, 11.05s/it][2022-12-20 17:28:31,321] [INFO] [timer.py:197:stop] 0/4118, RunningAvgSamplesPerSec=5.869585258911873, CurrSamplesPerSec=5.343393829919865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2035/5000 [7:06:11<9:05:29, 11.04s/it][2022-12-20 17:28:42,262] [INFO] [logging.py:68:log_dist] [Rank 0] step=2060, skipped=3, lr=[6.5422222222222235e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:28:42,264] [INFO] [timer.py:197:stop] 0/4120, RunningAvgSamplesPerSec=5.869606882394255, CurrSamplesPerSec=5.391906828586765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2036/5000 [7:06:22<9:04:14, 11.02s/it][2022-12-20 17:28:53,318] [INFO] [timer.py:197:stop] 0/4122, RunningAvgSamplesPerSec=5.869599794371608, CurrSamplesPerSec=5.315206043359304, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2037/5000 [7:06:33<9:03:55, 11.01s/it][2022-12-20 17:29:04,503] [INFO] [timer.py:197:stop] 0/4124, RunningAvgSamplesPerSec=5.869558529313979, CurrSamplesPerSec=5.163125641915442, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2038/5000 [7:06:45<9:06:58, 11.08s/it][2022-12-20 17:29:15,558] [INFO] [timer.py:197:stop] 0/4126, RunningAvgSamplesPerSec=5.8695514107354, CurrSamplesPerSec=5.316132990019348, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2039/5000 [7:06:56<9:05:54, 11.06s/it][2022-12-20 17:29:26,563] [INFO] [timer.py:197:stop] 0/4128, RunningAvgSamplesPerSec=5.869556915876289, CurrSamplesPerSec=5.327956222211054, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2040/5000 [7:07:07<9:05:13, 11.05s/it][2022-12-20 17:29:37,613] [INFO] [timer.py:197:stop] 0/4130, RunningAvgSamplesPerSec=5.8695506649490925, CurrSamplesPerSec=5.309514405792698, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2041/5000 [7:07:18<9:05:00, 11.05s/it][2022-12-20 17:29:48,625] [INFO] [timer.py:197:stop] 0/4132, RunningAvgSamplesPerSec=5.86955452873535, CurrSamplesPerSec=5.342272776320921, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2042/5000 [7:07:29<9:04:17, 11.04s/it][2022-12-20 17:29:59,638] [INFO] [timer.py:197:stop] 0/4134, RunningAvgSamplesPerSec=5.869558156887463, CurrSamplesPerSec=5.345172614523018, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2043/5000 [7:07:40<9:03:32, 11.03s/it][2022-12-20 17:30:10,689] [INFO] [timer.py:197:stop] 0/4136, RunningAvgSamplesPerSec=5.869552224078155, CurrSamplesPerSec=5.302943918581562, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2044/5000 [7:07:51<9:03:48, 11.04s/it][2022-12-20 17:30:21,690] [INFO] [timer.py:197:stop] 0/4138, RunningAvgSamplesPerSec=5.8695625811830645, CurrSamplesPerSec=5.353986727297278, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2045/5000 [7:08:02<9:02:57, 11.02s/it][2022-12-20 17:30:32,739] [INFO] [logging.py:68:log_dist] [Rank 0] step=2070, skipped=3, lr=[6.520000000000001e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:30:32,740] [INFO] [timer.py:197:stop] 0/4140, RunningAvgSamplesPerSec=5.869560732376997, CurrSamplesPerSec=5.316873642806605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2046/5000 [7:08:13<9:03:22, 11.04s/it][2022-12-20 17:30:43,829] [INFO] [timer.py:197:stop] 0/4142, RunningAvgSamplesPerSec=5.869544485222814, CurrSamplesPerSec=5.280961092956325, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2047/5000 [7:08:24<9:03:48, 11.05s/it][2022-12-20 17:30:54,862] [INFO] [timer.py:197:stop] 0/4144, RunningAvgSamplesPerSec=5.86954288194538, CurrSamplesPerSec=5.321929993984476, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2048/5000 [7:08:35<9:03:14, 11.04s/it][2022-12-20 17:31:05,924] [INFO] [timer.py:197:stop] 0/4146, RunningAvgSamplesPerSec=5.869533864641028, CurrSamplesPerSec=5.287930065763785, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2049/5000 [7:08:46<9:03:51, 11.06s/it][2022-12-20 17:31:17,158] [INFO] [timer.py:197:stop] 0/4148, RunningAvgSamplesPerSec=5.8694834639432605, CurrSamplesPerSec=5.181297246311236, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2050/5000 [7:08:57<9:08:30, 11.16s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.511111111111112e-06, 'epoch': 49.99}
+
 41%|████      | 2050/5000 [7:08:57<9:08:30, 11.16s/it][2022-12-20 17:31:27,473] [INFO] [timer.py:197:stop] 0/4150, RunningAvgSamplesPerSec=5.8697079344926655, CurrSamplesPerSec=6.239696643891063, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 17:31:38,466] [INFO] [timer.py:197:stop] 0/4152, RunningAvgSamplesPerSec=5.8697160700537525, CurrSamplesPerSec=5.350369764565395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2051/5000 [7:09:14<10:22:20, 12.66s/it][2022-12-20 17:31:49,495] [INFO] [timer.py:197:stop] 0/4154, RunningAvgSamplesPerSec=5.8697156034390545, CurrSamplesPerSec=5.302351045047399, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2052/5000 [7:09:25<9:58:03, 12.17s/it] [2022-12-20 17:32:00,602] [INFO] [timer.py:197:stop] 0/4156, RunningAvgSamplesPerSec=5.869694646721922, CurrSamplesPerSec=5.338214675430894, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2053/5000 [7:09:36<9:42:09, 11.85s/it][2022-12-20 17:32:11,641] [INFO] [timer.py:197:stop] 0/4158, RunningAvgSamplesPerSec=5.86969148264135, CurrSamplesPerSec=5.290477350090553, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2054/5000 [7:09:47<9:29:58, 11.61s/it][2022-12-20 17:32:22,663] [INFO] [logging.py:68:log_dist] [Rank 0] step=2080, skipped=3, lr=[6.497777777777779e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:32:22,665] [INFO] [timer.py:197:stop] 0/4160, RunningAvgSamplesPerSec=5.869692209595589, CurrSamplesPerSec=5.329147236245231, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2055/5000 [7:09:58<9:21:10, 11.43s/it][2022-12-20 17:32:33,753] [INFO] [timer.py:197:stop] 0/4162, RunningAvgSamplesPerSec=5.86967641522077, CurrSamplesPerSec=5.338710054012893, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2056/5000 [7:10:09<9:15:54, 11.33s/it][2022-12-20 17:32:44,769] [INFO] [timer.py:197:stop] 0/4164, RunningAvgSamplesPerSec=5.869679360293293, CurrSamplesPerSec=5.312447958640891, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2057/5000 [7:10:20<9:11:06, 11.24s/it][2022-12-20 17:32:55,779] [INFO] [timer.py:197:stop] 0/4166, RunningAvgSamplesPerSec=5.869683787419864, CurrSamplesPerSec=5.353371710050705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2058/5000 [7:10:31<9:07:35, 11.17s/it][2022-12-20 17:33:06,788] [INFO] [timer.py:197:stop] 0/4168, RunningAvgSamplesPerSec=5.869688566079263, CurrSamplesPerSec=5.343523171623795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2059/5000 [7:10:42<9:05:04, 11.12s/it][2022-12-20 17:33:17,843] [INFO] [timer.py:197:stop] 0/4170, RunningAvgSamplesPerSec=5.869681574305433, CurrSamplesPerSec=5.309039970327098, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2060/5000 [7:10:53<9:03:55, 11.10s/it][2022-12-20 17:33:28,864] [INFO] [timer.py:197:stop] 0/4172, RunningAvgSamplesPerSec=5.869683134701962, CurrSamplesPerSec=5.324704695455814, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2061/5000 [7:11:04<9:02:34, 11.08s/it][2022-12-20 17:33:39,971] [INFO] [timer.py:197:stop] 0/4174, RunningAvgSamplesPerSec=5.869662549136632, CurrSamplesPerSec=5.330508775134229, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████      | 2062/5000 [7:11:15<9:02:50, 11.09s/it][2022-12-20 17:33:50,954] [INFO] [timer.py:197:stop] 0/4176, RunningAvgSamplesPerSec=5.86967388188391, CurrSamplesPerSec=5.33916028488574, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2063/5000 [7:11:26<9:01:08, 11.06s/it][2022-12-20 17:34:02,016] [INFO] [timer.py:197:stop] 0/4178, RunningAvgSamplesPerSec=5.869665145304002, CurrSamplesPerSec=5.353101189922566, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2064/5000 [7:11:37<9:01:03, 11.06s/it][2022-12-20 17:34:13,118] [INFO] [logging.py:68:log_dist] [Rank 0] step=2090, skipped=3, lr=[6.475555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:34:13,120] [INFO] [timer.py:197:stop] 0/4180, RunningAvgSamplesPerSec=5.869645570805209, CurrSamplesPerSec=5.302852569708583, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2065/5000 [7:11:48<9:01:33, 11.07s/it][2022-12-20 17:34:24,192] [INFO] [timer.py:197:stop] 0/4182, RunningAvgSamplesPerSec=5.869634215174035, CurrSamplesPerSec=5.314287836552514, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2066/5000 [7:11:59<9:01:23, 11.07s/it][2022-12-20 17:34:35,265] [INFO] [timer.py:197:stop] 0/4184, RunningAvgSamplesPerSec=5.869622344729002, CurrSamplesPerSec=5.2766248403265195, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2067/5000 [7:12:10<9:01:13, 11.07s/it][2022-12-20 17:34:46,238] [INFO] [timer.py:197:stop] 0/4186, RunningAvgSamplesPerSec=5.869636679265411, CurrSamplesPerSec=5.3299302530564425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2068/5000 [7:12:21<8:59:35, 11.04s/it][2022-12-20 17:34:57,222] [INFO] [timer.py:197:stop] 0/4188, RunningAvgSamplesPerSec=5.86964808370703, CurrSamplesPerSec=5.367462580490307, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2069/5000 [7:12:32<8:58:33, 11.02s/it][2022-12-20 17:35:08,234] [INFO] [timer.py:197:stop] 0/4190, RunningAvgSamplesPerSec=5.8696522614952, CurrSamplesPerSec=5.348995290875572, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2070/5000 [7:12:43<8:58:11, 11.02s/it][2022-12-20 17:35:19,554] [INFO] [timer.py:197:stop] 0/4192, RunningAvgSamplesPerSec=5.869577219664674, CurrSamplesPerSec=5.345937990211932, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2071/5000 [7:12:55<9:02:23, 11.11s/it][2022-12-20 17:35:30,545] [INFO] [timer.py:197:stop] 0/4194, RunningAvgSamplesPerSec=5.869586753379504, CurrSamplesPerSec=5.326354798856263, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2072/5000 [7:13:06<9:00:26, 11.07s/it][2022-12-20 17:35:41,542] [INFO] [timer.py:197:stop] 0/4196, RunningAvgSamplesPerSec=5.869594752712661, CurrSamplesPerSec=5.313466917308831, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2073/5000 [7:13:17<8:59:07, 11.05s/it][2022-12-20 17:35:52,612] [INFO] [timer.py:197:stop] 0/4198, RunningAvgSamplesPerSec=5.869583978725496, CurrSamplesPerSec=5.298326573661605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 41%|████▏     | 2074/5000 [7:13:28<8:59:12, 11.06s/it][2022-12-20 17:36:03,607] [INFO] [logging.py:68:log_dist] [Rank 0] step=2100, skipped=3, lr=[6.453333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:36:03,608] [INFO] [timer.py:197:stop] 0/4200, RunningAvgSamplesPerSec=5.869591912806813, CurrSamplesPerSec=5.348805998925758, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2075/5000 [7:13:39<8:58:08, 11.04s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.453333333333334e-06, 'epoch': 50.6}
+
 42%|████▏     | 2075/5000 [7:13:39<8:58:08, 11.04s/it][2022-12-20 17:36:14,626] [INFO] [timer.py:197:stop] 0/4202, RunningAvgSamplesPerSec=5.869594295978162, CurrSamplesPerSec=5.324807150009601, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2076/5000 [7:13:50<8:57:39, 11.03s/it][2022-12-20 17:36:25,803] [INFO] [timer.py:197:stop] 0/4204, RunningAvgSamplesPerSec=5.869556074177701, CurrSamplesPerSec=5.3490257749533265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2077/5000 [7:14:01<8:59:34, 11.08s/it][2022-12-20 17:36:36,843] [INFO] [timer.py:197:stop] 0/4206, RunningAvgSamplesPerSec=5.86955279584577, CurrSamplesPerSec=5.311405429106456, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2078/5000 [7:14:12<8:58:51, 11.06s/it][2022-12-20 17:36:47,774] [INFO] [timer.py:197:stop] 0/4208, RunningAvgSamplesPerSec=5.869577425901459, CurrSamplesPerSec=5.386259492479917, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2079/5000 [7:14:23<8:56:43, 11.02s/it][2022-12-20 17:36:58,749] [INFO] [timer.py:197:stop] 0/4210, RunningAvgSamplesPerSec=5.869590692885201, CurrSamplesPerSec=5.365074389402667, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2080/5000 [7:14:34<8:55:49, 11.01s/it][2022-12-20 17:37:09,799] [INFO] [timer.py:197:stop] 0/4212, RunningAvgSamplesPerSec=5.8695851040138844, CurrSamplesPerSec=5.3009906606274955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2081/5000 [7:14:45<8:56:13, 11.02s/it][2022-12-20 17:37:20,877] [INFO] [timer.py:197:stop] 0/4214, RunningAvgSamplesPerSec=5.869572173135971, CurrSamplesPerSec=5.287375953645529, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2082/5000 [7:14:56<8:56:51, 11.04s/it][2022-12-20 17:37:31,891] [INFO] [timer.py:197:stop] 0/4216, RunningAvgSamplesPerSec=5.869575762964075, CurrSamplesPerSec=5.340546920733833, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2083/5000 [7:15:07<8:56:18, 11.03s/it][2022-12-20 17:37:42,909] [INFO] [timer.py:197:stop] 0/4218, RunningAvgSamplesPerSec=5.8695779670188255, CurrSamplesPerSec=5.308307796573596, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2084/5000 [7:15:18<8:55:55, 11.03s/it][2022-12-20 17:37:53,944] [INFO] [logging.py:68:log_dist] [Rank 0] step=2110, skipped=3, lr=[6.4311111111111116e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:37:53,945] [INFO] [timer.py:197:stop] 0/4220, RunningAvgSamplesPerSec=5.8695766717611395, CurrSamplesPerSec=5.322517545035062, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2085/5000 [7:15:29<8:55:52, 11.03s/it][2022-12-20 17:38:04,983] [INFO] [timer.py:197:stop] 0/4222, RunningAvgSamplesPerSec=5.869574562234351, CurrSamplesPerSec=5.306864827768524, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2086/5000 [7:15:40<8:55:48, 11.03s/it][2022-12-20 17:38:15,999] [INFO] [timer.py:197:stop] 0/4224, RunningAvgSamplesPerSec=5.869577307558116, CurrSamplesPerSec=5.307349367899886, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2087/5000 [7:15:51<8:55:22, 11.03s/it][2022-12-20 17:38:27,007] [INFO] [timer.py:197:stop] 0/4226, RunningAvgSamplesPerSec=5.869582315865934, CurrSamplesPerSec=5.345965245459716, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2088/5000 [7:16:02<8:54:54, 11.02s/it][2022-12-20 17:38:38,058] [INFO] [timer.py:197:stop] 0/4228, RunningAvgSamplesPerSec=5.869576059396264, CurrSamplesPerSec=5.3052649420324185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2089/5000 [7:16:13<8:55:09, 11.03s/it][2022-12-20 17:38:49,117] [INFO] [timer.py:197:stop] 0/4230, RunningAvgSamplesPerSec=5.869567975143271, CurrSamplesPerSec=5.294105433290934, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2090/5000 [7:16:24<8:55:23, 11.04s/it][2022-12-20 17:39:00,145] [INFO] [timer.py:197:stop] 0/4232, RunningAvgSamplesPerSec=5.869567600646852, CurrSamplesPerSec=5.319191911425831, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2091/5000 [7:16:35<8:55:02, 11.04s/it][2022-12-20 17:39:10,260] [INFO] [timer.py:197:stop] 0/4234, RunningAvgSamplesPerSec=5.86979676285488, CurrSamplesPerSec=5.327729714798093, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2092/5000 [7:16:50<9:53:39, 12.25s/it][2022-12-20 17:39:21,339] [INFO] [timer.py:197:stop] 0/4236, RunningAvgSamplesPerSec=5.869783350627567, CurrSamplesPerSec=5.288677051016541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2093/5000 [7:17:01<9:36:08, 11.89s/it][2022-12-20 17:39:32,329] [INFO] [timer.py:197:stop] 0/4238, RunningAvgSamplesPerSec=5.869792256322182, CurrSamplesPerSec=5.348110551324939, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2094/5000 [7:17:12<9:23:02, 11.63s/it][2022-12-20 17:39:43,297] [INFO] [logging.py:68:log_dist] [Rank 0] step=2120, skipped=3, lr=[6.408888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:39:43,298] [INFO] [timer.py:197:stop] 0/4240, RunningAvgSamplesPerSec=5.86980670245358, CurrSamplesPerSec=5.378837255398425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2095/5000 [7:17:23<9:15:09, 11.47s/it][2022-12-20 17:39:54,431] [INFO] [timer.py:197:stop] 0/4242, RunningAvgSamplesPerSec=5.869779694916928, CurrSamplesPerSec=5.346042328211769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2096/5000 [7:17:34<9:08:09, 11.33s/it][2022-12-20 17:40:05,494] [INFO] [timer.py:197:stop] 0/4244, RunningAvgSamplesPerSec=5.869770364546634, CurrSamplesPerSec=5.288226125886814, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2097/5000 [7:17:46<9:04:16, 11.25s/it][2022-12-20 17:40:16,546] [INFO] [timer.py:197:stop] 0/4246, RunningAvgSamplesPerSec=5.869763862379358, CurrSamplesPerSec=5.305539457228038, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2098/5000 [7:17:57<9:04:25, 11.26s/it][2022-12-20 17:40:27,819] [INFO] [timer.py:197:stop] 0/4248, RunningAvgSamplesPerSec=5.8697012104858315, CurrSamplesPerSec=5.305066360894888, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2099/5000 [7:18:08<9:00:51, 11.19s/it][2022-12-20 17:40:38,806] [INFO] [timer.py:197:stop] 0/4250, RunningAvgSamplesPerSec=5.869711315650282, CurrSamplesPerSec=5.336993714631181, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2100/5000 [7:18:19<8:58:20, 11.14s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.397777777777778e-06, 'epoch': 51.22}
+
 42%|████▏     | 2100/5000 [7:18:19<8:58:20, 11.14s/it][2022-12-20 17:40:49,803] [INFO] [timer.py:197:stop] 0/4252, RunningAvgSamplesPerSec=5.86971910690254, CurrSamplesPerSec=5.3637182874079326, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2101/5000 [7:18:30<8:59:27, 11.16s/it][2022-12-20 17:41:01,112] [INFO] [timer.py:197:stop] 0/4254, RunningAvgSamplesPerSec=5.869647633610892, CurrSamplesPerSec=5.290069277747488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2102/5000 [7:18:41<8:57:33, 11.13s/it][2022-12-20 17:41:12,152] [INFO] [timer.py:197:stop] 0/4256, RunningAvgSamplesPerSec=5.869644173440285, CurrSamplesPerSec=5.2962623941129126, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2103/5000 [7:18:52<8:56:03, 11.10s/it][2022-12-20 17:41:23,193] [INFO] [timer.py:197:stop] 0/4258, RunningAvgSamplesPerSec=5.869640370871527, CurrSamplesPerSec=5.2940626252218665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2104/5000 [7:19:03<8:55:24, 11.09s/it][2022-12-20 17:41:34,251] [INFO] [logging.py:68:log_dist] [Rank 0] step=2130, skipped=3, lr=[6.386666666666668e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:41:34,253] [INFO] [timer.py:197:stop] 0/4260, RunningAvgSamplesPerSec=5.869632039320196, CurrSamplesPerSec=5.302904738748303, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2105/5000 [7:19:14<8:54:15, 11.07s/it][2022-12-20 17:41:45,269] [INFO] [timer.py:197:stop] 0/4262, RunningAvgSamplesPerSec=5.86963465097371, CurrSamplesPerSec=5.312507466016119, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2106/5000 [7:19:25<8:53:17, 11.06s/it][2022-12-20 17:41:56,298] [INFO] [timer.py:197:stop] 0/4264, RunningAvgSamplesPerSec=5.869634075739578, CurrSamplesPerSec=5.302530569080753, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2107/5000 [7:19:36<8:53:14, 11.06s/it][2022-12-20 17:42:07,305] [INFO] [timer.py:197:stop] 0/4266, RunningAvgSamplesPerSec=5.869643806783693, CurrSamplesPerSec=5.370656734958366, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2108/5000 [7:19:47<8:51:26, 11.03s/it][2022-12-20 17:42:18,265] [INFO] [timer.py:197:stop] 0/4268, RunningAvgSamplesPerSec=5.869660726214251, CurrSamplesPerSec=5.3446251700586815, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2109/5000 [7:19:58<8:50:24, 11.01s/it][2022-12-20 17:42:29,254] [INFO] [timer.py:197:stop] 0/4270, RunningAvgSamplesPerSec=5.869670177358915, CurrSamplesPerSec=5.325067000887526, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2110/5000 [7:20:09<8:50:48, 11.02s/it][2022-12-20 17:42:40,329] [INFO] [timer.py:197:stop] 0/4272, RunningAvgSamplesPerSec=5.869658165513329, CurrSamplesPerSec=5.3021093247715445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2111/5000 [7:20:20<8:50:28, 11.02s/it][2022-12-20 17:42:51,352] [INFO] [timer.py:197:stop] 0/4274, RunningAvgSamplesPerSec=5.869659146614219, CurrSamplesPerSec=5.289885384270645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2112/5000 [7:20:31<8:51:01, 11.03s/it][2022-12-20 17:43:02,370] [INFO] [timer.py:197:stop] 0/4276, RunningAvgSamplesPerSec=5.869661398917182, CurrSamplesPerSec=5.334349295941013, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2113/5000 [7:20:42<8:51:01, 11.04s/it][2022-12-20 17:43:13,473] [INFO] [timer.py:197:stop] 0/4278, RunningAvgSamplesPerSec=5.869646096751135, CurrSamplesPerSec=5.296525945800256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2114/5000 [7:20:54<8:51:23, 11.05s/it][2022-12-20 17:43:24,511] [INFO] [logging.py:68:log_dist] [Rank 0] step=2140, skipped=3, lr=[6.364444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:43:24,513] [INFO] [timer.py:197:stop] 0/4280, RunningAvgSamplesPerSec=5.8696426152506636, CurrSamplesPerSec=5.31262795676282, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2115/5000 [7:21:05<8:51:13, 11.05s/it][2022-12-20 17:43:35,577] [INFO] [timer.py:197:stop] 0/4282, RunningAvgSamplesPerSec=5.8696329184626155, CurrSamplesPerSec=5.299705680115532, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2116/5000 [7:21:16<8:51:37, 11.06s/it][2022-12-20 17:43:46,652] [INFO] [timer.py:197:stop] 0/4284, RunningAvgSamplesPerSec=5.869620942547407, CurrSamplesPerSec=5.312044689371326, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2117/5000 [7:21:27<8:51:15, 11.06s/it][2022-12-20 17:43:57,685] [INFO] [timer.py:197:stop] 0/4286, RunningAvgSamplesPerSec=5.869619288311004, CurrSamplesPerSec=5.324626959373504, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2118/5000 [7:21:38<8:50:04, 11.04s/it][2022-12-20 17:44:08,685] [INFO] [timer.py:197:stop] 0/4288, RunningAvgSamplesPerSec=5.869626023887719, CurrSamplesPerSec=5.3130315249165285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2119/5000 [7:21:49<8:50:19, 11.04s/it][2022-12-20 17:44:19,804] [INFO] [timer.py:197:stop] 0/4290, RunningAvgSamplesPerSec=5.8696028069119865, CurrSamplesPerSec=5.266956575244587, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2120/5000 [7:22:00<8:50:56, 11.06s/it][2022-12-20 17:44:30,838] [INFO] [timer.py:197:stop] 0/4292, RunningAvgSamplesPerSec=5.869601414518833, CurrSamplesPerSec=5.324822148842129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2121/5000 [7:22:11<8:49:55, 11.04s/it][2022-12-20 17:44:41,825] [INFO] [timer.py:197:stop] 0/4294, RunningAvgSamplesPerSec=5.869611346944094, CurrSamplesPerSec=5.339806243947262, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2122/5000 [7:22:22<8:49:28, 11.04s/it][2022-12-20 17:44:52,817] [INFO] [timer.py:197:stop] 0/4296, RunningAvgSamplesPerSec=5.869619988490275, CurrSamplesPerSec=5.369450533729697, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2123/5000 [7:22:33<8:48:30, 11.02s/it][2022-12-20 17:45:03,818] [INFO] [timer.py:197:stop] 0/4298, RunningAvgSamplesPerSec=5.869626629431279, CurrSamplesPerSec=5.353618340621456, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▏     | 2124/5000 [7:22:44<8:47:31, 11.01s/it][2022-12-20 17:45:14,796] [INFO] [logging.py:68:log_dist] [Rank 0] step=2150, skipped=3, lr=[6.342222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:45:14,797] [INFO] [timer.py:197:stop] 0/4300, RunningAvgSamplesPerSec=5.869638326558733, CurrSamplesPerSec=5.341478688961967, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 42%|████▎     | 2125/5000 [7:22:55<8:46:59, 11.00s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.342222222222223e-06, 'epoch': 51.82}
+
 42%|████▎     | 2125/5000 [7:22:55<8:46:59, 11.00s/it][2022-12-20 17:45:25,809] [INFO] [timer.py:197:stop] 0/4302, RunningAvgSamplesPerSec=5.869642185507423, CurrSamplesPerSec=5.315709792343204, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2126/5000 [7:23:06<8:47:33, 11.01s/it][2022-12-20 17:45:36,888] [INFO] [timer.py:197:stop] 0/4304, RunningAvgSamplesPerSec=5.869629242509666, CurrSamplesPerSec=5.289866620349006, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2127/5000 [7:23:17<8:48:11, 11.03s/it][2022-12-20 17:45:47,875] [INFO] [timer.py:197:stop] 0/4306, RunningAvgSamplesPerSec=5.869639180685798, CurrSamplesPerSec=5.363519378649219, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2128/5000 [7:23:28<8:47:09, 11.01s/it][2022-12-20 17:45:58,840] [INFO] [timer.py:197:stop] 0/4308, RunningAvgSamplesPerSec=5.869654527111631, CurrSamplesPerSec=5.36942540134854, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2129/5000 [7:23:39<8:45:48, 10.99s/it][2022-12-20 17:46:09,833] [INFO] [timer.py:197:stop] 0/4310, RunningAvgSamplesPerSec=5.869662953994062, CurrSamplesPerSec=5.315573372827606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2130/5000 [7:23:50<8:45:43, 10.99s/it][2022-12-20 17:46:20,861] [INFO] [timer.py:197:stop] 0/4312, RunningAvgSamplesPerSec=5.86966269078074, CurrSamplesPerSec=5.286640996889408, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2131/5000 [7:24:01<8:47:12, 11.03s/it][2022-12-20 17:46:31,922] [INFO] [timer.py:197:stop] 0/4314, RunningAvgSamplesPerSec=5.869654094542675, CurrSamplesPerSec=5.326235798299472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2132/5000 [7:24:12<8:46:49, 11.02s/it][2022-12-20 17:46:42,001] [INFO] [timer.py:197:stop] 0/4316, RunningAvgSamplesPerSec=5.869887935151488, CurrSamplesPerSec=6.293578115870945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 17:46:52,967] [INFO] [timer.py:197:stop] 0/4318, RunningAvgSamplesPerSec=5.8699023388172, CurrSamplesPerSec=5.347754265774817, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2133/5000 [7:24:28<9:59:30, 12.55s/it][2022-12-20 17:47:03,915] [INFO] [logging.py:68:log_dist] [Rank 0] step=2160, skipped=3, lr=[6.3200000000000005e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:47:03,916] [INFO] [timer.py:197:stop] 0/4320, RunningAvgSamplesPerSec=5.869921522704897, CurrSamplesPerSec=5.353709097849645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2134/5000 [7:24:39<9:36:24, 12.07s/it][2022-12-20 17:47:14,920] [INFO] [timer.py:197:stop] 0/4322, RunningAvgSamplesPerSec=5.869927275788899, CurrSamplesPerSec=5.30320394487293, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2135/5000 [7:24:50<9:20:58, 11.75s/it][2022-12-20 17:47:25,916] [INFO] [timer.py:197:stop] 0/4324, RunningAvgSamplesPerSec=5.869934934749594, CurrSamplesPerSec=5.34697899735475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2136/5000 [7:25:01<9:10:00, 11.52s/it][2022-12-20 17:47:36,941] [INFO] [timer.py:197:stop] 0/4326, RunningAvgSamplesPerSec=5.869935279665259, CurrSamplesPerSec=5.33774380066359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2137/5000 [7:25:12<9:02:41, 11.37s/it][2022-12-20 17:47:47,959] [INFO] [timer.py:197:stop] 0/4328, RunningAvgSamplesPerSec=5.869937445561894, CurrSamplesPerSec=5.330176633495267, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2138/5000 [7:25:23<8:57:25, 11.27s/it][2022-12-20 17:47:58,931] [INFO] [timer.py:197:stop] 0/4330, RunningAvgSamplesPerSec=5.8699513153885015, CurrSamplesPerSec=5.351406524313069, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2139/5000 [7:25:34<8:53:00, 11.18s/it][2022-12-20 17:48:09,925] [INFO] [timer.py:197:stop] 0/4332, RunningAvgSamplesPerSec=5.869959632934795, CurrSamplesPerSec=5.319912963133473, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2140/5000 [7:25:45<8:50:11, 11.12s/it][2022-12-20 17:48:20,985] [INFO] [timer.py:197:stop] 0/4334, RunningAvgSamplesPerSec=5.869951054808173, CurrSamplesPerSec=5.308060494721827, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2141/5000 [7:25:56<8:49:06, 11.10s/it][2022-12-20 17:48:31,969] [INFO] [timer.py:197:stop] 0/4336, RunningAvgSamplesPerSec=5.869961320350865, CurrSamplesPerSec=5.332545714679501, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2142/5000 [7:26:07<8:47:12, 11.07s/it][2022-12-20 17:48:42,953] [INFO] [timer.py:197:stop] 0/4338, RunningAvgSamplesPerSec=5.869971946443739, CurrSamplesPerSec=5.340183355764844, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2143/5000 [7:26:18<8:45:48, 11.04s/it][2022-12-20 17:48:53,975] [INFO] [logging.py:68:log_dist] [Rank 0] step=2170, skipped=3, lr=[6.297777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:48:53,977] [INFO] [timer.py:197:stop] 0/4340, RunningAvgSamplesPerSec=5.869972284016424, CurrSamplesPerSec=5.3027461396701865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2144/5000 [7:26:29<8:45:21, 11.04s/it][2022-12-20 17:49:04,990] [INFO] [timer.py:197:stop] 0/4342, RunningAvgSamplesPerSec=5.869975692059363, CurrSamplesPerSec=5.302321509585663, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2145/5000 [7:26:40<8:44:50, 11.03s/it][2022-12-20 17:49:16,029] [INFO] [timer.py:197:stop] 0/4344, RunningAvgSamplesPerSec=5.869972887456117, CurrSamplesPerSec=5.305878813210336, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2146/5000 [7:26:51<8:44:46, 11.03s/it][2022-12-20 17:49:27,127] [INFO] [timer.py:197:stop] 0/4346, RunningAvgSamplesPerSec=5.869959062222257, CurrSamplesPerSec=5.284625444551737, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2147/5000 [7:27:02<8:45:32, 11.05s/it][2022-12-20 17:49:38,127] [INFO] [timer.py:197:stop] 0/4348, RunningAvgSamplesPerSec=5.869965526700936, CurrSamplesPerSec=5.346384116915314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2148/5000 [7:27:13<8:44:36, 11.04s/it][2022-12-20 17:49:49,132] [INFO] [timer.py:197:stop] 0/4350, RunningAvgSamplesPerSec=5.869970863714276, CurrSamplesPerSec=5.317595540846101, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2149/5000 [7:27:24<8:43:57, 11.03s/it][2022-12-20 17:50:00,078] [INFO] [timer.py:197:stop] 0/4352, RunningAvgSamplesPerSec=5.869990840164583, CurrSamplesPerSec=5.375280292652161, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2150/5000 [7:27:35<8:42:37, 11.00s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.284444444444445e-06, 'epoch': 52.43}
+
 43%|████▎     | 2150/5000 [7:27:35<8:42:37, 11.00s/it][2022-12-20 17:50:11,118] [INFO] [timer.py:197:stop] 0/4354, RunningAvgSamplesPerSec=5.869987759194347, CurrSamplesPerSec=5.327790199486686, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2151/5000 [7:27:46<8:42:58, 11.01s/it][2022-12-20 17:50:22,097] [INFO] [timer.py:197:stop] 0/4356, RunningAvgSamplesPerSec=5.8699993219459765, CurrSamplesPerSec=5.341547776878396, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2152/5000 [7:27:57<8:42:17, 11.00s/it][2022-12-20 17:50:33,107] [INFO] [timer.py:197:stop] 0/4358, RunningAvgSamplesPerSec=5.870004415816592, CurrSamplesPerSec=5.3410822651618055, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2153/5000 [7:28:08<8:42:12, 11.01s/it][2022-12-20 17:50:44,168] [INFO] [logging.py:68:log_dist] [Rank 0] step=2180, skipped=3, lr=[6.275555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:50:44,170] [INFO] [timer.py:197:stop] 0/4360, RunningAvgSamplesPerSec=5.86999523356314, CurrSamplesPerSec=5.284802729842509, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2154/5000 [7:28:19<8:42:50, 11.02s/it][2022-12-20 17:50:55,194] [INFO] [timer.py:197:stop] 0/4362, RunningAvgSamplesPerSec=5.869995830137972, CurrSamplesPerSec=5.332496774305306, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2155/5000 [7:28:30<8:42:40, 11.02s/it][2022-12-20 17:51:06,184] [INFO] [timer.py:197:stop] 0/4364, RunningAvgSamplesPerSec=5.8700047786994265, CurrSamplesPerSec=5.334004804580248, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2156/5000 [7:28:41<8:42:01, 11.01s/it][2022-12-20 17:51:17,164] [INFO] [timer.py:197:stop] 0/4366, RunningAvgSamplesPerSec=5.8700162725724425, CurrSamplesPerSec=5.324683148814321, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2157/5000 [7:28:52<8:41:22, 11.00s/it][2022-12-20 17:51:28,170] [INFO] [timer.py:197:stop] 0/4368, RunningAvgSamplesPerSec=5.870021390200435, CurrSamplesPerSec=5.337694552556263, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2158/5000 [7:29:03<8:41:13, 11.00s/it][2022-12-20 17:51:39,236] [INFO] [timer.py:197:stop] 0/4370, RunningAvgSamplesPerSec=5.870011348349186, CurrSamplesPerSec=5.286158564834461, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2159/5000 [7:29:14<8:41:55, 11.02s/it][2022-12-20 17:51:50,270] [INFO] [timer.py:197:stop] 0/4372, RunningAvgSamplesPerSec=5.870009402718063, CurrSamplesPerSec=5.340187817688265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2160/5000 [7:29:25<8:41:53, 11.03s/it][2022-12-20 17:52:01,280] [INFO] [timer.py:197:stop] 0/4374, RunningAvgSamplesPerSec=5.87001347311125, CurrSamplesPerSec=5.333873591573434, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2161/5000 [7:29:36<8:41:29, 11.02s/it][2022-12-20 17:52:12,269] [INFO] [timer.py:197:stop] 0/4376, RunningAvgSamplesPerSec=5.8700222858667574, CurrSamplesPerSec=5.3644795497438, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2162/5000 [7:29:47<8:40:51, 11.01s/it][2022-12-20 17:52:23,275] [INFO] [timer.py:197:stop] 0/4378, RunningAvgSamplesPerSec=5.870027375099405, CurrSamplesPerSec=5.342871208550299, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2163/5000 [7:29:58<8:40:34, 11.01s/it][2022-12-20 17:52:34,313] [INFO] [logging.py:68:log_dist] [Rank 0] step=2190, skipped=3, lr=[6.253333333333333e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:52:34,314] [INFO] [timer.py:197:stop] 0/4380, RunningAvgSamplesPerSec=5.870023951950112, CurrSamplesPerSec=5.309192015665981, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2164/5000 [7:30:09<8:40:49, 11.02s/it][2022-12-20 17:52:45,355] [INFO] [timer.py:197:stop] 0/4382, RunningAvgSamplesPerSec=5.8700228460972586, CurrSamplesPerSec=5.300255263224404, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2165/5000 [7:30:20<8:40:56, 11.03s/it][2022-12-20 17:52:56,417] [INFO] [timer.py:197:stop] 0/4384, RunningAvgSamplesPerSec=5.870014232238921, CurrSamplesPerSec=5.304841376154955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████��     | 2166/5000 [7:30:32<8:41:16, 11.04s/it][2022-12-20 17:53:07,469] [INFO] [timer.py:197:stop] 0/4386, RunningAvgSamplesPerSec=5.8700076871468205, CurrSamplesPerSec=5.312893139767952, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2167/5000 [7:30:43<8:41:19, 11.04s/it][2022-12-20 17:53:18,485] [INFO] [timer.py:197:stop] 0/4388, RunningAvgSamplesPerSec=5.870010227981493, CurrSamplesPerSec=5.301837258729139, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2168/5000 [7:30:54<8:40:46, 11.03s/it][2022-12-20 17:53:29,512] [INFO] [timer.py:197:stop] 0/4390, RunningAvgSamplesPerSec=5.870010098330011, CurrSamplesPerSec=5.337043586474473, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2169/5000 [7:31:05<8:40:30, 11.03s/it][2022-12-20 17:53:40,511] [INFO] [timer.py:197:stop] 0/4392, RunningAvgSamplesPerSec=5.870016919672246, CurrSamplesPerSec=5.308867354821755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2170/5000 [7:31:16<8:39:51, 11.02s/it][2022-12-20 17:53:51,528] [INFO] [timer.py:197:stop] 0/4394, RunningAvgSamplesPerSec=5.870018937985506, CurrSamplesPerSec=5.314870333896878, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2171/5000 [7:31:27<8:39:36, 11.02s/it][2022-12-20 17:54:02,543] [INFO] [timer.py:197:stop] 0/4396, RunningAvgSamplesPerSec=5.870021515999309, CurrSamplesPerSec=5.295017722947612, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2172/5000 [7:31:38<8:39:21, 11.02s/it][2022-12-20 17:54:13,563] [INFO] [timer.py:197:stop] 0/4398, RunningAvgSamplesPerSec=5.870023207670243, CurrSamplesPerSec=5.33698098154396, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2173/5000 [7:31:49<8:39:11, 11.02s/it][2022-12-20 17:54:23,695] [INFO] [logging.py:68:log_dist] [Rank 0] step=2200, skipped=3, lr=[6.231111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:54:23,697] [INFO] [timer.py:197:stop] 0/4400, RunningAvgSamplesPerSec=5.870239400872252, CurrSamplesPerSec=5.32010949416207, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 43%|████▎     | 2174/5000 [7:32:04<9:36:10, 12.23s/it][2022-12-20 17:54:34,701] [INFO] [timer.py:197:stop] 0/4402, RunningAvgSamplesPerSec=5.870244794134129, CurrSamplesPerSec=5.325983653056489, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2175/5000 [7:32:15<9:18:17, 11.86s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.22888888888889e-06, 'epoch': 53.05}
+
 44%|████▎     | 2175/5000 [7:32:15<9:18:17, 11.86s/it][2022-12-20 17:54:45,798] [INFO] [timer.py:197:stop] 0/4404, RunningAvgSamplesPerSec=5.870227706485792, CurrSamplesPerSec=5.226620921471748, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2176/5000 [7:32:26<9:08:35, 11.66s/it][2022-12-20 17:54:56,831] [INFO] [timer.py:197:stop] 0/4406, RunningAvgSamplesPerSec=5.8702261649024825, CurrSamplesPerSec=5.35868922853292, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2177/5000 [7:32:37<8:59:14, 11.46s/it][2022-12-20 17:55:07,875] [INFO] [timer.py:197:stop] 0/4408, RunningAvgSamplesPerSec=5.870221953911768, CurrSamplesPerSec=5.325334483644098, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2178/5000 [7:32:48<8:52:39, 11.33s/it][2022-12-20 17:55:18,908] [INFO] [timer.py:197:stop] 0/4410, RunningAvgSamplesPerSec=5.870220297986947, CurrSamplesPerSec=5.302666948437348, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2179/5000 [7:32:59<8:48:04, 11.23s/it][2022-12-20 17:55:29,917] [INFO] [timer.py:197:stop] 0/4412, RunningAvgSamplesPerSec=5.8702244810109026, CurrSamplesPerSec=5.306502896880908, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2180/5000 [7:33:10<8:44:46, 11.17s/it][2022-12-20 17:55:40,944] [INFO] [timer.py:197:stop] 0/4414, RunningAvgSamplesPerSec=5.870224377539624, CurrSamplesPerSec=5.2935157869495395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2181/5000 [7:33:21<8:43:20, 11.14s/it][2022-12-20 17:55:51,996] [INFO] [timer.py:197:stop] 0/4416, RunningAvgSamplesPerSec=5.870217996225358, CurrSamplesPerSec=5.314867808343779, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2182/5000 [7:33:32<8:42:11, 11.12s/it][2022-12-20 17:56:03,045] [INFO] [timer.py:197:stop] 0/4418, RunningAvgSamplesPerSec=5.870216377594884, CurrSamplesPerSec=5.333658450129131, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2183/5000 [7:33:43<8:40:40, 11.09s/it][2022-12-20 17:56:14,080] [INFO] [logging.py:68:log_dist] [Rank 0] step=2210, skipped=3, lr=[6.20888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:56:14,081] [INFO] [timer.py:197:stop] 0/4420, RunningAvgSamplesPerSec=5.870213945107222, CurrSamplesPerSec=5.3229291612897445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2184/5000 [7:33:54<8:39:52, 11.08s/it][2022-12-20 17:56:25,142] [INFO] [timer.py:197:stop] 0/4422, RunningAvgSamplesPerSec=5.87020532669836, CurrSamplesPerSec=5.309951322644149, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2185/5000 [7:34:05<8:38:14, 11.05s/it][2022-12-20 17:56:36,128] [INFO] [timer.py:197:stop] 0/4424, RunningAvgSamplesPerSec=5.870215394173533, CurrSamplesPerSec=5.300248146797007, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2186/5000 [7:34:16<8:38:30, 11.06s/it][2022-12-20 17:56:47,198] [INFO] [timer.py:197:stop] 0/4426, RunningAvgSamplesPerSec=5.8702046824193035, CurrSamplesPerSec=5.306991148024579, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▎     | 2187/5000 [7:34:27<8:37:52, 11.05s/it][2022-12-20 17:56:58,210] [INFO] [timer.py:197:stop] 0/4428, RunningAvgSamplesPerSec=5.870208108012052, CurrSamplesPerSec=5.316995384847527, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2188/5000 [7:34:38<8:37:12, 11.04s/it][2022-12-20 17:57:09,244] [INFO] [timer.py:197:stop] 0/4430, RunningAvgSamplesPerSec=5.870206100544846, CurrSamplesPerSec=5.297439279018229, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2189/5000 [7:34:49<8:36:48, 11.03s/it][2022-12-20 17:57:20,201] [INFO] [timer.py:197:stop] 0/4432, RunningAvgSamplesPerSec=5.870223037162975, CurrSamplesPerSec=5.3535448829064745, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2190/5000 [7:35:00<8:35:43, 11.01s/it][2022-12-20 17:57:31,178] [INFO] [timer.py:197:stop] 0/4434, RunningAvgSamplesPerSec=5.8702345546739, CurrSamplesPerSec=5.34425424013747, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2191/5000 [7:35:11<8:35:27, 11.01s/it][2022-12-20 17:57:42,179] [INFO] [timer.py:197:stop] 0/4436, RunningAvgSamplesPerSec=5.870240225084986, CurrSamplesPerSec=5.3483601073324305, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2192/5000 [7:35:22<8:34:15, 10.99s/it][2022-12-20 17:57:53,146] [INFO] [timer.py:197:stop] 0/4438, RunningAvgSamplesPerSec=5.870255408221334, CurrSamplesPerSec=5.324532960892702, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2193/5000 [7:35:33<8:34:14, 10.99s/it][2022-12-20 17:58:04,109] [INFO] [logging.py:68:log_dist] [Rank 0] step=2220, skipped=3, lr=[6.186666666666668e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:58:04,111] [INFO] [timer.py:197:stop] 0/4440, RunningAvgSamplesPerSec=5.870270424616973, CurrSamplesPerSec=5.354673236326013, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2194/5000 [7:35:44<8:33:26, 10.98s/it][2022-12-20 17:58:15,076] [INFO] [timer.py:197:stop] 0/4442, RunningAvgSamplesPerSec=5.870285074820494, CurrSamplesPerSec=5.339975565876111, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2195/5000 [7:35:55<8:33:09, 10.98s/it][2022-12-20 17:58:26,004] [INFO] [timer.py:197:stop] 0/4444, RunningAvgSamplesPerSec=5.870308704588686, CurrSamplesPerSec=5.377953822621086, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2196/5000 [7:36:06<8:32:56, 10.98s/it][2022-12-20 17:58:37,065] [INFO] [timer.py:197:stop] 0/4446, RunningAvgSamplesPerSec=5.870304264304532, CurrSamplesPerSec=5.315643476426314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2197/5000 [7:36:17<8:33:29, 10.99s/it][2022-12-20 17:58:48,033] [INFO] [timer.py:197:stop] 0/4448, RunningAvgSamplesPerSec=5.870318229776591, CurrSamplesPerSec=5.354227647566181, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2198/5000 [7:36:28<8:32:44, 10.98s/it][2022-12-20 17:58:59,035] [INFO] [timer.py:197:stop] 0/4450, RunningAvgSamplesPerSec=5.870323706572499, CurrSamplesPerSec=5.308686561565392, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2199/5000 [7:36:39<8:33:24, 11.00s/it][2022-12-20 17:59:10,047] [INFO] [timer.py:197:stop] 0/4452, RunningAvgSamplesPerSec=5.870327037311967, CurrSamplesPerSec=5.334483712637762, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2200/5000 [7:36:50<8:33:22, 11.00s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.173333333333333e-06, 'epoch': 53.65}
+
 44%|████▍     | 2200/5000 [7:36:50<8:33:22, 11.00s/it][2022-12-20 17:59:21,036] [INFO] [timer.py:197:stop] 0/4454, RunningAvgSamplesPerSec=5.870336345240717, CurrSamplesPerSec=5.353205594214484, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2201/5000 [7:37:01<8:33:09, 11.00s/it][2022-12-20 17:59:32,064] [INFO] [timer.py:197:stop] 0/4456, RunningAvgSamplesPerSec=5.870335691973032, CurrSamplesPerSec=5.324774828868988, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2202/5000 [7:37:12<8:33:24, 11.01s/it][2022-12-20 17:59:43,110] [INFO] [timer.py:197:stop] 0/4458, RunningAvgSamplesPerSec=5.870330194346564, CurrSamplesPerSec=5.310666298899504, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2203/5000 [7:37:23<8:33:59, 11.03s/it][2022-12-20 17:59:54,192] [INFO] [logging.py:68:log_dist] [Rank 0] step=2230, skipped=3, lr=[6.1644444444444455e-06], mom=[[0.9, 0.999]]
+[2022-12-20 17:59:54,194] [INFO] [timer.py:197:stop] 0/4460, RunningAvgSamplesPerSec=5.870315993061552, CurrSamplesPerSec=5.2943798390137315, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2204/5000 [7:37:34<8:34:26, 11.04s/it][2022-12-20 18:00:05,205] [INFO] [timer.py:197:stop] 0/4462, RunningAvgSamplesPerSec=5.87031917879999, CurrSamplesPerSec=5.347559309030412, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2205/5000 [7:37:45<8:33:33, 11.02s/it][2022-12-20 18:00:16,227] [INFO] [timer.py:197:stop] 0/4464, RunningAvgSamplesPerSec=5.870320164045288, CurrSamplesPerSec=5.319510246413381, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2206/5000 [7:37:56<8:33:21, 11.02s/it][2022-12-20 18:00:27,236] [INFO] [timer.py:197:stop] 0/4466, RunningAvgSamplesPerSec=5.870324146437861, CurrSamplesPerSec=5.331035968260933, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2207/5000 [7:38:07<8:32:58, 11.02s/it][2022-12-20 18:00:38,284] [INFO] [timer.py:197:stop] 0/4468, RunningAvgSamplesPerSec=5.870318796931328, CurrSamplesPerSec=5.298296664712766, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2208/5000 [7:38:18<8:33:16, 11.03s/it][2022-12-20 18:00:49,284] [INFO] [timer.py:197:stop] 0/4470, RunningAvgSamplesPerSec=5.870324975772341, CurrSamplesPerSec=5.346333005545231, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2209/5000 [7:38:29<8:32:43, 11.02s/it][2022-12-20 18:01:00,332] [INFO] [timer.py:197:stop] 0/4472, RunningAvgSamplesPerSec=5.870319495239341, CurrSamplesPerSec=5.306339886746351, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2210/5000 [7:38:40<8:32:52, 11.03s/it][2022-12-20 18:01:11,376] [INFO] [timer.py:197:stop] 0/4474, RunningAvgSamplesPerSec=5.87031502853921, CurrSamplesPerSec=5.309616066564138, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2211/5000 [7:38:51<8:32:50, 11.03s/it][2022-12-20 18:01:22,434] [INFO] [timer.py:197:stop] 0/4476, RunningAvgSamplesPerSec=5.870307104465001, CurrSamplesPerSec=5.29404383165413, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2212/5000 [7:39:02<8:32:55, 11.04s/it][2022-12-20 18:01:33,479] [INFO] [timer.py:197:stop] 0/4478, RunningAvgSamplesPerSec=5.870306279161913, CurrSamplesPerSec=5.299571545565143, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2213/5000 [7:39:14<8:32:37, 11.04s/it][2022-12-20 18:01:44,431] [INFO] [logging.py:68:log_dist] [Rank 0] step=2240, skipped=3, lr=[6.142222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:01:44,433] [INFO] [timer.py:197:stop] 0/4480, RunningAvgSamplesPerSec=5.87032324136822, CurrSamplesPerSec=5.366578373722497, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2214/5000 [7:39:24<8:31:01, 11.01s/it][2022-12-20 18:01:54,495] [INFO] [timer.py:197:stop] 0/4482, RunningAvgSamplesPerSec=5.870552127590959, CurrSamplesPerSec=6.273965263184023, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 18:02:05,535] [INFO] [timer.py:197:stop] 0/4484, RunningAvgSamplesPerSec=5.870551357029141, CurrSamplesPerSec=5.322212989988344, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2215/5000 [7:39:41<9:43:04, 12.56s/it][2022-12-20 18:02:16,518] [INFO] [timer.py:197:stop] 0/4486, RunningAvgSamplesPerSec=5.8705615437381935, CurrSamplesPerSec=5.318573691468843, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2216/5000 [7:39:52<9:20:53, 12.09s/it][2022-12-20 18:02:27,553] [INFO] [timer.py:197:stop] 0/4488, RunningAvgSamplesPerSec=5.870559359701557, CurrSamplesPerSec=5.308922162178412, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2217/5000 [7:40:03<9:06:02, 11.77s/it][2022-12-20 18:02:38,552] [INFO] [timer.py:197:stop] 0/4490, RunningAvgSamplesPerSec=5.870565793698353, CurrSamplesPerSec=5.349227447604921, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2218/5000 [7:40:14<8:55:04, 11.54s/it][2022-12-20 18:02:49,573] [INFO] [timer.py:197:stop] 0/4492, RunningAvgSamplesPerSec=5.870566941645717, CurrSamplesPerSec=5.338899269152701, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2219/5000 [7:40:25<8:47:41, 11.38s/it][2022-12-20 18:03:00,590] [INFO] [timer.py:197:stop] 0/4494, RunningAvgSamplesPerSec=5.87057296030001, CurrSamplesPerSec=5.30826664797813, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2220/5000 [7:40:36<8:42:21, 11.27s/it][2022-12-20 18:03:11,635] [INFO] [timer.py:197:stop] 0/4496, RunningAvgSamplesPerSec=5.870567957636121, CurrSamplesPerSec=5.302303704679044, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2221/5000 [7:40:47<8:39:00, 11.21s/it][2022-12-20 18:03:22,663] [INFO] [timer.py:197:stop] 0/4498, RunningAvgSamplesPerSec=5.8705674549640685, CurrSamplesPerSec=5.314936630524195, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2222/5000 [7:40:58<8:36:20, 11.15s/it][2022-12-20 18:03:33,662] [INFO] [logging.py:68:log_dist] [Rank 0] step=2250, skipped=3, lr=[6.120000000000001e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:03:33,664] [INFO] [timer.py:197:stop] 0/4500, RunningAvgSamplesPerSec=5.87057328942282, CurrSamplesPerSec=5.357323521758618, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2223/5000 [7:41:09<8:34:03, 11.11s/it][2022-12-20 18:03:44,671] [INFO] [timer.py:197:stop] 0/4502, RunningAvgSamplesPerSec=5.870577443053673, CurrSamplesPerSec=5.349235335744927, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2224/5000 [7:41:20<8:32:29, 11.08s/it][2022-12-20 18:03:55,730] [INFO] [timer.py:197:stop] 0/4504, RunningAvgSamplesPerSec=5.870569751496985, CurrSamplesPerSec=5.299186338843811, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 44%|████▍     | 2225/5000 [7:41:31<8:32:03, 11.07s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.1155555555555555e-06, 'epoch': 54.27}
+
 44%|████▍     | 2225/5000 [7:41:31<8:32:03, 11.07s/it][2022-12-20 18:04:06,826] [INFO] [timer.py:197:stop] 0/4506, RunningAvgSamplesPerSec=5.870553284526691, CurrSamplesPerSec=5.28072173446042, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2226/5000 [7:41:42<8:32:12, 11.08s/it][2022-12-20 18:04:17,807] [INFO] [timer.py:197:stop] 0/4508, RunningAvgSamplesPerSec=5.870563717541872, CurrSamplesPerSec=5.341550753013409, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2227/5000 [7:41:53<8:30:40, 11.05s/it][2022-12-20 18:04:28,839] [INFO] [timer.py:197:stop] 0/4510, RunningAvgSamplesPerSec=5.870562004227285, CurrSamplesPerSec=5.312123950707151, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2228/5000 [7:42:04<8:30:14, 11.04s/it][2022-12-20 18:04:39,852] [INFO] [timer.py:197:stop] 0/4512, RunningAvgSamplesPerSec=5.87056507052006, CurrSamplesPerSec=5.3347385717229825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2229/5000 [7:42:15<8:29:37, 11.03s/it][2022-12-20 18:04:50,784] [INFO] [timer.py:197:stop] 0/4514, RunningAvgSamplesPerSec=5.870586964962464, CurrSamplesPerSec=5.34263088383506, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2230/5000 [7:42:26<8:28:01, 11.00s/it][2022-12-20 18:05:01,863] [INFO] [timer.py:197:stop] 0/4516, RunningAvgSamplesPerSec=5.8705742430316725, CurrSamplesPerSec=5.2931830204812815, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2231/5000 [7:42:37<8:28:52, 11.03s/it][2022-12-20 18:05:12,839] [INFO] [timer.py:197:stop] 0/4518, RunningAvgSamplesPerSec=5.87058590799604, CurrSamplesPerSec=5.328780144078012, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2232/5000 [7:42:48<8:27:59, 11.01s/it][2022-12-20 18:05:23,841] [INFO] [logging.py:68:log_dist] [Rank 0] step=2260, skipped=3, lr=[6.097777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:05:23,842] [INFO] [timer.py:197:stop] 0/4520, RunningAvgSamplesPerSec=5.870591104634569, CurrSamplesPerSec=5.342439064932352, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2233/5000 [7:42:59<8:27:41, 11.01s/it][2022-12-20 18:05:34,886] [INFO] [timer.py:197:stop] 0/4522, RunningAvgSamplesPerSec=5.870586754854745, CurrSamplesPerSec=5.304477624799513, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2234/5000 [7:43:10<8:27:59, 11.02s/it][2022-12-20 18:05:45,894] [INFO] [timer.py:197:stop] 0/4524, RunningAvgSamplesPerSec=5.87059072204096, CurrSamplesPerSec=5.3213124032765124, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2235/5000 [7:43:21<8:27:39, 11.02s/it][2022-12-20 18:05:56,953] [INFO] [timer.py:197:stop] 0/4526, RunningAvgSamplesPerSec=5.870582519940331, CurrSamplesPerSec=5.290731358429899, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2236/5000 [7:43:32<8:28:03, 11.03s/it][2022-12-20 18:06:07,944] [INFO] [timer.py:197:stop] 0/4528, RunningAvgSamplesPerSec=5.87059032690477, CurrSamplesPerSec=5.349775408863965, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2237/5000 [7:43:43<8:27:21, 11.02s/it][2022-12-20 18:06:18,924] [INFO] [timer.py:197:stop] 0/4530, RunningAvgSamplesPerSec=5.870601055139825, CurrSamplesPerSec=5.341148792232082, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2238/5000 [7:43:54<8:26:39, 11.01s/it][2022-12-20 18:06:29,916] [INFO] [timer.py:197:stop] 0/4532, RunningAvgSamplesPerSec=5.870609208945358, CurrSamplesPerSec=5.317973103961121, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2239/5000 [7:44:05<8:26:16, 11.00s/it][2022-12-20 18:06:40,900] [INFO] [timer.py:197:stop] 0/4534, RunningAvgSamplesPerSec=5.870618986275708, CurrSamplesPerSec=5.323115781631799, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2240/5000 [7:44:16<8:25:50, 11.00s/it][2022-12-20 18:06:51,890] [INFO] [timer.py:197:stop] 0/4536, RunningAvgSamplesPerSec=5.870627128086081, CurrSamplesPerSec=5.3152138314802375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2241/5000 [7:44:27<8:25:34, 10.99s/it][2022-12-20 18:07:02,851] [INFO] [timer.py:197:stop] 0/4538, RunningAvgSamplesPerSec=5.870642433326336, CurrSamplesPerSec=5.347074216072864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2242/5000 [7:44:38<8:24:55, 10.98s/it][2022-12-20 18:07:13,865] [INFO] [logging.py:68:log_dist] [Rank 0] step=2270, skipped=3, lr=[6.075555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:07:13,868] [INFO] [timer.py:197:stop] 0/4540, RunningAvgSamplesPerSec=5.8706444135745235, CurrSamplesPerSec=5.313394346954689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2243/5000 [7:44:49<8:25:10, 10.99s/it][2022-12-20 18:07:24,926] [INFO] [timer.py:197:stop] 0/4542, RunningAvgSamplesPerSec=5.8706363597859665, CurrSamplesPerSec=5.328197978259616, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2244/5000 [7:45:00<8:25:53, 11.01s/it][2022-12-20 18:07:35,924] [INFO] [timer.py:197:stop] 0/4544, RunningAvgSamplesPerSec=5.870642901433451, CurrSamplesPerSec=5.344200190390852, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2245/5000 [7:45:11<8:25:29, 11.01s/it][2022-12-20 18:07:46,929] [INFO] [timer.py:197:stop] 0/4546, RunningAvgSamplesPerSec=5.870647615742826, CurrSamplesPerSec=5.340968556594532, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2246/5000 [7:45:22<8:25:15, 11.01s/it][2022-12-20 18:07:57,866] [INFO] [timer.py:197:stop] 0/4548, RunningAvgSamplesPerSec=5.870668448988069, CurrSamplesPerSec=5.343126444309703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2247/5000 [7:45:33<8:24:05, 10.99s/it][2022-12-20 18:08:08,904] [INFO] [timer.py:197:stop] 0/4550, RunningAvgSamplesPerSec=5.870665197017616, CurrSamplesPerSec=5.313265197662393, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2248/5000 [7:45:44<8:24:37, 11.00s/it][2022-12-20 18:08:19,925] [INFO] [timer.py:197:stop] 0/4552, RunningAvgSamplesPerSec=5.870666064435402, CurrSamplesPerSec=5.321725099182347, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▍     | 2249/5000 [7:45:55<8:24:42, 11.01s/it][2022-12-20 18:08:30,912] [INFO] [timer.py:197:stop] 0/4554, RunningAvgSamplesPerSec=5.870675043040991, CurrSamplesPerSec=5.33792679081686, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2250/5000 [7:46:06<8:24:14, 11.00s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.0600000000000004e-06, 'epoch': 54.87}
+
 45%|████▌     | 2250/5000 [7:46:06<8:24:14, 11.00s/it][2022-12-20 18:08:41,862] [INFO] [timer.py:197:stop] 0/4556, RunningAvgSamplesPerSec=5.870693082485557, CurrSamplesPerSec=5.3985375515628995, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2251/5000 [7:46:17<8:23:20, 10.99s/it][2022-12-20 18:08:52,903] [INFO] [timer.py:197:stop] 0/4558, RunningAvgSamplesPerSec=5.8706893069688215, CurrSamplesPerSec=5.310890096822702, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2252/5000 [7:46:28<8:23:54, 11.00s/it][2022-12-20 18:09:03,916] [INFO] [logging.py:68:log_dist] [Rank 0] step=2280, skipped=3, lr=[6.0533333333333335e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:09:03,917] [INFO] [timer.py:197:stop] 0/4560, RunningAvgSamplesPerSec=5.8706915175928085, CurrSamplesPerSec=5.307560713025237, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2253/5000 [7:46:39<8:23:53, 11.01s/it][2022-12-20 18:09:14,968] [INFO] [timer.py:197:stop] 0/4562, RunningAvgSamplesPerSec=5.870685772162035, CurrSamplesPerSec=5.310349231600586, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2254/5000 [7:46:50<8:24:18, 11.02s/it][2022-12-20 18:09:26,012] [INFO] [timer.py:197:stop] 0/4564, RunningAvgSamplesPerSec=5.870681150043331, CurrSamplesPerSec=5.317140303056774, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2255/5000 [7:47:01<8:24:28, 11.03s/it][2022-12-20 18:09:36,100] [INFO] [timer.py:197:stop] 0/4566, RunningAvgSamplesPerSec=5.870899919603606, CurrSamplesPerSec=5.3357597364753175, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2256/5000 [7:47:16<9:19:15, 12.23s/it][2022-12-20 18:09:47,143] [INFO] [timer.py:197:stop] 0/4568, RunningAvgSamplesPerSec=5.870895442935629, CurrSamplesPerSec=5.302241074251752, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2257/5000 [7:47:27<9:02:57, 11.88s/it][2022-12-20 18:09:58,184] [INFO] [timer.py:197:stop] 0/4570, RunningAvgSamplesPerSec=5.87089147654617, CurrSamplesPerSec=5.315417804178863, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2258/5000 [7:47:38<8:51:34, 11.63s/it][2022-12-20 18:10:09,202] [INFO] [timer.py:197:stop] 0/4572, RunningAvgSamplesPerSec=5.870892897873652, CurrSamplesPerSec=5.353499826991517, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2259/5000 [7:47:49<8:42:36, 11.44s/it][2022-12-20 18:10:20,207] [INFO] [timer.py:197:stop] 0/4574, RunningAvgSamplesPerSec=5.870899033808796, CurrSamplesPerSec=5.341302894846542, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2260/5000 [7:48:00<8:35:46, 11.29s/it][2022-12-20 18:10:31,130] [INFO] [timer.py:197:stop] 0/4576, RunningAvgSamplesPerSec=5.870922645187618, CurrSamplesPerSec=5.369947644724779, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2261/5000 [7:48:11<8:30:45, 11.19s/it][2022-12-20 18:10:42,116] [INFO] [timer.py:197:stop] 0/4578, RunningAvgSamplesPerSec=5.8709316067586546, CurrSamplesPerSec=5.330570381480216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2262/5000 [7:48:22<8:28:10, 11.14s/it][2022-12-20 18:10:53,131] [INFO] [logging.py:68:log_dist] [Rank 0] step=2290, skipped=3, lr=[6.031111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:10:53,133] [INFO] [timer.py:197:stop] 0/4580, RunningAvgSamplesPerSec=5.870933116073864, CurrSamplesPerSec=5.327448457889522, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2263/5000 [7:48:33<8:25:20, 11.08s/it][2022-12-20 18:11:04,100] [INFO] [timer.py:197:stop] 0/4582, RunningAvgSamplesPerSec=5.8709464092326655, CurrSamplesPerSec=5.305196999188752, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2264/5000 [7:48:44<8:24:02, 11.05s/it][2022-12-20 18:11:15,085] [INFO] [timer.py:197:stop] 0/4584, RunningAvgSamplesPerSec=5.870955350393718, CurrSamplesPerSec=5.315038919993688, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2265/5000 [7:48:55<8:23:57, 11.06s/it][2022-12-20 18:11:26,125] [INFO] [timer.py:197:stop] 0/4586, RunningAvgSamplesPerSec=5.870951370397516, CurrSamplesPerSec=5.334038085803798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2266/5000 [7:49:06<8:23:09, 11.04s/it][2022-12-20 18:11:37,186] [INFO] [timer.py:197:stop] 0/4588, RunningAvgSamplesPerSec=5.870942316040723, CurrSamplesPerSec=5.2892047560615465, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2267/5000 [7:49:17<8:23:26, 11.05s/it][2022-12-20 18:11:48,232] [INFO] [timer.py:197:stop] 0/4590, RunningAvgSamplesPerSec=5.870937003251864, CurrSamplesPerSec=5.315885169074902, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2268/5000 [7:49:28<8:22:38, 11.04s/it][2022-12-20 18:11:59,202] [INFO] [timer.py:197:stop] 0/4592, RunningAvgSamplesPerSec=5.870949679704978, CurrSamplesPerSec=5.350317723843352, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2269/5000 [7:49:39<8:21:54, 11.03s/it][2022-12-20 18:12:10,222] [INFO] [timer.py:197:stop] 0/4594, RunningAvgSamplesPerSec=5.870950831891253, CurrSamplesPerSec=5.33158402088243, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2270/5000 [7:49:50<8:21:14, 11.02s/it][2022-12-20 18:12:21,218] [INFO] [timer.py:197:stop] 0/4596, RunningAvgSamplesPerSec=5.870957431204611, CurrSamplesPerSec=5.327416950506657, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2271/5000 [7:50:01<8:21:17, 11.02s/it][2022-12-20 18:12:32,202] [INFO] [timer.py:197:stop] 0/4598, RunningAvgSamplesPerSec=5.870966682666349, CurrSamplesPerSec=5.371038217012425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2272/5000 [7:50:12<8:20:35, 11.01s/it][2022-12-20 18:12:43,263] [INFO] [logging.py:68:log_dist] [Rank 0] step=2300, skipped=3, lr=[6.00888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:12:43,265] [INFO] [timer.py:197:stop] 0/4600, RunningAvgSamplesPerSec=5.870957457904608, CurrSamplesPerSec=5.300088660113958, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2273/5000 [7:50:23<8:21:01, 11.02s/it][2022-12-20 18:12:54,321] [INFO] [timer.py:197:stop] 0/4602, RunningAvgSamplesPerSec=5.870950026559844, CurrSamplesPerSec=5.301217203297933, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 45%|████▌     | 2274/5000 [7:50:34<8:20:58, 11.03s/it][2022-12-20 18:13:05,333] [INFO] [timer.py:197:stop] 0/4604, RunningAvgSamplesPerSec=5.870952996175184, CurrSamplesPerSec=5.320126786213654, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2275/5000 [7:50:45<8:20:25, 11.02s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 6.004444444444445e-06, 'epoch': 55.48}
+
 46%|████▌     | 2275/5000 [7:50:45<8:20:25, 11.02s/it][2022-12-20 18:13:16,356] [INFO] [timer.py:197:stop] 0/4606, RunningAvgSamplesPerSec=5.870954273438248, CurrSamplesPerSec=5.302377648191757, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2276/5000 [7:50:56<8:20:08, 11.02s/it][2022-12-20 18:13:27,415] [INFO] [timer.py:197:stop] 0/4608, RunningAvgSamplesPerSec=5.870947788452088, CurrSamplesPerSec=5.262222986141363, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2277/5000 [7:51:07<8:21:24, 11.05s/it][2022-12-20 18:13:38,499] [INFO] [timer.py:197:stop] 0/4610, RunningAvgSamplesPerSec=5.870933892110724, CurrSamplesPerSec=5.291357307374993, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2278/5000 [7:51:19<8:21:30, 11.05s/it][2022-12-20 18:13:49,553] [INFO] [timer.py:197:stop] 0/4612, RunningAvgSamplesPerSec=5.870927525029095, CurrSamplesPerSec=5.304561482358754, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2279/5000 [7:51:30<8:20:59, 11.05s/it][2022-12-20 18:14:00,542] [INFO] [timer.py:197:stop] 0/4614, RunningAvgSamplesPerSec=5.870937168649814, CurrSamplesPerSec=5.341187689018548, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2280/5000 [7:51:41<8:20:28, 11.04s/it][2022-12-20 18:14:11,614] [INFO] [timer.py:197:stop] 0/4616, RunningAvgSamplesPerSec=5.8709257256057805, CurrSamplesPerSec=5.296230418571072, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2281/5000 [7:51:52<8:20:15, 11.04s/it][2022-12-20 18:14:22,644] [INFO] [timer.py:197:stop] 0/4618, RunningAvgSamplesPerSec=5.870924313676961, CurrSamplesPerSec=5.30355369006467, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2282/5000 [7:52:03<8:19:56, 11.04s/it][2022-12-20 18:14:33,697] [INFO] [logging.py:68:log_dist] [Rank 0] step=2310, skipped=3, lr=[5.986666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:14:33,698] [INFO] [timer.py:197:stop] 0/4620, RunningAvgSamplesPerSec=5.870917361602176, CurrSamplesPerSec=5.282387928496305, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2283/5000 [7:52:14<8:20:29, 11.05s/it][2022-12-20 18:14:44,751] [INFO] [timer.py:197:stop] 0/4622, RunningAvgSamplesPerSec=5.870912329417045, CurrSamplesPerSec=5.315249194046155, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2284/5000 [7:52:25<8:19:52, 11.04s/it][2022-12-20 18:14:55,752] [INFO] [timer.py:197:stop] 0/4624, RunningAvgSamplesPerSec=5.870918348324344, CurrSamplesPerSec=5.334660966461219, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2285/5000 [7:52:36<8:18:52, 11.03s/it][2022-12-20 18:15:06,771] [INFO] [timer.py:197:stop] 0/4626, RunningAvgSamplesPerSec=5.870919689238294, CurrSamplesPerSec=5.301260336536591, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2286/5000 [7:52:47<8:19:08, 11.03s/it][2022-12-20 18:15:17,787] [INFO] [timer.py:197:stop] 0/4628, RunningAvgSamplesPerSec=5.870921556314186, CurrSamplesPerSec=5.338307034374673, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2287/5000 [7:52:58<8:18:48, 11.03s/it][2022-12-20 18:15:28,918] [INFO] [timer.py:197:stop] 0/4630, RunningAvgSamplesPerSec=5.870896769145393, CurrSamplesPerSec=5.243830854016576, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2288/5000 [7:53:09<8:20:05, 11.06s/it][2022-12-20 18:15:39,944] [INFO] [timer.py:197:stop] 0/4632, RunningAvgSamplesPerSec=5.8708967228023345, CurrSamplesPerSec=5.343863574451488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2289/5000 [7:53:20<8:19:16, 11.05s/it][2022-12-20 18:15:50,982] [INFO] [timer.py:197:stop] 0/4634, RunningAvgSamplesPerSec=5.870893957067914, CurrSamplesPerSec=5.325957023854675, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2290/5000 [7:53:31<8:17:47, 11.02s/it][2022-12-20 18:16:01,965] [INFO] [timer.py:197:stop] 0/4636, RunningAvgSamplesPerSec=5.870903716027733, CurrSamplesPerSec=5.299895698245635, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2291/5000 [7:53:42<8:18:01, 11.03s/it][2022-12-20 18:16:13,023] [INFO] [timer.py:197:stop] 0/4638, RunningAvgSamplesPerSec=5.8708973809534974, CurrSamplesPerSec=5.300044290197461, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2292/5000 [7:53:53<8:17:39, 11.03s/it][2022-12-20 18:16:24,058] [INFO] [logging.py:68:log_dist] [Rank 0] step=2320, skipped=3, lr=[5.964444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:16:24,059] [INFO] [timer.py:197:stop] 0/4640, RunningAvgSamplesPerSec=5.870894541327961, CurrSamplesPerSec=5.2769881011772455, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2293/5000 [7:54:04<8:18:23, 11.05s/it][2022-12-20 18:16:35,116] [INFO] [timer.py:197:stop] 0/4642, RunningAvgSamplesPerSec=5.8708870822757655, CurrSamplesPerSec=5.310068546224846, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2294/5000 [7:54:15<8:18:07, 11.05s/it][2022-12-20 18:16:46,120] [INFO] [timer.py:197:stop] 0/4644, RunningAvgSamplesPerSec=5.870892285797219, CurrSamplesPerSec=5.34450301074386, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2295/5000 [7:54:26<8:17:21, 11.03s/it][2022-12-20 18:16:57,142] [INFO] [timer.py:197:stop] 0/4646, RunningAvgSamplesPerSec=5.870894343718299, CurrSamplesPerSec=5.327156658802541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2296/5000 [7:54:37<8:17:01, 11.03s/it][2022-12-20 18:17:07,242] [INFO] [timer.py:197:stop] 0/4648, RunningAvgSamplesPerSec=5.87110609209428, CurrSamplesPerSec=6.275430511545547, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 18:17:18,222] [INFO] [timer.py:197:stop] 0/4650, RunningAvgSamplesPerSec=5.8711192627637, CurrSamplesPerSec=5.344714984127457, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2297/5000 [7:54:53<9:25:49, 12.56s/it][2022-12-20 18:17:29,263] [INFO] [timer.py:197:stop] 0/4652, RunningAvgSamplesPerSec=5.871115269475744, CurrSamplesPerSec=5.303019975204742, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2298/5000 [7:55:04<9:05:05, 12.10s/it][2022-12-20 18:17:40,278] [INFO] [timer.py:197:stop] 0/4654, RunningAvgSamplesPerSec=5.8711173808462425, CurrSamplesPerSec=5.343142822770184, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2299/5000 [7:55:15<8:50:10, 11.78s/it][2022-12-20 18:17:51,404] [INFO] [timer.py:197:stop] 0/4656, RunningAvgSamplesPerSec=5.871097705050236, CurrSamplesPerSec=5.299646040670556, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2300/5000 [7:55:26<8:41:11, 11.58s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.946666666666668e-06, 'epoch': 56.1}
+
 46%|████▌     | 2300/5000 [7:55:27<8:41:11, 11.58s/it][2022-12-20 18:18:02,413] [INFO] [timer.py:197:stop] 0/4658, RunningAvgSamplesPerSec=5.871103440017517, CurrSamplesPerSec=5.326089115764955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2301/5000 [7:55:38<8:33:16, 11.41s/it][2022-12-20 18:18:13,424] [INFO] [logging.py:68:log_dist] [Rank 0] step=2330, skipped=3, lr=[5.9422222222222225e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:18:13,426] [INFO] [timer.py:197:stop] 0/4660, RunningAvgSamplesPerSec=5.871106384133189, CurrSamplesPerSec=5.327966585756051, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2302/5000 [7:55:49<8:27:42, 11.29s/it][2022-12-20 18:18:24,600] [INFO] [timer.py:197:stop] 0/4662, RunningAvgSamplesPerSec=5.871076016291257, CurrSamplesPerSec=5.295388743821942, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2303/5000 [7:56:00<8:25:57, 11.26s/it][2022-12-20 18:18:35,585] [INFO] [timer.py:197:stop] 0/4664, RunningAvgSamplesPerSec=5.8710852238829885, CurrSamplesPerSec=5.352689590593273, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2304/5000 [7:56:11<8:22:06, 11.17s/it][2022-12-20 18:18:46,618] [INFO] [timer.py:197:stop] 0/4666, RunningAvgSamplesPerSec=5.871083356925433, CurrSamplesPerSec=5.310553881707882, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2305/5000 [7:56:22<8:20:01, 11.13s/it][2022-12-20 18:18:57,552] [INFO] [timer.py:197:stop] 0/4668, RunningAvgSamplesPerSec=5.871104311904402, CurrSamplesPerSec=5.380907637614472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2306/5000 [7:56:33<8:17:09, 11.07s/it][2022-12-20 18:19:08,457] [INFO] [timer.py:197:stop] 0/4670, RunningAvgSamplesPerSec=5.871131886742229, CurrSamplesPerSec=5.364129440848581, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2307/5000 [7:56:44<8:14:43, 11.02s/it][2022-12-20 18:19:19,525] [INFO] [timer.py:197:stop] 0/4672, RunningAvgSamplesPerSec=5.871121644112818, CurrSamplesPerSec=5.315448327755886, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2308/5000 [7:56:55<8:15:09, 11.04s/it][2022-12-20 18:19:30,518] [INFO] [timer.py:197:stop] 0/4674, RunningAvgSamplesPerSec=5.8711287193253865, CurrSamplesPerSec=5.347486656655464, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2309/5000 [7:57:06<8:14:23, 11.02s/it][2022-12-20 18:19:41,518] [INFO] [timer.py:197:stop] 0/4676, RunningAvgSamplesPerSec=5.871134100577123, CurrSamplesPerSec=5.351351049627345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2310/5000 [7:57:17<8:13:53, 11.02s/it][2022-12-20 18:19:52,568] [INFO] [timer.py:197:stop] 0/4678, RunningAvgSamplesPerSec=5.871128114351411, CurrSamplesPerSec=5.321487094957284, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2311/5000 [7:57:28<8:14:09, 11.03s/it][2022-12-20 18:20:03,554] [INFO] [logging.py:68:log_dist] [Rank 0] step=2340, skipped=3, lr=[5.92e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:20:03,555] [INFO] [timer.py:197:stop] 0/4680, RunningAvgSamplesPerSec=5.8711365683614485, CurrSamplesPerSec=5.312783782743919, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▌     | 2312/5000 [7:57:39<8:13:27, 11.01s/it][2022-12-20 18:20:14,533] [INFO] [timer.py:197:stop] 0/4682, RunningAvgSamplesPerSec=5.871146973454374, CurrSamplesPerSec=5.3725575904531935, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2313/5000 [7:57:50<8:12:47, 11.00s/it][2022-12-20 18:20:25,521] [INFO] [timer.py:197:stop] 0/4684, RunningAvgSamplesPerSec=5.871156271139281, CurrSamplesPerSec=5.351185912563284, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2314/5000 [7:58:01<8:12:22, 11.00s/it][2022-12-20 18:20:36,520] [INFO] [timer.py:197:stop] 0/4686, RunningAvgSamplesPerSec=5.871162687798975, CurrSamplesPerSec=5.348481803636464, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2315/5000 [7:58:12<8:12:11, 11.00s/it][2022-12-20 18:20:47,538] [INFO] [timer.py:197:stop] 0/4688, RunningAvgSamplesPerSec=5.8711639430197415, CurrSamplesPerSec=5.329260653619353, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2316/5000 [7:58:23<8:12:16, 11.00s/it][2022-12-20 18:20:58,593] [INFO] [timer.py:197:stop] 0/4690, RunningAvgSamplesPerSec=5.871156808925758, CurrSamplesPerSec=5.307966030657009, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2317/5000 [7:58:34<8:12:46, 11.02s/it][2022-12-20 18:21:09,564] [INFO] [timer.py:197:stop] 0/4692, RunningAvgSamplesPerSec=5.871169346033693, CurrSamplesPerSec=5.333274206181227, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2318/5000 [7:58:45<8:11:56, 11.01s/it][2022-12-20 18:21:20,614] [INFO] [timer.py:197:stop] 0/4694, RunningAvgSamplesPerSec=5.871164018137679, CurrSamplesPerSec=5.305311286760566, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2319/5000 [7:58:56<8:12:20, 11.02s/it][2022-12-20 18:21:31,594] [INFO] [timer.py:197:stop] 0/4696, RunningAvgSamplesPerSec=5.871174517765798, CurrSamplesPerSec=5.368504257877519, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2320/5000 [7:59:07<8:11:38, 11.01s/it][2022-12-20 18:21:42,598] [INFO] [timer.py:197:stop] 0/4698, RunningAvgSamplesPerSec=5.871179260390596, CurrSamplesPerSec=5.351455812402581, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2321/5000 [7:59:18<8:11:25, 11.01s/it][2022-12-20 18:21:53,552] [INFO] [logging.py:68:log_dist] [Rank 0] step=2350, skipped=3, lr=[5.897777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:21:53,553] [INFO] [timer.py:197:stop] 0/4700, RunningAvgSamplesPerSec=5.871195088016474, CurrSamplesPerSec=5.311177804678387, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2322/5000 [7:59:29<8:10:33, 10.99s/it][2022-12-20 18:22:04,587] [INFO] [timer.py:197:stop] 0/4702, RunningAvgSamplesPerSec=5.871192897721099, CurrSamplesPerSec=5.311430021278926, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2323/5000 [7:59:40<8:10:57, 11.00s/it][2022-12-20 18:22:15,591] [INFO] [timer.py:197:stop] 0/4704, RunningAvgSamplesPerSec=5.871197949589899, CurrSamplesPerSec=5.310524254791887, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2324/5000 [7:59:51<8:10:45, 11.00s/it][2022-12-20 18:22:26,617] [INFO] [timer.py:197:stop] 0/4706, RunningAvgSamplesPerSec=5.871198485794172, CurrSamplesPerSec=5.337180898024717, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 46%|████▋     | 2325/5000 [8:00:02<8:10:53, 11.01s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.891111111111112e-06, 'epoch': 56.7}
+
 46%|████▋     | 2325/5000 [8:00:02<8:10:53, 11.01s/it][2022-12-20 18:22:37,646] [INFO] [timer.py:197:stop] 0/4708, RunningAvgSamplesPerSec=5.8711978476400235, CurrSamplesPerSec=5.302749072724235, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2326/5000 [8:00:13<8:10:57, 11.02s/it][2022-12-20 18:22:48,684] [INFO] [timer.py:197:stop] 0/4710, RunningAvgSamplesPerSec=5.871195166364643, CurrSamplesPerSec=5.297592751576862, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2327/5000 [8:00:24<8:11:03, 11.02s/it][2022-12-20 18:22:59,692] [INFO] [timer.py:197:stop] 0/4712, RunningAvgSamplesPerSec=5.871202369437678, CurrSamplesPerSec=5.334848410351626, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2328/5000 [8:00:35<8:10:40, 11.02s/it][2022-12-20 18:23:10,747] [INFO] [timer.py:197:stop] 0/4714, RunningAvgSamplesPerSec=5.871195319756936, CurrSamplesPerSec=5.31674285004424, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2329/5000 [8:00:46<8:10:59, 11.03s/it][2022-12-20 18:23:21,764] [INFO] [timer.py:197:stop] 0/4716, RunningAvgSamplesPerSec=5.871197301076524, CurrSamplesPerSec=5.346963660428676, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2330/5000 [8:00:57<8:10:37, 11.03s/it][2022-12-20 18:23:32,838] [INFO] [timer.py:197:stop] 0/4718, RunningAvgSamplesPerSec=5.871186023619932, CurrSamplesPerSec=5.286369891301856, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2331/5000 [8:01:08<8:11:06, 11.04s/it][2022-12-20 18:23:44,171] [INFO] [logging.py:68:log_dist] [Rank 0] step=2360, skipped=3, lr=[5.875555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:23:44,173] [INFO] [timer.py:197:stop] 0/4720, RunningAvgSamplesPerSec=5.871114862906311, CurrSamplesPerSec=5.04660795426858, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2332/5000 [8:01:19<8:14:50, 11.13s/it][2022-12-20 18:23:55,241] [INFO] [timer.py:197:stop] 0/4722, RunningAvgSamplesPerSec=5.87110466036547, CurrSamplesPerSec=5.296698177726609, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2333/5000 [8:01:30<8:13:51, 11.11s/it][2022-12-20 18:24:06,270] [INFO] [timer.py:197:stop] 0/4724, RunningAvgSamplesPerSec=5.87110379322603, CurrSamplesPerSec=5.336869994039957, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2334/5000 [8:01:41<8:12:35, 11.09s/it][2022-12-20 18:24:17,310] [INFO] [timer.py:197:stop] 0/4726, RunningAvgSamplesPerSec=5.871100236188354, CurrSamplesPerSec=5.294814895116352, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2335/5000 [8:01:52<8:11:47, 11.07s/it][2022-12-20 18:24:28,332] [INFO] [timer.py:197:stop] 0/4728, RunningAvgSamplesPerSec=5.871100753572674, CurrSamplesPerSec=5.295850503445754, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2336/5000 [8:02:03<8:10:55, 11.06s/it][2022-12-20 18:24:39,407] [INFO] [timer.py:197:stop] 0/4730, RunningAvgSamplesPerSec=5.871089200046865, CurrSamplesPerSec=5.291451807036544, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2337/5000 [8:02:15<8:10:59, 11.06s/it][2022-12-20 18:24:49,459] [INFO] [timer.py:197:stop] 0/4732, RunningAvgSamplesPerSec=5.871308701399774, CurrSamplesPerSec=5.382618451994167, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2338/5000 [8:02:30<9:03:16, 12.25s/it][2022-12-20 18:25:00,466] [INFO] [timer.py:197:stop] 0/4734, RunningAvgSamplesPerSec=5.871312538646093, CurrSamplesPerSec=5.341324151077426, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2339/5000 [8:02:40<8:46:08, 11.86s/it][2022-12-20 18:25:11,474] [INFO] [timer.py:197:stop] 0/4736, RunningAvgSamplesPerSec=5.871316042301735, CurrSamplesPerSec=5.310430543461063, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2340/5000 [8:02:52<8:35:37, 11.63s/it][2022-12-20 18:25:22,572] [INFO] [timer.py:197:stop] 0/4738, RunningAvgSamplesPerSec=5.871308931966556, CurrSamplesPerSec=5.301230394456609, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2341/5000 [8:03:03<8:27:41, 11.46s/it][2022-12-20 18:25:33,647] [INFO] [logging.py:68:log_dist] [Rank 0] step=2370, skipped=3, lr=[5.853333333333335e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:25:33,648] [INFO] [timer.py:197:stop] 0/4740, RunningAvgSamplesPerSec=5.871296781156275, CurrSamplesPerSec=5.276104412658258, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2342/5000 [8:03:14<8:21:57, 11.33s/it][2022-12-20 18:25:44,669] [INFO] [timer.py:197:stop] 0/4742, RunningAvgSamplesPerSec=5.8712971890803205, CurrSamplesPerSec=5.291811687642197, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2343/5000 [8:03:25<8:17:39, 11.24s/it][2022-12-20 18:25:55,642] [INFO] [timer.py:197:stop] 0/4744, RunningAvgSamplesPerSec=5.871308739807625, CurrSamplesPerSec=5.335759948595829, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2344/5000 [8:03:36<8:14:24, 11.17s/it][2022-12-20 18:26:06,699] [INFO] [timer.py:197:stop] 0/4746, RunningAvgSamplesPerSec=5.871300974582938, CurrSamplesPerSec=5.291530663642193, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2345/5000 [8:03:47<8:12:54, 11.14s/it][2022-12-20 18:26:17,772] [INFO] [timer.py:197:stop] 0/4748, RunningAvgSamplesPerSec=5.871289917901719, CurrSamplesPerSec=5.289992758160313, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2346/5000 [8:03:58<8:11:56, 11.12s/it][2022-12-20 18:26:28,780] [INFO] [timer.py:197:stop] 0/4750, RunningAvgSamplesPerSec=5.871293634939237, CurrSamplesPerSec=5.354359650311551, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2347/5000 [8:04:09<8:10:13, 11.09s/it][2022-12-20 18:26:39,852] [INFO] [timer.py:197:stop] 0/4752, RunningAvgSamplesPerSec=5.871287929908939, CurrSamplesPerSec=5.3035438404076505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2348/5000 [8:04:20<8:10:13, 11.09s/it][2022-12-20 18:26:50,952] [INFO] [timer.py:197:stop] 0/4754, RunningAvgSamplesPerSec=5.871278136640009, CurrSamplesPerSec=5.309591491186154, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2349/5000 [8:04:31<8:09:55, 11.09s/it][2022-12-20 18:27:01,952] [INFO] [timer.py:197:stop] 0/4756, RunningAvgSamplesPerSec=5.871284974427918, CurrSamplesPerSec=5.368724367177203, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2350/5000 [8:04:42<8:09:02, 11.07s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.8355555555555565e-06, 'epoch': 57.31}
+
 47%|████▋     | 2350/5000 [8:04:42<8:09:02, 11.07s/it][2022-12-20 18:27:13,046] [INFO] [timer.py:197:stop] 0/4758, RunningAvgSamplesPerSec=5.8712767712992, CurrSamplesPerSec=5.331145654472759, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2351/5000 [8:04:53<8:08:48, 11.07s/it][2022-12-20 18:27:24,113] [INFO] [logging.py:68:log_dist] [Rank 0] step=2380, skipped=3, lr=[5.831111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:27:24,114] [INFO] [timer.py:197:stop] 0/4760, RunningAvgSamplesPerSec=5.871270703559603, CurrSamplesPerSec=5.3177923220746, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2352/5000 [8:05:04<8:07:54, 11.06s/it][2022-12-20 18:27:35,086] [INFO] [timer.py:197:stop] 0/4762, RunningAvgSamplesPerSec=5.871286168924748, CurrSamplesPerSec=5.357933458545512, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2353/5000 [8:05:15<8:06:22, 11.02s/it][2022-12-20 18:27:46,025] [INFO] [timer.py:197:stop] 0/4764, RunningAvgSamplesPerSec=5.871308360278715, CurrSamplesPerSec=5.37100146342459, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2354/5000 [8:05:26<8:05:33, 11.01s/it][2022-12-20 18:27:57,057] [INFO] [timer.py:197:stop] 0/4766, RunningAvgSamplesPerSec=5.871306616009069, CurrSamplesPerSec=5.322218055067927, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2355/5000 [8:05:37<8:05:31, 11.01s/it][2022-12-20 18:28:08,102] [INFO] [timer.py:197:stop] 0/4768, RunningAvgSamplesPerSec=5.871301614768655, CurrSamplesPerSec=5.300375408148057, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2356/5000 [8:05:48<8:05:53, 11.03s/it][2022-12-20 18:28:19,056] [INFO] [timer.py:197:stop] 0/4770, RunningAvgSamplesPerSec=5.87131752311646, CurrSamplesPerSec=5.3925441652102295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2357/5000 [8:05:59<8:04:19, 10.99s/it][2022-12-20 18:28:30,022] [INFO] [timer.py:197:stop] 0/4772, RunningAvgSamplesPerSec=5.871330422922975, CurrSamplesPerSec=5.3519854499454285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2358/5000 [8:06:10<8:03:45, 10.99s/it][2022-12-20 18:28:40,993] [INFO] [timer.py:197:stop] 0/4774, RunningAvgSamplesPerSec=5.871342457862554, CurrSamplesPerSec=5.347253159751435, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2359/5000 [8:06:21<8:03:25, 10.98s/it][2022-12-20 18:28:51,986] [INFO] [timer.py:197:stop] 0/4776, RunningAvgSamplesPerSec=5.871349608737935, CurrSamplesPerSec=5.331870586294818, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2360/5000 [8:06:32<8:03:33, 10.99s/it][2022-12-20 18:29:03,011] [INFO] [timer.py:197:stop] 0/4778, RunningAvgSamplesPerSec=5.8713492541808066, CurrSamplesPerSec=5.315323499065961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2361/5000 [8:06:43<8:03:51, 11.00s/it][2022-12-20 18:29:14,035] [INFO] [logging.py:68:log_dist] [Rank 0] step=2390, skipped=3, lr=[5.80888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:29:14,036] [INFO] [timer.py:197:stop] 0/4780, RunningAvgSamplesPerSec=5.871349326091083, CurrSamplesPerSec=5.31751337740555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2362/5000 [8:06:54<8:03:23, 10.99s/it][2022-12-20 18:29:24,962] [INFO] [timer.py:197:stop] 0/4782, RunningAvgSamplesPerSec=5.871371212931361, CurrSamplesPerSec=5.363026885754006, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2363/5000 [8:07:05<8:02:22, 10.98s/it][2022-12-20 18:29:35,898] [INFO] [timer.py:197:stop] 0/4784, RunningAvgSamplesPerSec=5.871390661734643, CurrSamplesPerSec=5.358502886721719, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2364/5000 [8:07:16<8:02:17, 10.98s/it][2022-12-20 18:29:46,887] [INFO] [timer.py:197:stop] 0/4786, RunningAvgSamplesPerSec=5.871398564487362, CurrSamplesPerSec=5.354393186026385, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2365/5000 [8:07:27<8:02:12, 10.98s/it][2022-12-20 18:29:57,855] [INFO] [timer.py:197:stop] 0/4788, RunningAvgSamplesPerSec=5.871410604610635, CurrSamplesPerSec=5.3689423468555475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2366/5000 [8:07:38<8:01:37, 10.97s/it][2022-12-20 18:30:08,850] [INFO] [timer.py:197:stop] 0/4790, RunningAvgSamplesPerSec=5.871416844215971, CurrSamplesPerSec=5.328237955783054, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2367/5000 [8:07:49<8:01:59, 10.98s/it][2022-12-20 18:30:19,786] [INFO] [timer.py:197:stop] 0/4792, RunningAvgSamplesPerSec=5.871436365759751, CurrSamplesPerSec=5.397679979801992, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2368/5000 [8:08:00<8:00:39, 10.96s/it][2022-12-20 18:30:30,752] [INFO] [timer.py:197:stop] 0/4794, RunningAvgSamplesPerSec=5.871449434694542, CurrSamplesPerSec=5.336274390450937, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2369/5000 [8:08:11<8:01:32, 10.98s/it][2022-12-20 18:30:41,798] [INFO] [timer.py:197:stop] 0/4796, RunningAvgSamplesPerSec=5.871444179956857, CurrSamplesPerSec=5.328559913051929, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2370/5000 [8:08:22<8:02:07, 11.00s/it][2022-12-20 18:30:52,812] [INFO] [timer.py:197:stop] 0/4798, RunningAvgSamplesPerSec=5.871446201639655, CurrSamplesPerSec=5.352123318007361, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2371/5000 [8:08:33<8:02:13, 11.01s/it][2022-12-20 18:31:03,844] [INFO] [logging.py:68:log_dist] [Rank 0] step=2400, skipped=3, lr=[5.7866666666666674e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:31:03,845] [INFO] [timer.py:197:stop] 0/4800, RunningAvgSamplesPerSec=5.871444561064582, CurrSamplesPerSec=5.339088710090611, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2372/5000 [8:08:44<8:01:32, 10.99s/it][2022-12-20 18:31:14,796] [INFO] [timer.py:197:stop] 0/4802, RunningAvgSamplesPerSec=5.871460546645895, CurrSamplesPerSec=5.35483217950346, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2373/5000 [8:08:55<8:01:19, 10.99s/it][2022-12-20 18:31:25,804] [INFO] [timer.py:197:stop] 0/4804, RunningAvgSamplesPerSec=5.8714638150683625, CurrSamplesPerSec=5.3399487965755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 47%|████▋     | 2374/5000 [8:09:06<8:01:26, 11.00s/it][2022-12-20 18:31:36,838] [INFO] [timer.py:197:stop] 0/4806, RunningAvgSamplesPerSec=5.871461308607143, CurrSamplesPerSec=5.324365672929574, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2375/5000 [8:09:17<8:01:53, 11.01s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.78e-06, 'epoch': 57.92}
+
 48%|████▊     | 2375/5000 [8:09:17<8:01:53, 11.01s/it][2022-12-20 18:31:47,938] [INFO] [timer.py:197:stop] 0/4808, RunningAvgSamplesPerSec=5.8714518389484756, CurrSamplesPerSec=5.295220775228394, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2376/5000 [8:09:28<8:02:18, 11.03s/it][2022-12-20 18:31:58,941] [INFO] [timer.py:197:stop] 0/4810, RunningAvgSamplesPerSec=5.871457473056553, CurrSamplesPerSec=5.330284379178751, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2377/5000 [8:09:39<8:01:56, 11.02s/it][2022-12-20 18:32:09,976] [INFO] [timer.py:197:stop] 0/4812, RunningAvgSamplesPerSec=5.871454481356718, CurrSamplesPerSec=5.310627004872787, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2378/5000 [8:09:50<8:02:04, 11.03s/it][2022-12-20 18:32:20,093] [INFO] [timer.py:197:stop] 0/4814, RunningAvgSamplesPerSec=5.871655140383373, CurrSamplesPerSec=6.271014257244144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 18:32:31,103] [INFO] [timer.py:197:stop] 0/4816, RunningAvgSamplesPerSec=5.871657608221426, CurrSamplesPerSec=5.331562630317045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2379/5000 [8:10:06<9:09:11, 12.57s/it][2022-12-20 18:32:42,115] [INFO] [timer.py:197:stop] 0/4818, RunningAvgSamplesPerSec=5.871660124734972, CurrSamplesPerSec=5.330284590864146, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2380/5000 [8:10:17<8:48:32, 12.10s/it][2022-12-20 18:32:53,122] [INFO] [logging.py:68:log_dist] [Rank 0] step=2410, skipped=3, lr=[5.764444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:32:53,123] [INFO] [timer.py:197:stop] 0/4820, RunningAvgSamplesPerSec=5.871663243961402, CurrSamplesPerSec=5.298652875039607, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2381/5000 [8:10:28<8:33:59, 11.78s/it][2022-12-20 18:33:04,101] [INFO] [timer.py:197:stop] 0/4822, RunningAvgSamplesPerSec=5.8716734776601935, CurrSamplesPerSec=5.347792193437963, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2382/5000 [8:10:39<8:23:21, 11.54s/it][2022-12-20 18:33:15,046] [INFO] [timer.py:197:stop] 0/4824, RunningAvgSamplesPerSec=5.871691034025172, CurrSamplesPerSec=5.365108059470483, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2383/5000 [8:10:50<8:15:25, 11.36s/it][2022-12-20 18:33:26,043] [INFO] [timer.py:197:stop] 0/4826, RunningAvgSamplesPerSec=5.871696988808957, CurrSamplesPerSec=5.344771385500332, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2384/5000 [8:11:01<8:10:30, 11.25s/it][2022-12-20 18:33:37,025] [INFO] [timer.py:197:stop] 0/4828, RunningAvgSamplesPerSec=5.871706102369788, CurrSamplesPerSec=5.3414876171459955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2385/5000 [8:11:12<8:06:49, 11.17s/it][2022-12-20 18:33:48,075] [INFO] [timer.py:197:stop] 0/4830, RunningAvgSamplesPerSec=5.871700183288573, CurrSamplesPerSec=5.318114493232603, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2386/5000 [8:11:23<8:05:03, 11.13s/it][2022-12-20 18:33:59,093] [INFO] [timer.py:197:stop] 0/4832, RunningAvgSamplesPerSec=5.871701511435076, CurrSamplesPerSec=5.314612950286145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2387/5000 [8:11:34<8:03:21, 11.10s/it][2022-12-20 18:34:10,082] [INFO] [timer.py:197:stop] 0/4834, RunningAvgSamplesPerSec=5.871710156019766, CurrSamplesPerSec=5.364259573968584, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2388/5000 [8:11:45<8:01:44, 11.07s/it][2022-12-20 18:34:21,101] [INFO] [timer.py:197:stop] 0/4836, RunningAvgSamplesPerSec=5.871711352634429, CurrSamplesPerSec=5.348038949460336, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2389/5000 [8:11:56<8:00:56, 11.05s/it][2022-12-20 18:34:32,084] [INFO] [timer.py:197:stop] 0/4838, RunningAvgSamplesPerSec=5.871720277587745, CurrSamplesPerSec=5.344100179963291, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2390/5000 [8:12:07<7:59:51, 11.03s/it][2022-12-20 18:34:43,109] [INFO] [logging.py:68:log_dist] [Rank 0] step=2420, skipped=3, lr=[5.742222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:34:43,110] [INFO] [timer.py:197:stop] 0/4840, RunningAvgSamplesPerSec=5.87172304075709, CurrSamplesPerSec=5.302100108836711, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2391/5000 [8:12:18<7:59:37, 11.03s/it][2022-12-20 18:34:54,179] [INFO] [timer.py:197:stop] 0/4842, RunningAvgSamplesPerSec=5.871720267588995, CurrSamplesPerSec=5.301456120011009, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2392/5000 [8:12:29<7:59:56, 11.04s/it][2022-12-20 18:35:05,215] [INFO] [timer.py:197:stop] 0/4844, RunningAvgSamplesPerSec=5.87172481922872, CurrSamplesPerSec=5.332558002879916, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2393/5000 [8:12:40<7:59:40, 11.04s/it][2022-12-20 18:35:16,359] [INFO] [timer.py:197:stop] 0/4846, RunningAvgSamplesPerSec=5.871705350974805, CurrSamplesPerSec=5.27148273585709, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2394/5000 [8:12:51<8:00:51, 11.07s/it][2022-12-20 18:35:27,443] [INFO] [timer.py:197:stop] 0/4848, RunningAvgSamplesPerSec=5.871696666494056, CurrSamplesPerSec=5.285085536985909, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2395/5000 [8:13:03<8:00:49, 11.07s/it][2022-12-20 18:35:38,502] [INFO] [timer.py:197:stop] 0/4850, RunningAvgSamplesPerSec=5.871696188018329, CurrSamplesPerSec=5.2976630090122185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2396/5000 [8:13:14<8:00:26, 11.07s/it][2022-12-20 18:35:49,544] [INFO] [timer.py:197:stop] 0/4852, RunningAvgSamplesPerSec=5.871691588914764, CurrSamplesPerSec=5.319879436221168, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2397/5000 [8:13:25<7:59:53, 11.06s/it][2022-12-20 18:36:00,493] [INFO] [timer.py:197:stop] 0/4854, RunningAvgSamplesPerSec=5.871707753738764, CurrSamplesPerSec=5.367595021977238, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2398/5000 [8:13:36<7:58:14, 11.03s/it][2022-12-20 18:36:11,480] [INFO] [timer.py:197:stop] 0/4856, RunningAvgSamplesPerSec=5.871715644285251, CurrSamplesPerSec=5.347717190938689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2399/5000 [8:13:47<7:57:31, 11.02s/it][2022-12-20 18:36:22,496] [INFO] [timer.py:197:stop] 0/4858, RunningAvgSamplesPerSec=5.871717154842123, CurrSamplesPerSec=5.310279897872513, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2400/5000 [8:13:58<7:57:20, 11.02s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.722222222222222e-06, 'epoch': 58.53}
+
 48%|████▊     | 2400/5000 [8:13:58<7:57:20, 11.02s/it][2022-12-20 18:36:33,595] [INFO] [logging.py:68:log_dist] [Rank 0] step=2430, skipped=3, lr=[5.72e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:36:33,596] [INFO] [timer.py:197:stop] 0/4860, RunningAvgSamplesPerSec=5.8717001262260275, CurrSamplesPerSec=5.255825138376957, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2401/5000 [8:14:09<7:58:15, 11.04s/it][2022-12-20 18:36:44,593] [INFO] [timer.py:197:stop] 0/4862, RunningAvgSamplesPerSec=5.871706038039523, CurrSamplesPerSec=5.3499120968494775, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2402/5000 [8:14:20<7:57:30, 11.03s/it][2022-12-20 18:36:55,607] [INFO] [timer.py:197:stop] 0/4864, RunningAvgSamplesPerSec=5.8717080901022936, CurrSamplesPerSec=5.313388667619308, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2403/5000 [8:14:31<7:57:08, 11.02s/it][2022-12-20 18:37:06,552] [INFO] [timer.py:197:stop] 0/4866, RunningAvgSamplesPerSec=5.871725462517289, CurrSamplesPerSec=5.359632261187743, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2404/5000 [8:14:42<7:55:56, 11.00s/it][2022-12-20 18:37:17,596] [INFO] [timer.py:197:stop] 0/4868, RunningAvgSamplesPerSec=5.871721030792308, CurrSamplesPerSec=5.314184523947395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2405/5000 [8:14:53<7:56:19, 11.01s/it][2022-12-20 18:37:28,592] [INFO] [timer.py:197:stop] 0/4870, RunningAvgSamplesPerSec=5.871727067661912, CurrSamplesPerSec=5.3435089182014055, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2406/5000 [8:15:04<7:55:55, 11.01s/it][2022-12-20 18:37:39,680] [INFO] [timer.py:197:stop] 0/4872, RunningAvgSamplesPerSec=5.871713148231342, CurrSamplesPerSec=5.30028372912509, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2407/5000 [8:15:15<7:56:45, 11.03s/it][2022-12-20 18:37:50,749] [INFO] [timer.py:197:stop] 0/4874, RunningAvgSamplesPerSec=5.871703509486168, CurrSamplesPerSec=5.297550096185987, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2408/5000 [8:15:26<7:57:03, 11.04s/it][2022-12-20 18:38:01,806] [INFO] [timer.py:197:stop] 0/4876, RunningAvgSamplesPerSec=5.871696029210158, CurrSamplesPerSec=5.305417400363923, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2409/5000 [8:15:37<7:57:03, 11.05s/it][2022-12-20 18:38:12,826] [INFO] [timer.py:197:stop] 0/4878, RunningAvgSamplesPerSec=5.871696849115344, CurrSamplesPerSec=5.296535142367608, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2410/5000 [8:15:48<7:56:30, 11.04s/it][2022-12-20 18:38:23,915] [INFO] [logging.py:68:log_dist] [Rank 0] step=2440, skipped=3, lr=[5.697777777777779e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:38:23,917] [INFO] [timer.py:197:stop] 0/4880, RunningAvgSamplesPerSec=5.87168224840771, CurrSamplesPerSec=5.26478087145876, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2411/5000 [8:15:59<7:57:01, 11.05s/it][2022-12-20 18:38:35,029] [INFO] [timer.py:197:stop] 0/4882, RunningAvgSamplesPerSec=5.871663782166975, CurrSamplesPerSec=5.271885253680091, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2412/5000 [8:16:10<7:57:34, 11.07s/it][2022-12-20 18:38:46,145] [INFO] [timer.py:197:stop] 0/4884, RunningAvgSamplesPerSec=5.871649099797516, CurrSamplesPerSec=5.302700049246964, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2413/5000 [8:16:21<7:57:57, 11.09s/it][2022-12-20 18:38:57,182] [INFO] [timer.py:197:stop] 0/4886, RunningAvgSamplesPerSec=5.87164695081476, CurrSamplesPerSec=5.3149526261486075, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2414/5000 [8:16:32<7:57:08, 11.07s/it][2022-12-20 18:39:08,227] [INFO] [timer.py:197:stop] 0/4888, RunningAvgSamplesPerSec=5.8716424759067785, CurrSamplesPerSec=5.35386221760711, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2415/5000 [8:16:43<7:56:37, 11.06s/it][2022-12-20 18:39:19,513] [INFO] [timer.py:197:stop] 0/4890, RunningAvgSamplesPerSec=5.871584188643315, CurrSamplesPerSec=5.10681120965177, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2416/5000 [8:16:55<7:59:19, 11.13s/it][2022-12-20 18:39:30,513] [INFO] [timer.py:197:stop] 0/4892, RunningAvgSamplesPerSec=5.871589347350429, CurrSamplesPerSec=5.321083717977313, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2417/5000 [8:17:06<7:57:28, 11.09s/it][2022-12-20 18:39:41,580] [INFO] [timer.py:197:stop] 0/4894, RunningAvgSamplesPerSec=5.871580876673432, CurrSamplesPerSec=5.300439878401381, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2418/5000 [8:17:17<7:56:58, 11.08s/it][2022-12-20 18:39:52,550] [INFO] [timer.py:197:stop] 0/4896, RunningAvgSamplesPerSec=5.871592482794839, CurrSamplesPerSec=5.312585269189001, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2419/5000 [8:17:28<7:55:18, 11.05s/it][2022-12-20 18:40:02,676] [INFO] [timer.py:197:stop] 0/4898, RunningAvgSamplesPerSec=5.871787780774248, CurrSamplesPerSec=5.304302999941984, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2420/5000 [8:17:43<8:46:30, 12.24s/it][2022-12-20 18:40:13,613] [INFO] [logging.py:68:log_dist] [Rank 0] step=2450, skipped=3, lr=[5.675555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:40:13,614] [INFO] [timer.py:197:stop] 0/4900, RunningAvgSamplesPerSec=5.871806095287539, CurrSamplesPerSec=5.361284384481619, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2421/5000 [8:17:54<8:29:43, 11.86s/it][2022-12-20 18:40:24,622] [INFO] [timer.py:197:stop] 0/4902, RunningAvgSamplesPerSec=5.871809425351564, CurrSamplesPerSec=5.319108644652957, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2422/5000 [8:18:05<8:19:08, 11.62s/it][2022-12-20 18:40:35,636] [INFO] [timer.py:197:stop] 0/4904, RunningAvgSamplesPerSec=5.8718118034632125, CurrSamplesPerSec=5.353556413926023, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2423/5000 [8:18:16<8:10:34, 11.42s/it][2022-12-20 18:40:46,613] [INFO] [timer.py:197:stop] 0/4906, RunningAvgSamplesPerSec=5.87182160179414, CurrSamplesPerSec=5.343865914871038, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2424/5000 [8:18:27<8:05:00, 11.30s/it][2022-12-20 18:40:57,662] [INFO] [timer.py:197:stop] 0/4908, RunningAvgSamplesPerSec=5.8718161706456735, CurrSamplesPerSec=5.305903983492042, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 48%|████▊     | 2425/5000 [8:18:38<8:01:16, 11.21s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.666666666666667e-06, 'epoch': 59.14}
+
 48%|████▊     | 2425/5000 [8:18:38<8:01:16, 11.21s/it][2022-12-20 18:41:08,722] [INFO] [timer.py:197:stop] 0/4910, RunningAvgSamplesPerSec=5.871808174117076, CurrSamplesPerSec=5.272714708738472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2426/5000 [8:18:49<7:58:58, 11.16s/it][2022-12-20 18:41:19,721] [INFO] [timer.py:197:stop] 0/4912, RunningAvgSamplesPerSec=5.871813841561986, CurrSamplesPerSec=5.316887543864966, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2427/5000 [8:19:00<7:57:34, 11.14s/it][2022-12-20 18:41:30,908] [INFO] [timer.py:197:stop] 0/4914, RunningAvgSamplesPerSec=5.871783117285293, CurrSamplesPerSec=5.233740172338104, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2428/5000 [8:19:11<7:57:34, 11.14s/it][2022-12-20 18:41:41,919] [INFO] [timer.py:197:stop] 0/4916, RunningAvgSamplesPerSec=5.871785734102461, CurrSamplesPerSec=5.336652064514544, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2429/5000 [8:19:22<7:55:42, 11.10s/it][2022-12-20 18:41:52,968] [INFO] [timer.py:197:stop] 0/4918, RunningAvgSamplesPerSec=5.871780106738351, CurrSamplesPerSec=5.303191162968575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2430/5000 [8:19:33<7:54:51, 11.09s/it][2022-12-20 18:42:04,016] [INFO] [logging.py:68:log_dist] [Rank 0] step=2460, skipped=3, lr=[5.653333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:42:04,017] [INFO] [timer.py:197:stop] 0/4920, RunningAvgSamplesPerSec=5.871774320787825, CurrSamplesPerSec=5.303234956962544, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2431/5000 [8:19:44<7:54:21, 11.08s/it][2022-12-20 18:42:15,055] [INFO] [timer.py:197:stop] 0/4922, RunningAvgSamplesPerSec=5.871770766148264, CurrSamplesPerSec=5.323737380849016, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2432/5000 [8:19:55<7:53:24, 11.06s/it][2022-12-20 18:42:26,149] [INFO] [timer.py:197:stop] 0/4924, RunningAvgSamplesPerSec=5.871755244367677, CurrSamplesPerSec=5.258758162928191, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2433/5000 [8:20:06<7:53:47, 11.07s/it][2022-12-20 18:42:37,396] [INFO] [timer.py:197:stop] 0/4926, RunningAvgSamplesPerSec=5.871707256697994, CurrSamplesPerSec=5.143848065126159, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2434/5000 [8:20:18<7:58:07, 11.18s/it][2022-12-20 18:42:48,567] [INFO] [timer.py:197:stop] 0/4928, RunningAvgSamplesPerSec=5.871713586246785, CurrSamplesPerSec=5.35856364428101, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2435/5000 [8:20:29<7:55:27, 11.12s/it][2022-12-20 18:42:59,715] [INFO] [timer.py:197:stop] 0/4930, RunningAvgSamplesPerSec=5.871696605304927, CurrSamplesPerSec=5.259028298337791, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2436/5000 [8:20:40<7:55:09, 11.12s/it][2022-12-20 18:43:10,720] [INFO] [timer.py:197:stop] 0/4932, RunningAvgSamplesPerSec=5.871700534574397, CurrSamplesPerSec=5.311321355023245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▊     | 2437/5000 [8:20:51<7:54:12, 11.10s/it][2022-12-20 18:43:21,764] [INFO] [timer.py:197:stop] 0/4934, RunningAvgSamplesPerSec=5.8716965673483665, CurrSamplesPerSec=5.326587742513263, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2438/5000 [8:21:02<7:52:31, 11.07s/it][2022-12-20 18:43:32,797] [INFO] [timer.py:197:stop] 0/4936, RunningAvgSamplesPerSec=5.8716945004175445, CurrSamplesPerSec=5.281912508136298, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2439/5000 [8:21:13<7:52:24, 11.07s/it][2022-12-20 18:43:43,851] [INFO] [timer.py:197:stop] 0/4938, RunningAvgSamplesPerSec=5.871688158893296, CurrSamplesPerSec=5.298625263366504, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2440/5000 [8:21:24<7:52:03, 11.06s/it][2022-12-20 18:43:54,923] [INFO] [logging.py:68:log_dist] [Rank 0] step=2470, skipped=3, lr=[5.631111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:43:54,925] [INFO] [timer.py:197:stop] 0/4940, RunningAvgSamplesPerSec=5.871679879289307, CurrSamplesPerSec=5.290846066412871, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2441/5000 [8:21:35<7:52:30, 11.08s/it][2022-12-20 18:44:05,936] [INFO] [timer.py:197:stop] 0/4942, RunningAvgSamplesPerSec=5.8716827241793474, CurrSamplesPerSec=5.372342112630245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2442/5000 [8:21:46<7:51:09, 11.05s/it][2022-12-20 18:44:16,953] [INFO] [timer.py:197:stop] 0/4944, RunningAvgSamplesPerSec=5.871684243741934, CurrSamplesPerSec=5.345195178792876, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2443/5000 [8:21:57<7:50:36, 11.04s/it][2022-12-20 18:44:28,023] [INFO] [timer.py:197:stop] 0/4946, RunningAvgSamplesPerSec=5.8716756842036855, CurrSamplesPerSec=5.303326319010396, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2444/5000 [8:22:08<7:50:44, 11.05s/it][2022-12-20 18:44:39,052] [INFO] [timer.py:197:stop] 0/4948, RunningAvgSamplesPerSec=5.871674754827284, CurrSamplesPerSec=5.337118926558458, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2445/5000 [8:22:19<7:49:46, 11.03s/it][2022-12-20 18:44:50,095] [INFO] [timer.py:197:stop] 0/4950, RunningAvgSamplesPerSec=5.8716705464871355, CurrSamplesPerSec=5.289995468625582, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2446/5000 [8:22:30<7:49:48, 11.04s/it][2022-12-20 18:45:01,183] [INFO] [timer.py:197:stop] 0/4952, RunningAvgSamplesPerSec=5.871658834274283, CurrSamplesPerSec=5.257137521243159, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2447/5000 [8:22:41<7:50:11, 11.05s/it][2022-12-20 18:45:12,438] [INFO] [timer.py:197:stop] 0/4954, RunningAvgSamplesPerSec=5.87160896367958, CurrSamplesPerSec=5.110452018413321, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2448/5000 [8:22:52<7:52:35, 11.11s/it][2022-12-20 18:45:23,493] [INFO] [timer.py:197:stop] 0/4956, RunningAvgSamplesPerSec=5.871602054551806, CurrSamplesPerSec=5.276547464550864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2449/5000 [8:23:04<7:51:31, 11.09s/it][2022-12-20 18:45:34,478] [INFO] [timer.py:197:stop] 0/4958, RunningAvgSamplesPerSec=5.871610235201246, CurrSamplesPerSec=5.326988571665799, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2450/5000 [8:23:14<7:49:53, 11.06s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.611111111111112e-06, 'epoch': 59.75}
+
 49%|████▉     | 2450/5000 [8:23:14<7:49:53, 11.06s/it][2022-12-20 18:45:45,463] [INFO] [logging.py:68:log_dist] [Rank 0] step=2480, skipped=3, lr=[5.608888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:45:45,465] [INFO] [timer.py:197:stop] 0/4960, RunningAvgSamplesPerSec=5.871618295347072, CurrSamplesPerSec=5.3191834792289345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2451/5000 [8:23:26<7:49:27, 11.05s/it][2022-12-20 18:45:56,513] [INFO] [timer.py:197:stop] 0/4962, RunningAvgSamplesPerSec=5.87161284404724, CurrSamplesPerSec=5.308229068950839, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2452/5000 [8:23:37<7:49:13, 11.05s/it][2022-12-20 18:46:07,542] [INFO] [timer.py:197:stop] 0/4964, RunningAvgSamplesPerSec=5.87161236718972, CurrSamplesPerSec=5.326316751918808, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2453/5000 [8:23:48<7:48:24, 11.03s/it][2022-12-20 18:46:18,548] [INFO] [timer.py:197:stop] 0/4966, RunningAvgSamplesPerSec=5.871616230619212, CurrSamplesPerSec=5.319873321295308, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2454/5000 [8:23:59<7:48:04, 11.03s/it][2022-12-20 18:46:29,588] [INFO] [timer.py:197:stop] 0/4968, RunningAvgSamplesPerSec=5.871613727343084, CurrSamplesPerSec=5.304967809778389, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2455/5000 [8:24:10<7:48:11, 11.04s/it][2022-12-20 18:46:40,579] [INFO] [timer.py:197:stop] 0/4970, RunningAvgSamplesPerSec=5.871620842556244, CurrSamplesPerSec=5.359499356306124, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2456/5000 [8:24:21<7:47:27, 11.03s/it][2022-12-20 18:46:51,638] [INFO] [timer.py:197:stop] 0/4972, RunningAvgSamplesPerSec=5.8716141009507945, CurrSamplesPerSec=5.305595034808724, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2457/5000 [8:24:32<7:47:14, 11.02s/it][2022-12-20 18:47:02,608] [INFO] [timer.py:197:stop] 0/4974, RunningAvgSamplesPerSec=5.871626238084978, CurrSamplesPerSec=5.349131299444035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2458/5000 [8:24:43<7:46:08, 11.00s/it][2022-12-20 18:47:13,577] [INFO] [timer.py:197:stop] 0/4976, RunningAvgSamplesPerSec=5.871638273895937, CurrSamplesPerSec=5.333032413084245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2459/5000 [8:24:54<7:45:48, 11.00s/it][2022-12-20 18:47:24,581] [INFO] [timer.py:197:stop] 0/4978, RunningAvgSamplesPerSec=5.871642987706822, CurrSamplesPerSec=5.3226904161080295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2460/5000 [8:25:05<7:46:01, 11.01s/it][2022-12-20 18:47:34,733] [INFO] [logging.py:68:log_dist] [Rank 0] step=2490, skipped=3, lr=[5.586666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:47:34,734] [INFO] [timer.py:197:stop] 0/4980, RunningAvgSamplesPerSec=5.871829592580814, CurrSamplesPerSec=6.221628139850031, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 18:47:45,724] [INFO] [timer.py:197:stop] 0/4982, RunningAvgSamplesPerSec=5.871836648371863, CurrSamplesPerSec=5.3329696903233295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2461/5000 [8:25:21<8:51:36, 12.56s/it][2022-12-20 18:47:56,738] [INFO] [timer.py:197:stop] 0/4984, RunningAvgSamplesPerSec=5.871839216278782, CurrSamplesPerSec=5.3476647756682, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2462/5000 [8:25:32<8:31:45, 12.10s/it][2022-12-20 18:48:07,770] [INFO] [timer.py:197:stop] 0/4986, RunningAvgSamplesPerSec=5.871837500025651, CurrSamplesPerSec=5.336332735480601, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2463/5000 [8:25:43<8:18:01, 11.78s/it][2022-12-20 18:48:18,789] [INFO] [timer.py:197:stop] 0/4988, RunningAvgSamplesPerSec=5.871838286040147, CurrSamplesPerSec=5.331701354363256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2464/5000 [8:25:54<8:08:12, 11.55s/it][2022-12-20 18:48:29,798] [INFO] [timer.py:197:stop] 0/4990, RunningAvgSamplesPerSec=5.8718422770787075, CurrSamplesPerSec=5.316900602501156, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2465/5000 [8:26:05<8:01:08, 11.39s/it][2022-12-20 18:48:40,783] [INFO] [timer.py:197:stop] 0/4992, RunningAvgSamplesPerSec=5.871850459880945, CurrSamplesPerSec=5.3382824050105135, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2466/5000 [8:26:16<7:55:50, 11.27s/it][2022-12-20 18:48:51,806] [INFO] [timer.py:197:stop] 0/4994, RunningAvgSamplesPerSec=5.871850797416019, CurrSamplesPerSec=5.301406911039251, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2467/5000 [8:26:27<7:52:34, 11.19s/it][2022-12-20 18:49:02,823] [INFO] [timer.py:197:stop] 0/4996, RunningAvgSamplesPerSec=5.871853261782841, CurrSamplesPerSec=5.310736693395954, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2468/5000 [8:26:38<7:50:08, 11.14s/it][2022-12-20 18:49:13,901] [INFO] [timer.py:197:stop] 0/4998, RunningAvgSamplesPerSec=5.871847210848844, CurrSamplesPerSec=5.302028476980958, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2469/5000 [8:26:49<7:49:09, 11.12s/it][2022-12-20 18:49:24,954] [INFO] [logging.py:68:log_dist] [Rank 0] step=2500, skipped=3, lr=[5.5644444444444444e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:49:24,955] [INFO] [timer.py:197:stop] 0/5000, RunningAvgSamplesPerSec=5.871841508671049, CurrSamplesPerSec=5.315770004418398, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2470/5000 [8:27:00<7:48:07, 11.10s/it][2022-12-20 18:49:36,007] [INFO] [timer.py:197:stop] 0/5002, RunningAvgSamplesPerSec=5.871835410737261, CurrSamplesPerSec=5.329846014798496, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2471/5000 [8:27:11<7:47:17, 11.09s/it][2022-12-20 18:49:47,045] [INFO] [timer.py:197:stop] 0/5004, RunningAvgSamplesPerSec=5.871831986780581, CurrSamplesPerSec=5.310777460017343, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2472/5000 [8:27:22<7:46:30, 11.07s/it][2022-12-20 18:49:58,028] [INFO] [timer.py:197:stop] 0/5006, RunningAvgSamplesPerSec=5.871840397442546, CurrSamplesPerSec=5.286396126111237, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2473/5000 [8:27:33<7:45:12, 11.05s/it][2022-12-20 18:50:09,070] [INFO] [timer.py:197:stop] 0/5008, RunningAvgSamplesPerSec=5.871840993962608, CurrSamplesPerSec=5.303263035788978, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 49%|████▉     | 2474/5000 [8:27:44<7:44:58, 11.04s/it][2022-12-20 18:50:20,093] [INFO] [timer.py:197:stop] 0/5010, RunningAvgSamplesPerSec=5.8718468332763765, CurrSamplesPerSec=5.333661205525595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2475/5000 [8:27:55<7:44:30, 11.04s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.553333333333334e-06, 'epoch': 60.36}
+
 50%|████▉     | 2475/5000 [8:27:55<7:44:30, 11.04s/it][2022-12-20 18:50:31,141] [INFO] [timer.py:197:stop] 0/5012, RunningAvgSamplesPerSec=5.871841746152169, CurrSamplesPerSec=5.3116582979523495, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2476/5000 [8:28:06<7:44:27, 11.04s/it][2022-12-20 18:50:42,142] [INFO] [timer.py:197:stop] 0/5014, RunningAvgSamplesPerSec=5.8718466335876816, CurrSamplesPerSec=5.324295550294023, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2477/5000 [8:28:17<7:43:45, 11.03s/it][2022-12-20 18:50:53,167] [INFO] [timer.py:197:stop] 0/5016, RunningAvgSamplesPerSec=5.871846155118186, CurrSamplesPerSec=5.335796857821585, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2478/5000 [8:28:28<7:43:31, 11.03s/it][2022-12-20 18:51:04,162] [INFO] [timer.py:197:stop] 0/5018, RunningAvgSamplesPerSec=5.871852185824686, CurrSamplesPerSec=5.360985655186286, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2479/5000 [8:28:39<7:42:55, 11.02s/it][2022-12-20 18:51:15,198] [INFO] [logging.py:68:log_dist] [Rank 0] step=2510, skipped=3, lr=[5.542222222222222e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:51:15,199] [INFO] [timer.py:197:stop] 0/5020, RunningAvgSamplesPerSec=5.87184903883532, CurrSamplesPerSec=5.3179115777435415, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2480/5000 [8:28:50<7:42:59, 11.02s/it][2022-12-20 18:51:26,190] [INFO] [timer.py:197:stop] 0/5022, RunningAvgSamplesPerSec=5.8718560005630245, CurrSamplesPerSec=5.322349961576678, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2481/5000 [8:29:01<7:42:23, 11.01s/it][2022-12-20 18:51:37,126] [INFO] [timer.py:197:stop] 0/5024, RunningAvgSamplesPerSec=5.871874402778357, CurrSamplesPerSec=5.3861767065062685, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2482/5000 [8:29:12<7:41:14, 10.99s/it][2022-12-20 18:51:48,160] [INFO] [timer.py:197:stop] 0/5026, RunningAvgSamplesPerSec=5.8718721569956145, CurrSamplesPerSec=5.312690201942249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2483/5000 [8:29:23<7:41:35, 11.00s/it][2022-12-20 18:51:59,114] [INFO] [timer.py:197:stop] 0/5028, RunningAvgSamplesPerSec=5.871886978181, CurrSamplesPerSec=5.356709020452382, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2484/5000 [8:29:34<7:40:47, 10.99s/it][2022-12-20 18:52:10,138] [INFO] [timer.py:197:stop] 0/5030, RunningAvgSamplesPerSec=5.871886662021788, CurrSamplesPerSec=5.323994804277007, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2485/5000 [8:29:45<7:41:03, 11.00s/it][2022-12-20 18:52:21,139] [INFO] [timer.py:197:stop] 0/5032, RunningAvgSamplesPerSec=5.87189140402149, CurrSamplesPerSec=5.30809744158186, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2486/5000 [8:29:56<7:40:53, 11.00s/it][2022-12-20 18:52:32,171] [INFO] [timer.py:197:stop] 0/5034, RunningAvgSamplesPerSec=5.8718893024745045, CurrSamplesPerSec=5.302125452734595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2487/5000 [8:30:07<7:41:06, 11.01s/it][2022-12-20 18:52:43,143] [INFO] [timer.py:197:stop] 0/5036, RunningAvgSamplesPerSec=5.871900254620668, CurrSamplesPerSec=5.34094475279706, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2488/5000 [8:30:18<7:40:27, 11.00s/it][2022-12-20 18:52:54,171] [INFO] [timer.py:197:stop] 0/5038, RunningAvgSamplesPerSec=5.871898983053507, CurrSamplesPerSec=5.317138828558471, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2489/5000 [8:30:29<7:40:39, 11.01s/it][2022-12-20 18:53:05,118] [INFO] [logging.py:68:log_dist] [Rank 0] step=2520, skipped=3, lr=[5.5200000000000005e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:53:05,119] [INFO] [timer.py:197:stop] 0/5040, RunningAvgSamplesPerSec=5.871914660040124, CurrSamplesPerSec=5.353910910244085, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2490/5000 [8:30:40<7:39:43, 10.99s/it][2022-12-20 18:53:16,171] [INFO] [timer.py:197:stop] 0/5042, RunningAvgSamplesPerSec=5.871908439166753, CurrSamplesPerSec=5.330472150730607, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2491/5000 [8:30:51<7:40:19, 11.01s/it][2022-12-20 18:53:27,136] [INFO] [timer.py:197:stop] 0/5044, RunningAvgSamplesPerSec=5.871920965391994, CurrSamplesPerSec=5.356706454980454, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2492/5000 [8:31:02<7:39:35, 11.00s/it][2022-12-20 18:53:38,098] [INFO] [timer.py:197:stop] 0/5046, RunningAvgSamplesPerSec=5.871933840949352, CurrSamplesPerSec=5.341326064146504, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2493/5000 [8:31:13<7:38:59, 10.99s/it][2022-12-20 18:53:49,146] [INFO] [timer.py:197:stop] 0/5048, RunningAvgSamplesPerSec=5.8719282825272625, CurrSamplesPerSec=5.301406073447639, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2494/5000 [8:31:24<7:39:36, 11.00s/it][2022-12-20 18:54:00,154] [INFO] [timer.py:197:stop] 0/5050, RunningAvgSamplesPerSec=5.871931482356053, CurrSamplesPerSec=5.323001359010395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2495/5000 [8:31:35<7:39:27, 11.01s/it][2022-12-20 18:54:11,197] [INFO] [timer.py:197:stop] 0/5052, RunningAvgSamplesPerSec=5.871927496777105, CurrSamplesPerSec=5.307984923200993, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2496/5000 [8:31:46<7:39:45, 11.02s/it][2022-12-20 18:54:22,196] [INFO] [timer.py:197:stop] 0/5054, RunningAvgSamplesPerSec=5.871932283640357, CurrSamplesPerSec=5.309838095461523, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2497/5000 [8:31:57<7:39:21, 11.01s/it][2022-12-20 18:54:33,218] [INFO] [timer.py:197:stop] 0/5056, RunningAvgSamplesPerSec=5.871932475939159, CurrSamplesPerSec=5.3038085364627205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2498/5000 [8:32:08<7:39:18, 11.01s/it][2022-12-20 18:54:44,177] [INFO] [timer.py:197:stop] 0/5058, RunningAvgSamplesPerSec=5.8719465345984885, CurrSamplesPerSec=5.3673674928214945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|████▉     | 2499/5000 [8:32:19<7:38:25, 11.00s/it][2022-12-20 18:54:55,182] [INFO] [logging.py:68:log_dist] [Rank 0] step=2530, skipped=3, lr=[5.497777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:54:55,184] [INFO] [timer.py:197:stop] 0/5060, RunningAvgSamplesPerSec=5.871950117571574, CurrSamplesPerSec=5.336104454561736, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2500/5000 [8:32:30<7:38:21, 11.00s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.497777777777778e-06, 'epoch': 60.96}
+
 50%|█████     | 2500/5000 [8:32:30<7:38:21, 11.00s/it][2022-12-20 18:55:06,176] [INFO] [timer.py:197:stop] 0/5062, RunningAvgSamplesPerSec=5.871957127467681, CurrSamplesPerSec=5.349127675301577, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2501/5000 [8:32:41<7:38:04, 11.00s/it][2022-12-20 18:55:16,326] [INFO] [timer.py:197:stop] 0/5064, RunningAvgSamplesPerSec=5.8721411534671635, CurrSamplesPerSec=5.306683961061043, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2502/5000 [8:32:56<8:28:49, 12.22s/it][2022-12-20 18:55:27,354] [INFO] [timer.py:197:stop] 0/5066, RunningAvgSamplesPerSec=5.872139924419976, CurrSamplesPerSec=5.300541191968221, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2503/5000 [8:33:07<8:13:31, 11.86s/it][2022-12-20 18:55:38,344] [INFO] [timer.py:197:stop] 0/5068, RunningAvgSamplesPerSec=5.872146819936066, CurrSamplesPerSec=5.320431946155144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2504/5000 [8:33:18<8:02:57, 11.61s/it][2022-12-20 18:55:49,392] [INFO] [timer.py:197:stop] 0/5070, RunningAvgSamplesPerSec=5.872141464133654, CurrSamplesPerSec=5.302871844896868, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2505/5000 [8:33:29<7:55:10, 11.43s/it][2022-12-20 18:56:00,381] [INFO] [timer.py:197:stop] 0/5072, RunningAvgSamplesPerSec=5.872148456066426, CurrSamplesPerSec=5.31271312367645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2506/5000 [8:33:40<7:49:40, 11.30s/it][2022-12-20 18:56:11,343] [INFO] [timer.py:197:stop] 0/5074, RunningAvgSamplesPerSec=5.872161557349289, CurrSamplesPerSec=5.349148354298003, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2507/5000 [8:33:51<7:45:10, 11.20s/it][2022-12-20 18:56:22,345] [INFO] [timer.py:197:stop] 0/5076, RunningAvgSamplesPerSec=5.87216604640883, CurrSamplesPerSec=5.305302269397074, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2508/5000 [8:34:02<7:43:09, 11.15s/it][2022-12-20 18:56:33,414] [INFO] [timer.py:197:stop] 0/5078, RunningAvgSamplesPerSec=5.872156162464104, CurrSamplesPerSec=5.28726243748894, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2509/5000 [8:34:13<7:41:45, 11.12s/it][2022-12-20 18:56:44,412] [INFO] [logging.py:68:log_dist] [Rank 0] step=2540, skipped=3, lr=[5.475555555555557e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:56:44,414] [INFO] [timer.py:197:stop] 0/5080, RunningAvgSamplesPerSec=5.87216086785783, CurrSamplesPerSec=5.3347377235658415, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2510/5000 [8:34:24<7:40:09, 11.09s/it][2022-12-20 18:56:55,464] [INFO] [timer.py:197:stop] 0/5082, RunningAvgSamplesPerSec=5.872155032888189, CurrSamplesPerSec=5.298654339303033, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2511/5000 [8:34:36<7:39:38, 11.08s/it][2022-12-20 18:57:06,502] [INFO] [timer.py:197:stop] 0/5084, RunningAvgSamplesPerSec=5.87215175607283, CurrSamplesPerSec=5.3195743396543484, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2512/5000 [8:34:47<7:39:04, 11.07s/it][2022-12-20 18:57:17,540] [INFO] [timer.py:197:stop] 0/5086, RunningAvgSamplesPerSec=5.872148105415894, CurrSamplesPerSec=5.328507238011519, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2513/5000 [8:34:58<7:38:20, 11.06s/it][2022-12-20 18:57:28,576] [INFO] [timer.py:197:stop] 0/5088, RunningAvgSamplesPerSec=5.872145086931538, CurrSamplesPerSec=5.3201925813886906, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2514/5000 [8:35:09<7:37:53, 11.05s/it][2022-12-20 18:57:39,656] [INFO] [timer.py:197:stop] 0/5090, RunningAvgSamplesPerSec=5.8721323002822405, CurrSamplesPerSec=5.281010338671472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2515/5000 [8:35:20<7:37:36, 11.05s/it][2022-12-20 18:57:50,649] [INFO] [timer.py:197:stop] 0/5092, RunningAvgSamplesPerSec=5.872138302153605, CurrSamplesPerSec=5.326531090093715, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2516/5000 [8:35:31<7:36:30, 11.03s/it][2022-12-20 18:58:01,651] [INFO] [timer.py:197:stop] 0/5094, RunningAvgSamplesPerSec=5.87214255921676, CurrSamplesPerSec=5.303132702256971, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2517/5000 [8:35:42<7:35:53, 11.02s/it][2022-12-20 18:58:12,630] [INFO] [timer.py:197:stop] 0/5096, RunningAvgSamplesPerSec=5.8721514506222, CurrSamplesPerSec=5.314450283232289, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2518/5000 [8:35:53<7:35:37, 11.01s/it][2022-12-20 18:58:23,633] [INFO] [timer.py:197:stop] 0/5098, RunningAvgSamplesPerSec=5.872155819642922, CurrSamplesPerSec=5.321056082985459, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2519/5000 [8:36:04<7:35:40, 11.02s/it][2022-12-20 18:58:34,659] [INFO] [logging.py:68:log_dist] [Rank 0] step=2550, skipped=3, lr=[5.453333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 18:58:34,660] [INFO] [timer.py:197:stop] 0/5100, RunningAvgSamplesPerSec=5.87215482207691, CurrSamplesPerSec=5.325843958384067, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2520/5000 [8:36:15<7:35:11, 11.01s/it][2022-12-20 18:58:45,627] [INFO] [timer.py:197:stop] 0/5102, RunningAvgSamplesPerSec=5.87216673274909, CurrSamplesPerSec=5.3520074315288735, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2521/5000 [8:36:26<7:34:17, 11.00s/it][2022-12-20 18:58:56,612] [INFO] [timer.py:197:stop] 0/5104, RunningAvgSamplesPerSec=5.872174699293519, CurrSamplesPerSec=5.325479645487349, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2522/5000 [8:36:37<7:33:51, 10.99s/it][2022-12-20 18:59:07,568] [INFO] [timer.py:197:stop] 0/5106, RunningAvgSamplesPerSec=5.8721887528437, CurrSamplesPerSec=5.342204094915323, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2523/5000 [8:36:48<7:33:29, 10.98s/it][2022-12-20 18:59:18,548] [INFO] [timer.py:197:stop] 0/5108, RunningAvgSamplesPerSec=5.872197665123957, CurrSamplesPerSec=5.337070963329383, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2524/5000 [8:36:59<7:33:19, 10.99s/it][2022-12-20 18:59:29,558] [INFO] [timer.py:197:stop] 0/5110, RunningAvgSamplesPerSec=5.872200169816816, CurrSamplesPerSec=5.316130463266113, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 50%|█████     | 2525/5000 [8:37:10<7:33:39, 11.00s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.442222222222223e-06, 'epoch': 61.58}
+
 50%|█████     | 2525/5000 [8:37:10<7:33:39, 11.00s/it][2022-12-20 18:59:40,556] [INFO] [timer.py:197:stop] 0/5112, RunningAvgSamplesPerSec=5.872205569878199, CurrSamplesPerSec=5.343376386287666, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2526/5000 [8:37:21<7:33:34, 11.00s/it][2022-12-20 18:59:51,582] [INFO] [timer.py:197:stop] 0/5114, RunningAvgSamplesPerSec=5.872204758302169, CurrSamplesPerSec=5.32411518308491, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2527/5000 [8:37:32<7:33:10, 10.99s/it][2022-12-20 19:00:02,515] [INFO] [timer.py:197:stop] 0/5116, RunningAvgSamplesPerSec=5.872223380910028, CurrSamplesPerSec=5.367832445604984, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2528/5000 [8:37:43<7:32:14, 10.98s/it][2022-12-20 19:00:13,487] [INFO] [timer.py:197:stop] 0/5118, RunningAvgSamplesPerSec=5.872234028180447, CurrSamplesPerSec=5.334728393855094, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2529/5000 [8:37:54<7:32:08, 10.98s/it][2022-12-20 19:00:24,492] [INFO] [logging.py:68:log_dist] [Rank 0] step=2560, skipped=3, lr=[5.431111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:00:24,494] [INFO] [timer.py:197:stop] 0/5120, RunningAvgSamplesPerSec=5.872238550734601, CurrSamplesPerSec=5.314313507517374, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2530/5000 [8:38:05<7:32:44, 11.00s/it][2022-12-20 19:00:35,566] [INFO] [timer.py:197:stop] 0/5122, RunningAvgSamplesPerSec=5.872228351084889, CurrSamplesPerSec=5.288056736269761, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2531/5000 [8:38:16<7:33:22, 11.02s/it][2022-12-20 19:00:46,631] [INFO] [timer.py:197:stop] 0/5124, RunningAvgSamplesPerSec=5.8722193601088515, CurrSamplesPerSec=5.286901301075573, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2532/5000 [8:38:27<7:34:00, 11.04s/it][2022-12-20 19:00:57,692] [INFO] [timer.py:197:stop] 0/5126, RunningAvgSamplesPerSec=5.872211210508516, CurrSamplesPerSec=5.306885810783267, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2533/5000 [8:38:38<7:34:09, 11.05s/it][2022-12-20 19:01:08,719] [INFO] [timer.py:197:stop] 0/5128, RunningAvgSamplesPerSec=5.872210355592526, CurrSamplesPerSec=5.340641485485858, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2534/5000 [8:38:49<7:33:32, 11.04s/it][2022-12-20 19:01:19,795] [INFO] [timer.py:197:stop] 0/5130, RunningAvgSamplesPerSec=5.872198833737548, CurrSamplesPerSec=5.282010828108723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2535/5000 [8:39:00<7:33:35, 11.04s/it][2022-12-20 19:01:30,821] [INFO] [timer.py:197:stop] 0/5132, RunningAvgSamplesPerSec=5.872198801741341, CurrSamplesPerSec=5.310039554965101, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2536/5000 [8:39:11<7:33:12, 11.04s/it][2022-12-20 19:01:41,820] [INFO] [timer.py:197:stop] 0/5134, RunningAvgSamplesPerSec=5.872203713309893, CurrSamplesPerSec=5.329619135046385, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2537/5000 [8:39:22<7:33:00, 11.04s/it][2022-12-20 19:01:52,889] [INFO] [timer.py:197:stop] 0/5136, RunningAvgSamplesPerSec=5.87219410798, CurrSamplesPerSec=5.299740208828777, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2538/5000 [8:39:33<7:33:04, 11.04s/it][2022-12-20 19:02:04,005] [INFO] [timer.py:197:stop] 0/5138, RunningAvgSamplesPerSec=5.872174337225317, CurrSamplesPerSec=5.2465735057703515, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2539/5000 [8:39:44<7:33:49, 11.06s/it][2022-12-20 19:02:15,037] [INFO] [logging.py:68:log_dist] [Rank 0] step=2570, skipped=3, lr=[5.4088888888888894e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:02:15,038] [INFO] [timer.py:197:stop] 0/5140, RunningAvgSamplesPerSec=5.872172191848566, CurrSamplesPerSec=5.320590128524832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2540/5000 [8:39:55<7:32:55, 11.05s/it][2022-12-20 19:02:26,013] [INFO] [timer.py:197:stop] 0/5142, RunningAvgSamplesPerSec=5.872182161851613, CurrSamplesPerSec=5.348704323983782, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2541/5000 [8:40:06<7:32:19, 11.04s/it][2022-12-20 19:02:37,052] [INFO] [timer.py:197:stop] 0/5144, RunningAvgSamplesPerSec=5.872179133580794, CurrSamplesPerSec=5.3260254994830625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2542/5000 [8:40:17<7:31:41, 11.03s/it][2022-12-20 19:02:47,132] [INFO] [timer.py:197:stop] 0/5146, RunningAvgSamplesPerSec=5.8723747005393445, CurrSamplesPerSec=6.277143622139676, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 19:02:58,124] [INFO] [timer.py:197:stop] 0/5148, RunningAvgSamplesPerSec=5.872380494293966, CurrSamplesPerSec=5.333861297309065, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2543/5000 [8:40:33<8:34:23, 12.56s/it][2022-12-20 19:03:09,173] [INFO] [timer.py:197:stop] 0/5150, RunningAvgSamplesPerSec=5.872375238368492, CurrSamplesPerSec=5.320558280384848, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2544/5000 [8:40:44<8:15:36, 12.11s/it][2022-12-20 19:03:20,092] [INFO] [timer.py:197:stop] 0/5152, RunningAvgSamplesPerSec=5.872396955559943, CurrSamplesPerSec=5.382331154271723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2545/5000 [8:40:55<8:00:48, 11.75s/it][2022-12-20 19:03:31,119] [INFO] [timer.py:197:stop] 0/5154, RunningAvgSamplesPerSec=5.872396192332469, CurrSamplesPerSec=5.3003130325775345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2546/5000 [8:41:06<7:51:44, 11.53s/it][2022-12-20 19:03:42,106] [INFO] [timer.py:197:stop] 0/5156, RunningAvgSamplesPerSec=5.872406038616784, CurrSamplesPerSec=5.374869580518407, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2547/5000 [8:41:17<7:44:50, 11.37s/it][2022-12-20 19:03:53,135] [INFO] [timer.py:197:stop] 0/5158, RunningAvgSamplesPerSec=5.872404705316606, CurrSamplesPerSec=5.298845118831041, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2548/5000 [8:41:28<7:40:27, 11.27s/it][2022-12-20 19:04:04,140] [INFO] [logging.py:68:log_dist] [Rank 0] step=2580, skipped=3, lr=[5.386666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:04:04,140] [INFO] [timer.py:197:stop] 0/5160, RunningAvgSamplesPerSec=5.87240817226454, CurrSamplesPerSec=5.335408063861249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2549/5000 [8:41:39<7:37:04, 11.19s/it][2022-12-20 19:04:15,150] [INFO] [timer.py:197:stop] 0/5162, RunningAvgSamplesPerSec=5.872412520415067, CurrSamplesPerSec=5.345415296852599, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2550/5000 [8:41:50<7:34:40, 11.13s/it]
                                                       
{'loss': 0.0002, 'learning_rate': 5.3844444444444445e-06, 'epoch': 62.19}
+
 51%|█████     | 2550/5000 [8:41:50<7:34:40, 11.13s/it][2022-12-20 19:04:26,209] [INFO] [timer.py:197:stop] 0/5164, RunningAvgSamplesPerSec=5.872405017978086, CurrSamplesPerSec=5.287594459931749, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2551/5000 [8:42:01<7:33:33, 11.11s/it][2022-12-20 19:04:37,237] [INFO] [timer.py:197:stop] 0/5166, RunningAvgSamplesPerSec=5.872403584942668, CurrSamplesPerSec=5.296938777725091, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2552/5000 [8:42:12<7:32:21, 11.09s/it][2022-12-20 19:04:48,203] [INFO] [timer.py:197:stop] 0/5168, RunningAvgSamplesPerSec=5.872415469603348, CurrSamplesPerSec=5.380284910590834, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2553/5000 [8:42:23<7:30:41, 11.05s/it][2022-12-20 19:04:59,223] [INFO] [timer.py:197:stop] 0/5170, RunningAvgSamplesPerSec=5.872416055550142, CurrSamplesPerSec=5.319675542653149, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2554/5000 [8:42:34<7:30:07, 11.04s/it][2022-12-20 19:05:10,222] [INFO] [timer.py:197:stop] 0/5172, RunningAvgSamplesPerSec=5.872420990808128, CurrSamplesPerSec=5.322553849139647, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2555/5000 [8:42:45<7:29:25, 11.03s/it][2022-12-20 19:05:21,240] [INFO] [timer.py:197:stop] 0/5174, RunningAvgSamplesPerSec=5.8724217157010745, CurrSamplesPerSec=5.329474805555454, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2556/5000 [8:42:56<7:29:06, 11.03s/it][2022-12-20 19:05:32,257] [INFO] [timer.py:197:stop] 0/5176, RunningAvgSamplesPerSec=5.8724226704020595, CurrSamplesPerSec=5.335125783622286, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2557/5000 [8:43:07<7:28:49, 11.02s/it][2022-12-20 19:05:43,208] [INFO] [timer.py:197:stop] 0/5178, RunningAvgSamplesPerSec=5.872437516115028, CurrSamplesPerSec=5.348096486521649, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2558/5000 [8:43:18<7:27:45, 11.00s/it][2022-12-20 19:05:54,247] [INFO] [logging.py:68:log_dist] [Rank 0] step=2590, skipped=3, lr=[5.364444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:05:54,249] [INFO] [timer.py:197:stop] 0/5180, RunningAvgSamplesPerSec=5.8724337866150425, CurrSamplesPerSec=5.324551337862783, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2559/5000 [8:43:29<7:28:03, 11.01s/it][2022-12-20 19:06:05,223] [INFO] [timer.py:197:stop] 0/5182, RunningAvgSamplesPerSec=5.872443655345555, CurrSamplesPerSec=5.3557300438118265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2560/5000 [8:43:40<7:27:24, 11.00s/it][2022-12-20 19:06:16,268] [INFO] [timer.py:197:stop] 0/5184, RunningAvgSamplesPerSec=5.872438911687643, CurrSamplesPerSec=5.328012693444929, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2561/5000 [8:43:51<7:27:44, 11.01s/it][2022-12-20 19:06:27,264] [INFO] [timer.py:197:stop] 0/5186, RunningAvgSamplesPerSec=5.87244440592142, CurrSamplesPerSec=5.323439865013492, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████     | 2562/5000 [8:44:02<7:27:19, 11.01s/it][2022-12-20 19:06:38,231] [INFO] [timer.py:197:stop] 0/5188, RunningAvgSamplesPerSec=5.872455876510898, CurrSamplesPerSec=5.331492529766117, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2563/5000 [8:44:13<7:26:38, 11.00s/it][2022-12-20 19:06:49,232] [INFO] [timer.py:197:stop] 0/5190, RunningAvgSamplesPerSec=5.872460355966199, CurrSamplesPerSec=5.335391096511791, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2564/5000 [8:44:24<7:26:30, 11.00s/it][2022-12-20 19:07:00,215] [INFO] [timer.py:197:stop] 0/5192, RunningAvgSamplesPerSec=5.87246847264832, CurrSamplesPerSec=5.361601994859694, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2565/5000 [8:44:35<7:26:08, 10.99s/it][2022-12-20 19:07:11,199] [INFO] [timer.py:197:stop] 0/5194, RunningAvgSamplesPerSec=5.8724761275646, CurrSamplesPerSec=5.316681352290014, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2566/5000 [8:44:46<7:25:51, 10.99s/it][2022-12-20 19:07:22,196] [INFO] [timer.py:197:stop] 0/5196, RunningAvgSamplesPerSec=5.8724814691814355, CurrSamplesPerSec=5.35660875508661, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2567/5000 [8:44:57<7:25:44, 10.99s/it][2022-12-20 19:07:33,216] [INFO] [timer.py:197:stop] 0/5198, RunningAvgSamplesPerSec=5.87248225048813, CurrSamplesPerSec=5.334542018516444, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2568/5000 [8:45:08<7:25:53, 11.00s/it][2022-12-20 19:07:44,242] [INFO] [logging.py:68:log_dist] [Rank 0] step=2600, skipped=3, lr=[5.342222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:07:44,243] [INFO] [timer.py:197:stop] 0/5200, RunningAvgSamplesPerSec=5.8724811897992995, CurrSamplesPerSec=5.299574265855183, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2569/5000 [8:45:19<7:26:02, 11.01s/it][2022-12-20 19:07:55,241] [INFO] [timer.py:197:stop] 0/5202, RunningAvgSamplesPerSec=5.872486257874632, CurrSamplesPerSec=5.350744317000331, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2570/5000 [8:45:30<7:25:43, 11.01s/it][2022-12-20 19:08:06,215] [INFO] [timer.py:197:stop] 0/5204, RunningAvgSamplesPerSec=5.872496423513133, CurrSamplesPerSec=5.332045336421326, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2571/5000 [8:45:41<7:25:08, 11.00s/it][2022-12-20 19:08:17,197] [INFO] [timer.py:197:stop] 0/5206, RunningAvgSamplesPerSec=5.872504351093058, CurrSamplesPerSec=5.355481509111017, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2572/5000 [8:45:52<7:24:48, 10.99s/it][2022-12-20 19:08:28,194] [INFO] [timer.py:197:stop] 0/5208, RunningAvgSamplesPerSec=5.872509629874759, CurrSamplesPerSec=5.308374349478798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2573/5000 [8:46:03<7:24:41, 10.99s/it][2022-12-20 19:08:39,237] [INFO] [timer.py:197:stop] 0/5210, RunningAvgSamplesPerSec=5.872505283142606, CurrSamplesPerSec=5.310614817538794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 51%|█████▏    | 2574/5000 [8:46:14<7:25:05, 11.01s/it][2022-12-20 19:08:50,202] [INFO] [timer.py:197:stop] 0/5212, RunningAvgSamplesPerSec=5.872516952704419, CurrSamplesPerSec=5.33588764820488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2575/5000 [8:46:25<7:24:23, 11.00s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 5.328888888888889e-06, 'epoch': 62.8}
+
 52%|█████▏    | 2575/5000 [8:46:25<7:24:23, 11.00s/it][2022-12-20 19:09:01,234] [INFO] [timer.py:197:stop] 0/5214, RunningAvgSamplesPerSec=5.87251514521287, CurrSamplesPerSec=5.318851271947023, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2576/5000 [8:46:36<7:24:39, 11.01s/it][2022-12-20 19:09:12,298] [INFO] [timer.py:197:stop] 0/5216, RunningAvgSamplesPerSec=5.8725063446183485, CurrSamplesPerSec=5.322440294925763, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2577/5000 [8:46:47<7:25:10, 11.02s/it][2022-12-20 19:09:23,330] [INFO] [timer.py:197:stop] 0/5218, RunningAvgSamplesPerSec=5.872504532882617, CurrSamplesPerSec=5.313965497677407, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2578/5000 [8:46:58<7:25:05, 11.03s/it][2022-12-20 19:09:34,319] [INFO] [logging.py:68:log_dist] [Rank 0] step=2610, skipped=3, lr=[5.320000000000001e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:09:34,320] [INFO] [timer.py:197:stop] 0/5220, RunningAvgSamplesPerSec=5.8725111269640085, CurrSamplesPerSec=5.316674402290827, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2579/5000 [8:47:09<7:24:28, 11.02s/it][2022-12-20 19:09:45,320] [INFO] [timer.py:197:stop] 0/5222, RunningAvgSamplesPerSec=5.87251572552731, CurrSamplesPerSec=5.358455821953807, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2580/5000 [8:47:20<7:24:05, 11.01s/it][2022-12-20 19:09:56,348] [INFO] [timer.py:197:stop] 0/5224, RunningAvgSamplesPerSec=5.872514471085712, CurrSamplesPerSec=5.319845909731444, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2581/5000 [8:47:31<7:24:07, 11.02s/it][2022-12-20 19:10:07,296] [INFO] [timer.py:197:stop] 0/5226, RunningAvgSamplesPerSec=5.872529894701253, CurrSamplesPerSec=5.358852475464828, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2582/5000 [8:47:42<7:23:07, 11.00s/it][2022-12-20 19:10:18,297] [INFO] [timer.py:197:stop] 0/5228, RunningAvgSamplesPerSec=5.8725341969919205, CurrSamplesPerSec=5.340574758548974, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2583/5000 [8:47:53<7:23:00, 11.00s/it][2022-12-20 19:10:28,414] [INFO] [timer.py:197:stop] 0/5230, RunningAvgSamplesPerSec=5.872718591619811, CurrSamplesPerSec=5.321307761866535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2584/5000 [8:48:08<8:11:53, 12.22s/it][2022-12-20 19:10:39,413] [INFO] [timer.py:197:stop] 0/5232, RunningAvgSamplesPerSec=5.872723067245804, CurrSamplesPerSec=5.339408156646932, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2585/5000 [8:48:19<7:56:52, 11.85s/it][2022-12-20 19:10:50,388] [INFO] [timer.py:197:stop] 0/5234, RunningAvgSamplesPerSec=5.8727324689966585, CurrSamplesPerSec=5.352125238820002, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2586/5000 [8:48:30<7:46:27, 11.59s/it][2022-12-20 19:11:01,435] [INFO] [timer.py:197:stop] 0/5236, RunningAvgSamplesPerSec=5.872727255673875, CurrSamplesPerSec=5.312134462998311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2587/5000 [8:48:41<7:39:45, 11.43s/it][2022-12-20 19:11:12,486] [INFO] [timer.py:197:stop] 0/5238, RunningAvgSamplesPerSec=5.872721115861741, CurrSamplesPerSec=5.314858548002951, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2588/5000 [8:48:53<7:34:45, 11.31s/it][2022-12-20 19:11:23,474] [INFO] [logging.py:68:log_dist] [Rank 0] step=2620, skipped=3, lr=[5.297777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:11:23,475] [INFO] [timer.py:197:stop] 0/5240, RunningAvgSamplesPerSec=5.872727751635108, CurrSamplesPerSec=5.35398736801392, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2589/5000 [8:49:04<7:30:46, 11.22s/it][2022-12-20 19:11:34,484] [INFO] [timer.py:197:stop] 0/5242, RunningAvgSamplesPerSec=5.872730459308229, CurrSamplesPerSec=5.344973801738098, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2590/5000 [8:49:14<7:27:29, 11.14s/it][2022-12-20 19:11:45,492] [INFO] [timer.py:197:stop] 0/5244, RunningAvgSamplesPerSec=5.872733275261813, CurrSamplesPerSec=5.302214891401318, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2591/5000 [8:49:26<7:26:24, 11.12s/it][2022-12-20 19:11:56,550] [INFO] [timer.py:197:stop] 0/5246, RunningAvgSamplesPerSec=5.872725738328682, CurrSamplesPerSec=5.3096190072228975, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2592/5000 [8:49:37<7:25:14, 11.09s/it][2022-12-20 19:12:07,540] [INFO] [timer.py:197:stop] 0/5248, RunningAvgSamplesPerSec=5.872732196179646, CurrSamplesPerSec=5.351492512348972, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2593/5000 [8:49:48<7:23:21, 11.05s/it][2022-12-20 19:12:18,566] [INFO] [timer.py:197:stop] 0/5250, RunningAvgSamplesPerSec=5.872731112776292, CurrSamplesPerSec=5.286961695310178, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2594/5000 [8:49:59<7:23:28, 11.06s/it][2022-12-20 19:12:29,613] [INFO] [timer.py:197:stop] 0/5252, RunningAvgSamplesPerSec=5.872725859377988, CurrSamplesPerSec=5.312414105216711, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2595/5000 [8:50:10<7:23:11, 11.06s/it][2022-12-20 19:12:40,654] [INFO] [timer.py:197:stop] 0/5254, RunningAvgSamplesPerSec=5.8727218447482805, CurrSamplesPerSec=5.322010183681454, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2596/5000 [8:50:21<7:22:45, 11.05s/it][2022-12-20 19:12:51,641] [INFO] [timer.py:197:stop] 0/5256, RunningAvgSamplesPerSec=5.8727288260653765, CurrSamplesPerSec=5.364719056554497, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2597/5000 [8:50:32<7:22:07, 11.04s/it][2022-12-20 19:13:02,699] [INFO] [timer.py:197:stop] 0/5258, RunningAvgSamplesPerSec=5.872721269421697, CurrSamplesPerSec=5.325807820713338, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2598/5000 [8:50:43<7:21:46, 11.04s/it][2022-12-20 19:13:13,727] [INFO] [logging.py:68:log_dist] [Rank 0] step=2630, skipped=3, lr=[5.275555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:13:13,729] [INFO] [timer.py:197:stop] 0/5260, RunningAvgSamplesPerSec=5.872719315890642, CurrSamplesPerSec=5.320843872690054, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2599/5000 [8:50:54<7:20:56, 11.02s/it][2022-12-20 19:13:24,684] [INFO] [timer.py:197:stop] 0/5262, RunningAvgSamplesPerSec=5.872732633905665, CurrSamplesPerSec=5.3437231530299885, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2600/5000 [8:51:05<7:19:59, 11.00s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 5.273333333333333e-06, 'epoch': 63.41}
+
 52%|█████▏    | 2600/5000 [8:51:05<7:19:59, 11.00s/it][2022-12-20 19:13:35,609] [INFO] [timer.py:197:stop] 0/5264, RunningAvgSamplesPerSec=5.872752614533249, CurrSamplesPerSec=5.3727328670988275, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2601/5000 [8:51:16<7:18:54, 10.98s/it][2022-12-20 19:13:46,582] [INFO] [timer.py:197:stop] 0/5266, RunningAvgSamplesPerSec=5.872762270962602, CurrSamplesPerSec=5.327367892773217, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2602/5000 [8:51:27<7:18:38, 10.98s/it][2022-12-20 19:13:57,509] [INFO] [timer.py:197:stop] 0/5268, RunningAvgSamplesPerSec=5.8727816184647095, CurrSamplesPerSec=5.3667060506336615, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2603/5000 [8:51:38<7:18:11, 10.97s/it][2022-12-20 19:14:08,523] [INFO] [timer.py:197:stop] 0/5270, RunningAvgSamplesPerSec=5.872783312205159, CurrSamplesPerSec=5.313508146566945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2604/5000 [8:51:49<7:18:27, 10.98s/it][2022-12-20 19:14:19,507] [INFO] [timer.py:197:stop] 0/5272, RunningAvgSamplesPerSec=5.872791127127035, CurrSamplesPerSec=5.332071391057764, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2605/5000 [8:52:00<7:18:46, 10.99s/it][2022-12-20 19:14:30,545] [INFO] [timer.py:197:stop] 0/5274, RunningAvgSamplesPerSec=5.872787866177656, CurrSamplesPerSec=5.318367158356488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2606/5000 [8:52:11<7:18:43, 11.00s/it][2022-12-20 19:14:41,566] [INFO] [timer.py:197:stop] 0/5276, RunningAvgSamplesPerSec=5.87278749320739, CurrSamplesPerSec=5.300960930956442, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2607/5000 [8:52:22<7:19:07, 11.01s/it][2022-12-20 19:14:52,597] [INFO] [timer.py:197:stop] 0/5278, RunningAvgSamplesPerSec=5.872785592058974, CurrSamplesPerSec=5.3137071498402895, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2608/5000 [8:52:33<7:19:23, 11.02s/it][2022-12-20 19:15:03,614] [INFO] [logging.py:68:log_dist] [Rank 0] step=2640, skipped=3, lr=[5.2533333333333336e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:15:03,615] [INFO] [timer.py:197:stop] 0/5280, RunningAvgSamplesPerSec=5.872786121908367, CurrSamplesPerSec=5.339306201465275, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2609/5000 [8:52:44<7:18:59, 11.02s/it][2022-12-20 19:15:14,644] [INFO] [timer.py:197:stop] 0/5282, RunningAvgSamplesPerSec=5.8727846037916835, CurrSamplesPerSec=5.317423000553502, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2610/5000 [8:52:55<7:18:51, 11.02s/it][2022-12-20 19:15:25,607] [INFO] [timer.py:197:stop] 0/5284, RunningAvgSamplesPerSec=5.872796393456776, CurrSamplesPerSec=5.368255609766087, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2611/5000 [8:53:06<7:17:50, 11.00s/it][2022-12-20 19:15:36,616] [INFO] [timer.py:197:stop] 0/5286, RunningAvgSamplesPerSec=5.872798888387285, CurrSamplesPerSec=5.314350541477422, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2612/5000 [8:53:17<7:18:15, 11.01s/it][2022-12-20 19:15:47,606] [INFO] [timer.py:197:stop] 0/5288, RunningAvgSamplesPerSec=5.872805041662639, CurrSamplesPerSec=5.364112719093264, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2613/5000 [8:53:28<7:17:30, 11.00s/it][2022-12-20 19:15:58,593] [INFO] [timer.py:197:stop] 0/5290, RunningAvgSamplesPerSec=5.872811978991787, CurrSamplesPerSec=5.343599971223219, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2614/5000 [8:53:39<7:16:50, 10.99s/it][2022-12-20 19:16:09,580] [INFO] [timer.py:197:stop] 0/5292, RunningAvgSamplesPerSec=5.872818754723418, CurrSamplesPerSec=5.316514978510569, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2615/5000 [8:53:50<7:17:30, 11.01s/it][2022-12-20 19:16:20,629] [INFO] [timer.py:197:stop] 0/5294, RunningAvgSamplesPerSec=5.872813005560405, CurrSamplesPerSec=5.322860343063403, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2616/5000 [8:54:01<7:17:13, 11.00s/it][2022-12-20 19:16:31,603] [INFO] [timer.py:197:stop] 0/5296, RunningAvgSamplesPerSec=5.872822677605095, CurrSamplesPerSec=5.345316518227611, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2617/5000 [8:54:12<7:16:55, 11.00s/it][2022-12-20 19:16:42,569] [INFO] [timer.py:197:stop] 0/5298, RunningAvgSamplesPerSec=5.872833789152779, CurrSamplesPerSec=5.369290721570961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2618/5000 [8:54:23<7:16:29, 10.99s/it][2022-12-20 19:16:53,592] [INFO] [logging.py:68:log_dist] [Rank 0] step=2650, skipped=3, lr=[5.231111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:16:53,593] [INFO] [timer.py:197:stop] 0/5300, RunningAvgSamplesPerSec=5.872833023221077, CurrSamplesPerSec=5.32964199145988, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2619/5000 [8:54:34<7:16:07, 10.99s/it][2022-12-20 19:17:04,583] [INFO] [timer.py:197:stop] 0/5302, RunningAvgSamplesPerSec=5.872839567170281, CurrSamplesPerSec=5.321767089725632, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2620/5000 [8:54:45<7:16:04, 10.99s/it][2022-12-20 19:17:15,561] [INFO] [timer.py:197:stop] 0/5304, RunningAvgSamplesPerSec=5.872848409600101, CurrSamplesPerSec=5.342748278693963, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2621/5000 [8:54:56<7:15:54, 10.99s/it][2022-12-20 19:17:26,564] [INFO] [timer.py:197:stop] 0/5306, RunningAvgSamplesPerSec=5.87285184991018, CurrSamplesPerSec=5.3343825814635855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2622/5000 [8:55:07<7:16:09, 11.00s/it][2022-12-20 19:17:37,613] [INFO] [timer.py:197:stop] 0/5308, RunningAvgSamplesPerSec=5.87284632523787, CurrSamplesPerSec=5.3181749705221035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2623/5000 [8:55:18<7:16:17, 11.01s/it][2022-12-20 19:17:48,596] [INFO] [timer.py:197:stop] 0/5310, RunningAvgSamplesPerSec=5.872853908669255, CurrSamplesPerSec=5.3613664070235965, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▏    | 2624/5000 [8:55:29<7:16:02, 11.01s/it][2022-12-20 19:17:58,745] [INFO] [timer.py:197:stop] 0/5312, RunningAvgSamplesPerSec=5.873028763706414, CurrSamplesPerSec=6.2490212401411664, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 19:18:09,793] [INFO] [timer.py:197:stop] 0/5314, RunningAvgSamplesPerSec=5.873022553566531, CurrSamplesPerSec=5.292588362031317, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 52%|█████▎    | 2625/5000 [8:55:45<8:17:44, 12.57s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 5.215555555555556e-06, 'epoch': 64.02}
+
 52%|█████▎    | 2625/5000 [8:55:45<8:17:44, 12.57s/it][2022-12-20 19:18:20,859] [INFO] [timer.py:197:stop] 0/5316, RunningAvgSamplesPerSec=5.873013301741487, CurrSamplesPerSec=5.29528491123225, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2626/5000 [8:55:56<7:59:37, 12.12s/it][2022-12-20 19:18:31,797] [INFO] [timer.py:197:stop] 0/5318, RunningAvgSamplesPerSec=5.873030148960379, CurrSamplesPerSec=5.369022456150541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2627/5000 [8:56:07<7:45:22, 11.77s/it][2022-12-20 19:18:42,872] [INFO] [logging.py:68:log_dist] [Rank 0] step=2660, skipped=3, lr=[5.208888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:18:42,874] [INFO] [timer.py:197:stop] 0/5320, RunningAvgSamplesPerSec=5.873018633455563, CurrSamplesPerSec=5.24457424332001, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2628/5000 [8:56:18<7:36:59, 11.56s/it][2022-12-20 19:18:53,901] [INFO] [timer.py:197:stop] 0/5322, RunningAvgSamplesPerSec=5.873017277272148, CurrSamplesPerSec=5.3174409071449436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2629/5000 [8:56:29<7:30:29, 11.40s/it][2022-12-20 19:19:04,963] [INFO] [timer.py:197:stop] 0/5324, RunningAvgSamplesPerSec=5.873008760019579, CurrSamplesPerSec=5.296574855184251, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2630/5000 [8:56:40<7:26:18, 11.30s/it][2022-12-20 19:19:16,039] [INFO] [timer.py:197:stop] 0/5326, RunningAvgSamplesPerSec=5.872997820805419, CurrSamplesPerSec=5.304639472268364, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2631/5000 [8:56:51<7:23:27, 11.23s/it][2022-12-20 19:19:27,078] [INFO] [timer.py:197:stop] 0/5328, RunningAvgSamplesPerSec=5.872994519269735, CurrSamplesPerSec=5.322823612591185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2632/5000 [8:57:02<7:20:59, 11.17s/it][2022-12-20 19:19:38,113] [INFO] [timer.py:197:stop] 0/5330, RunningAvgSamplesPerSec=5.8729917629827995, CurrSamplesPerSec=5.332183450079617, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2633/5000 [8:57:13<7:19:10, 11.13s/it][2022-12-20 19:19:49,105] [INFO] [timer.py:197:stop] 0/5332, RunningAvgSamplesPerSec=5.87299790650301, CurrSamplesPerSec=5.298354809692404, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2634/5000 [8:57:24<7:17:19, 11.09s/it][2022-12-20 19:20:00,116] [INFO] [timer.py:197:stop] 0/5334, RunningAvgSamplesPerSec=5.872999848092622, CurrSamplesPerSec=5.341615165890961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2635/5000 [8:57:35<7:16:12, 11.07s/it][2022-12-20 19:20:11,157] [INFO] [timer.py:197:stop] 0/5336, RunningAvgSamplesPerSec=5.8729957919291245, CurrSamplesPerSec=5.308615381536392, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2636/5000 [8:57:46<7:15:43, 11.06s/it][2022-12-20 19:20:22,228] [INFO] [timer.py:197:stop] 0/5338, RunningAvgSamplesPerSec=5.8729860320114735, CurrSamplesPerSec=5.309025060272417, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2637/5000 [8:57:57<7:15:40, 11.06s/it][2022-12-20 19:20:33,212] [INFO] [logging.py:68:log_dist] [Rank 0] step=2670, skipped=3, lr=[5.186666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:20:33,214] [INFO] [timer.py:197:stop] 0/5340, RunningAvgSamplesPerSec=5.872993424695448, CurrSamplesPerSec=5.3231822841240914, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2638/5000 [8:58:08<7:14:35, 11.04s/it][2022-12-20 19:20:44,225] [INFO] [timer.py:197:stop] 0/5342, RunningAvgSamplesPerSec=5.872995579110673, CurrSamplesPerSec=5.313117966714269, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2639/5000 [8:58:19<7:14:04, 11.03s/it][2022-12-20 19:20:55,207] [INFO] [timer.py:197:stop] 0/5344, RunningAvgSamplesPerSec=5.873003614027115, CurrSamplesPerSec=5.349213163734925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2640/5000 [8:58:30<7:13:18, 11.02s/it][2022-12-20 19:21:06,238] [INFO] [timer.py:197:stop] 0/5346, RunningAvgSamplesPerSec=5.873001452918266, CurrSamplesPerSec=5.304556241183625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2641/5000 [8:58:41<7:13:17, 11.02s/it][2022-12-20 19:21:17,291] [INFO] [timer.py:197:stop] 0/5348, RunningAvgSamplesPerSec=5.872994980954503, CurrSamplesPerSec=5.302913328929573, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2642/5000 [8:58:52<7:13:29, 11.03s/it][2022-12-20 19:21:28,334] [INFO] [timer.py:197:stop] 0/5350, RunningAvgSamplesPerSec=5.872990571074898, CurrSamplesPerSec=5.312529124508324, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2643/5000 [8:59:03<7:13:27, 11.03s/it][2022-12-20 19:21:39,400] [INFO] [timer.py:197:stop] 0/5352, RunningAvgSamplesPerSec=5.8729814620583305, CurrSamplesPerSec=5.300411201504138, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2644/5000 [8:59:14<7:13:38, 11.04s/it][2022-12-20 19:21:50,400] [INFO] [timer.py:197:stop] 0/5354, RunningAvgSamplesPerSec=5.872985804174806, CurrSamplesPerSec=5.3324870287103385, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2645/5000 [8:59:25<7:12:56, 11.03s/it][2022-12-20 19:22:01,447] [INFO] [timer.py:197:stop] 0/5356, RunningAvgSamplesPerSec=5.872980512400144, CurrSamplesPerSec=5.290296556057971, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2646/5000 [8:59:37<7:12:57, 11.04s/it][2022-12-20 19:22:12,424] [INFO] [timer.py:197:stop] 0/5358, RunningAvgSamplesPerSec=5.872989312307629, CurrSamplesPerSec=5.353349290216511, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2647/5000 [8:59:48<7:12:05, 11.02s/it][2022-12-20 19:22:23,405] [INFO] [logging.py:68:log_dist] [Rank 0] step=2680, skipped=3, lr=[5.164444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:22:23,407] [INFO] [timer.py:197:stop] 0/5360, RunningAvgSamplesPerSec=5.872997299604377, CurrSamplesPerSec=5.3312633921859645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2648/5000 [8:59:59<7:11:29, 11.01s/it][2022-12-20 19:22:34,445] [INFO] [timer.py:197:stop] 0/5362, RunningAvgSamplesPerSec=5.872993868370962, CurrSamplesPerSec=5.305449906368411, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2649/5000 [9:00:10<7:11:40, 11.02s/it][2022-12-20 19:22:45,443] [INFO] [timer.py:197:stop] 0/5364, RunningAvgSamplesPerSec=5.87299853236085, CurrSamplesPerSec=5.331731853442185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2650/5000 [9:00:21<7:11:16, 11.01s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 5.1600000000000006e-06, 'epoch': 64.63}
+
 53%|█████▎    | 2650/5000 [9:00:21<7:11:16, 11.01s/it][2022-12-20 19:22:56,439] [INFO] [timer.py:197:stop] 0/5366, RunningAvgSamplesPerSec=5.873003617885527, CurrSamplesPerSec=5.319256629427013, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2651/5000 [9:00:32<7:10:54, 11.01s/it][2022-12-20 19:23:07,423] [INFO] [timer.py:197:stop] 0/5368, RunningAvgSamplesPerSec=5.873011208773891, CurrSamplesPerSec=5.34459665153082, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2652/5000 [9:00:43<7:10:27, 11.00s/it][2022-12-20 19:23:18,492] [INFO] [timer.py:197:stop] 0/5370, RunningAvgSamplesPerSec=5.873001512588647, CurrSamplesPerSec=5.290541162699469, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2653/5000 [9:00:54<7:11:05, 11.02s/it][2022-12-20 19:23:29,543] [INFO] [timer.py:197:stop] 0/5372, RunningAvgSamplesPerSec=5.872995480082908, CurrSamplesPerSec=5.304801958612517, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2654/5000 [9:01:05<7:11:15, 11.03s/it][2022-12-20 19:23:40,531] [INFO] [timer.py:197:stop] 0/5374, RunningAvgSamplesPerSec=5.873001862037056, CurrSamplesPerSec=5.329954593752459, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2655/5000 [9:01:16<7:10:35, 11.02s/it][2022-12-20 19:23:51,565] [INFO] [timer.py:197:stop] 0/5376, RunningAvgSamplesPerSec=5.872999266143301, CurrSamplesPerSec=5.328287664122435, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2656/5000 [9:01:27<7:10:35, 11.02s/it][2022-12-20 19:24:02,611] [INFO] [timer.py:197:stop] 0/5378, RunningAvgSamplesPerSec=5.872993929326478, CurrSamplesPerSec=5.3147495309620485, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2657/5000 [9:01:38<7:10:41, 11.03s/it][2022-12-20 19:24:13,686] [INFO] [logging.py:68:log_dist] [Rank 0] step=2690, skipped=3, lr=[5.1422222222222225e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:24:13,688] [INFO] [timer.py:197:stop] 0/5380, RunningAvgSamplesPerSec=5.87298256364583, CurrSamplesPerSec=5.278240500087736, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2658/5000 [9:01:49<7:11:03, 11.04s/it][2022-12-20 19:24:24,677] [INFO] [timer.py:197:stop] 0/5382, RunningAvgSamplesPerSec=5.872988673437353, CurrSamplesPerSec=5.343458074523034, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2659/5000 [9:02:00<7:10:15, 11.03s/it][2022-12-20 19:24:35,634] [INFO] [timer.py:197:stop] 0/5384, RunningAvgSamplesPerSec=5.873001617676269, CurrSamplesPerSec=5.353229507427144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2660/5000 [9:02:11<7:09:14, 11.01s/it][2022-12-20 19:24:46,706] [INFO] [timer.py:197:stop] 0/5386, RunningAvgSamplesPerSec=5.872991035787454, CurrSamplesPerSec=5.290466923339986, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2661/5000 [9:02:22<7:09:49, 11.03s/it][2022-12-20 19:24:57,775] [INFO] [timer.py:197:stop] 0/5388, RunningAvgSamplesPerSec=5.872981542600957, CurrSamplesPerSec=5.292540569733379, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2662/5000 [9:02:33<7:10:08, 11.04s/it][2022-12-20 19:25:08,835] [INFO] [timer.py:197:stop] 0/5390, RunningAvgSamplesPerSec=5.872973786269931, CurrSamplesPerSec=5.308869244711767, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2663/5000 [9:02:44<7:10:12, 11.05s/it][2022-12-20 19:25:19,883] [INFO] [timer.py:197:stop] 0/5392, RunningAvgSamplesPerSec=5.872968507954164, CurrSamplesPerSec=5.317162209985033, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2664/5000 [9:02:55<7:10:03, 11.05s/it][2022-12-20 19:25:30,920] [INFO] [timer.py:197:stop] 0/5394, RunningAvgSamplesPerSec=5.8729655928815925, CurrSamplesPerSec=5.3190289639543495, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2665/5000 [9:03:06<7:09:45, 11.04s/it][2022-12-20 19:25:41,046] [INFO] [timer.py:197:stop] 0/5396, RunningAvgSamplesPerSec=5.873142360979769, CurrSamplesPerSec=5.3161761557581215, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2666/5000 [9:03:21<7:56:42, 12.25s/it][2022-12-20 19:25:52,015] [INFO] [timer.py:197:stop] 0/5398, RunningAvgSamplesPerSec=5.873152696212998, CurrSamplesPerSec=5.3781859138350505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2667/5000 [9:03:32<7:40:53, 11.85s/it][2022-12-20 19:26:02,981] [INFO] [logging.py:68:log_dist] [Rank 0] step=2700, skipped=3, lr=[5.12e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:26:02,982] [INFO] [timer.py:197:stop] 0/5400, RunningAvgSamplesPerSec=5.873163369617966, CurrSamplesPerSec=5.333012706111894, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2668/5000 [9:03:43<7:30:40, 11.60s/it][2022-12-20 19:26:13,998] [INFO] [timer.py:197:stop] 0/5402, RunningAvgSamplesPerSec=5.873164243362551, CurrSamplesPerSec=5.313742702734375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2669/5000 [9:03:54<7:23:41, 11.42s/it][2022-12-20 19:26:24,956] [INFO] [timer.py:197:stop] 0/5404, RunningAvgSamplesPerSec=5.873176910215417, CurrSamplesPerSec=5.362685322830571, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2670/5000 [9:04:05<7:17:53, 11.28s/it][2022-12-20 19:26:35,948] [INFO] [timer.py:197:stop] 0/5406, RunningAvgSamplesPerSec=5.873182388507867, CurrSamplesPerSec=5.3148699129711945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2671/5000 [9:04:16<7:15:00, 11.21s/it][2022-12-20 19:26:46,990] [INFO] [timer.py:197:stop] 0/5408, RunningAvgSamplesPerSec=5.873177998257727, CurrSamplesPerSec=5.316868798515467, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2672/5000 [9:04:27<7:12:28, 11.15s/it][2022-12-20 19:26:57,989] [INFO] [timer.py:197:stop] 0/5410, RunningAvgSamplesPerSec=5.873182089474428, CurrSamplesPerSec=5.322459712775036, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2673/5000 [9:04:38<7:10:45, 11.11s/it][2022-12-20 19:27:08,981] [INFO] [timer.py:197:stop] 0/5412, RunningAvgSamplesPerSec=5.87318763531925, CurrSamplesPerSec=5.342952668647412, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 53%|█████▎    | 2674/5000 [9:04:49<7:09:02, 11.07s/it][2022-12-20 19:27:19,959] [INFO] [timer.py:197:stop] 0/5414, RunningAvgSamplesPerSec=5.873195970017013, CurrSamplesPerSec=5.340773244206991, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2675/5000 [9:05:00<7:08:15, 11.05s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 5.1044444444444455e-06, 'epoch': 65.24}
+
 54%|█████▎    | 2675/5000 [9:05:00<7:08:15, 11.05s/it][2022-12-20 19:27:31,007] [INFO] [timer.py:197:stop] 0/5416, RunningAvgSamplesPerSec=5.873190868120821, CurrSamplesPerSec=5.314474272314122, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2676/5000 [9:05:11<7:08:03, 11.05s/it][2022-12-20 19:27:42,050] [INFO] [timer.py:197:stop] 0/5418, RunningAvgSamplesPerSec=5.87318633516062, CurrSamplesPerSec=5.318414785998138, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2677/5000 [9:05:22<7:07:48, 11.05s/it][2022-12-20 19:27:53,120] [INFO] [logging.py:68:log_dist] [Rank 0] step=2710, skipped=3, lr=[5.0977777777777786e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:27:53,122] [INFO] [timer.py:197:stop] 0/5420, RunningAvgSamplesPerSec=5.873175921260329, CurrSamplesPerSec=5.294825547890776, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2678/5000 [9:05:33<7:07:23, 11.04s/it][2022-12-20 19:28:04,091] [INFO] [timer.py:197:stop] 0/5422, RunningAvgSamplesPerSec=5.873185786736362, CurrSamplesPerSec=5.347557178434134, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2679/5000 [9:05:44<7:06:40, 11.03s/it][2022-12-20 19:28:15,101] [INFO] [timer.py:197:stop] 0/5424, RunningAvgSamplesPerSec=5.8731878712614325, CurrSamplesPerSec=5.337774368911436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2680/5000 [9:05:55<7:06:00, 11.02s/it][2022-12-20 19:28:26,061] [INFO] [timer.py:197:stop] 0/5426, RunningAvgSamplesPerSec=5.873199868641912, CurrSamplesPerSec=5.362978670031498, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2681/5000 [9:06:06<7:04:47, 10.99s/it][2022-12-20 19:28:37,043] [INFO] [timer.py:197:stop] 0/5428, RunningAvgSamplesPerSec=5.873207361822999, CurrSamplesPerSec=5.31523930089995, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2682/5000 [9:06:17<7:04:34, 10.99s/it][2022-12-20 19:28:48,008] [INFO] [timer.py:197:stop] 0/5430, RunningAvgSamplesPerSec=5.873218476979349, CurrSamplesPerSec=5.335861132002502, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2683/5000 [9:06:28<7:04:18, 10.99s/it][2022-12-20 19:28:58,974] [INFO] [timer.py:197:stop] 0/5432, RunningAvgSamplesPerSec=5.873229123034713, CurrSamplesPerSec=5.3508159913250095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2684/5000 [9:06:39<7:03:36, 10.97s/it][2022-12-20 19:29:09,964] [INFO] [timer.py:197:stop] 0/5434, RunningAvgSamplesPerSec=5.873234898687209, CurrSamplesPerSec=5.307958053845484, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2685/5000 [9:06:50<7:04:04, 10.99s/it][2022-12-20 19:29:20,944] [INFO] [timer.py:197:stop] 0/5436, RunningAvgSamplesPerSec=5.873242852156995, CurrSamplesPerSec=5.353249577609812, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2686/5000 [9:07:01<7:03:44, 10.99s/it][2022-12-20 19:29:31,933] [INFO] [timer.py:197:stop] 0/5438, RunningAvgSamplesPerSec=5.873249210420044, CurrSamplesPerSec=5.344848220984026, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▎    | 2687/5000 [9:07:12<7:03:36, 10.99s/it][2022-12-20 19:29:42,926] [INFO] [logging.py:68:log_dist] [Rank 0] step=2720, skipped=3, lr=[5.075555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:29:42,927] [INFO] [timer.py:197:stop] 0/5440, RunningAvgSamplesPerSec=5.873254239134911, CurrSamplesPerSec=5.3420050776574, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2688/5000 [9:07:23<7:03:37, 10.99s/it][2022-12-20 19:29:53,919] [INFO] [timer.py:197:stop] 0/5442, RunningAvgSamplesPerSec=5.8732595932174725, CurrSamplesPerSec=5.35391048311184, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2689/5000 [9:07:34<7:03:16, 10.99s/it][2022-12-20 19:30:04,920] [INFO] [timer.py:197:stop] 0/5444, RunningAvgSamplesPerSec=5.873263264122879, CurrSamplesPerSec=5.33441311122531, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2690/5000 [9:07:45<7:02:46, 10.98s/it][2022-12-20 19:30:15,888] [INFO] [timer.py:197:stop] 0/5446, RunningAvgSamplesPerSec=5.873273570455514, CurrSamplesPerSec=5.32995946191834, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2691/5000 [9:07:56<7:02:59, 10.99s/it][2022-12-20 19:30:26,921] [INFO] [timer.py:197:stop] 0/5448, RunningAvgSamplesPerSec=5.873270970160009, CurrSamplesPerSec=5.314747215976855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2692/5000 [9:08:07<7:02:38, 10.99s/it][2022-12-20 19:30:37,853] [INFO] [timer.py:197:stop] 0/5450, RunningAvgSamplesPerSec=5.873288320206676, CurrSamplesPerSec=5.354985363510395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2693/5000 [9:08:18<7:02:32, 10.99s/it][2022-12-20 19:30:48,910] [INFO] [timer.py:197:stop] 0/5452, RunningAvgSamplesPerSec=5.873280674117597, CurrSamplesPerSec=5.298236638680514, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2694/5000 [9:08:29<7:03:22, 11.02s/it][2022-12-20 19:30:59,919] [INFO] [timer.py:197:stop] 0/5454, RunningAvgSamplesPerSec=5.87328285064709, CurrSamplesPerSec=5.360196056622872, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2695/5000 [9:08:40<7:02:34, 11.00s/it][2022-12-20 19:31:10,925] [INFO] [timer.py:197:stop] 0/5456, RunningAvgSamplesPerSec=5.8732855811203954, CurrSamplesPerSec=5.321309660624183, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2696/5000 [9:08:51<7:02:48, 11.01s/it][2022-12-20 19:31:21,995] [INFO] [timer.py:197:stop] 0/5458, RunningAvgSamplesPerSec=5.873276216977695, CurrSamplesPerSec=5.292462100408798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2697/5000 [9:09:02<7:03:06, 11.02s/it][2022-12-20 19:31:33,004] [INFO] [logging.py:68:log_dist] [Rank 0] step=2730, skipped=3, lr=[5.053333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:31:33,006] [INFO] [timer.py:197:stop] 0/5460, RunningAvgSamplesPerSec=5.873278132955492, CurrSamplesPerSec=5.328033844027922, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2698/5000 [9:09:13<7:02:25, 11.01s/it][2022-12-20 19:31:43,965] [INFO] [timer.py:197:stop] 0/5462, RunningAvgSamplesPerSec=5.873290029596865, CurrSamplesPerSec=5.345760412042352, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2699/5000 [9:09:24<7:02:05, 11.01s/it][2022-12-20 19:31:54,989] [INFO] [timer.py:197:stop] 0/5464, RunningAvgSamplesPerSec=5.87328938015648, CurrSamplesPerSec=5.322120976054332, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2700/5000 [9:09:35<7:02:09, 11.01s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 5.0488888888888895e-06, 'epoch': 65.84}
+
 54%|█████▍    | 2700/5000 [9:09:35<7:02:09, 11.01s/it][2022-12-20 19:32:06,017] [INFO] [timer.py:197:stop] 0/5466, RunningAvgSamplesPerSec=5.873288041484228, CurrSamplesPerSec=5.32390420697494, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2701/5000 [9:09:46<7:02:05, 11.02s/it][2022-12-20 19:32:17,026] [INFO] [timer.py:197:stop] 0/5468, RunningAvgSamplesPerSec=5.873290530472754, CurrSamplesPerSec=5.335470419797562, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2702/5000 [9:09:57<7:01:49, 11.01s/it][2022-12-20 19:32:28,003] [INFO] [timer.py:197:stop] 0/5470, RunningAvgSamplesPerSec=5.873298845353902, CurrSamplesPerSec=5.363090960433488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2703/5000 [9:10:08<7:01:16, 11.00s/it][2022-12-20 19:32:39,028] [INFO] [timer.py:197:stop] 0/5472, RunningAvgSamplesPerSec=5.873297875966904, CurrSamplesPerSec=5.32386007094475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2704/5000 [9:10:19<7:01:21, 11.01s/it][2022-12-20 19:32:50,057] [INFO] [timer.py:197:stop] 0/5474, RunningAvgSamplesPerSec=5.873297006721445, CurrSamplesPerSec=5.3260383916964775, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2705/5000 [9:10:30<7:01:19, 11.01s/it][2022-12-20 19:33:01,104] [INFO] [timer.py:197:stop] 0/5476, RunningAvgSamplesPerSec=5.8732918151526246, CurrSamplesPerSec=5.302880434971567, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2706/5000 [9:10:41<7:01:03, 11.01s/it][2022-12-20 19:33:11,167] [INFO] [timer.py:197:stop] 0/5478, RunningAvgSamplesPerSec=5.873478404380549, CurrSamplesPerSec=6.275279408249182, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 19:33:22,199] [INFO] [logging.py:68:log_dist] [Rank 0] step=2740, skipped=3, lr=[5.031111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:33:22,201] [INFO] [timer.py:197:stop] 0/5480, RunningAvgSamplesPerSec=5.87347490645655, CurrSamplesPerSec=5.310503663229394, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2707/5000 [9:10:57<8:00:12, 12.57s/it][2022-12-20 19:33:33,236] [INFO] [timer.py:197:stop] 0/5482, RunningAvgSamplesPerSec=5.873471734203726, CurrSamplesPerSec=5.320749374944079, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2708/5000 [9:11:08<7:42:27, 12.11s/it][2022-12-20 19:33:44,209] [INFO] [timer.py:197:stop] 0/5484, RunningAvgSamplesPerSec=5.873480807556759, CurrSamplesPerSec=5.371699437642947, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2709/5000 [9:11:19<7:29:16, 11.77s/it][2022-12-20 19:33:55,189] [INFO] [timer.py:197:stop] 0/5486, RunningAvgSamplesPerSec=5.8734888259086375, CurrSamplesPerSec=5.3289934110657455, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2710/5000 [9:11:30<7:20:04, 11.53s/it][2022-12-20 19:34:06,207] [INFO] [timer.py:197:stop] 0/5488, RunningAvgSamplesPerSec=5.873489169950928, CurrSamplesPerSec=5.325164609964567, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2711/5000 [9:11:41<7:14:01, 11.38s/it][2022-12-20 19:34:17,236] [INFO] [timer.py:197:stop] 0/5490, RunningAvgSamplesPerSec=5.873487250218843, CurrSamplesPerSec=5.33543584812903, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2712/5000 [9:11:52<7:09:51, 11.27s/it][2022-12-20 19:34:28,231] [INFO] [timer.py:197:stop] 0/5492, RunningAvgSamplesPerSec=5.873492282770016, CurrSamplesPerSec=5.319760724828281, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2713/5000 [9:12:03<7:06:29, 11.19s/it][2022-12-20 19:34:39,234] [INFO] [timer.py:197:stop] 0/5494, RunningAvgSamplesPerSec=5.873495409961955, CurrSamplesPerSec=5.341343494394558, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2714/5000 [9:12:14<7:04:10, 11.13s/it][2022-12-20 19:34:50,210] [INFO] [timer.py:197:stop] 0/5496, RunningAvgSamplesPerSec=5.873504014481066, CurrSamplesPerSec=5.3942010352523955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2715/5000 [9:12:25<7:02:11, 11.09s/it][2022-12-20 19:35:01,213] [INFO] [timer.py:197:stop] 0/5498, RunningAvgSamplesPerSec=5.873507213848976, CurrSamplesPerSec=5.323364699489747, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2716/5000 [9:12:36<7:01:03, 11.06s/it][2022-12-20 19:35:12,211] [INFO] [logging.py:68:log_dist] [Rank 0] step=2750, skipped=3, lr=[5.00888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:35:12,213] [INFO] [timer.py:197:stop] 0/5500, RunningAvgSamplesPerSec=5.873511101389101, CurrSamplesPerSec=5.32293718316199, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2717/5000 [9:12:47<7:00:10, 11.04s/it][2022-12-20 19:35:23,217] [INFO] [timer.py:197:stop] 0/5502, RunningAvgSamplesPerSec=5.873514018967947, CurrSamplesPerSec=5.358672112785867, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2718/5000 [9:12:58<6:59:33, 11.03s/it][2022-12-20 19:35:34,268] [INFO] [timer.py:197:stop] 0/5504, RunningAvgSamplesPerSec=5.87350800158869, CurrSamplesPerSec=5.313978962748543, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2719/5000 [9:13:09<6:59:35, 11.04s/it][2022-12-20 19:35:45,257] [INFO] [timer.py:197:stop] 0/5506, RunningAvgSamplesPerSec=5.873513930115178, CurrSamplesPerSec=5.360979659526894, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2720/5000 [9:13:20<6:58:52, 11.02s/it][2022-12-20 19:35:56,259] [INFO] [timer.py:197:stop] 0/5508, RunningAvgSamplesPerSec=5.873517267736895, CurrSamplesPerSec=5.338539962961313, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2721/5000 [9:13:31<6:58:26, 11.02s/it][2022-12-20 19:36:07,261] [INFO] [timer.py:197:stop] 0/5510, RunningAvgSamplesPerSec=5.873520697482273, CurrSamplesPerSec=5.341154105967314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2722/5000 [9:13:42<6:58:05, 11.01s/it][2022-12-20 19:36:18,197] [INFO] [timer.py:197:stop] 0/5512, RunningAvgSamplesPerSec=5.873536885184952, CurrSamplesPerSec=5.3780313996281155, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2723/5000 [9:13:53<6:57:02, 10.99s/it][2022-12-20 19:36:29,187] [INFO] [timer.py:197:stop] 0/5514, RunningAvgSamplesPerSec=5.8735426417794665, CurrSamplesPerSec=5.36003765108807, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 54%|█████▍    | 2724/5000 [9:14:04<6:56:52, 10.99s/it][2022-12-20 19:36:40,204] [INFO] [timer.py:197:stop] 0/5516, RunningAvgSamplesPerSec=5.873543391599133, CurrSamplesPerSec=5.30801200274857, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2725/5000 [9:14:15<6:56:59, 11.00s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.991111111111112e-06, 'epoch': 66.46}
+
 55%|█████▍    | 2725/5000 [9:14:15<6:56:59, 11.00s/it][2022-12-20 19:36:51,205] [INFO] [timer.py:197:stop] 0/5518, RunningAvgSamplesPerSec=5.873547739027401, CurrSamplesPerSec=5.347540133725031, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2726/5000 [9:14:26<6:56:50, 11.00s/it][2022-12-20 19:37:02,176] [INFO] [logging.py:68:log_dist] [Rank 0] step=2760, skipped=3, lr=[4.986666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:37:02,177] [INFO] [timer.py:197:stop] 0/5520, RunningAvgSamplesPerSec=5.873556916619805, CurrSamplesPerSec=5.35864750909103, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2727/5000 [9:14:37<6:56:22, 10.99s/it][2022-12-20 19:37:13,201] [INFO] [timer.py:197:stop] 0/5522, RunningAvgSamplesPerSec=5.873556369180423, CurrSamplesPerSec=5.342890137698586, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2728/5000 [9:14:48<6:56:33, 11.00s/it][2022-12-20 19:37:24,195] [INFO] [timer.py:197:stop] 0/5524, RunningAvgSamplesPerSec=5.873561513267471, CurrSamplesPerSec=5.3282817413780075, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2729/5000 [9:14:59<6:56:18, 11.00s/it][2022-12-20 19:37:35,216] [INFO] [timer.py:197:stop] 0/5526, RunningAvgSamplesPerSec=5.873561310421968, CurrSamplesPerSec=5.326268771278203, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2730/5000 [9:15:10<6:56:21, 11.01s/it][2022-12-20 19:37:46,285] [INFO] [timer.py:197:stop] 0/5528, RunningAvgSamplesPerSec=5.873551749990324, CurrSamplesPerSec=5.288224459023446, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2731/5000 [9:15:21<6:56:54, 11.02s/it][2022-12-20 19:37:57,350] [INFO] [timer.py:197:stop] 0/5530, RunningAvgSamplesPerSec=5.873542956496493, CurrSamplesPerSec=5.294936255731051, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2732/5000 [9:15:32<6:57:11, 11.04s/it][2022-12-20 19:38:08,326] [INFO] [timer.py:197:stop] 0/5532, RunningAvgSamplesPerSec=5.873551340368789, CurrSamplesPerSec=5.360058414523632, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2733/5000 [9:15:43<6:56:19, 11.02s/it][2022-12-20 19:38:19,330] [INFO] [timer.py:197:stop] 0/5534, RunningAvgSamplesPerSec=5.873554694401584, CurrSamplesPerSec=5.342884395133809, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2734/5000 [9:15:54<6:55:57, 11.01s/it][2022-12-20 19:38:30,356] [INFO] [timer.py:197:stop] 0/5536, RunningAvgSamplesPerSec=5.873553591374784, CurrSamplesPerSec=5.322258576051651, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2735/5000 [9:16:05<6:55:55, 11.02s/it][2022-12-20 19:38:41,339] [INFO] [timer.py:197:stop] 0/5538, RunningAvgSamplesPerSec=5.873560905327284, CurrSamplesPerSec=5.321241306202499, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2736/5000 [9:16:16<6:55:20, 11.01s/it][2022-12-20 19:38:52,328] [INFO] [logging.py:68:log_dist] [Rank 0] step=2770, skipped=3, lr=[4.964444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:38:52,329] [INFO] [timer.py:197:stop] 0/5540, RunningAvgSamplesPerSec=5.873566896299324, CurrSamplesPerSec=5.314301092839466, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2737/5000 [9:16:27<6:54:57, 11.00s/it][2022-12-20 19:39:03,302] [INFO] [timer.py:197:stop] 0/5542, RunningAvgSamplesPerSec=5.8735761746976, CurrSamplesPerSec=5.361571795625663, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2738/5000 [9:16:38<6:54:27, 10.99s/it][2022-12-20 19:39:14,345] [INFO] [timer.py:197:stop] 0/5544, RunningAvgSamplesPerSec=5.873572219641492, CurrSamplesPerSec=5.3165741558389445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2739/5000 [9:16:49<6:54:49, 11.01s/it][2022-12-20 19:39:25,320] [INFO] [timer.py:197:stop] 0/5546, RunningAvgSamplesPerSec=5.873581127670186, CurrSamplesPerSec=5.361694950665539, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2740/5000 [9:17:00<6:54:16, 11.00s/it][2022-12-20 19:39:36,334] [INFO] [timer.py:197:stop] 0/5548, RunningAvgSamplesPerSec=5.873582498875161, CurrSamplesPerSec=5.332232172862595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2741/5000 [9:17:11<6:54:15, 11.00s/it][2022-12-20 19:39:47,396] [INFO] [timer.py:197:stop] 0/5550, RunningAvgSamplesPerSec=5.873574789539594, CurrSamplesPerSec=5.286789887746182, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2742/5000 [9:17:22<6:54:44, 11.02s/it][2022-12-20 19:39:58,400] [INFO] [timer.py:197:stop] 0/5552, RunningAvgSamplesPerSec=5.873578084719662, CurrSamplesPerSec=5.354993695946194, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2743/5000 [9:17:33<6:54:22, 11.02s/it][2022-12-20 19:40:09,365] [INFO] [timer.py:197:stop] 0/5554, RunningAvgSamplesPerSec=5.873589161204107, CurrSamplesPerSec=5.336683893422925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2744/5000 [9:17:44<6:53:36, 11.00s/it][2022-12-20 19:40:20,409] [INFO] [timer.py:197:stop] 0/5556, RunningAvgSamplesPerSec=5.873584460360476, CurrSamplesPerSec=5.3223045849999355, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2745/5000 [9:17:56<6:53:55, 11.01s/it][2022-12-20 19:40:31,433] [INFO] [timer.py:197:stop] 0/5558, RunningAvgSamplesPerSec=5.873584007540353, CurrSamplesPerSec=5.334101681653499, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2746/5000 [9:18:07<6:53:51, 11.02s/it][2022-12-20 19:40:42,391] [INFO] [logging.py:68:log_dist] [Rank 0] step=2780, skipped=3, lr=[4.942222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:40:42,393] [INFO] [timer.py:197:stop] 0/5560, RunningAvgSamplesPerSec=5.873595856104104, CurrSamplesPerSec=5.374349177000311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2747/5000 [9:18:17<6:53:02, 11.00s/it][2022-12-20 19:40:52,434] [INFO] [timer.py:197:stop] 0/5562, RunningAvgSamplesPerSec=5.873784061792545, CurrSamplesPerSec=5.390280375228017, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2748/5000 [9:18:32<7:37:54, 12.20s/it][2022-12-20 19:41:03,464] [INFO] [timer.py:197:stop] 0/5564, RunningAvgSamplesPerSec=5.873782154540114, CurrSamplesPerSec=5.328951517988772, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▍    | 2749/5000 [9:18:43<7:23:53, 11.83s/it][2022-12-20 19:41:14,411] [INFO] [timer.py:197:stop] 0/5566, RunningAvgSamplesPerSec=5.873796367150424, CurrSamplesPerSec=5.351536680975152, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2750/5000 [9:18:54<7:14:24, 11.58s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.935555555555556e-06, 'epoch': 67.07}
+
 55%|█████▌    | 2750/5000 [9:18:54<7:14:24, 11.58s/it][2022-12-20 19:41:25,456] [INFO] [timer.py:197:stop] 0/5568, RunningAvgSamplesPerSec=5.873791776318774, CurrSamplesPerSec=5.318304780080689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2751/5000 [9:19:06<7:08:14, 11.42s/it][2022-12-20 19:41:36,508] [INFO] [timer.py:197:stop] 0/5570, RunningAvgSamplesPerSec=5.87378551825661, CurrSamplesPerSec=5.31663207086998, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2752/5000 [9:19:17<7:03:21, 11.30s/it][2022-12-20 19:41:47,473] [INFO] [timer.py:197:stop] 0/5572, RunningAvgSamplesPerSec=5.873796096979941, CurrSamplesPerSec=5.354570483729123, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2753/5000 [9:19:28<6:59:44, 11.21s/it][2022-12-20 19:41:58,533] [INFO] [timer.py:197:stop] 0/5574, RunningAvgSamplesPerSec=5.873788494292831, CurrSamplesPerSec=5.297263653758482, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2754/5000 [9:19:39<6:57:50, 11.16s/it][2022-12-20 19:42:09,535] [INFO] [timer.py:197:stop] 0/5576, RunningAvgSamplesPerSec=5.8737919261518385, CurrSamplesPerSec=5.344961881955423, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2755/5000 [9:19:50<6:55:52, 11.11s/it][2022-12-20 19:42:20,556] [INFO] [timer.py:197:stop] 0/5578, RunningAvgSamplesPerSec=5.873791728788294, CurrSamplesPerSec=5.328777816853849, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2756/5000 [9:20:01<6:54:50, 11.09s/it][2022-12-20 19:42:31,607] [INFO] [logging.py:68:log_dist] [Rank 0] step=2790, skipped=3, lr=[4.92e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:42:31,609] [INFO] [timer.py:197:stop] 0/5580, RunningAvgSamplesPerSec=5.873785311314344, CurrSamplesPerSec=5.316008129170332, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2757/5000 [9:20:12<6:53:43, 11.07s/it][2022-12-20 19:42:42,576] [INFO] [timer.py:197:stop] 0/5582, RunningAvgSamplesPerSec=5.873795462857526, CurrSamplesPerSec=5.354707416858103, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2758/5000 [9:20:23<6:52:25, 11.04s/it][2022-12-20 19:42:53,554] [INFO] [timer.py:197:stop] 0/5584, RunningAvgSamplesPerSec=5.873803465972992, CurrSamplesPerSec=5.344407457959762, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2759/5000 [9:20:34<6:51:45, 11.02s/it][2022-12-20 19:43:04,578] [INFO] [timer.py:197:stop] 0/5586, RunningAvgSamplesPerSec=5.873802423431712, CurrSamplesPerSec=5.317430373841255, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2760/5000 [9:20:45<6:51:47, 11.03s/it][2022-12-20 19:43:15,608] [INFO] [timer.py:197:stop] 0/5588, RunningAvgSamplesPerSec=5.87380048815207, CurrSamplesPerSec=5.330129006634499, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2761/5000 [9:20:56<6:51:14, 11.02s/it][2022-12-20 19:43:26,648] [INFO] [timer.py:197:stop] 0/5590, RunningAvgSamplesPerSec=5.8737966535704285, CurrSamplesPerSec=5.293396370329786, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2762/5000 [9:21:07<6:51:39, 11.04s/it][2022-12-20 19:43:37,708] [INFO] [timer.py:197:stop] 0/5592, RunningAvgSamplesPerSec=5.8737886375716055, CurrSamplesPerSec=5.304641778456514, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2763/5000 [9:21:18<6:51:34, 11.04s/it][2022-12-20 19:43:48,700] [INFO] [timer.py:197:stop] 0/5594, RunningAvgSamplesPerSec=5.873793766613712, CurrSamplesPerSec=5.351837772532358, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2764/5000 [9:21:29<6:50:29, 11.01s/it][2022-12-20 19:43:59,682] [INFO] [timer.py:197:stop] 0/5596, RunningAvgSamplesPerSec=5.873801199347313, CurrSamplesPerSec=5.331814245059606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2765/5000 [9:21:40<6:50:03, 11.01s/it][2022-12-20 19:44:10,643] [INFO] [timer.py:197:stop] 0/5598, RunningAvgSamplesPerSec=5.873812509289567, CurrSamplesPerSec=5.359181567010169, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2766/5000 [9:21:51<6:49:19, 10.99s/it][2022-12-20 19:44:21,657] [INFO] [logging.py:68:log_dist] [Rank 0] step=2800, skipped=3, lr=[4.897777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:44:21,659] [INFO] [timer.py:197:stop] 0/5600, RunningAvgSamplesPerSec=5.873813294912358, CurrSamplesPerSec=5.310260568792263, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2767/5000 [9:22:02<6:49:44, 11.01s/it][2022-12-20 19:44:32,627] [INFO] [timer.py:197:stop] 0/5602, RunningAvgSamplesPerSec=5.873823350634759, CurrSamplesPerSec=5.379686477851786, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2768/5000 [9:22:13<6:49:02, 11.00s/it][2022-12-20 19:44:43,652] [INFO] [timer.py:197:stop] 0/5604, RunningAvgSamplesPerSec=5.873822587688678, CurrSamplesPerSec=5.324985662723378, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2769/5000 [9:22:24<6:48:59, 11.00s/it][2022-12-20 19:44:54,665] [INFO] [timer.py:197:stop] 0/5606, RunningAvgSamplesPerSec=5.87382371538216, CurrSamplesPerSec=5.320041592315165, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2770/5000 [9:22:35<6:49:27, 11.02s/it][2022-12-20 19:45:05,733] [INFO] [timer.py:197:stop] 0/5608, RunningAvgSamplesPerSec=5.873814666235967, CurrSamplesPerSec=5.311608689198078, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2771/5000 [9:22:46<6:49:46, 11.03s/it][2022-12-20 19:45:16,836] [INFO] [timer.py:197:stop] 0/5610, RunningAvgSamplesPerSec=5.873798613325374, CurrSamplesPerSec=5.275306650567709, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2772/5000 [9:22:57<6:50:07, 11.04s/it][2022-12-20 19:45:27,857] [INFO] [timer.py:197:stop] 0/5612, RunningAvgSamplesPerSec=5.8737985908181045, CurrSamplesPerSec=5.325901652874853, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2773/5000 [9:23:08<6:49:50, 11.04s/it][2022-12-20 19:45:38,911] [INFO] [timer.py:197:stop] 0/5614, RunningAvgSamplesPerSec=5.873791739166662, CurrSamplesPerSec=5.3083934548797505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 55%|█████▌    | 2774/5000 [9:23:19<6:49:26, 11.04s/it][2022-12-20 19:45:49,891] [INFO] [timer.py:197:stop] 0/5616, RunningAvgSamplesPerSec=5.873799138445663, CurrSamplesPerSec=5.34574167546553, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2775/5000 [9:23:30<6:48:50, 11.02s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.880000000000001e-06, 'epoch': 67.67}
+
 56%|█████▌    | 2775/5000 [9:23:30<6:48:50, 11.02s/it][2022-12-20 19:46:00,921] [INFO] [timer.py:197:stop] 0/5618, RunningAvgSamplesPerSec=5.873797035386725, CurrSamplesPerSec=5.318684762336322, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2776/5000 [9:23:41<6:48:36, 11.02s/it][2022-12-20 19:46:11,915] [INFO] [logging.py:68:log_dist] [Rank 0] step=2810, skipped=3, lr=[4.875555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:46:11,917] [INFO] [timer.py:197:stop] 0/5620, RunningAvgSamplesPerSec=5.873801293978187, CurrSamplesPerSec=5.340453634293534, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2777/5000 [9:23:52<6:48:24, 11.02s/it][2022-12-20 19:46:22,960] [INFO] [timer.py:197:stop] 0/5622, RunningAvgSamplesPerSec=5.873796478560907, CurrSamplesPerSec=5.321919864931496, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2778/5000 [9:24:03<6:48:13, 11.02s/it][2022-12-20 19:46:33,973] [INFO] [timer.py:197:stop] 0/5624, RunningAvgSamplesPerSec=5.8737972428438425, CurrSamplesPerSec=5.329389735235788, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2779/5000 [9:24:14<6:47:27, 11.01s/it][2022-12-20 19:46:44,939] [INFO] [timer.py:197:stop] 0/5626, RunningAvgSamplesPerSec=5.87380723818, CurrSamplesPerSec=5.334365620635847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2780/5000 [9:24:25<6:46:43, 10.99s/it][2022-12-20 19:46:55,862] [INFO] [timer.py:197:stop] 0/5628, RunningAvgSamplesPerSec=5.873825495623536, CurrSamplesPerSec=5.36617585570204, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2781/5000 [9:24:36<6:46:21, 10.99s/it][2022-12-20 19:47:06,848] [INFO] [timer.py:197:stop] 0/5630, RunningAvgSamplesPerSec=5.873831663148455, CurrSamplesPerSec=5.357923191980688, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2782/5000 [9:24:47<6:46:04, 10.98s/it][2022-12-20 19:47:17,798] [INFO] [timer.py:197:stop] 0/5632, RunningAvgSamplesPerSec=5.873844738981575, CurrSamplesPerSec=5.382513113129933, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2783/5000 [9:24:58<6:45:36, 10.98s/it][2022-12-20 19:47:28,810] [INFO] [timer.py:197:stop] 0/5634, RunningAvgSamplesPerSec=5.8738460673213515, CurrSamplesPerSec=5.336412511113242, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2784/5000 [9:25:09<6:45:30, 10.98s/it][2022-12-20 19:47:39,800] [INFO] [timer.py:197:stop] 0/5636, RunningAvgSamplesPerSec=5.873851531051227, CurrSamplesPerSec=5.3320199175092124, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2785/5000 [9:25:20<6:45:47, 10.99s/it][2022-12-20 19:47:50,826] [INFO] [timer.py:197:stop] 0/5638, RunningAvgSamplesPerSec=5.8738499112889535, CurrSamplesPerSec=5.326702953576216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2786/5000 [9:25:31<6:45:49, 11.00s/it][2022-12-20 19:48:01,833] [INFO] [logging.py:68:log_dist] [Rank 0] step=2820, skipped=3, lr=[4.853333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:48:01,834] [INFO] [timer.py:197:stop] 0/5640, RunningAvgSamplesPerSec=5.873851931605211, CurrSamplesPerSec=5.330478078345613, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2787/5000 [9:25:42<6:45:31, 10.99s/it][2022-12-20 19:48:12,806] [INFO] [timer.py:197:stop] 0/5642, RunningAvgSamplesPerSec=5.873860829070926, CurrSamplesPerSec=5.344046771672473, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2788/5000 [9:25:53<6:45:26, 11.00s/it][2022-12-20 19:48:22,907] [INFO] [timer.py:197:stop] 0/5644, RunningAvgSamplesPerSec=5.874034392041914, CurrSamplesPerSec=6.280781027546057, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 19:48:33,912] [INFO] [timer.py:197:stop] 0/5646, RunningAvgSamplesPerSec=5.874037323258759, CurrSamplesPerSec=5.3153839128937115, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2789/5000 [9:26:09<7:42:15, 12.54s/it][2022-12-20 19:48:44,878] [INFO] [timer.py:197:stop] 0/5648, RunningAvgSamplesPerSec=5.874047515144264, CurrSamplesPerSec=5.363802313643479, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2790/5000 [9:26:20<7:24:36, 12.07s/it][2022-12-20 19:48:55,930] [INFO] [timer.py:197:stop] 0/5650, RunningAvgSamplesPerSec=5.874041232593043, CurrSamplesPerSec=5.3037232356404855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2791/5000 [9:26:31<7:13:09, 11.77s/it][2022-12-20 19:49:06,909] [INFO] [timer.py:197:stop] 0/5652, RunningAvgSamplesPerSec=5.874048922961252, CurrSamplesPerSec=5.357295936736377, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2792/5000 [9:26:42<7:04:16, 11.53s/it][2022-12-20 19:49:17,841] [INFO] [timer.py:197:stop] 0/5654, RunningAvgSamplesPerSec=5.874065682504712, CurrSamplesPerSec=5.365903825480039, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2793/5000 [9:26:53<6:57:30, 11.35s/it][2022-12-20 19:49:28,803] [INFO] [timer.py:197:stop] 0/5656, RunningAvgSamplesPerSec=5.874076387249856, CurrSamplesPerSec=5.314600744650652, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2794/5000 [9:27:04<6:53:01, 11.23s/it][2022-12-20 19:49:39,850] [INFO] [timer.py:197:stop] 0/5658, RunningAvgSamplesPerSec=5.874071170541584, CurrSamplesPerSec=5.266305597013842, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2795/5000 [9:27:15<6:50:46, 11.18s/it][2022-12-20 19:49:50,921] [INFO] [logging.py:68:log_dist] [Rank 0] step=2830, skipped=3, lr=[4.831111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:49:50,923] [INFO] [timer.py:197:stop] 0/5660, RunningAvgSamplesPerSec=5.87406074746877, CurrSamplesPerSec=5.293036274159943, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2796/5000 [9:27:26<6:49:26, 11.15s/it][2022-12-20 19:50:01,950] [INFO] [timer.py:197:stop] 0/5662, RunningAvgSamplesPerSec=5.8740592631947175, CurrSamplesPerSec=5.322980037257525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2797/5000 [9:27:37<6:47:56, 11.11s/it][2022-12-20 19:50:12,988] [INFO] [timer.py:197:stop] 0/5664, RunningAvgSamplesPerSec=5.874055553027704, CurrSamplesPerSec=5.313397081453832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2798/5000 [9:27:48<6:46:57, 11.09s/it][2022-12-20 19:50:23,957] [INFO] [timer.py:197:stop] 0/5666, RunningAvgSamplesPerSec=5.8740650716204605, CurrSamplesPerSec=5.335225670509933, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2799/5000 [9:27:59<6:45:27, 11.05s/it][2022-12-20 19:50:34,903] [INFO] [timer.py:197:stop] 0/5668, RunningAvgSamplesPerSec=5.874079108070821, CurrSamplesPerSec=5.353138552953794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2800/5000 [9:28:10<6:44:05, 11.02s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.822222222222222e-06, 'epoch': 68.29}
+
 56%|█████▌    | 2800/5000 [9:28:10<6:44:05, 11.02s/it][2022-12-20 19:50:45,907] [INFO] [timer.py:197:stop] 0/5670, RunningAvgSamplesPerSec=5.874081955942947, CurrSamplesPerSec=5.340098580636496, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2801/5000 [9:28:21<6:43:43, 11.02s/it][2022-12-20 19:50:56,877] [INFO] [timer.py:197:stop] 0/5672, RunningAvgSamplesPerSec=5.874091433743807, CurrSamplesPerSec=5.371057131335535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2802/5000 [9:28:32<6:43:02, 11.00s/it][2022-12-20 19:51:07,908] [INFO] [timer.py:197:stop] 0/5674, RunningAvgSamplesPerSec=5.874088896813064, CurrSamplesPerSec=5.314274580331696, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2803/5000 [9:28:43<6:43:10, 11.01s/it][2022-12-20 19:51:18,909] [INFO] [timer.py:197:stop] 0/5676, RunningAvgSamplesPerSec=5.874092199611981, CurrSamplesPerSec=5.31110361552119, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2804/5000 [9:28:54<6:42:53, 11.01s/it][2022-12-20 19:51:29,935] [INFO] [timer.py:197:stop] 0/5678, RunningAvgSamplesPerSec=5.874090597320321, CurrSamplesPerSec=5.315743056188142, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2805/5000 [9:29:05<6:42:54, 11.01s/it][2022-12-20 19:51:40,985] [INFO] [logging.py:68:log_dist] [Rank 0] step=2840, skipped=3, lr=[4.808888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:51:40,987] [INFO] [timer.py:197:stop] 0/5680, RunningAvgSamplesPerSec=5.874084296384431, CurrSamplesPerSec=5.313309789216538, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2806/5000 [9:29:16<6:43:08, 11.02s/it][2022-12-20 19:51:51,990] [INFO] [timer.py:197:stop] 0/5682, RunningAvgSamplesPerSec=5.874087462115047, CurrSamplesPerSec=5.324057949674623, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2807/5000 [9:29:27<6:42:42, 11.02s/it][2022-12-20 19:52:02,981] [INFO] [timer.py:197:stop] 0/5684, RunningAvgSamplesPerSec=5.874092943791963, CurrSamplesPerSec=5.333381229406907, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2808/5000 [9:29:38<6:42:13, 11.01s/it][2022-12-20 19:52:13,994] [INFO] [timer.py:197:stop] 0/5686, RunningAvgSamplesPerSec=5.87409428822226, CurrSamplesPerSec=5.344357661203878, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2809/5000 [9:29:49<6:42:04, 11.01s/it][2022-12-20 19:52:25,039] [INFO] [timer.py:197:stop] 0/5688, RunningAvgSamplesPerSec=5.874089613519568, CurrSamplesPerSec=5.325526766680763, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2810/5000 [9:30:00<6:42:16, 11.02s/it][2022-12-20 19:52:36,059] [INFO] [timer.py:197:stop] 0/5690, RunningAvgSamplesPerSec=5.874089342463575, CurrSamplesPerSec=5.351310084443843, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2811/5000 [9:30:11<6:42:04, 11.02s/it][2022-12-20 19:52:47,093] [INFO] [timer.py:197:stop] 0/5692, RunningAvgSamplesPerSec=5.874086672425911, CurrSamplesPerSec=5.314015360860494, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▌    | 2812/5000 [9:30:22<6:42:02, 11.02s/it][2022-12-20 19:52:58,160] [INFO] [timer.py:197:stop] 0/5694, RunningAvgSamplesPerSec=5.874077738030854, CurrSamplesPerSec=5.307387773955233, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2813/5000 [9:30:33<6:42:19, 11.04s/it][2022-12-20 19:53:09,186] [INFO] [timer.py:197:stop] 0/5696, RunningAvgSamplesPerSec=5.874076500856282, CurrSamplesPerSec=5.321241306202499, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2814/5000 [9:30:44<6:42:00, 11.03s/it][2022-12-20 19:53:20,201] [INFO] [timer.py:197:stop] 0/5698, RunningAvgSamplesPerSec=5.874077279176397, CurrSamplesPerSec=5.316346719924513, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2815/5000 [9:30:55<6:41:36, 11.03s/it][2022-12-20 19:53:31,221] [INFO] [logging.py:68:log_dist] [Rank 0] step=2850, skipped=3, lr=[4.786666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:53:31,222] [INFO] [timer.py:197:stop] 0/5700, RunningAvgSamplesPerSec=5.874076700486542, CurrSamplesPerSec=5.3059660712079655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2816/5000 [9:31:06<6:41:21, 11.03s/it][2022-12-20 19:53:42,239] [INFO] [timer.py:197:stop] 0/5702, RunningAvgSamplesPerSec=5.8740770406572125, CurrSamplesPerSec=5.299960156829668, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2817/5000 [9:31:17<6:41:04, 11.02s/it][2022-12-20 19:53:53,321] [INFO] [timer.py:197:stop] 0/5704, RunningAvgSamplesPerSec=5.874065554450859, CurrSamplesPerSec=5.306221778839163, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2818/5000 [9:31:28<6:41:31, 11.04s/it][2022-12-20 19:54:04,303] [INFO] [timer.py:197:stop] 0/5706, RunningAvgSamplesPerSec=5.874072414392515, CurrSamplesPerSec=5.3202275884592, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2819/5000 [9:31:39<6:40:41, 11.02s/it][2022-12-20 19:54:15,297] [INFO] [timer.py:197:stop] 0/5708, RunningAvgSamplesPerSec=5.874077129048862, CurrSamplesPerSec=5.357236490909987, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2820/5000 [9:31:50<6:40:11, 11.01s/it][2022-12-20 19:54:26,249] [INFO] [timer.py:197:stop] 0/5710, RunningAvgSamplesPerSec=5.874089910910046, CurrSamplesPerSec=5.362374225584863, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2821/5000 [9:32:01<6:39:19, 11.00s/it][2022-12-20 19:54:37,267] [INFO] [timer.py:197:stop] 0/5712, RunningAvgSamplesPerSec=5.874090321363082, CurrSamplesPerSec=5.351450051410196, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2822/5000 [9:32:12<6:39:22, 11.00s/it][2022-12-20 19:54:48,297] [INFO] [timer.py:197:stop] 0/5714, RunningAvgSamplesPerSec=5.8740880477430695, CurrSamplesPerSec=5.316479809690409, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2823/5000 [9:32:23<6:39:30, 11.01s/it][2022-12-20 19:54:59,365] [INFO] [timer.py:197:stop] 0/5716, RunningAvgSamplesPerSec=5.874078942640688, CurrSamplesPerSec=5.310364149093905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2824/5000 [9:32:34<6:39:56, 11.03s/it][2022-12-20 19:55:10,353] [INFO] [timer.py:197:stop] 0/5718, RunningAvgSamplesPerSec=5.874084770268875, CurrSamplesPerSec=5.359720226014547, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 56%|█████▋    | 2825/5000 [9:32:45<6:39:19, 11.02s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.766666666666667e-06, 'epoch': 68.89}
+
 56%|█████▋    | 2825/5000 [9:32:45<6:39:19, 11.02s/it][2022-12-20 19:55:21,306] [INFO] [logging.py:68:log_dist] [Rank 0] step=2860, skipped=3, lr=[4.7644444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:55:21,308] [INFO] [timer.py:197:stop] 0/5720, RunningAvgSamplesPerSec=5.8740969873446165, CurrSamplesPerSec=5.323027536445529, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2826/5000 [9:32:56<6:38:28, 11.00s/it][2022-12-20 19:55:32,352] [INFO] [timer.py:197:stop] 0/5722, RunningAvgSamplesPerSec=5.874092365053945, CurrSamplesPerSec=5.311949242519917, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2827/5000 [9:33:07<6:38:48, 11.01s/it][2022-12-20 19:55:43,381] [INFO] [timer.py:197:stop] 0/5724, RunningAvgSamplesPerSec=5.874090600049113, CurrSamplesPerSec=5.29813353093577, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2828/5000 [9:33:18<6:38:48, 11.02s/it][2022-12-20 19:55:54,379] [INFO] [timer.py:197:stop] 0/5726, RunningAvgSamplesPerSec=5.874094592327143, CurrSamplesPerSec=5.348733738978411, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2829/5000 [9:33:29<6:38:25, 11.01s/it][2022-12-20 19:56:04,496] [INFO] [timer.py:197:stop] 0/5728, RunningAvgSamplesPerSec=5.874262808502337, CurrSamplesPerSec=5.310570901575081, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2830/5000 [9:33:45<7:22:20, 12.23s/it][2022-12-20 19:56:15,520] [INFO] [timer.py:197:stop] 0/5730, RunningAvgSamplesPerSec=5.874261923899835, CurrSamplesPerSec=5.332497621750029, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2831/5000 [9:33:56<7:08:31, 11.85s/it][2022-12-20 19:56:26,464] [INFO] [timer.py:197:stop] 0/5732, RunningAvgSamplesPerSec=5.874275811394911, CurrSamplesPerSec=5.359821035972468, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2832/5000 [9:34:07<6:58:54, 11.59s/it][2022-12-20 19:56:37,461] [INFO] [timer.py:197:stop] 0/5734, RunningAvgSamplesPerSec=5.874279911569508, CurrSamplesPerSec=5.349697365502477, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2833/5000 [9:34:18<6:52:27, 11.42s/it][2022-12-20 19:56:48,449] [INFO] [timer.py:197:stop] 0/5736, RunningAvgSamplesPerSec=5.874285631114197, CurrSamplesPerSec=5.374792094701099, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2834/5000 [9:34:28<6:46:55, 11.27s/it][2022-12-20 19:56:59,403] [INFO] [timer.py:197:stop] 0/5738, RunningAvgSamplesPerSec=5.874297596453894, CurrSamplesPerSec=5.350599054392578, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2835/5000 [9:34:39<6:43:42, 11.19s/it][2022-12-20 19:57:10,404] [INFO] [logging.py:68:log_dist] [Rank 0] step=2870, skipped=3, lr=[4.742222222222222e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:57:10,405] [INFO] [timer.py:197:stop] 0/5740, RunningAvgSamplesPerSec=5.874300721749914, CurrSamplesPerSec=5.342114365213371, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2836/5000 [9:34:50<6:41:12, 11.12s/it][2022-12-20 19:57:21,381] [INFO] [timer.py:197:stop] 0/5742, RunningAvgSamplesPerSec=5.874308922431549, CurrSamplesPerSec=5.3413607121903, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2837/5000 [9:35:01<6:39:23, 11.08s/it][2022-12-20 19:57:32,373] [INFO] [timer.py:197:stop] 0/5744, RunningAvgSamplesPerSec=5.87431381226725, CurrSamplesPerSec=5.324671530599606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2838/5000 [9:35:12<6:38:36, 11.06s/it][2022-12-20 19:57:43,367] [INFO] [timer.py:197:stop] 0/5746, RunningAvgSamplesPerSec=5.874318580181258, CurrSamplesPerSec=5.351462640261471, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2839/5000 [9:35:23<6:37:41, 11.04s/it][2022-12-20 19:57:54,396] [INFO] [timer.py:197:stop] 0/5748, RunningAvgSamplesPerSec=5.874316732306779, CurrSamplesPerSec=5.320914326318702, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2840/5000 [9:35:34<6:36:32, 11.02s/it][2022-12-20 19:58:05,308] [INFO] [timer.py:197:stop] 0/5750, RunningAvgSamplesPerSec=5.874336907249694, CurrSamplesPerSec=5.3576851470414475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2841/5000 [9:35:45<6:36:18, 11.01s/it][2022-12-20 19:58:16,316] [INFO] [timer.py:197:stop] 0/5752, RunningAvgSamplesPerSec=5.874339099919703, CurrSamplesPerSec=5.3583129212701515, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2842/5000 [9:35:56<6:35:55, 11.01s/it][2022-12-20 19:58:27,337] [INFO] [timer.py:197:stop] 0/5754, RunningAvgSamplesPerSec=5.87433871573882, CurrSamplesPerSec=5.335525565717599, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2843/5000 [9:36:07<6:35:48, 11.01s/it][2022-12-20 19:58:38,328] [INFO] [timer.py:197:stop] 0/5756, RunningAvgSamplesPerSec=5.874344109108672, CurrSamplesPerSec=5.3569016516554875, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2844/5000 [9:36:18<6:35:00, 10.99s/it][2022-12-20 19:58:49,304] [INFO] [timer.py:197:stop] 0/5758, RunningAvgSamplesPerSec=5.874352030077367, CurrSamplesPerSec=5.3351187853181345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2845/5000 [9:36:29<6:34:55, 11.00s/it][2022-12-20 19:59:00,297] [INFO] [logging.py:68:log_dist] [Rank 0] step=2880, skipped=3, lr=[4.7200000000000005e-06], mom=[[0.9, 0.999]]
+[2022-12-20 19:59:00,299] [INFO] [timer.py:197:stop] 0/5760, RunningAvgSamplesPerSec=5.874356041967765, CurrSamplesPerSec=5.341083752969541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2846/5000 [9:36:40<6:34:31, 10.99s/it][2022-12-20 19:59:11,270] [INFO] [timer.py:197:stop] 0/5762, RunningAvgSamplesPerSec=5.8743645345354185, CurrSamplesPerSec=5.3454372244994195, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2847/5000 [9:36:51<6:34:10, 10.98s/it][2022-12-20 19:59:22,234] [INFO] [timer.py:197:stop] 0/5764, RunningAvgSamplesPerSec=5.8743746694006695, CurrSamplesPerSec=5.354211628258398, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2848/5000 [9:37:02<6:34:01, 10.99s/it][2022-12-20 19:59:33,276] [INFO] [timer.py:197:stop] 0/5766, RunningAvgSamplesPerSec=5.874369968611366, CurrSamplesPerSec=5.30543942051699, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2849/5000 [9:37:13<6:34:22, 11.00s/it][2022-12-20 19:59:44,240] [INFO] [timer.py:197:stop] 0/5768, RunningAvgSamplesPerSec=5.8743798423328535, CurrSamplesPerSec=5.3689341857210655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2850/5000 [9:37:24<6:33:37, 10.98s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.711111111111111e-06, 'epoch': 69.51}
+
 57%|█████▋    | 2850/5000 [9:37:24<6:33:37, 10.98s/it][2022-12-20 19:59:55,218] [INFO] [timer.py:197:stop] 0/5770, RunningAvgSamplesPerSec=5.874387325776765, CurrSamplesPerSec=5.342818037502982, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2851/5000 [9:37:35<6:33:40, 10.99s/it][2022-12-20 20:00:06,284] [INFO] [timer.py:197:stop] 0/5772, RunningAvgSamplesPerSec=5.8743785660706545, CurrSamplesPerSec=5.29163810433456, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2852/5000 [9:37:46<6:34:14, 11.01s/it][2022-12-20 20:00:17,332] [INFO] [timer.py:197:stop] 0/5774, RunningAvgSamplesPerSec=5.87437272401746, CurrSamplesPerSec=5.301045096100744, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2853/5000 [9:37:57<6:34:30, 11.02s/it][2022-12-20 20:00:28,380] [INFO] [timer.py:197:stop] 0/5776, RunningAvgSamplesPerSec=5.874367000237943, CurrSamplesPerSec=5.307387354213918, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2854/5000 [9:38:08<6:34:42, 11.04s/it][2022-12-20 20:00:39,434] [INFO] [timer.py:197:stop] 0/5778, RunningAvgSamplesPerSec=5.874360095593777, CurrSamplesPerSec=5.3126557145680495, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2855/5000 [9:38:19<6:34:06, 11.02s/it][2022-12-20 20:00:50,466] [INFO] [logging.py:68:log_dist] [Rank 0] step=2890, skipped=3, lr=[4.697777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:00:50,468] [INFO] [timer.py:197:stop] 0/5780, RunningAvgSamplesPerSec=5.874356978811214, CurrSamplesPerSec=5.280061948740124, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2856/5000 [9:38:30<6:34:10, 11.03s/it][2022-12-20 20:01:01,490] [INFO] [timer.py:197:stop] 0/5782, RunningAvgSamplesPerSec=5.874355798659134, CurrSamplesPerSec=5.301830766347144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2857/5000 [9:38:42<6:34:15, 11.04s/it][2022-12-20 20:01:12,481] [INFO] [timer.py:197:stop] 0/5784, RunningAvgSamplesPerSec=5.874360896035313, CurrSamplesPerSec=5.360804934771974, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2858/5000 [9:38:53<6:33:12, 11.01s/it][2022-12-20 20:01:23,486] [INFO] [timer.py:197:stop] 0/5786, RunningAvgSamplesPerSec=5.8743632233404925, CurrSamplesPerSec=5.318876565458222, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2859/5000 [9:39:04<6:33:29, 11.03s/it][2022-12-20 20:01:34,551] [INFO] [timer.py:197:stop] 0/5788, RunningAvgSamplesPerSec=5.874354271060342, CurrSamplesPerSec=5.312288157445873, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2860/5000 [9:39:15<6:33:12, 11.02s/it][2022-12-20 20:01:45,564] [INFO] [timer.py:197:stop] 0/5790, RunningAvgSamplesPerSec=5.874354954865868, CurrSamplesPerSec=5.315766214807008, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2861/5000 [9:39:26<6:32:54, 11.02s/it][2022-12-20 20:01:56,574] [INFO] [timer.py:197:stop] 0/5792, RunningAvgSamplesPerSec=5.874356617019064, CurrSamplesPerSec=5.3200865085602045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2862/5000 [9:39:37<6:32:46, 11.02s/it][2022-12-20 20:02:07,617] [INFO] [timer.py:197:stop] 0/5794, RunningAvgSamplesPerSec=5.874352035993619, CurrSamplesPerSec=5.305779602675689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2863/5000 [9:39:48<6:32:24, 11.02s/it][2022-12-20 20:02:18,587] [INFO] [timer.py:197:stop] 0/5796, RunningAvgSamplesPerSec=5.874360896817245, CurrSamplesPerSec=5.3368224596949165, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2864/5000 [9:39:59<6:32:07, 11.01s/it][2022-12-20 20:02:29,619] [INFO] [timer.py:197:stop] 0/5798, RunningAvgSamplesPerSec=5.874358246246645, CurrSamplesPerSec=5.315409383943446, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2865/5000 [9:40:10<6:31:54, 11.01s/it][2022-12-20 20:02:40,662] [INFO] [logging.py:68:log_dist] [Rank 0] step=2900, skipped=3, lr=[4.675555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:02:40,664] [INFO] [timer.py:197:stop] 0/5800, RunningAvgSamplesPerSec=5.874353288484792, CurrSamplesPerSec=5.286806755630848, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2866/5000 [9:40:21<6:31:51, 11.02s/it][2022-12-20 20:02:51,658] [INFO] [timer.py:197:stop] 0/5802, RunningAvgSamplesPerSec=5.874357731685904, CurrSamplesPerSec=5.315508744421285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2867/5000 [9:40:32<6:31:49, 11.02s/it][2022-12-20 20:03:02,648] [INFO] [timer.py:197:stop] 0/5804, RunningAvgSamplesPerSec=5.874362961404516, CurrSamplesPerSec=5.351406524313069, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2868/5000 [9:40:43<6:31:04, 11.01s/it][2022-12-20 20:03:13,604] [INFO] [timer.py:197:stop] 0/5806, RunningAvgSamplesPerSec=5.874374559195766, CurrSamplesPerSec=5.363802527999487, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2869/5000 [9:40:54<6:30:23, 10.99s/it][2022-12-20 20:03:24,593] [INFO] [timer.py:197:stop] 0/5808, RunningAvgSamplesPerSec=5.874379684424819, CurrSamplesPerSec=5.33495931378749, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2870/5000 [9:41:05<6:30:13, 10.99s/it][2022-12-20 20:03:34,670] [INFO] [timer.py:197:stop] 0/5810, RunningAvgSamplesPerSec=5.874552452548534, CurrSamplesPerSec=6.285439165149174, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 20:03:45,726] [INFO] [timer.py:197:stop] 0/5812, RunningAvgSamplesPerSec=5.874545059490975, CurrSamplesPerSec=5.301394347192858, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2871/5000 [9:41:21<7:25:28, 12.55s/it][2022-12-20 20:03:56,774] [INFO] [timer.py:197:stop] 0/5814, RunningAvgSamplesPerSec=5.874539375564316, CurrSamplesPerSec=5.308503471073308, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2872/5000 [9:41:32<7:09:14, 12.10s/it][2022-12-20 20:04:07,759] [INFO] [timer.py:197:stop] 0/5816, RunningAvgSamplesPerSec=5.874545262649012, CurrSamplesPerSec=5.3318873194473895, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2873/5000 [9:41:43<6:57:09, 11.77s/it][2022-12-20 20:04:18,713] [INFO] [timer.py:197:stop] 0/5818, RunningAvgSamplesPerSec=5.874557316502126, CurrSamplesPerSec=5.364406436885103, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▋    | 2874/5000 [9:41:54<6:48:18, 11.52s/it][2022-12-20 20:04:29,731] [INFO] [logging.py:68:log_dist] [Rank 0] step=2910, skipped=3, lr=[4.653333333333333e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:04:29,733] [INFO] [timer.py:197:stop] 0/5820, RunningAvgSamplesPerSec=5.874556681895778, CurrSamplesPerSec=5.322935072140634, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 57%|█████▊    | 2875/5000 [9:42:05<6:42:46, 11.37s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.653333333333333e-06, 'epoch': 70.12}
+
 57%|█████▊    | 2875/5000 [9:42:05<6:42:46, 11.37s/it][2022-12-20 20:04:40,690] [INFO] [timer.py:197:stop] 0/5822, RunningAvgSamplesPerSec=5.874568113956616, CurrSamplesPerSec=5.360720574200327, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2876/5000 [9:42:16<6:38:10, 11.25s/it][2022-12-20 20:04:51,734] [INFO] [timer.py:197:stop] 0/5824, RunningAvgSamplesPerSec=5.874563092787816, CurrSamplesPerSec=5.32181709942815, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2877/5000 [9:42:27<6:35:49, 11.19s/it][2022-12-20 20:05:02,715] [INFO] [timer.py:197:stop] 0/5826, RunningAvgSamplesPerSec=5.874569945836367, CurrSamplesPerSec=5.335487175552704, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2878/5000 [9:42:38<6:33:26, 11.12s/it][2022-12-20 20:05:13,744] [INFO] [timer.py:197:stop] 0/5828, RunningAvgSamplesPerSec=5.874567711884025, CurrSamplesPerSec=5.321493424581322, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2879/5000 [9:42:49<6:32:15, 11.10s/it][2022-12-20 20:05:24,761] [INFO] [timer.py:197:stop] 0/5830, RunningAvgSamplesPerSec=5.8745678265348635, CurrSamplesPerSec=5.336573979200925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2880/5000 [9:43:00<6:31:13, 11.07s/it][2022-12-20 20:05:35,758] [INFO] [timer.py:197:stop] 0/5832, RunningAvgSamplesPerSec=5.87457184003875, CurrSamplesPerSec=5.309804695415316, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2881/5000 [9:43:11<6:30:14, 11.05s/it][2022-12-20 20:05:46,727] [INFO] [timer.py:197:stop] 0/5834, RunningAvgSamplesPerSec=5.874580827909812, CurrSamplesPerSec=5.341647054028996, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2882/5000 [9:43:22<6:29:12, 11.03s/it][2022-12-20 20:05:57,713] [INFO] [timer.py:197:stop] 0/5836, RunningAvgSamplesPerSec=5.874586477108707, CurrSamplesPerSec=5.312242111254212, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2883/5000 [9:43:33<6:28:35, 11.01s/it][2022-12-20 20:06:08,755] [INFO] [timer.py:197:stop] 0/5838, RunningAvgSamplesPerSec=5.874582069666755, CurrSamplesPerSec=5.30614521080266, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2884/5000 [9:43:44<6:28:42, 11.02s/it][2022-12-20 20:06:19,739] [INFO] [logging.py:68:log_dist] [Rank 0] step=2920, skipped=3, lr=[4.631111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:06:19,740] [INFO] [timer.py:197:stop] 0/5840, RunningAvgSamplesPerSec=5.874587815959889, CurrSamplesPerSec=5.316077823119983, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2885/5000 [9:43:55<6:28:08, 11.01s/it][2022-12-20 20:06:30,750] [INFO] [timer.py:197:stop] 0/5842, RunningAvgSamplesPerSec=5.874589267807861, CurrSamplesPerSec=5.331193087746084, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2886/5000 [9:44:06<6:27:56, 11.01s/it][2022-12-20 20:06:41,728] [INFO] [timer.py:197:stop] 0/5844, RunningAvgSamplesPerSec=5.874596574107854, CurrSamplesPerSec=5.365968397981047, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2887/5000 [9:44:17<6:27:24, 11.00s/it][2022-12-20 20:06:52,735] [INFO] [timer.py:197:stop] 0/5846, RunningAvgSamplesPerSec=5.874598268007782, CurrSamplesPerSec=5.340028680362362, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2888/5000 [9:44:28<6:27:18, 11.00s/it][2022-12-20 20:07:03,743] [INFO] [timer.py:197:stop] 0/5848, RunningAvgSamplesPerSec=5.874600249676044, CurrSamplesPerSec=5.335969956175687, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2889/5000 [9:44:39<6:27:09, 11.00s/it][2022-12-20 20:07:14,707] [INFO] [timer.py:197:stop] 0/5850, RunningAvgSamplesPerSec=5.874609959319911, CurrSamplesPerSec=5.354375029695503, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2890/5000 [9:44:50<6:26:33, 10.99s/it][2022-12-20 20:07:25,693] [INFO] [timer.py:197:stop] 0/5852, RunningAvgSamplesPerSec=5.874615827677498, CurrSamplesPerSec=5.336775987202972, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2891/5000 [9:45:01<6:26:18, 10.99s/it][2022-12-20 20:07:36,671] [INFO] [timer.py:197:stop] 0/5854, RunningAvgSamplesPerSec=5.87462319179014, CurrSamplesPerSec=5.342575378279608, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2892/5000 [9:45:12<6:25:59, 10.99s/it][2022-12-20 20:07:47,672] [INFO] [timer.py:197:stop] 0/5856, RunningAvgSamplesPerSec=5.8746263533278364, CurrSamplesPerSec=5.3509781189051955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2893/5000 [9:45:23<6:25:58, 10.99s/it][2022-12-20 20:07:58,676] [INFO] [timer.py:197:stop] 0/5858, RunningAvgSamplesPerSec=5.874629115116678, CurrSamplesPerSec=5.308491713399877, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2894/5000 [9:45:34<6:25:55, 10.99s/it][2022-12-20 20:08:09,649] [INFO] [logging.py:68:log_dist] [Rank 0] step=2930, skipped=3, lr=[4.6088888888888895e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:08:09,650] [INFO] [timer.py:197:stop] 0/5860, RunningAvgSamplesPerSec=5.874637289752536, CurrSamplesPerSec=5.337249874807918, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2895/5000 [9:45:45<6:25:31, 10.99s/it][2022-12-20 20:08:20,574] [INFO] [timer.py:197:stop] 0/5862, RunningAvgSamplesPerSec=5.87465474384245, CurrSamplesPerSec=5.375662647147338, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2896/5000 [9:45:56<6:24:39, 10.97s/it][2022-12-20 20:08:31,598] [INFO] [timer.py:197:stop] 0/5864, RunningAvgSamplesPerSec=5.874653585468551, CurrSamplesPerSec=5.344068475311295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2897/5000 [9:46:07<6:25:02, 10.99s/it][2022-12-20 20:08:42,624] [INFO] [timer.py:197:stop] 0/5866, RunningAvgSamplesPerSec=5.874652133087903, CurrSamplesPerSec=5.316332189957959, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2898/5000 [9:46:18<6:25:17, 11.00s/it][2022-12-20 20:08:53,681] [INFO] [timer.py:197:stop] 0/5868, RunningAvgSamplesPerSec=5.874644986601471, CurrSamplesPerSec=5.306703473929222, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2899/5000 [9:46:29<6:25:43, 11.02s/it][2022-12-20 20:09:04,706] [INFO] [timer.py:197:stop] 0/5870, RunningAvgSamplesPerSec=5.874643603436264, CurrSamplesPerSec=5.333013765622832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2900/5000 [9:46:40<6:25:38, 11.02s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.597777777777778e-06, 'epoch': 70.72}
+
 58%|█████▊    | 2900/5000 [9:46:40<6:25:38, 11.02s/it][2022-12-20 20:09:15,760] [INFO] [timer.py:197:stop] 0/5872, RunningAvgSamplesPerSec=5.874637328605862, CurrSamplesPerSec=5.29676130436869, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2901/5000 [9:46:51<6:25:49, 11.03s/it][2022-12-20 20:09:26,755] [INFO] [timer.py:197:stop] 0/5874, RunningAvgSamplesPerSec=5.87464149795436, CurrSamplesPerSec=5.330367149449948, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2902/5000 [9:47:02<6:25:17, 11.02s/it][2022-12-20 20:09:37,817] [INFO] [timer.py:197:stop] 0/5876, RunningAvgSamplesPerSec=5.874633583260283, CurrSamplesPerSec=5.286907756945132, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2903/5000 [9:47:13<6:25:33, 11.03s/it][2022-12-20 20:09:48,862] [INFO] [timer.py:197:stop] 0/5878, RunningAvgSamplesPerSec=5.874629010129903, CurrSamplesPerSec=5.31441198532244, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2904/5000 [9:47:24<6:25:30, 11.04s/it][2022-12-20 20:09:59,894] [INFO] [logging.py:68:log_dist] [Rank 0] step=2940, skipped=3, lr=[4.586666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:09:59,895] [INFO] [timer.py:197:stop] 0/5880, RunningAvgSamplesPerSec=5.874626239363037, CurrSamplesPerSec=5.324101033012881, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2905/5000 [9:47:35<6:25:18, 11.04s/it][2022-12-20 20:10:10,904] [INFO] [timer.py:197:stop] 0/5882, RunningAvgSamplesPerSec=5.874628065696858, CurrSamplesPerSec=5.338482631341345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2906/5000 [9:47:46<6:24:51, 11.03s/it][2022-12-20 20:10:21,951] [INFO] [timer.py:197:stop] 0/5884, RunningAvgSamplesPerSec=5.874622932751447, CurrSamplesPerSec=5.32160419543948, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2907/5000 [9:47:57<6:24:52, 11.03s/it][2022-12-20 20:10:32,962] [INFO] [timer.py:197:stop] 0/5886, RunningAvgSamplesPerSec=5.874624177323628, CurrSamplesPerSec=5.333100858861519, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2908/5000 [9:48:08<6:24:27, 11.03s/it][2022-12-20 20:10:44,036] [INFO] [timer.py:197:stop] 0/5888, RunningAvgSamplesPerSec=5.874614211948825, CurrSamplesPerSec=5.298494530184896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2909/5000 [9:48:19<6:24:46, 11.04s/it][2022-12-20 20:10:55,048] [INFO] [timer.py:197:stop] 0/5890, RunningAvgSamplesPerSec=5.874615239935691, CurrSamplesPerSec=5.301244632606311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2910/5000 [9:48:30<6:24:17, 11.03s/it][2022-12-20 20:11:06,001] [INFO] [timer.py:197:stop] 0/5892, RunningAvgSamplesPerSec=5.874627084469442, CurrSamplesPerSec=5.355306928879785, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2911/5000 [9:48:41<6:23:16, 11.01s/it][2022-12-20 20:11:16,106] [INFO] [timer.py:197:stop] 0/5894, RunningAvgSamplesPerSec=5.874792395484535, CurrSamplesPerSec=5.3433325649823855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2912/5000 [9:48:56<7:05:18, 12.22s/it][2022-12-20 20:11:27,167] [INFO] [timer.py:197:stop] 0/5896, RunningAvgSamplesPerSec=5.874784515501794, CurrSamplesPerSec=5.289426540318353, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2913/5000 [9:49:07<6:53:07, 11.88s/it][2022-12-20 20:11:38,205] [INFO] [timer.py:197:stop] 0/5898, RunningAvgSamplesPerSec=5.874781057310203, CurrSamplesPerSec=5.321837567805312, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2914/5000 [9:49:18<6:43:54, 11.62s/it][2022-12-20 20:11:49,180] [INFO] [logging.py:68:log_dist] [Rank 0] step=2950, skipped=3, lr=[4.564444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:11:49,183] [INFO] [timer.py:197:stop] 0/5900, RunningAvgSamplesPerSec=5.874788303180257, CurrSamplesPerSec=5.352282109839765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2915/5000 [9:49:29<6:37:10, 11.43s/it][2022-12-20 20:12:00,188] [INFO] [timer.py:197:stop] 0/5902, RunningAvgSamplesPerSec=5.874790439555649, CurrSamplesPerSec=5.338493035877214, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2916/5000 [9:49:40<6:32:47, 11.31s/it][2022-12-20 20:12:11,252] [INFO] [timer.py:197:stop] 0/5904, RunningAvgSamplesPerSec=5.874781919178066, CurrSamplesPerSec=5.305462279726399, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2917/5000 [9:49:51<6:29:22, 11.22s/it][2022-12-20 20:12:22,217] [INFO] [timer.py:197:stop] 0/5906, RunningAvgSamplesPerSec=5.874791483180663, CurrSamplesPerSec=5.335204250713688, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2918/5000 [9:50:02<6:27:22, 11.16s/it][2022-12-20 20:12:33,278] [INFO] [timer.py:197:stop] 0/5908, RunningAvgSamplesPerSec=5.874783358807379, CurrSamplesPerSec=5.3183357582982715, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2919/5000 [9:50:13<6:25:49, 11.12s/it][2022-12-20 20:12:44,306] [INFO] [timer.py:197:stop] 0/5910, RunningAvgSamplesPerSec=5.874781700411822, CurrSamplesPerSec=5.323015503237727, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2920/5000 [9:50:24<6:24:42, 11.10s/it][2022-12-20 20:12:55,381] [INFO] [timer.py:197:stop] 0/5912, RunningAvgSamplesPerSec=5.8747712202171, CurrSamplesPerSec=5.287040626407139, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2921/5000 [9:50:35<6:24:02, 11.08s/it][2022-12-20 20:13:06,403] [INFO] [timer.py:197:stop] 0/5914, RunningAvgSamplesPerSec=5.874770505718344, CurrSamplesPerSec=5.312512933191664, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2922/5000 [9:50:46<6:23:09, 11.06s/it][2022-12-20 20:13:17,433] [INFO] [timer.py:197:stop] 0/5916, RunningAvgSamplesPerSec=5.87476816641745, CurrSamplesPerSec=5.300320777115527, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2923/5000 [9:50:57<6:22:37, 11.05s/it][2022-12-20 20:13:28,467] [INFO] [timer.py:197:stop] 0/5918, RunningAvgSamplesPerSec=5.874764854642008, CurrSamplesPerSec=5.295299953125821, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2924/5000 [9:51:08<6:22:19, 11.05s/it][2022-12-20 20:13:39,452] [INFO] [logging.py:68:log_dist] [Rank 0] step=2960, skipped=3, lr=[4.542222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:13:39,454] [INFO] [timer.py:197:stop] 0/5920, RunningAvgSamplesPerSec=5.874772209248993, CurrSamplesPerSec=5.345957366961097, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 58%|█████▊    | 2925/5000 [9:51:20<6:21:42, 11.04s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.542222222222223e-06, 'epoch': 71.34}
+
 58%|█████▊    | 2925/5000 [9:51:20<6:21:42, 11.04s/it][2022-12-20 20:13:50,522] [INFO] [timer.py:197:stop] 0/5922, RunningAvgSamplesPerSec=5.874763098552551, CurrSamplesPerSec=5.292787471110896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2926/5000 [9:51:31<6:21:53, 11.05s/it][2022-12-20 20:14:01,559] [INFO] [timer.py:197:stop] 0/5924, RunningAvgSamplesPerSec=5.874759759457521, CurrSamplesPerSec=5.324651462893609, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2927/5000 [9:51:42<6:21:39, 11.05s/it][2022-12-20 20:14:12,623] [INFO] [timer.py:197:stop] 0/5926, RunningAvgSamplesPerSec=5.87475105028978, CurrSamplesPerSec=5.305039101709108, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2928/5000 [9:51:53<6:21:13, 11.04s/it][2022-12-20 20:14:23,632] [INFO] [timer.py:197:stop] 0/5928, RunningAvgSamplesPerSec=5.874752551978114, CurrSamplesPerSec=5.316676508349269, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2929/5000 [9:52:04<6:20:34, 11.03s/it][2022-12-20 20:14:34,622] [INFO] [timer.py:197:stop] 0/5930, RunningAvgSamplesPerSec=5.874757505869775, CurrSamplesPerSec=5.321209239301193, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2930/5000 [9:52:15<6:20:12, 11.02s/it][2022-12-20 20:14:45,614] [INFO] [timer.py:197:stop] 0/5932, RunningAvgSamplesPerSec=5.874762211949463, CurrSamplesPerSec=5.335565653274986, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2931/5000 [9:52:26<6:20:03, 11.02s/it][2022-12-20 20:14:56,627] [INFO] [timer.py:197:stop] 0/5934, RunningAvgSamplesPerSec=5.874763046653146, CurrSamplesPerSec=5.3460261449082, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2932/5000 [9:52:37<6:19:38, 11.01s/it][2022-12-20 20:15:07,619] [INFO] [timer.py:197:stop] 0/5936, RunningAvgSamplesPerSec=5.87476734348689, CurrSamplesPerSec=5.35145453218098, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2933/5000 [9:52:48<6:18:46, 11.00s/it][2022-12-20 20:15:18,613] [INFO] [timer.py:197:stop] 0/5938, RunningAvgSamplesPerSec=5.874771674772431, CurrSamplesPerSec=5.3119469299735655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2934/5000 [9:52:59<6:18:37, 11.00s/it][2022-12-20 20:15:29,603] [INFO] [logging.py:68:log_dist] [Rank 0] step=2970, skipped=3, lr=[4.520000000000001e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:15:29,605] [INFO] [timer.py:197:stop] 0/5940, RunningAvgSamplesPerSec=5.874776328146612, CurrSamplesPerSec=5.316928194346642, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2935/5000 [9:53:10<6:18:52, 11.01s/it][2022-12-20 20:15:40,599] [INFO] [timer.py:197:stop] 0/5942, RunningAvgSamplesPerSec=5.874780488742046, CurrSamplesPerSec=5.3559589383896276, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2936/5000 [9:53:21<6:18:41, 11.01s/it][2022-12-20 20:15:51,620] [INFO] [timer.py:197:stop] 0/5944, RunningAvgSamplesPerSec=5.874779672772005, CurrSamplesPerSec=5.344396179126073, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▊    | 2937/5000 [9:53:32<6:18:51, 11.02s/it][2022-12-20 20:16:02,701] [INFO] [timer.py:197:stop] 0/5946, RunningAvgSamplesPerSec=5.874768220669111, CurrSamplesPerSec=5.311476473781591, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2938/5000 [9:53:43<6:18:58, 11.03s/it][2022-12-20 20:16:13,754] [INFO] [timer.py:197:stop] 0/5948, RunningAvgSamplesPerSec=5.874761910084056, CurrSamplesPerSec=5.305818825001748, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2939/5000 [9:53:54<6:19:10, 11.04s/it][2022-12-20 20:16:24,798] [INFO] [timer.py:197:stop] 0/5950, RunningAvgSamplesPerSec=5.874757065495042, CurrSamplesPerSec=5.3242602784617015, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2940/5000 [9:54:05<6:18:57, 11.04s/it][2022-12-20 20:16:35,863] [INFO] [timer.py:197:stop] 0/5952, RunningAvgSamplesPerSec=5.874748328657431, CurrSamplesPerSec=5.297274525460245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2941/5000 [9:54:16<6:18:34, 11.03s/it][2022-12-20 20:16:46,843] [INFO] [timer.py:197:stop] 0/5954, RunningAvgSamplesPerSec=5.874755061416279, CurrSamplesPerSec=5.3316952122298495, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2942/5000 [9:54:27<6:18:20, 11.03s/it][2022-12-20 20:16:57,861] [INFO] [timer.py:197:stop] 0/5956, RunningAvgSamplesPerSec=5.874754928720397, CurrSamplesPerSec=5.339744211499144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2943/5000 [9:54:38<6:18:08, 11.03s/it][2022-12-20 20:17:08,882] [INFO] [timer.py:197:stop] 0/5958, RunningAvgSamplesPerSec=5.874754514321736, CurrSamplesPerSec=5.347652843880279, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2944/5000 [9:54:49<6:17:51, 11.03s/it][2022-12-20 20:17:19,912] [INFO] [logging.py:68:log_dist] [Rank 0] step=2980, skipped=3, lr=[4.497777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:17:19,914] [INFO] [timer.py:197:stop] 0/5960, RunningAvgSamplesPerSec=5.874751959958992, CurrSamplesPerSec=5.33694617808222, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2945/5000 [9:55:00<6:17:19, 11.02s/it][2022-12-20 20:17:30,910] [INFO] [timer.py:197:stop] 0/5962, RunningAvgSamplesPerSec=5.874755915365653, CurrSamplesPerSec=5.334873008038894, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2946/5000 [9:55:11<6:17:18, 11.02s/it][2022-12-20 20:17:41,956] [INFO] [timer.py:197:stop] 0/5964, RunningAvgSamplesPerSec=5.87475062471221, CurrSamplesPerSec=5.321819209562847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2947/5000 [9:55:22<6:16:27, 11.00s/it][2022-12-20 20:17:52,913] [INFO] [timer.py:197:stop] 0/5966, RunningAvgSamplesPerSec=5.874761532254081, CurrSamplesPerSec=5.322460557032565, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2948/5000 [9:55:33<6:16:35, 11.01s/it][2022-12-20 20:18:03,980] [INFO] [timer.py:197:stop] 0/5968, RunningAvgSamplesPerSec=5.8747528738851384, CurrSamplesPerSec=5.292777870119689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2949/5000 [9:55:44<6:16:06, 11.00s/it][2022-12-20 20:18:14,893] [INFO] [timer.py:197:stop] 0/5970, RunningAvgSamplesPerSec=5.874771870257053, CurrSamplesPerSec=5.355048818866976, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2950/5000 [9:55:55<6:15:56, 11.00s/it]
                                                       
{'loss': 0.0001, 'learning_rate': 4.486666666666667e-06, 'epoch': 71.94}
+
 59%|█████▉    | 2950/5000 [9:55:55<6:15:56, 11.00s/it][2022-12-20 20:18:25,896] [INFO] [timer.py:197:stop] 0/5972, RunningAvgSamplesPerSec=5.874774743899718, CurrSamplesPerSec=5.356706882558939, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2951/5000 [9:56:06<6:15:34, 11.00s/it][2022-12-20 20:18:36,871] [INFO] [timer.py:197:stop] 0/5974, RunningAvgSamplesPerSec=5.874782800833587, CurrSamplesPerSec=5.366402211017528, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2952/5000 [9:56:17<6:15:18, 11.00s/it][2022-12-20 20:18:46,982] [INFO] [timer.py:197:stop] 0/5976, RunningAvgSamplesPerSec=5.874944533486637, CurrSamplesPerSec=6.279710194562957, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 20:18:57,978] [INFO] [timer.py:197:stop] 0/5978, RunningAvgSamplesPerSec=5.874947750348053, CurrSamplesPerSec=5.319260002395333, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2953/5000 [9:56:33<7:07:50, 12.54s/it][2022-12-20 20:19:08,966] [INFO] [logging.py:68:log_dist] [Rank 0] step=2990, skipped=3, lr=[4.475555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:19:08,968] [INFO] [timer.py:197:stop] 0/5980, RunningAvgSamplesPerSec=5.874952473030796, CurrSamplesPerSec=5.349581584054581, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2954/5000 [9:56:44<6:51:46, 12.08s/it][2022-12-20 20:19:19,929] [INFO] [timer.py:197:stop] 0/5982, RunningAvgSamplesPerSec=5.874962476942582, CurrSamplesPerSec=5.3563727507902, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2955/5000 [9:56:55<6:40:10, 11.74s/it][2022-12-20 20:19:30,851] [INFO] [timer.py:197:stop] 0/5984, RunningAvgSamplesPerSec=5.874979645858327, CurrSamplesPerSec=5.370755807623169, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2956/5000 [9:57:06<6:31:36, 11.50s/it][2022-12-20 20:19:41,853] [INFO] [timer.py:197:stop] 0/5986, RunningAvgSamplesPerSec=5.874982423680192, CurrSamplesPerSec=5.3164063146762155, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2957/5000 [9:57:17<6:26:22, 11.35s/it][2022-12-20 20:19:52,834] [INFO] [timer.py:197:stop] 0/5988, RunningAvgSamplesPerSec=5.874988703275025, CurrSamplesPerSec=5.3557657338357245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2958/5000 [9:57:28<6:22:27, 11.24s/it][2022-12-20 20:20:03,840] [INFO] [timer.py:197:stop] 0/5990, RunningAvgSamplesPerSec=5.874990853164031, CurrSamplesPerSec=5.320454723836691, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2959/5000 [9:57:39<6:19:53, 11.17s/it][2022-12-20 20:20:14,878] [INFO] [timer.py:197:stop] 0/5992, RunningAvgSamplesPerSec=5.874986959762335, CurrSamplesPerSec=5.30235565344699, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2960/5000 [9:57:50<6:18:23, 11.13s/it][2022-12-20 20:20:25,875] [INFO] [timer.py:197:stop] 0/5994, RunningAvgSamplesPerSec=5.8749905872983765, CurrSamplesPerSec=5.338349711655825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2961/5000 [9:58:01<6:16:51, 11.09s/it][2022-12-20 20:20:36,919] [INFO] [timer.py:197:stop] 0/5996, RunningAvgSamplesPerSec=5.874985603502789, CurrSamplesPerSec=5.325078832100223, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2962/5000 [9:58:12<6:16:12, 11.08s/it][2022-12-20 20:20:47,905] [INFO] [timer.py:197:stop] 0/5998, RunningAvgSamplesPerSec=5.874991275074586, CurrSamplesPerSec=5.368897246159506, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2963/5000 [9:58:23<6:15:06, 11.05s/it][2022-12-20 20:20:58,907] [INFO] [logging.py:68:log_dist] [Rank 0] step=3000, skipped=3, lr=[4.453333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:20:58,909] [INFO] [timer.py:197:stop] 0/6000, RunningAvgSamplesPerSec=5.874993508244673, CurrSamplesPerSec=5.311423085002095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2964/5000 [9:58:34<6:14:27, 11.04s/it][2022-12-20 20:21:09,903] [INFO] [timer.py:197:stop] 0/6002, RunningAvgSamplesPerSec=5.874997682394482, CurrSamplesPerSec=5.32820115105702, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2965/5000 [9:58:45<6:13:51, 11.02s/it][2022-12-20 20:21:20,945] [INFO] [timer.py:197:stop] 0/6004, RunningAvgSamplesPerSec=5.874993009041949, CurrSamplesPerSec=5.3093995170034, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2966/5000 [9:58:56<6:13:52, 11.03s/it][2022-12-20 20:21:31,974] [INFO] [timer.py:197:stop] 0/6006, RunningAvgSamplesPerSec=5.874990890584703, CurrSamplesPerSec=5.318863075555646, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2967/5000 [9:59:07<6:13:41, 11.03s/it][2022-12-20 20:21:43,014] [INFO] [timer.py:197:stop] 0/6008, RunningAvgSamplesPerSec=5.874986751495474, CurrSamplesPerSec=5.300674328907552, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2968/5000 [9:59:18<6:13:37, 11.03s/it][2022-12-20 20:21:54,113] [INFO] [timer.py:197:stop] 0/6010, RunningAvgSamplesPerSec=5.874972077451004, CurrSamplesPerSec=5.234501525994954, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2969/5000 [9:59:29<6:14:07, 11.05s/it][2022-12-20 20:22:05,142] [INFO] [timer.py:197:stop] 0/6012, RunningAvgSamplesPerSec=5.874969949476336, CurrSamplesPerSec=5.294270825115543, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2970/5000 [9:59:40<6:13:41, 11.05s/it][2022-12-20 20:22:16,185] [INFO] [timer.py:197:stop] 0/6014, RunningAvgSamplesPerSec=5.8749651629196205, CurrSamplesPerSec=5.315109430459475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2971/5000 [9:59:51<6:13:29, 11.04s/it][2022-12-20 20:22:27,221] [INFO] [timer.py:197:stop] 0/6016, RunningAvgSamplesPerSec=5.874961833134361, CurrSamplesPerSec=5.3111216897424605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2972/5000 [10:00:02<6:13:12, 11.04s/it][2022-12-20 20:22:38,162] [INFO] [timer.py:197:stop] 0/6018, RunningAvgSamplesPerSec=5.874975347528325, CurrSamplesPerSec=5.4081553397432325, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2973/5000 [10:00:13<6:12:00, 11.01s/it][2022-12-20 20:22:49,132] [INFO] [logging.py:68:log_dist] [Rank 0] step=3010, skipped=3, lr=[4.431111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:22:49,134] [INFO] [timer.py:197:stop] 0/6020, RunningAvgSamplesPerSec=5.874983345465486, CurrSamplesPerSec=5.36519255818966, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 59%|█████▉    | 2974/5000 [10:00:24<6:11:25, 11.00s/it][2022-12-20 20:23:00,138] [INFO] [timer.py:197:stop] 0/6022, RunningAvgSamplesPerSec=5.874985678743617, CurrSamplesPerSec=5.358299444503174, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2975/5000 [10:00:35<6:11:17, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.4288888888888895e-06, 'epoch': 72.55}
+
 60%|█████▉    | 2975/5000 [10:00:35<6:11:17, 11.00s/it][2022-12-20 20:23:11,080] [INFO] [timer.py:197:stop] 0/6024, RunningAvgSamplesPerSec=5.874999499314156, CurrSamplesPerSec=5.329793102681461, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2976/5000 [10:00:46<6:10:30, 10.98s/it][2022-12-20 20:23:22,148] [INFO] [timer.py:197:stop] 0/6026, RunningAvgSamplesPerSec=5.874990329985133, CurrSamplesPerSec=5.304369452464169, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2977/5000 [10:00:57<6:11:10, 11.01s/it][2022-12-20 20:23:33,138] [INFO] [timer.py:197:stop] 0/6028, RunningAvgSamplesPerSec=5.874995148556201, CurrSamplesPerSec=5.336265691846711, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2978/5000 [10:01:08<6:10:48, 11.00s/it][2022-12-20 20:23:44,175] [INFO] [timer.py:197:stop] 0/6030, RunningAvgSamplesPerSec=5.874991843398066, CurrSamplesPerSec=5.315187730840546, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2979/5000 [10:01:19<6:10:57, 11.01s/it][2022-12-20 20:23:55,219] [INFO] [timer.py:197:stop] 0/6032, RunningAvgSamplesPerSec=5.874987017091406, CurrSamplesPerSec=5.3069004989923165, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2980/5000 [10:01:30<6:11:05, 11.02s/it][2022-12-20 20:24:06,258] [INFO] [timer.py:197:stop] 0/6034, RunningAvgSamplesPerSec=5.874983300693197, CurrSamplesPerSec=5.3164844426820235, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2981/5000 [10:01:41<6:11:04, 11.03s/it][2022-12-20 20:24:17,223] [INFO] [timer.py:197:stop] 0/6036, RunningAvgSamplesPerSec=5.874992357534083, CurrSamplesPerSec=5.356019424538606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2982/5000 [10:01:52<6:10:15, 11.01s/it][2022-12-20 20:24:28,250] [INFO] [timer.py:197:stop] 0/6038, RunningAvgSamplesPerSec=5.874990588761148, CurrSamplesPerSec=5.320033790040056, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2983/5000 [10:02:03<6:10:15, 11.01s/it][2022-12-20 20:24:39,196] [INFO] [logging.py:68:log_dist] [Rank 0] step=3020, skipped=3, lr=[4.408888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:24:39,198] [INFO] [timer.py:197:stop] 0/6040, RunningAvgSamplesPerSec=5.875002771617135, CurrSamplesPerSec=5.355393042308162, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2984/5000 [10:02:14<6:09:24, 10.99s/it][2022-12-20 20:24:50,168] [INFO] [timer.py:197:stop] 0/6042, RunningAvgSamplesPerSec=5.8750110276213015, CurrSamplesPerSec=5.369174520047398, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2985/5000 [10:02:25<6:08:58, 10.99s/it][2022-12-20 20:25:01,208] [INFO] [timer.py:197:stop] 0/6044, RunningAvgSamplesPerSec=5.875006816225193, CurrSamplesPerSec=5.305942997900634, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2986/5000 [10:02:36<6:09:19, 11.00s/it][2022-12-20 20:25:12,203] [INFO] [timer.py:197:stop] 0/6046, RunningAvgSamplesPerSec=5.875010622193367, CurrSamplesPerSec=5.310001740754221, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2987/5000 [10:02:47<6:09:04, 11.00s/it][2022-12-20 20:25:23,231] [INFO] [timer.py:197:stop] 0/6048, RunningAvgSamplesPerSec=5.875008800570498, CurrSamplesPerSec=5.3236824783347405, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2988/5000 [10:02:58<6:09:09, 11.01s/it][2022-12-20 20:25:34,286] [INFO] [timer.py:197:stop] 0/6050, RunningAvgSamplesPerSec=5.875002121418948, CurrSamplesPerSec=5.308097651508671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2989/5000 [10:03:09<6:09:26, 11.02s/it][2022-12-20 20:25:45,274] [INFO] [timer.py:197:stop] 0/6052, RunningAvgSamplesPerSec=5.875007438801493, CurrSamplesPerSec=5.3362031051645795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2990/5000 [10:03:20<6:08:54, 11.01s/it][2022-12-20 20:25:56,339] [INFO] [timer.py:197:stop] 0/6054, RunningAvgSamplesPerSec=5.874999018904711, CurrSamplesPerSec=5.308771392175493, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2991/5000 [10:03:31<6:09:15, 11.03s/it][2022-12-20 20:26:07,263] [INFO] [timer.py:197:stop] 0/6056, RunningAvgSamplesPerSec=5.87501571571107, CurrSamplesPerSec=5.356558089367755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2992/5000 [10:03:42<6:08:01, 11.00s/it][2022-12-20 20:26:18,268] [INFO] [timer.py:197:stop] 0/6058, RunningAvgSamplesPerSec=5.875017984622778, CurrSamplesPerSec=5.313190739848801, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2993/5000 [10:03:53<6:07:55, 11.00s/it][2022-12-20 20:26:28,398] [INFO] [logging.py:68:log_dist] [Rank 0] step=3030, skipped=3, lr=[4.3866666666666665e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:26:28,400] [INFO] [timer.py:197:stop] 0/6060, RunningAvgSamplesPerSec=5.875174020225311, CurrSamplesPerSec=5.311957021099694, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2994/5000 [10:04:08<6:48:39, 12.22s/it][2022-12-20 20:26:39,442] [INFO] [timer.py:197:stop] 0/6062, RunningAvgSamplesPerSec=5.875169564795225, CurrSamplesPerSec=5.306426530525911, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2995/5000 [10:04:19<6:36:26, 11.86s/it][2022-12-20 20:26:50,464] [INFO] [timer.py:197:stop] 0/6064, RunningAvgSamplesPerSec=5.8751683152183585, CurrSamplesPerSec=5.307507402935565, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2996/5000 [10:04:31<6:28:03, 11.62s/it][2022-12-20 20:27:01,500] [INFO] [timer.py:197:stop] 0/6066, RunningAvgSamplesPerSec=5.875164760788702, CurrSamplesPerSec=5.317560568404629, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2997/5000 [10:04:41<6:21:25, 11.43s/it][2022-12-20 20:27:12,479] [INFO] [timer.py:197:stop] 0/6068, RunningAvgSamplesPerSec=5.875171212836504, CurrSamplesPerSec=5.313532127144026, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2998/5000 [10:04:53<6:17:19, 11.31s/it][2022-12-20 20:27:23,512] [INFO] [timer.py:197:stop] 0/6070, RunningAvgSamplesPerSec=5.875168091074225, CurrSamplesPerSec=5.315863693751322, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|█████▉    | 2999/5000 [10:05:04<6:14:04, 11.22s/it][2022-12-20 20:27:34,534] [INFO] [timer.py:197:stop] 0/6072, RunningAvgSamplesPerSec=5.875167211684567, CurrSamplesPerSec=5.298936120484576, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3000/5000 [10:05:15<6:12:15, 11.17s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.3733333333333335e-06, 'epoch': 73.17}
+
 60%|██████    | 3000/5000 [10:05:15<6:12:15, 11.17s/it][INFO|trainer.py:2955] 2022-12-20 20:27:39,491 >> ***** Running Evaluation *****
+[INFO|trainer.py:2957] 2022-12-20 20:27:39,492 >>   Num examples = 1325
+[INFO|trainer.py:2960] 2022-12-20 20:27:39,492 >>   Batch size = 8
+
+
  0%|          | 0/166 [00:00<?, ?it/s]
+
  1%|          | 2/166 [00:05<07:53,  2.89s/it]
+
  2%|▏         | 3/166 [00:11<11:09,  4.11s/it]
+
  2%|▏         | 4/166 [00:17<12:56,  4.79s/it]
+
  3%|▎         | 5/166 [00:24<15:07,  5.64s/it]
+
  4%|▎         | 6/166 [00:32<16:32,  6.20s/it]
+
  4%|▍         | 7/166 [00:38<16:32,  6.24s/it]
+
  5%|▍         | 8/166 [00:44<16:08,  6.13s/it]
+
  5%|▌         | 9/166 [00:50<15:52,  6.07s/it]
+
  6%|▌         | 10/166 [00:56<15:32,  5.98s/it]
+
  7%|▋         | 11/166 [01:01<15:09,  5.87s/it]
+
  7%|▋         | 12/166 [01:07<15:09,  5.91s/it]
+
  8%|▊         | 13/166 [01:14<15:30,  6.08s/it]
+
  8%|▊         | 14/166 [01:21<16:04,  6.34s/it]
+
  9%|▉         | 15/166 [01:26<15:27,  6.14s/it]
+
 10%|▉         | 16/166 [01:34<16:15,  6.50s/it]
+
 10%|█         | 17/166 [01:40<16:18,  6.57s/it]
+
 11%|█         | 18/166 [01:47<16:00,  6.49s/it]
+
 11%|█▏        | 19/166 [01:52<15:27,  6.31s/it]
+
 12%|█▏        | 20/166 [02:03<18:32,  7.62s/it]
+
 13%|█▎        | 21/166 [02:10<18:07,  7.50s/it]
+
 13%|█▎        | 22/166 [02:18<17:52,  7.45s/it]
+
 14%|█▍        | 23/166 [02:25<17:50,  7.49s/it]
+
 14%|█▍        | 24/166 [02:32<17:00,  7.19s/it]
+
 15%|█▌        | 25/166 [02:39<16:54,  7.20s/it]
+
 16%|█▌        | 26/166 [02:46<16:41,  7.15s/it]
+
 16%|█▋        | 27/166 [02:53<16:44,  7.22s/it]
+
 17%|█▋        | 28/166 [02:59<15:47,  6.87s/it]
+
 17%|█▋        | 29/166 [03:08<16:36,  7.27s/it]
+
 18%|█▊        | 30/166 [03:15<16:12,  7.15s/it]
+
 19%|█▊        | 31/166 [03:22<16:10,  7.19s/it]
+
 19%|█▉        | 32/166 [03:30<16:23,  7.34s/it]
+
 20%|█▉        | 33/166 [03:37<16:18,  7.36s/it]
+
 20%|██        | 34/166 [03:43<15:34,  7.08s/it]
+
 21%|██        | 35/166 [03:51<16:01,  7.34s/it]
+
 22%|██▏       | 36/166 [03:59<15:58,  7.37s/it]
+
 22%|██▏       | 37/166 [04:05<15:16,  7.10s/it]
+
 23%|██▎       | 38/166 [04:14<16:04,  7.53s/it]
+
 23%|██▎       | 39/166 [04:21<15:31,  7.33s/it]
+
 24%|██▍       | 40/166 [04:28<15:37,  7.44s/it]
+
 25%|██▍       | 41/166 [04:35<15:02,  7.22s/it]
+
 25%|██▌       | 42/166 [04:42<14:35,  7.06s/it]
+
 26%|██▌       | 43/166 [04:48<14:05,  6.87s/it]
+
 27%|██▋       | 44/166 [04:57<14:58,  7.37s/it]
+
 27%|██▋       | 45/166 [05:05<15:17,  7.58s/it]
+
 28%|██▊       | 46/166 [05:15<16:37,  8.31s/it]
+
 28%|██▊       | 47/166 [05:23<16:25,  8.28s/it]
+
 29%|██▉       | 48/166 [05:30<15:35,  7.93s/it]
+
 30%|██▉       | 49/166 [05:36<14:15,  7.31s/it]
+
 30%|███       | 50/166 [05:44<14:50,  7.68s/it]
+
 31%|███       | 51/166 [05:52<14:34,  7.61s/it]
+
 31%|███▏      | 52/166 [06:00<14:28,  7.61s/it]
+
 32%|███▏      | 53/166 [06:07<14:04,  7.48s/it]
+
 33%|███▎      | 54/166 [06:15<14:13,  7.62s/it]
+
 33%|███▎      | 55/166 [06:21<13:09,  7.11s/it]
+
 34%|███▎      | 56/166 [06:27<12:31,  6.84s/it]
+
 34%|███▍      | 57/166 [06:35<13:07,  7.23s/it]
+
 35%|███▍      | 58/166 [06:40<12:06,  6.73s/it]
+
 36%|███▌      | 59/166 [06:47<12:07,  6.80s/it]
+
 36%|███▌      | 60/166 [06:55<12:10,  6.89s/it]
+
 37%|███▋      | 61/166 [07:04<13:13,  7.56s/it]
+
 37%|███▋      | 62/166 [07:11<12:59,  7.49s/it]
+
 38%|███▊      | 63/166 [07:18<12:41,  7.39s/it]
+
 39%|███▊      | 64/166 [07:27<13:28,  7.93s/it]
+
 39%|███▉      | 65/166 [07:36<13:37,  8.09s/it]
+
 40%|███▉      | 66/166 [07:43<12:52,  7.72s/it]
+
 40%|████      | 67/166 [07:50<12:21,  7.49s/it]
+
 41%|████      | 68/166 [07:55<11:04,  6.78s/it]
+
 42%|████▏     | 69/166 [08:02<11:00,  6.81s/it]
+
 42%|████▏     | 70/166 [08:10<11:28,  7.17s/it]
+
 43%|████▎     | 71/166 [08:18<11:41,  7.39s/it]
+
 43%|████▎     | 72/166 [08:24<11:08,  7.12s/it]
+
 44%|████▍     | 73/166 [08:30<10:42,  6.91s/it]
+
 45%|████▍     | 74/166 [08:37<10:19,  6.73s/it]
+
 45%|████▌     | 75/166 [08:43<09:59,  6.59s/it]
+
 46%|████▌     | 76/166 [08:50<10:13,  6.81s/it]
+
 46%|████▋     | 77/166 [08:56<09:38,  6.50s/it]
+
 47%|████▋     | 78/166 [09:03<09:48,  6.68s/it]
+
 48%|████▊     | 79/166 [09:11<10:14,  7.07s/it]
+
 48%|████▊     | 80/166 [09:19<10:39,  7.43s/it]
+
 49%|████▉     | 81/166 [09:27<10:42,  7.56s/it]
+
 49%|████▉     | 82/166 [09:34<10:19,  7.37s/it]
+
 50%|█████     | 83/166 [09:40<09:21,  6.77s/it]
+
 51%|█████     | 84/166 [09:46<09:06,  6.67s/it]
+
 51%|█████     | 85/166 [09:53<09:14,  6.85s/it]
+
 52%|█████▏    | 86/166 [10:02<09:42,  7.28s/it]
+
 52%|█████▏    | 87/166 [10:09<09:42,  7.37s/it]
+
 53%|█████▎    | 88/166 [10:16<09:19,  7.18s/it]
+
 54%|█████▎    | 89/166 [10:23<09:16,  7.23s/it]
+
 54%|█████▍    | 90/166 [10:30<08:53,  7.02s/it]
+
 55%|█████▍    | 91/166 [10:37<08:39,  6.93s/it]
+
 55%|█████▌    | 92/166 [10:43<08:11,  6.65s/it]
+
 56%|█████▌    | 93/166 [10:51<08:36,  7.08s/it]
+
 57%|█████▋    | 94/166 [11:01<09:36,  8.01s/it]
+
 57%|█████▋    | 95/166 [11:06<08:30,  7.18s/it]
+
 58%|█████▊    | 96/166 [11:13<08:08,  6.98s/it]
+
 58%|█████▊    | 97/166 [11:23<09:09,  7.96s/it]
+
 59%|█████▉    | 98/166 [11:33<09:40,  8.54s/it]
+
 60%|█████▉    | 99/166 [11:40<09:10,  8.22s/it]
+
 60%|██████    | 100/166 [11:46<08:22,  7.61s/it]
+
 61%|██████    | 101/166 [11:53<07:55,  7.31s/it]
+
 61%|██████▏   | 102/166 [12:02<08:17,  7.78s/it]
+
 62%|██████▏   | 103/166 [12:11<08:32,  8.13s/it]
+
 63%|██████▎   | 104/166 [12:18<08:08,  7.88s/it]
+
 63%|██████▎   | 105/166 [12:26<08:05,  7.96s/it]
+
 64%|██████▍   | 106/166 [12:36<08:32,  8.55s/it]
+
 64%|██████▍   | 107/166 [12:46<08:41,  8.84s/it]
+
 65%|██████▌   | 108/166 [12:51<07:34,  7.83s/it]
+
 66%|██████▌   | 109/166 [12:58<07:01,  7.39s/it]
+
 66%|██████▋   | 110/166 [13:05<06:55,  7.41s/it]
+
 67%|██████▋   | 111/166 [13:11<06:24,  6.99s/it]
+
 67%|██████▋   | 112/166 [13:18<06:23,  7.09s/it]
+
 68%|██████▊   | 113/166 [13:27<06:39,  7.53s/it]
+
 69%|██████▊   | 114/166 [13:34<06:30,  7.51s/it]
+
 69%|██████▉   | 115/166 [13:41<06:05,  7.17s/it]
+
 70%|██████▉   | 116/166 [13:47<05:44,  6.90s/it]
+
 70%|███████   | 117/166 [13:54<05:47,  7.09s/it]
+
 71%|███████   | 118/166 [14:02<05:50,  7.29s/it]
+
 72%|███████▏  | 119/166 [14:10<05:48,  7.42s/it]
+
 72%|███████▏  | 120/166 [14:18<05:52,  7.66s/it]
+
 73%|███████▎  | 121/166 [14:27<05:54,  7.87s/it]
+
 73%|███████▎  | 122/166 [14:34<05:43,  7.80s/it]
+
 74%|███████▍  | 123/166 [14:42<05:29,  7.67s/it]
+
 75%|███████▍  | 124/166 [14:48<05:10,  7.40s/it]
+
 75%|███████▌  | 125/166 [14:56<05:04,  7.42s/it]
+
 76%|███████▌  | 126/166 [15:04<05:09,  7.74s/it]
+
 77%|███████▋  | 127/166 [15:09<04:29,  6.92s/it]
+
 77%|███████▋  | 128/166 [15:19<04:59,  7.88s/it]
+
 78%|███████▊  | 129/166 [15:27<04:51,  7.88s/it]
+
 78%|███████▊  | 130/166 [15:34<04:32,  7.58s/it]
+
 79%|███████▉  | 131/166 [15:41<04:19,  7.43s/it]
+
 80%|███████▉  | 132/166 [15:49<04:19,  7.63s/it]
+
 80%|████████  | 133/166 [15:57<04:13,  7.68s/it]
+
 81%|████████  | 134/166 [16:04<03:54,  7.34s/it]
+
 81%|████████▏ | 135/166 [16:10<03:36,  6.99s/it]
+
 82%|████████▏ | 136/166 [16:20<04:01,  8.03s/it]
+
 83%|████████▎ | 137/166 [16:29<04:01,  8.33s/it]
+
 83%|████████▎ | 138/166 [16:39<04:08,  8.86s/it]
+
 84%|████████▎ | 139/166 [16:48<03:54,  8.69s/it]
+
 84%|████████▍ | 140/166 [16:54<03:27,  7.98s/it]
+
 85%|████████▍ | 141/166 [17:00<03:04,  7.37s/it]
+
 86%|████████▌ | 142/166 [17:07<02:55,  7.31s/it]
+
 86%|████████▌ | 143/166 [17:13<02:35,  6.77s/it]
+
 87%|████████▋ | 144/166 [17:21<02:42,  7.38s/it]
+
 87%|████████▋ | 145/166 [17:31<02:49,  8.09s/it]
+
 88%|████████▊ | 146/166 [17:41<02:49,  8.47s/it]
+
 89%|████████▊ | 147/166 [17:49<02:40,  8.47s/it]
+
 89%|████████▉ | 148/166 [17:57<02:32,  8.45s/it]
+
 90%|████████▉ | 149/166 [18:07<02:27,  8.68s/it]
+
 90%|█████████ | 150/166 [18:16<02:23,  8.98s/it]
+
 91%|█████████ | 151/166 [18:24<02:10,  8.69s/it]
+
 92%|█████████▏| 152/166 [18:30<01:49,  7.81s/it]
+
 92%|█████████▏| 153/166 [18:37<01:36,  7.41s/it]
+
 93%|█████████▎| 154/166 [18:44<01:28,  7.39s/it]
+
 93%|█████████▎| 155/166 [18:52<01:22,  7.51s/it]
+
 94%|█████████▍| 156/166 [18:58<01:10,  7.02s/it]
+
 95%|█████████▍| 157/166 [19:04<01:02,  6.92s/it]
+
 95%|█████████▌| 158/166 [19:10<00:52,  6.53s/it]
+
 96%|█████████▌| 159/166 [19:15<00:41,  6.00s/it]
+
 96%|█████████▋| 160/166 [19:20<00:34,  5.71s/it]
+
 97%|█████████▋| 161/166 [19:26<00:29,  5.83s/it]
+
 98%|█████████▊| 162/166 [19:33<00:24,  6.16s/it]
+
 98%|█████████▊| 163/166 [19:40<00:19,  6.46s/it]
+
 99%|█████████▉| 164/166 [19:46<00:12,  6.37s/it]
+
 99%|█████████▉| 165/166 [19:52<00:06,  6.24s/it]
+
100%|██████████| 166/166 [19:56<00:00,  5.62s/it]
                                                        
+
                                                 
{'eval_loss': 0.275634765625, 'eval_wer': 8.451041417285133, 'eval_runtime': 1273.9374, 'eval_samples_per_second': 1.04, 'eval_steps_per_second': 0.13, 'epoch': 73.17}
+
 60%|██████    | 3000/5000 [10:26:29<6:12:15, 11.17s/it]
+
100%|██████████| 166/166 [21:08<00:00,  5.62s/it]
+
                                                 [INFO|trainer.py:2700] 2022-12-20 20:48:53,431 >> Saving model checkpoint to ./checkpoint-3000
+[INFO|configuration_utils.py:447] 2022-12-20 20:48:53,432 >> Configuration saved in ./checkpoint-3000/config.json
+[INFO|modeling_utils.py:1680] 2022-12-20 20:48:56,985 >> Model weights saved in ./checkpoint-3000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:368] 2022-12-20 20:48:57,000 >> Feature extractor saved in ./checkpoint-3000/preprocessor_config.json
+[2022-12-20 20:48:57,001] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step3036 is begin to save!
+/home/milan/hf_env/lib/python3.8/site-packages/torch/nn/modules/module.py:1428: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2022-12-20 20:48:57,011] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-3000/global_step3036/mp_rank_00_model_states.pt
+[2022-12-20 20:48:57,011] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-3000/global_step3036/mp_rank_00_model_states.pt...
+[2022-12-20 20:49:00,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-3000/global_step3036/mp_rank_00_model_states.pt.
+[2022-12-20 20:49:00,513] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-3000/global_step3036/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2022-12-20 20:49:16,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-3000/global_step3036/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2022-12-20 20:49:16,059] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-3000/global_step3036/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2022-12-20 20:49:16,059] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3036 is ready now!
+[INFO|feature_extraction_utils.py:368] 2022-12-20 20:49:20,274 >> Feature extractor saved in ./preprocessor_config.json