diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,2595 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 128 + | > Num. of Torch Threads: 1 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: True + > Start Tensorboard: tensorboard --logdir=/checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > Model has 532793243 parameters + + > EPOCH: 0/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 13:48:13)  + + --> TIME: 2024-12-23 13:48:15 -- STEP: 0/1935 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.048484861850738525 (0.048484861850738525) + | > loss_mel_ce: 2.963723659515381 (2.963723659515381) + | > loss: 0.3765260577201843 (0.3765260577201843) + | > current_lr: 5e-06 + | > step_time: 0.7765 (0.7765481472015381) + | > loader_time: 1.7836 (1.7836356163024902) + + + --> TIME: 2024-12-23 13:48:42 -- STEP: 50/1935 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.04716834798455238 (0.04877231724560261) + | > loss_mel_ce: 2.9513986110687256 (3.055670919418335) + | > loss: 0.37482085824012756 (0.3880554038286209) + | > current_lr: 5e-06 + | > step_time: 0.2411 (0.32908405303955085) + | > loader_time: 0.0399 (0.04178998947143555) + + + --> TIME: 2024-12-23 13:49:07 -- STEP: 100/1935 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.04889584332704544 (0.04885890308767557) + | > loss_mel_ce: 3.1541595458984375 (3.064870767593385) + | > loss: 0.4003819227218628 (0.3892162078619003) + | > current_lr: 5e-06 + | > step_time: 0.2945 (0.3245132756233214) + | > loader_time: 0.0435 (0.04086430549621582) + + + --> TIME: 2024-12-23 13:49:34 -- STEP: 150/1935 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.049205563962459564 (0.04880821858843168) + | > loss_mel_ce: 3.0107314586639404 (3.064979111353557) + | > loss: 0.3824921250343323 (0.38922341624895734) + | > current_lr: 5e-06 + | > step_time: 0.3326 (0.32392709573109923) + | > loader_time: 0.0337 (0.04009609699249267) + + + --> TIME: 2024-12-23 13:50:00 -- STEP: 200/1935 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.04918166622519493 (0.048815217260271326) + | > loss_mel_ce: 3.051337957382202 (3.065576879978181) + | > loss: 0.3875649571418762 (0.3892990125715734) + | > current_lr: 5e-06 + | > step_time: 0.311 (0.32360471606254565) + | > loader_time: 0.0348 (0.03926839232444765) + + + --> TIME: 2024-12-23 13:50:25 -- STEP: 250/1935 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.04876905679702759 (0.048791956141591084) + | > loss_mel_ce: 3.100797414779663 (3.0649013662338267) + | > loss: 0.39369580149650574 (0.3892116659879686) + | > current_lr: 5e-06 + | > step_time: 0.2708 (0.3232893781661986) + | > loader_time: 0.0366 (0.03852042961120607) + + + --> TIME: 2024-12-23 13:50:52 -- STEP: 300/1935 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.04906318709254265 (0.04881820795436704) + | > loss_mel_ce: 3.0893259048461914 (3.062608893712364) + | > loss: 0.3922986388206482 (0.3889283883571625) + | > current_lr: 5e-06 + | > step_time: 0.3402 (0.32346174796422306) + | > loader_time: 0.0341 (0.037925741672515896) + + + --> TIME: 2024-12-23 13:51:18 -- STEP: 350/1935 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.04838623106479645 (0.04885426042335379) + | > loss_mel_ce: 3.1003923416137695 (3.06265419346946) + | > loss: 0.3935973346233368 (0.38893855725015913) + | > current_lr: 5e-06 + | > step_time: 0.3373 (0.32409573078155507) + | > loader_time: 0.0334 (0.037450039046151344) + + + --> TIME: 2024-12-23 13:51:44 -- STEP: 400/1935 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.04886322468519211 (0.048860768433660315) + | > loss_mel_ce: 2.999774217605591 (3.061940974593166) + | > loss: 0.38107967376708984 (0.3888502187281847) + | > current_lr: 5e-06 + | > step_time: 0.2944 (0.32505339741706846) + | > loader_time: 0.0358 (0.03699498057365423) + + + --> TIME: 2024-12-23 13:52:11 -- STEP: 450/1935 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.04528723657131195 (0.04885781806376249) + | > loss_mel_ce: 3.0138306617736816 (3.060745453834537) + | > loss: 0.38238972425460815 (0.3887004095978207) + | > current_lr: 5e-06 + | > step_time: 0.3206 (0.3256325006484985) + | > loader_time: 0.0359 (0.03668879403008361) + + + --> TIME: 2024-12-23 13:52:36 -- STEP: 500/1935 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.04905926063656807 (0.048838755525648626) + | > loss_mel_ce: 3.0934154987335205 (3.060965664386752) + | > loss: 0.39280933141708374 (0.3887255531549454) + | > current_lr: 5e-06 + | > step_time: 0.2978 (0.325043158531189) + | > loader_time: 0.0405 (0.036398919582367004) + + + --> TIME: 2024-12-23 13:53:03 -- STEP: 550/1935 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.05033809691667557 (0.04883514852009039) + | > loss_mel_ce: 3.1703782081604004 (3.061156575896526) + | > loss: 0.4025895297527313 (0.38874896618452925) + | > current_lr: 5e-06 + | > step_time: 0.3417 (0.3251233473691073) + | > loader_time: 0.0352 (0.03630216208371254) + + + --> TIME: 2024-12-23 13:53:29 -- STEP: 600/1935 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.050040196627378464 (0.048837046499053675) + | > loss_mel_ce: 3.0834808349609375 (3.0622244052092253) + | > loss: 0.39169013500213623 (0.3888826821744442) + | > current_lr: 5e-06 + | > step_time: 0.2723 (0.32509735186894745) + | > loader_time: 0.0322 (0.03618204355239873) + + + --> TIME: 2024-12-23 13:53:55 -- STEP: 650/1935 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.05012405291199684 (0.04886887281559984) + | > loss_mel_ce: 3.062023162841797 (3.062164648496188) + | > loss: 0.3890184164047241 (0.38887919068336496) + | > current_lr: 5e-06 + | > step_time: 0.2732 (0.3251722324811494) + | > loader_time: 0.0344 (0.036035626851595406) + + + --> TIME: 2024-12-23 13:54:21 -- STEP: 700/1935 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.04747210443019867 (0.04885584952575822) + | > loss_mel_ce: 3.090380907058716 (3.0625877986635492) + | > loss: 0.39223161339759827 (0.3889304565106119) + | > current_lr: 5e-06 + | > step_time: 0.3467 (0.3253873753547668) + | > loader_time: 0.0351 (0.03593453032629835) + + + --> TIME: 2024-12-23 13:54:47 -- STEP: 750/1935 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.046776752918958664 (0.048846308996280056) + | > loss_mel_ce: 3.039609670639038 (3.061336757977805) + | > loss: 0.38579830527305603 (0.38877288389205933) + | > current_lr: 5e-06 + | > step_time: 0.3476 (0.3252053645451865) + | > loader_time: 0.0352 (0.035917364438374864) + + + --> TIME: 2024-12-23 13:55:13 -- STEP: 800/1935 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.048612408339977264 (0.04881562740541996) + | > loss_mel_ce: 3.0142743587493896 (3.059753600060941) + | > loss: 0.38286083936691284 (0.38857115399092435) + | > current_lr: 5e-06 + | > step_time: 0.3165 (0.3250488302111629) + | > loader_time: 0.0342 (0.03585750341415411) + + + --> TIME: 2024-12-23 13:55:39 -- STEP: 850/1935 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.04962390288710594 (0.04880598318050891) + | > loss_mel_ce: 2.926607370376587 (3.0585575159858265) + | > loss: 0.3720289170742035 (0.38842043799512527) + | > current_lr: 5e-06 + | > step_time: 0.2865 (0.3249300227445717) + | > loader_time: 0.0341 (0.03580501921036669) + + + --> TIME: 2024-12-23 13:56:05 -- STEP: 900/1935 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.04705502465367317 (0.04881204303354027) + | > loss_mel_ce: 3.075460910797119 (3.0581919932365427) + | > loss: 0.3903144896030426 (0.38837550507651436) + | > current_lr: 5e-06 + | > step_time: 0.3433 (0.3250894806120133) + | > loader_time: 0.0408 (0.035765853457980716) + + + --> TIME: 2024-12-23 13:56:31 -- STEP: 950/1935 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.04889689385890961 (0.048789394301803506) + | > loss_mel_ce: 2.9737796783447266 (3.0580288252077614) + | > loss: 0.3778345584869385 (0.38835227790631743) + | > current_lr: 5e-06 + | > step_time: 0.3097 (0.32490947698292005) + | > loader_time: 0.0339 (0.035661616827312295) + + + --> TIME: 2024-12-23 13:56:58 -- STEP: 1000/1935 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.04901577904820442 (0.04878763642162086) + | > loss_mel_ce: 2.956685781478882 (3.057054414987565) + | > loss: 0.37571269273757935 (0.3882302568256855) + | > current_lr: 5e-06 + | > step_time: 0.326 (0.3251318409442904) + | > loader_time: 0.0345 (0.03563417029380801) + + + --> TIME: 2024-12-23 13:57:23 -- STEP: 1050/1935 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.04646603390574455 (0.04878681848801319) + | > loss_mel_ce: 3.073881149291992 (3.0567272036416204) + | > loss: 0.3900434076786041 (0.38818925318263825) + | > current_lr: 5e-06 + | > step_time: 0.3363 (0.32484138670421797) + | > loader_time: 0.0208 (0.03520682789030532) + + + --> TIME: 2024-12-23 13:57:48 -- STEP: 1100/1935 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.04837711527943611 (0.04876129861243749) + | > loss_mel_ce: 3.056731700897217 (3.0561991232091743) + | > loss: 0.3881385922431946 (0.38812005308541386) + | > current_lr: 5e-06 + | > step_time: 0.3082 (0.32459382664073605) + | > loader_time: 0.0198 (0.034535746357657715) + + + --> TIME: 2024-12-23 13:58:14 -- STEP: 1150/1935 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.04810037463903427 (0.048761509342686) + | > loss_mel_ce: 3.0411484241485596 (3.055262425464134) + | > loss: 0.3861561119556427 (0.38800299211688655) + | > current_lr: 5e-06 + | > step_time: 0.3331 (0.3245797387413358) + | > loader_time: 0.0354 (0.034230740795964804) + + + --> TIME: 2024-12-23 13:58:40 -- STEP: 1200/1935 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.050180647522211075 (0.04874666290668151) + | > loss_mel_ce: 3.0309717655181885 (3.055426581104599) + | > loss: 0.38514405488967896 (0.38802165577808995) + | > current_lr: 5e-06 + | > step_time: 0.2882 (0.32437579194704713) + | > loader_time: 0.0339 (0.0342412646611532) + + + --> TIME: 2024-12-23 13:59:06 -- STEP: 1250/1935 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.04949814826250076 (0.04873536627292635) + | > loss_mel_ce: 3.050408363342285 (3.054716278457646) + | > loss: 0.38748830556869507 (0.3879314558982848) + | > current_lr: 5e-06 + | > step_time: 0.3413 (0.3244034450531005) + | > loader_time: 0.0345 (0.03426026802062992) + + + --> TIME: 2024-12-23 13:59:32 -- STEP: 1300/1935 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.0494474433362484 (0.04872833207536204) + | > loss_mel_ce: 3.036895751953125 (3.0547914030001753) + | > loss: 0.38579291105270386 (0.38793996714628654) + | > current_lr: 5e-06 + | > step_time: 0.2751 (0.32442978748908424) + | > loader_time: 0.0315 (0.034274254028613775) + + + --> TIME: 2024-12-23 13:59:58 -- STEP: 1350/1935 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.049786943942308426 (0.04871109149798201) + | > loss_mel_ce: 3.066476345062256 (3.0547594771561815) + | > loss: 0.3895329236984253 (0.3879338213691005) + | > current_lr: 5e-06 + | > step_time: 0.2985 (0.32460051412935625) + | > loader_time: 0.0335 (0.034267605852197694) + + + --> TIME: 2024-12-23 14:00:25 -- STEP: 1400/1935 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.04598052799701691 (0.04870288125106269) + | > loss_mel_ce: 3.0070011615753174 (3.0546189362662193) + | > loss: 0.38162270188331604 (0.38791522745575224) + | > current_lr: 5e-06 + | > step_time: 0.2867 (0.32484992350850805) + | > loader_time: 0.0241 (0.03425889338765823) + + + --> TIME: 2024-12-23 14:00:51 -- STEP: 1450/1935 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.04855700209736824 (0.04869894386622414) + | > loss_mel_ce: 3.0983471870422363 (3.0546749458641855) + | > loss: 0.39336302876472473 (0.3879217364459202) + | > current_lr: 5e-06 + | > step_time: 0.294 (0.3249078796649803) + | > loader_time: 0.033 (0.03426244801488416) + + + --> TIME: 2024-12-23 14:01:17 -- STEP: 1500/1935 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.04666290059685707 (0.048685953301688054) + | > loss_mel_ce: 3.032254457473755 (3.054543261845908) + | > loss: 0.3848646581172943 (0.3879036520520846) + | > current_lr: 5e-06 + | > step_time: 0.297 (0.3247092202504479) + | > loader_time: 0.0333 (0.034279950777689645) + + + --> TIME: 2024-12-23 14:01:43 -- STEP: 1550/1935 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.048993390053510666 (0.04867116837491914) + | > loss_mel_ce: 3.120128631591797 (3.0542121367300723) + | > loss: 0.3961402475833893 (0.38786041327061194) + | > current_lr: 5e-06 + | > step_time: 0.3759 (0.32470860404353025) + | > loader_time: 0.0256 (0.034296156206438666) + + + --> TIME: 2024-12-23 14:02:09 -- STEP: 1600/1935 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.04808104410767555 (0.048650734436232614) + | > loss_mel_ce: 3.1108145713806152 (3.0540801016986383) + | > loss: 0.39486196637153625 (0.38784135464578867) + | > current_lr: 5e-06 + | > step_time: 0.3237 (0.32470914393663447) + | > loader_time: 0.0312 (0.034284185320139014) + + + --> TIME: 2024-12-23 14:02:35 -- STEP: 1650/1935 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.04787340387701988 (0.048630762642080154) + | > loss_mel_ce: 3.0693271160125732 (3.053848094217706) + | > loss: 0.3896500766277313 (0.3878098572022987) + | > current_lr: 5e-06 + | > step_time: 0.3084 (0.3247887443773678) + | > loader_time: 0.0354 (0.03430282737269554) + + + --> TIME: 2024-12-23 14:03:01 -- STEP: 1700/1935 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.04830382764339447 (0.048621377070598756) + | > loss_mel_ce: 3.0122296810150146 (3.0534499444681065) + | > loss: 0.3825666904449463 (0.3877589153542238) + | > current_lr: 5e-06 + | > step_time: 0.3513 (0.3246864762025725) + | > loader_time: 0.0362 (0.03433487387264484) + + + --> TIME: 2024-12-23 14:03:26 -- STEP: 1750/1935 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.04746025800704956 (0.04861441391493595) + | > loss_mel_ce: 3.199918031692505 (3.0530372720445915) + | > loss: 0.4059222936630249 (0.3877064609357289) + | > current_lr: 5e-06 + | > step_time: 0.3462 (0.32439903954097243) + | > loader_time: 0.036 (0.03435148416246694) + + + --> TIME: 2024-12-23 14:03:53 -- STEP: 1800/1935 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.049650609493255615 (0.04859975409797498) + | > loss_mel_ce: 3.020087718963623 (3.052954106463327) + | > loss: 0.38371729850769043 (0.3876942327618599) + | > current_lr: 5e-06 + | > step_time: 0.295 (0.32441471325026655) + | > loader_time: 0.0338 (0.034366212156083896) + + + --> TIME: 2024-12-23 14:04:19 -- STEP: 1850/1935 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.047807686030864716 (0.04858852054621724) + | > loss_mel_ce: 3.17718505859375 (3.0527022205816743) + | > loss: 0.4031240940093994 (0.3876613428463807) + | > current_lr: 5e-06 + | > step_time: 0.3534 (0.3243858708562081) + | > loader_time: 0.0352 (0.03437732696533209) + + + --> TIME: 2024-12-23 14:04:45 -- STEP: 1900/1935 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.048592619597911835 (0.04858046021508545) + | > loss_mel_ce: 3.000420570373535 (3.052105881289433) + | > loss: 0.38112664222717285 (0.3875857929179543) + | > current_lr: 5e-06 + | > step_time: 0.3438 (0.3244300226161358) + | > loader_time: 0.0352 (0.03438263604515483) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11681033384920367 (+0) + | > avg_loss_text_ce: 0.04640609580985825 (+0) + | > avg_loss_mel_ce: 3.053221529180352 (+0) + | > avg_loss: 3.099627628470912 (+0) + + + > EPOCH: 1/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 14:06:10)  + + --> TIME: 2024-12-23 14:06:19 -- STEP: 15/1935 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.04682457819581032 (0.04751730759938558) + | > loss_mel_ce: 3.0126616954803467 (3.066989310582479) + | > loss: 0.38243579864501953 (0.3893133262793223) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3675 (0.3278784434000651) + | > loader_time: 0.0209 (0.02107516924540202) + + + --> TIME: 2024-12-23 14:06:44 -- STEP: 65/1935 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.04994269087910652 (0.048098181417355175) + | > loss_mel_ce: 3.0578720569610596 (3.0436349391937254) + | > loss: 0.3884768486022949 (0.3864666402339935) + | > current_lr: 5e-06 + | > step_time: 0.3131 (0.3211583687708928) + | > loader_time: 0.0194 (0.020395044180063104) + + + --> TIME: 2024-12-23 14:07:09 -- STEP: 115/1935 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.04792413115501404 (0.048036837739788965) + | > loss_mel_ce: 3.06571888923645 (3.040794028406558) + | > loss: 0.38920536637306213 (0.3861038583776225) + | > current_lr: 5e-06 + | > step_time: 0.2964 (0.3197436747343644) + | > loader_time: 0.0325 (0.023244714736938475) + + + --> TIME: 2024-12-23 14:07:36 -- STEP: 165/1935 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.04841485992074013 (0.047972710453199625) + | > loss_mel_ce: 3.0510172843933105 (3.0418782436486453) + | > loss: 0.38742902874946594 (0.38623136950261666) + | > current_lr: 5e-06 + | > step_time: 0.3464 (0.3228368701356831) + | > loader_time: 0.0347 (0.02640452818437056) + + + --> TIME: 2024-12-23 14:08:02 -- STEP: 215/1935 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.04614334926009178 (0.04797993006054745) + | > loss_mel_ce: 3.0164847373962402 (3.0408678265505062) + | > loss: 0.3828285038471222 (0.3861059697561486) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3618 (0.3235211560892506) + | > loader_time: 0.0351 (0.02811103754265364) + + + --> TIME: 2024-12-23 14:08:28 -- STEP: 265/1935 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.046806544065475464 (0.04800461082244819) + | > loss_mel_ce: 2.9780654907226562 (3.0404158664199548) + | > loss: 0.37810900807380676 (0.38605256024396645) + | > current_lr: 5e-06 + | > step_time: 0.3452 (0.3239777520017805) + | > loader_time: 0.0348 (0.028078299648356887) + + + --> TIME: 2024-12-23 14:08:53 -- STEP: 315/1935 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.04559674486517906 (0.04800315546610998) + | > loss_mel_ce: 3.0037600994110107 (3.0398695998721674) + | > loss: 0.3811696171760559 (0.3859840949376424) + | > current_lr: 5e-06 + | > step_time: 0.3086 (0.32322167441958466) + | > loader_time: 0.0329 (0.02898303062196762) + + + --> TIME: 2024-12-23 14:09:19 -- STEP: 365/1935 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.04821033775806427 (0.048038647936223304) + | > loss_mel_ce: 3.0074496269226074 (3.039886034351507) + | > loss: 0.3819575011730194 (0.3859905857745914) + | > current_lr: 5e-06 + | > step_time: 0.3355 (0.3234470426219783) + | > loader_time: 0.034 (0.02963611132478061) + + + --> TIME: 2024-12-23 14:09:45 -- STEP: 415/1935 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.048171743750572205 (0.04803182745733893) + | > loss_mel_ce: 3.0369713306427 (3.0389429322208277) + | > loss: 0.3856428861618042 (0.3858718451247156) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3619 (0.3233780717275228) + | > loader_time: 0.0353 (0.030100111214511366) + + + --> TIME: 2024-12-23 14:10:11 -- STEP: 465/1935 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.050772231072187424 (0.04804540381476443) + | > loss_mel_ce: 3.034914255142212 (3.0385963368159477) + | > loss: 0.38571080565452576 (0.38583021766395964) + | > current_lr: 5e-06 + | > step_time: 0.314 (0.32368178265069114) + | > loader_time: 0.0329 (0.03047481967556861) + + + --> TIME: 2024-12-23 14:10:37 -- STEP: 515/1935 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.04532027244567871 (0.04801256084471073) + | > loss_mel_ce: 3.1261773109436035 (3.0388571179028863) + | > loss: 0.3964371979236603 (0.3858587100089174) + | > current_lr: 5e-06 + | > step_time: 0.3423 (0.32383544421890403) + | > loader_time: 0.0179 (0.0297312130048437) + + + --> TIME: 2024-12-23 14:11:03 -- STEP: 565/1935 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.04762207344174385 (0.04801485281338733) + | > loss_mel_ce: 3.072577714920044 (3.039450567380517) + | > loss: 0.39002496004104614 (0.3859331777137992) + | > current_lr: 5e-06 + | > step_time: 0.3207 (0.3238205112187209) + | > loader_time: 0.0331 (0.030110644028250096) + + + --> TIME: 2024-12-23 14:11:29 -- STEP: 615/1935 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.05004260316491127 (0.04802167219965441) + | > loss_mel_ce: 3.0967187881469727 (3.038378193707971) + | > loss: 0.3933451771736145 (0.3857999833134131) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3666 (0.32339924796810005) + | > loader_time: 0.0338 (0.030393308934157457) + + + --> TIME: 2024-12-23 14:11:54 -- STEP: 665/1935 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.0486750490963459 (0.048023825743816875) + | > loss_mel_ce: 3.074293851852417 (3.037391762625903) + | > loss: 0.3903711140155792 (0.38567694873738073) + | > current_lr: 5e-06 + | > step_time: 0.3322 (0.32319568117758385) + | > loader_time: 0.0347 (0.030690451672202662) + + + --> TIME: 2024-12-23 14:12:20 -- STEP: 715/1935 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.04908235743641853 (0.048011532655129086) + | > loss_mel_ce: 3.0824837684631348 (3.0369232481176196) + | > loss: 0.39144575595855713 (0.38561684776852984) + | > current_lr: 5e-06 + | > step_time: 0.338 (0.32320604924555435) + | > loader_time: 0.0344 (0.03091127155544041) + + + --> TIME: 2024-12-23 14:12:46 -- STEP: 765/1935 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.049253348261117935 (0.048017984954944644) + | > loss_mel_ce: 2.992136240005493 (3.036933480057061) + | > loss: 0.3801737129688263 (0.38561893331459113) + | > current_lr: 5e-06 + | > step_time: 0.2993 (0.32301096573374644) + | > loader_time: 0.0374 (0.031117801105274873) + + + --> TIME: 2024-12-23 14:13:12 -- STEP: 815/1935 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.04843335971236229 (0.04800001530925191) + | > loss_mel_ce: 2.9226155281066895 (3.0366044591540935) + | > loss: 0.37138110399246216 (0.38557555960731243) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2531 (0.3230474173657003) + | > loader_time: 0.0366 (0.03133568354179523) + + + --> TIME: 2024-12-23 14:13:38 -- STEP: 865/1935 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.04699007421731949 (0.04798694352021798) + | > loss_mel_ce: 3.0061213970184326 (3.0362503842811357) + | > loss: 0.38163894414901733 (0.38552966624326074) + | > current_lr: 5e-06 + | > step_time: 0.2963 (0.32297360194211766) + | > loader_time: 0.0344 (0.03157113053206074) + + + --> TIME: 2024-12-23 14:14:05 -- STEP: 915/1935 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.04804626852273941 (0.047974914194293374) + | > loss_mel_ce: 3.008852958679199 (3.0361809394398667) + | > loss: 0.38211241364479065 (0.38551948200809477) + | > current_lr: 5e-06 + | > step_time: 0.3535 (0.3233073099063395) + | > loader_time: 0.0349 (0.03175156728817464) + + + --> TIME: 2024-12-23 14:14:30 -- STEP: 965/1935 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.04598081484436989 (0.04796331190923954) + | > loss_mel_ce: 3.01567006111145 (3.035714185052585) + | > loss: 0.3827063739299774 (0.38545968751833204) + | > current_lr: 5e-06 + | > step_time: 0.3192 (0.32329529208840496) + | > loader_time: 0.0343 (0.031897090259611245) + + + --> TIME: 2024-12-23 14:14:57 -- STEP: 1015/1935 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.047597307711839676 (0.047946559700179) + | > loss_mel_ce: 3.031179666519165 (3.036226991362172) + | > loss: 0.38484713435173035 (0.3855216942103623) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3763 (0.32337245964651634) + | > loader_time: 0.0357 (0.03202209496145762) + + + --> TIME: 2024-12-23 14:15:22 -- STEP: 1065/1935 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.047674842178821564 (0.04795225066408306) + | > loss_mel_ce: 3.1451895236968994 (3.036377919112013) + | > loss: 0.39910805225372314 (0.3855412716037229) + | > current_lr: 5e-06 + | > step_time: 0.3093 (0.32331296356631006) + | > loader_time: 0.034 (0.03212036325338297) + + + --> TIME: 2024-12-23 14:15:48 -- STEP: 1115/1935 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.045350927859544754 (0.04793827032227689) + | > loss_mel_ce: 3.0556588172912598 (3.036260541137558) + | > loss: 0.3876262307167053 (0.38552485188561153) + | > current_lr: 5e-06 + | > step_time: 0.2977 (0.323188810818933) + | > loader_time: 0.0336 (0.03222707889539773) + + + --> TIME: 2024-12-23 14:16:14 -- STEP: 1165/1935 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.04504968971014023 (0.04790943554670514) + | > loss_mel_ce: 2.96246600151062 (3.035988942134021) + | > loss: 0.3759394586086273 (0.3854872975994074) + | > current_lr: 5e-06 + | > step_time: 0.3356 (0.3233119019111339) + | > loader_time: 0.0344 (0.03232124373636527) + + + --> TIME: 2024-12-23 14:16:41 -- STEP: 1215/1935 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.04611687734723091 (0.04790149948601859) + | > loss_mel_ce: 3.1123151779174805 (3.035983410391788) + | > loss: 0.3948040008544922 (0.38548561418007415) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3804 (0.3235542611330135) + | > loader_time: 0.0356 (0.03241714705165027) + + + --> TIME: 2024-12-23 14:17:07 -- STEP: 1265/1935 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.05035349726676941 (0.04789985087019181) + | > loss_mel_ce: 3.0730206966400146 (3.035821714702801) + | > loss: 0.3904217779636383 (0.38546519611663926) + | > current_lr: 5e-06 + | > step_time: 0.31 (0.32360030257183575) + | > loader_time: 0.0338 (0.03249282309189143) + + + --> TIME: 2024-12-23 14:17:33 -- STEP: 1315/1935 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.04675517603754997 (0.047895938548978306) + | > loss_mel_ce: 3.03436279296875 (3.0357821714742093) + | > loss: 0.38513973355293274 (0.3854597641941254) + | > current_lr: 5e-06 + | > step_time: 0.3401 (0.32387351155734356) + | > loader_time: 0.036 (0.03256722457508619) + + + --> TIME: 2024-12-23 14:17:59 -- STEP: 1365/1935 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.049878623336553574 (0.04789476262016612) + | > loss_mel_ce: 3.0771992206573486 (3.0349639871618237) + | > loss: 0.39088472723960876 (0.38535734419420914) + | > current_lr: 5e-06 + | > step_time: 0.3343 (0.32381033862466785) + | > loader_time: 0.0341 (0.032644964050460606) + + + --> TIME: 2024-12-23 14:18:25 -- STEP: 1415/1935 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.04995732381939888 (0.047885298157643946) + | > loss_mel_ce: 2.9244000911712646 (3.0347504363043134) + | > loss: 0.3717946708202362 (0.385329467226675) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2964 (0.32392827613614905) + | > loader_time: 0.0326 (0.03271657940355709) + + + --> TIME: 2024-12-23 14:18:51 -- STEP: 1465/1935 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.04748201370239258 (0.04787293264178287) + | > loss_mel_ce: 3.0313687324523926 (3.035078872185924) + | > loss: 0.38485634326934814 (0.385368976027485) + | > current_lr: 5e-06 + | > step_time: 0.3101 (0.32409046013607523) + | > loader_time: 0.0203 (0.03258669498430581) + + + --> TIME: 2024-12-23 14:19:17 -- STEP: 1515/1935 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.045949723571538925 (0.047864113981672646) + | > loss_mel_ce: 3.00260591506958 (3.034453283127384) + | > loss: 0.38106945157051086 (0.3852896750170008) + | > current_lr: 5e-06 + | > step_time: 0.3205 (0.3240391309505251) + | > loader_time: 0.0335 (0.03252452261770518) + + + --> TIME: 2024-12-23 14:19:43 -- STEP: 1565/1935 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.045999445021152496 (0.047854878532048624) + | > loss_mel_ce: 3.073770523071289 (3.0343988954830463) + | > loss: 0.3899712562561035 (0.3852817221761888) + | > current_lr: 5e-06 + | > step_time: 0.3152 (0.32396603224757414) + | > loader_time: 0.0347 (0.032599070734871054) + + + --> TIME: 2024-12-23 14:20:09 -- STEP: 1615/1935 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.04656260088086128 (0.04784842622307802) + | > loss_mel_ce: 3.042388677597046 (3.0340807482185) + | > loss: 0.38611891865730286 (0.3852411472391414) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3665 (0.32387068529985297) + | > loader_time: 0.035 (0.032675652902561775) + + + --> TIME: 2024-12-23 14:20:35 -- STEP: 1665/1935 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.04458149895071983 (0.04782807438357457) + | > loss_mel_ce: 2.946211099624634 (3.0340070130230776) + | > loss: 0.3738490641117096 (0.3852293862952838) + | > current_lr: 5e-06 + | > step_time: 0.3453 (0.3238763766245802) + | > loader_time: 0.0344 (0.032745355886739956) + + + --> TIME: 2024-12-23 14:21:01 -- STEP: 1715/1935 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.04732364043593407 (0.047820701791141895) + | > loss_mel_ce: 3.1099092960357666 (3.033924167566326) + | > loss: 0.39465412497520447 (0.38521810903841097) + | > current_lr: 5e-06 + | > step_time: 0.3365 (0.32396617825455315) + | > loader_time: 0.032 (0.0328310705134889) + + + --> TIME: 2024-12-23 14:21:27 -- STEP: 1765/1935 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.049297742545604706 (0.047814841906173376) + | > loss_mel_ce: 2.9933347702026367 (3.0340752146399383) + | > loss: 0.38032907247543335 (0.38523625746664814) + | > current_lr: 5e-06 + | > step_time: 0.3428 (0.3240574001928229) + | > loader_time: 0.0357 (0.03288016508388785) + + + --> TIME: 2024-12-23 14:21:53 -- STEP: 1815/1935 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.04804185405373573 (0.047810997079144504) + | > loss_mel_ce: 3.0184240341186523 (3.0342263135043046) + | > loss: 0.3833082318305969 (0.385254664230609) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4044 (0.3238016065487194) + | > loader_time: 0.0356 (0.03290405969646019) + + + --> TIME: 2024-12-23 14:22:19 -- STEP: 1865/1935 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.04804936796426773 (0.047809535360847664) + | > loss_mel_ce: 3.056434154510498 (3.0345904823282757) + | > loss: 0.38806045055389404 (0.3853000026126326) + | > current_lr: 5e-06 + | > step_time: 0.3348 (0.3239634215991557) + | > loader_time: 0.036 (0.03295199966942011) + + + --> TIME: 2024-12-23 14:22:45 -- STEP: 1915/1935 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.046916067600250244 (0.04780652612914617) + | > loss_mel_ce: 3.077486991882324 (3.0347899907562805) + | > loss: 0.3905503749847412 (0.3853245650196819) + | > current_lr: 5e-06 + | > step_time: 0.3077 (0.3239941188623017) + | > loader_time: 0.0355 (0.03299403962516282) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11639625014680798 (-0.0004140837023956928) + | > avg_loss_text_ce: 0.04566926845923216 (-0.0007368273506260931) + | > avg_loss_mel_ce: 3.047976731049893 (-0.005244798130459305) + | > avg_loss: 3.093646004946544 (-0.0059816235243679294) + + + > EPOCH: 2/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 14:23:44)  + + --> TIME: 2024-12-23 14:24:00 -- STEP: 30/1935 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.046569958329200745 (0.04747733722130457) + | > loss_mel_ce: 3.0569539070129395 (3.042770520846049) + | > loss: 0.38794049620628357 (0.3862809826930364) + | > current_lr: 5e-06 + | > step_time: 0.3065 (0.3203617254892986) + | > loader_time: 0.0209 (0.02007877826690674) + + + --> TIME: 2024-12-23 14:24:26 -- STEP: 80/1935 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.04828063026070595 (0.047617848543450234) + | > loss_mel_ce: 3.058103084564209 (3.0411094993352887) + | > loss: 0.388297975063324 (0.3860909197479487) + | > current_lr: 5e-06 + | > step_time: 0.3234 (0.32434071004390724) + | > loader_time: 0.0346 (0.023385760188102726) + + + --> TIME: 2024-12-23 14:24:52 -- STEP: 130/1935 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.04855756089091301 (0.04753803676710679) + | > loss_mel_ce: 3.0861330032348633 (3.038886207800645) + | > loss: 0.3918363153934479 (0.38580303054589493) + | > current_lr: 5e-06 + | > step_time: 0.3139 (0.32554260217226483) + | > loader_time: 0.0342 (0.02721488292400654) + + + --> TIME: 2024-12-23 14:25:18 -- STEP: 180/1935 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.045230425894260406 (0.0474036502962311) + | > loss_mel_ce: 3.0465891361236572 (3.0372081690364414) + | > loss: 0.38647744059562683 (0.38557647748125906) + | > current_lr: 5e-06 + | > step_time: 0.3128 (0.326296016905043) + | > loader_time: 0.0247 (0.02904347048865425) + + + --> TIME: 2024-12-23 14:25:44 -- STEP: 230/1935 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.04762619361281395 (0.047462043198554404) + | > loss_mel_ce: 3.025028944015503 (3.0364872476328975) + | > loss: 0.3840819001197815 (0.38549366126889756) + | > current_lr: 5e-06 + | > step_time: 0.3091 (0.3256417492161629) + | > loader_time: 0.0335 (0.03003951259281325) + + + --> TIME: 2024-12-23 14:26:10 -- STEP: 280/1935 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.04679354652762413 (0.047431157582572527) + | > loss_mel_ce: 3.0018036365509033 (3.0357903718948367) + | > loss: 0.3810746371746063 (0.3854026910449776) + | > current_lr: 5e-06 + | > step_time: 0.2853 (0.32520232626370044) + | > loader_time: 0.0334 (0.030496045521327435) + + + --> TIME: 2024-12-23 14:26:36 -- STEP: 330/1935 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.045256663113832474 (0.04746424832353087) + | > loss_mel_ce: 2.9778473377227783 (3.03571424628749) + | > loss: 0.3778879940509796 (0.3853973115935469) + | > current_lr: 5e-06 + | > step_time: 0.3089 (0.32477523702563676) + | > loader_time: 0.034 (0.030874917001435256) + + + --> TIME: 2024-12-23 14:27:02 -- STEP: 380/1935 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.04860033467411995 (0.047446018016259923) + | > loss_mel_ce: 2.955089807510376 (3.032538982441551) + | > loss: 0.37546128034591675 (0.38499812499472963) + | > current_lr: 5e-06 + | > step_time: 0.3523 (0.3242428974101418) + | > loader_time: 0.0339 (0.03136002954683807) + + + --> TIME: 2024-12-23 14:27:28 -- STEP: 430/1935 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.0470288023352623 (0.047439429813692736) + | > loss_mel_ce: 3.053946018218994 (3.0322590922200403) + | > loss: 0.38762184977531433 (0.3849623148524483) + | > current_lr: 5e-06 + | > step_time: 0.3442 (0.32472999872163283) + | > loader_time: 0.0346 (0.03169467560080595) + + + --> TIME: 2024-12-23 14:27:54 -- STEP: 480/1935 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.0464167483150959 (0.04741991442472984) + | > loss_mel_ce: 3.0036301612854004 (3.0322215581933656) + | > loss: 0.3812558650970459 (0.3849551837270458) + | > current_lr: 5e-06 + | > step_time: 0.2947 (0.3248450756072998) + | > loader_time: 0.0341 (0.03196723908185958) + + + --> TIME: 2024-12-23 14:28:20 -- STEP: 530/1935 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.04625662788748741 (0.04740155640778677) + | > loss_mel_ce: 3.0525946617126465 (3.0327341934420025) + | > loss: 0.38735640048980713 (0.38501696839647476) + | > current_lr: 5e-06 + | > step_time: 0.3464 (0.3244723050099499) + | > loader_time: 0.0336 (0.032035891065057706) + + + --> TIME: 2024-12-23 14:28:46 -- STEP: 580/1935 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.04746858403086662 (0.047364525952986604) + | > loss_mel_ce: 3.1647441387176514 (3.0323326538349007) + | > loss: 0.4015266001224518 (0.3849621471659889) + | > current_lr: 5e-06 + | > step_time: 0.3145 (0.32481397554792224) + | > loader_time: 0.0351 (0.03222234372434945) + + + --> TIME: 2024-12-23 14:29:12 -- STEP: 630/1935 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.046994298696517944 (0.04736886111989856) + | > loss_mel_ce: 3.0362794399261475 (3.032538139252435) + | > loss: 0.3854092061519623 (0.3849883746533167) + | > current_lr: 5e-06 + | > step_time: 0.2704 (0.3247921992862038) + | > loader_time: 0.0329 (0.032367500426277285) + + + --> TIME: 2024-12-23 14:29:39 -- STEP: 680/1935 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.04694829508662224 (0.04734858476063786) + | > loss_mel_ce: 3.001399278640747 (3.03188066482544) + | > loss: 0.3810434341430664 (0.3849036558147737) + | > current_lr: 5e-06 + | > step_time: 0.3239 (0.3247747284524584) + | > loader_time: 0.0342 (0.03248542091425727) + + + --> TIME: 2024-12-23 14:30:04 -- STEP: 730/1935 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.04591499641537666 (0.047337413665978885) + | > loss_mel_ce: 2.9233903884887695 (3.0310176333335987) + | > loss: 0.37116315960884094 (0.38479438052601994) + | > current_lr: 5e-06 + | > step_time: 0.286 (0.32415385801498237) + | > loader_time: 0.0345 (0.03255613340090397) + + + --> TIME: 2024-12-23 14:30:30 -- STEP: 780/1935 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.0471564345061779 (0.047321527479932866) + | > loss_mel_ce: 3.1001241207122803 (3.030833731553494) + | > loss: 0.3934100568294525 (0.38476940699112705) + | > current_lr: 5e-06 + | > step_time: 0.3144 (0.3242030003131969) + | > loader_time: 0.0339 (0.03271252191983736) + + + --> TIME: 2024-12-23 14:30:57 -- STEP: 830/1935 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.04575686529278755 (0.04730658993484027) + | > loss_mel_ce: 3.0586254596710205 (3.029968851158418) + | > loss: 0.3880477845668793 (0.3846594298101332) + | > current_lr: 5e-06 + | > step_time: 0.3147 (0.32447637247752037) + | > loader_time: 0.0335 (0.0328446727201163) + + + --> TIME: 2024-12-23 14:31:23 -- STEP: 880/1935 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.04593326896429062 (0.04730646557343955) + | > loss_mel_ce: 2.979840040206909 (3.0301890795881103) + | > loss: 0.37822166085243225 (0.3846869426694782) + | > current_lr: 5e-06 + | > step_time: 0.3136 (0.3245611708272591) + | > loader_time: 0.0358 (0.032968741926279944) + + + --> TIME: 2024-12-23 14:31:49 -- STEP: 930/1935 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.04761926457285881 (0.0473127178447221) + | > loss_mel_ce: 3.077510118484497 (3.0295113053373117) + | > loss: 0.3906411826610565 (0.3846030024110629) + | > current_lr: 5e-06 + | > step_time: 0.3203 (0.32425803446000656) + | > loader_time: 0.034 (0.03307311381063155) + + + --> TIME: 2024-12-23 14:32:14 -- STEP: 980/1935 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.04557160660624504 (0.04729944341340844) + | > loss_mel_ce: 3.014876365661621 (3.029948886073366) + | > loss: 0.3825559914112091 (0.38465604067456954) + | > current_lr: 5e-06 + | > step_time: 0.3201 (0.3239781722730523) + | > loader_time: 0.033 (0.03314712607130713) + + + --> TIME: 2024-12-23 14:32:40 -- STEP: 1030/1935 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.048397891223430634 (0.04729220064230336) + | > loss_mel_ce: 2.9658143520355225 (3.029239436029231) + | > loss: 0.3767765164375305 (0.38456645414088525) + | > current_lr: 5e-06 + | > step_time: 0.3179 (0.32378674016415526) + | > loader_time: 0.0338 (0.03320532909874778) + + + --> TIME: 2024-12-23 14:33:06 -- STEP: 1080/1935 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.046093981713056564 (0.04728129201879105) + | > loss_mel_ce: 3.07924485206604 (3.0293126455059762) + | > loss: 0.3906673491001129 (0.3845742417706382) + | > current_lr: 5e-06 + | > step_time: 0.3539 (0.32391282562856355) + | > loader_time: 0.0289 (0.033272572358449305) + + + --> TIME: 2024-12-23 14:33:33 -- STEP: 1130/1935 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.04798826575279236 (0.047263219313020216) + | > loss_mel_ce: 2.917863130569458 (3.0295304975678445) + | > loss: 0.3707314133644104 (0.3845992141856555) + | > current_lr: 5e-06 + | > step_time: 0.2926 (0.32397083877462224) + | > loader_time: 0.0321 (0.03331534672627408) + + + --> TIME: 2024-12-23 14:33:58 -- STEP: 1180/1935 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.045079298317432404 (0.04725463690899189) + | > loss_mel_ce: 3.1071059703826904 (3.0295678259962706) + | > loss: 0.3940231502056122 (0.3846028074622154) + | > current_lr: 5e-06 + | > step_time: 0.2711 (0.32388695498644327) + | > loader_time: 0.0335 (0.03335248494552353) + + + --> TIME: 2024-12-23 14:34:24 -- STEP: 1230/1935 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.048000697046518326 (0.0472494449496754) + | > loss_mel_ce: 3.0206570625305176 (3.029137997898629) + | > loss: 0.3835822343826294 (0.3845484299872951) + | > current_lr: 5e-06 + | > step_time: 0.3457 (0.3238488129483976) + | > loader_time: 0.0358 (0.033390944372347714) + + + --> TIME: 2024-12-23 14:34:50 -- STEP: 1280/1935 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.04777815192937851 (0.047248846112052) + | > loss_mel_ce: 3.0264508724212646 (3.0294862199574712) + | > loss: 0.38427862524986267 (0.3845918828155847) + | > current_lr: 5e-06 + | > step_time: 0.3094 (0.3238208975642923) + | > loader_time: 0.0338 (0.033433062024414546) + + + --> TIME: 2024-12-23 14:35:16 -- STEP: 1330/1935 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.04709232226014137 (0.04725046957233792) + | > loss_mel_ce: 2.944483995437622 (3.029918057577951) + | > loss: 0.37394705414772034 (0.38464606540991836) + | > current_lr: 5e-06 + | > step_time: 0.2947 (0.32381621213783923) + | > loader_time: 0.0335 (0.033472122644123276) + + + --> TIME: 2024-12-23 14:35:43 -- STEP: 1380/1935 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.04735272377729416 (0.04724914103854391) + | > loss_mel_ce: 2.969967842102051 (3.029983685845914) + | > loss: 0.3771650791168213 (0.38465410289557067) + | > current_lr: 5e-06 + | > step_time: 0.3436 (0.32388747187628286) + | > loader_time: 0.035 (0.03350939819778222) + + + --> TIME: 2024-12-23 14:36:09 -- STEP: 1430/1935 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.04644821211695671 (0.04724257976546152) + | > loss_mel_ce: 3.052719831466675 (3.0302793184360426) + | > loss: 0.3873960077762604 (0.3846902367743577) + | > current_lr: 5e-06 + | > step_time: 0.2928 (0.323880115589062) + | > loader_time: 0.04 (0.03355187719518487) + + + --> TIME: 2024-12-23 14:36:35 -- STEP: 1480/1935 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.046090543270111084 (0.04724128037640773) + | > loss_mel_ce: 2.9954116344451904 (3.03032065404428) + | > loss: 0.3801877796649933 (0.38469524127808763) + | > current_lr: 5e-06 + | > step_time: 0.2887 (0.3240796485462706) + | > loader_time: 0.0342 (0.033599907159805285) + + + --> TIME: 2024-12-23 14:37:01 -- STEP: 1530/1935 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.04977583885192871 (0.047236942901622996) + | > loss_mel_ce: 3.078113079071045 (3.030368542982862) + | > loss: 0.3909861147403717 (0.38470068517463646) + | > current_lr: 5e-06 + | > step_time: 0.3532 (0.32397448798410283) + | > loader_time: 0.0353 (0.03364368575850339) + + + --> TIME: 2024-12-23 14:37:27 -- STEP: 1580/1935 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.04680688679218292 (0.04722740368658227) + | > loss_mel_ce: 3.0956227779388428 (3.0304730774481086) + | > loss: 0.39280369877815247 (0.3847125595506232) + | > current_lr: 5e-06 + | > step_time: 0.3359 (0.32415395114995277) + | > loader_time: 0.0359 (0.03366971664790864) + + + --> TIME: 2024-12-23 14:37:53 -- STEP: 1630/1935 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.049750398844480515 (0.047237373801645305) + | > loss_mel_ce: 3.0709433555603027 (3.030493935778097) + | > loss: 0.39008671045303345 (0.3847164130832519) + | > current_lr: 5e-06 + | > step_time: 0.3165 (0.3240363226346445) + | > loader_time: 0.0342 (0.033719058563372824) + + + --> TIME: 2024-12-23 14:38:19 -- STEP: 1680/1935 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.04645149037241936 (0.04722758015795121) + | > loss_mel_ce: 3.0959460735321045 (3.0302338202794394) + | > loss: 0.3927997052669525 (0.3846826744576294) + | > current_lr: 5e-06 + | > step_time: 0.3126 (0.32387846282550287) + | > loader_time: 0.0356 (0.0337660543975376) + + + --> TIME: 2024-12-23 14:38:45 -- STEP: 1730/1935 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.04635382443666458 (0.04722400170917799) + | > loss_mel_ce: 2.9942543506622314 (3.030415204770303) + | > loss: 0.38007602095603943 (0.3847049002185721) + | > current_lr: 5e-06 + | > step_time: 0.3409 (0.32387556891909935) + | > loader_time: 0.0347 (0.03379897313310922) + + + --> TIME: 2024-12-23 14:39:11 -- STEP: 1780/1935 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.049229927361011505 (0.04721489133459797) + | > loss_mel_ce: 3.068938732147217 (3.030099646562941) + | > loss: 0.38977107405662537 (0.3846643166930487) + | > current_lr: 5e-06 + | > step_time: 0.3462 (0.3239618170127442) + | > loader_time: 0.035 (0.03382197149683921) + + + --> TIME: 2024-12-23 14:39:37 -- STEP: 1830/1935 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.04955437406897545 (0.04721143935995348) + | > loss_mel_ce: 3.1241583824157715 (3.0298851294595686) + | > loss: 0.39671409130096436 (0.3846370705815611) + | > current_lr: 5e-06 + | > step_time: 0.2957 (0.32395007649406077) + | > loader_time: 0.0363 (0.03385478251618766) + + + --> TIME: 2024-12-23 14:40:04 -- STEP: 1880/1935 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.04947727918624878 (0.04720309412780593) + | > loss_mel_ce: 3.013029098510742 (3.0295817635160813) + | > loss: 0.38281330466270447 (0.3845981067006891) + | > current_lr: 5e-06 + | > step_time: 0.3249 (0.3241104444290731) + | > loader_time: 0.0345 (0.03388497766028058) + + + --> TIME: 2024-12-23 14:40:30 -- STEP: 1930/1935 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.04704469442367554 (0.04719520541284367) + | > loss_mel_ce: 3.0635883808135986 (3.029633112027855) + | > loss: 0.38882914185523987 (0.3846035392018796) + | > current_lr: 5e-06 + | > step_time: 0.3464 (0.3241607202767092) + | > loader_time: 0.0345 (0.03390919285117033) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11536145812333228 (-0.0010347920234757002) + | > avg_loss_text_ce: 0.045125745989457525 (-0.0005435224697746335) + | > avg_loss_mel_ce: 3.037335312727726 (-0.010641418322166807) + | > avg_loss: 3.0824610608996768 (-0.011184944046867429) + + + > EPOCH: 3/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 14:41:20)  + + --> TIME: 2024-12-23 14:41:45 -- STEP: 45/1935 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.04662612825632095 (0.047238440066576) + | > loss_mel_ce: 3.0652759075164795 (3.0404409991370307) + | > loss: 0.3889877498149872 (0.38595992922782896) + | > current_lr: 5e-06 + | > step_time: 0.2913 (0.32614472177293563) + | > loader_time: 0.0329 (0.03350101576911078) + + + --> TIME: 2024-12-23 14:42:11 -- STEP: 95/1935 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.04758873209357262 (0.047241077964243136) + | > loss_mel_ce: 3.0739831924438477 (3.036978877218146) + | > loss: 0.3901965022087097 (0.3855274940791883) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3687 (0.3247476151115014) + | > loader_time: 0.0362 (0.033244097860235924) + + + --> TIME: 2024-12-23 14:42:37 -- STEP: 145/1935 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.04703277349472046 (0.04709234124627607) + | > loss_mel_ce: 3.0416674613952637 (3.035797434839709) + | > loss: 0.3860875368118286 (0.3853612221520522) + | > current_lr: 5e-06 + | > step_time: 0.313 (0.3230001992192761) + | > loader_time: 0.0338 (0.03352182815814841) + + + --> TIME: 2024-12-23 14:43:03 -- STEP: 195/1935 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.04723095893859863 (0.04700583613071686) + | > loss_mel_ce: 3.059126138687134 (3.033048458588429) + | > loss: 0.38829463720321655 (0.385006786768253) + | > current_lr: 5e-06 + | > step_time: 0.3824 (0.325670672685672) + | > loader_time: 0.0345 (0.033641631786639856) + + + > CHECKPOINT : /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000/checkpoint_6000.pth + + --> TIME: 2024-12-23 14:43:33 -- STEP: 245/1935 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.04654159024357796 (0.04697688521475208) + | > loss_mel_ce: 3.033561944961548 (3.0303755867237947) + | > loss: 0.3850129544734955 (0.384669059150073) + | > current_lr: 5e-06 + | > step_time: 0.3095 (0.3240194262290487) + | > loader_time: 0.0197 (0.030717847785171205) + + + --> TIME: 2024-12-23 14:43:58 -- STEP: 295/1935 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.04525383189320564 (0.04691589075629994) + | > loss_mel_ce: 3.043050765991211 (3.030360499883103) + | > loss: 0.38603806495666504 (0.3846595495434133) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3297 (0.3236504150649249) + | > loader_time: 0.0332 (0.029881210650427873) + + + --> TIME: 2024-12-23 14:44:24 -- STEP: 345/1935 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.04823765158653259 (0.046937987856242966) + | > loss_mel_ce: 3.0117740631103516 (3.030658046750056) + | > loss: 0.3825014531612396 (0.38469950496286603) + | > current_lr: 5e-06 + | > step_time: 0.3447 (0.3233511620673581) + | > loader_time: 0.0346 (0.030455238231714227) + + + --> TIME: 2024-12-23 14:44:50 -- STEP: 395/1935 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.04760703444480896 (0.04695892498085771) + | > loss_mel_ce: 2.9917383193969727 (3.02944598137578) + | > loss: 0.3799181580543518 (0.38455061391939105) + | > current_lr: 5e-06 + | > step_time: 0.3054 (0.3228668864769273) + | > loader_time: 0.0335 (0.030744864669027212) + + + --> TIME: 2024-12-23 14:45:16 -- STEP: 445/1935 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.0476718544960022 (0.046960198795527555) + | > loss_mel_ce: 2.9119491577148438 (3.029101172993693) + | > loss: 0.36995261907577515 (0.38450767217057485) + | > current_lr: 5e-06 + | > step_time: 0.32 (0.3228750159231466) + | > loader_time: 0.0332 (0.031069085839089388) + + + --> TIME: 2024-12-23 14:45:41 -- STEP: 495/1935 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.04830111563205719 (0.046942483731592544) + | > loss_mel_ce: 2.9911117553710938 (3.0282850949451188) + | > loss: 0.3799266219139099 (0.38440344803261045) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3373 (0.32246378503664586) + | > loader_time: 0.0338 (0.03080096533804229) + + + --> TIME: 2024-12-23 14:46:07 -- STEP: 545/1935 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.04711706563830376 (0.04692596980464568) + | > loss_mel_ce: 3.108562469482422 (3.0268669443392984) + | > loss: 0.39445993304252625 (0.384224114833622) + | > current_lr: 5e-06 + | > step_time: 0.3133 (0.3222955384385696) + | > loader_time: 0.0333 (0.031070045593681685) + + + --> TIME: 2024-12-23 14:46:33 -- STEP: 595/1935 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.048065535724163055 (0.046874424808916924) + | > loss_mel_ce: 3.000256299972534 (3.026866900019285) + | > loss: 0.38104021549224854 (0.3842176662272767) + | > current_lr: 5e-06 + | > step_time: 0.2932 (0.32294906768478293) + | > loader_time: 0.0322 (0.03131199363900833) + + + --> TIME: 2024-12-23 14:46:59 -- STEP: 645/1935 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.047648001462221146 (0.046855126020982275) + | > loss_mel_ce: 3.0237720012664795 (3.025825890459755) + | > loss: 0.38392749428749084 (0.38408512768819364) + | > current_lr: 5e-06 + | > step_time: 0.2958 (0.3224999420402589) + | > loader_time: 0.0331 (0.03156319514725559) + + + --> TIME: 2024-12-23 14:47:25 -- STEP: 695/1935 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.04429589956998825 (0.04683951003838784) + | > loss_mel_ce: 2.9399120807647705 (3.024969707804618) + | > loss: 0.3730259835720062 (0.38397615277509906) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3099 (0.3229230829280059) + | > loader_time: 0.0333 (0.031750899253131674) + + + --> TIME: 2024-12-23 14:47:51 -- STEP: 745/1935 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.046721331775188446 (0.04683151427131369) + | > loss_mel_ce: 3.0494015216827393 (3.0254352227153394) + | > loss: 0.38701534271240234 (0.38403334249586074) + | > current_lr: 5e-06 + | > step_time: 0.3354 (0.32286609419240236) + | > loader_time: 0.0346 (0.031881621059955394) + + + --> TIME: 2024-12-23 14:48:17 -- STEP: 795/1935 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.04813070222735405 (0.046806647811295836) + | > loss_mel_ce: 2.9966886043548584 (3.0247475657073206) + | > loss: 0.3806024193763733 (0.38394427704361256) + | > current_lr: 5e-06 + | > step_time: 0.2995 (0.32253951576520795) + | > loader_time: 0.0329 (0.032013253745792836) + + + --> TIME: 2024-12-23 14:48:43 -- STEP: 845/1935 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.04520048573613167 (0.046785880497397736) + | > loss_mel_ce: 3.069641590118408 (3.024582298549675) + | > loss: 0.38935527205467224 (0.3839210227396362) + | > current_lr: 5e-06 + | > step_time: 0.3456 (0.32296205187690313) + | > loader_time: 0.0338 (0.032135252980790945) + + + --> TIME: 2024-12-23 14:49:10 -- STEP: 895/1935 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.043410904705524445 (0.04676555203242673) + | > loss_mel_ce: 3.0356550216674805 (3.0238109823045782) + | > loss: 0.38488325476646423 (0.38382206712355166) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3638 (0.3233080094087056) + | > loader_time: 0.0297 (0.03221922693305843) + + + --> TIME: 2024-12-23 14:49:36 -- STEP: 945/1935 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.04635459929704666 (0.04675147313053014) + | > loss_mel_ce: 2.9445762634277344 (3.0234543934070244) + | > loss: 0.37386634945869446 (0.3837757336399543) + | > current_lr: 5e-06 + | > step_time: 0.3158 (0.32342646992395785) + | > loader_time: 0.034 (0.032305059988031966) + + + --> TIME: 2024-12-23 14:50:01 -- STEP: 995/1935 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.047098807990550995 (0.04674550828771973) + | > loss_mel_ce: 3.037498712539673 (3.0235637897223087) + | > loss: 0.38557469844818115 (0.3837886625498384) + | > current_lr: 5e-06 + | > step_time: 0.3499 (0.3231602108059216) + | > loader_time: 0.0354 (0.032378821157330887) + + + --> TIME: 2024-12-23 14:50:27 -- STEP: 1045/1935 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.048804450780153275 (0.0467541703660237) + | > loss_mel_ce: 3.0066299438476562 (3.0234230684891843) + | > loss: 0.38192930817604065 (0.3837721551149086) + | > current_lr: 5e-06 + | > step_time: 0.3428 (0.3231798867859908) + | > loader_time: 0.0346 (0.03245187079507202) + + + --> TIME: 2024-12-23 14:50:53 -- STEP: 1095/1935 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.04787115380167961 (0.04676147828635559) + | > loss_mel_ce: 3.0491087436676025 (3.023141640615245) + | > loss: 0.3871224820613861 (0.38373789008893916) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3195 (0.3230561720181817) + | > loader_time: 0.0323 (0.03251313357592721) + + + --> TIME: 2024-12-23 14:51:19 -- STEP: 1145/1935 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.04473523050546646 (0.04675172909638765) + | > loss_mel_ce: 3.066603660583496 (3.0230553533312547) + | > loss: 0.38891735672950745 (0.38372588558488524) + | > current_lr: 5e-06 + | > step_time: 0.3196 (0.32307440307983654) + | > loader_time: 0.034 (0.03257089764790768) + + + --> TIME: 2024-12-23 14:51:45 -- STEP: 1195/1935 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.04809545353055 (0.04673746681487706) + | > loss_mel_ce: 2.9392240047454834 (3.022834018084794) + | > loss: 0.37341493368148804 (0.38369643583457325) + | > current_lr: 5e-06 + | > step_time: 0.3132 (0.32314548731847775) + | > loader_time: 0.035 (0.032641009207051194) + + + --> TIME: 2024-12-23 14:52:11 -- STEP: 1245/1935 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.046945471316576004 (0.046720920111161114) + | > loss_mel_ce: 3.0668957233428955 (3.022918067112507) + | > loss: 0.3892301619052887 (0.38370487359632927) + | > current_lr: 5e-06 + | > step_time: 0.3135 (0.3231403931077702) + | > loader_time: 0.0357 (0.032709459320129676) + + + --> TIME: 2024-12-23 14:52:38 -- STEP: 1295/1935 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.047104720026254654 (0.04672397648965986) + | > loss_mel_ce: 3.0036659240722656 (3.0230997880913595) + | > loss: 0.38134634494781494 (0.3837279707768711) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3362 (0.32335530119052697) + | > loader_time: 0.0347 (0.032799165681522365) + + + --> TIME: 2024-12-23 14:53:04 -- STEP: 1345/1935 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.04722139611840248 (0.04672136700341687) + | > loss_mel_ce: 2.9893884658813477 (3.022592818781348) + | > loss: 0.3795762360095978 (0.3836642734401733) + | > current_lr: 5e-06 + | > step_time: 0.3163 (0.32338880287227123) + | > loader_time: 0.034 (0.032876515831646025) + + + --> TIME: 2024-12-23 14:53:30 -- STEP: 1395/1935 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.04613299295306206 (0.046716808882879025) + | > loss_mel_ce: 2.886112689971924 (3.0225163827233015) + | > loss: 0.36653071641921997 (0.3836541491810995) + | > current_lr: 5e-06 + | > step_time: 0.3413 (0.3232971560570502) + | > loader_time: 0.0352 (0.03290224400045202) + + + --> TIME: 2024-12-23 14:53:56 -- STEP: 1445/1935 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.04719839617609978 (0.046722410799722704) + | > loss_mel_ce: 2.9456710815429688 (3.022840172856739) + | > loss: 0.3741086721420288 (0.3836953232032619) + | > current_lr: 5e-06 + | > step_time: 0.2985 (0.3233001509339754) + | > loader_time: 0.034 (0.03294653232534873) + + + --> TIME: 2024-12-23 14:54:22 -- STEP: 1495/1935 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.04721018299460411 (0.04671090932245232) + | > loss_mel_ce: 3.0674946308135986 (3.0229832496132762) + | > loss: 0.38933810591697693 (0.38371177014698565) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3316 (0.32335270415979084) + | > loader_time: 0.0221 (0.03298746734159859) + + + --> TIME: 2024-12-23 14:54:47 -- STEP: 1545/1935 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.04717496410012245 (0.04671174508178883) + | > loss_mel_ce: 2.9214885234832764 (3.022740678108242) + | > loss: 0.37108293175697327 (0.38368155317784913) + | > current_lr: 5e-06 + | > step_time: 0.3363 (0.32312934684136163) + | > loader_time: 0.0347 (0.0330355008443198) + + + --> TIME: 2024-12-23 14:55:13 -- STEP: 1595/1935 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.045992184430360794 (0.0467096457123569) + | > loss_mel_ce: 3.005341053009033 (3.0225329343801777) + | > loss: 0.3814166486263275 (0.3836553228125678) + | > current_lr: 5e-06 + | > step_time: 0.319 (0.32310510802792203) + | > loader_time: 0.0359 (0.033088867940872915) + + + --> TIME: 2024-12-23 14:55:39 -- STEP: 1645/1935 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.04826631024479866 (0.046699395109998355) + | > loss_mel_ce: 2.915411949157715 (3.0223319891375997) + | > loss: 0.37045979499816895 (0.3836289232806233) + | > current_lr: 5e-06 + | > step_time: 0.3425 (0.3231173351542932) + | > loader_time: 0.0355 (0.03313456697666911) + + + --> TIME: 2024-12-23 14:56:05 -- STEP: 1695/1935 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.04455701634287834 (0.046695974403250476) + | > loss_mel_ce: 2.983562469482422 (3.0223877005169193) + | > loss: 0.37851494550704956 (0.38363545955458234) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3643 (0.3232267696245586) + | > loader_time: 0.0363 (0.03318760345819087) + + + --> TIME: 2024-12-23 14:56:32 -- STEP: 1745/1935 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.046495579183101654 (0.046689832334361255) + | > loss_mel_ce: 2.98921537399292 (3.022378781463494) + | > loss: 0.37946388125419617 (0.3836335768501534) + | > current_lr: 5e-06 + | > step_time: 0.2959 (0.32332206742469805) + | > loader_time: 0.0344 (0.033277651232088326) + + + --> TIME: 2024-12-23 14:56:58 -- STEP: 1795/1935 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.04555884748697281 (0.04668064832811895) + | > loss_mel_ce: 3.0242936611175537 (3.022213784433011) + | > loss: 0.38373157382011414 (0.38361180418049073) + | > current_lr: 5e-06 + | > step_time: 0.349 (0.32334737206568) + | > loader_time: 0.036 (0.03332623213446588) + + + --> TIME: 2024-12-23 14:57:24 -- STEP: 1845/1935 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.047004204243421555 (0.04666851790455293) + | > loss_mel_ce: 3.039909601211548 (3.022377026048781) + | > loss: 0.3858642280101776 (0.3836306931526682) + | > current_lr: 5e-06 + | > step_time: 0.3115 (0.32339971214129065) + | > loader_time: 0.0348 (0.03338168914402092) + + + --> TIME: 2024-12-23 14:57:50 -- STEP: 1895/1935 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.04632427915930748 (0.0466568069399818) + | > loss_mel_ce: 3.0092227458953857 (3.022011957193744) + | > loss: 0.38194337487220764 (0.383583595664646) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2939 (0.3234971842853882) + | > loader_time: 0.0255 (0.033424089200264025) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11699432796902125 (+0.0016328698456889723) + | > avg_loss_text_ce: 0.04461457644297619 (-0.0005111695464813312) + | > avg_loss_mel_ce: 3.0283503628740407 (-0.008984949853685364) + | > avg_loss: 3.072964940408264 (-0.009496120491412796) + + + > EPOCH: 4/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 14:58:59)  + + --> TIME: 2024-12-23 14:59:06 -- STEP: 10/1935 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.04876476153731346 (0.04687039740383625) + | > loss_mel_ce: 3.032423496246338 (3.02239933013916) + | > loss: 0.3851485252380371 (0.3836587190628052) + | > current_lr: 5e-06 + | > step_time: 0.3162 (0.3151905059814453) + | > loader_time: 0.0351 (0.0341094970703125) + + + --> TIME: 2024-12-23 14:59:32 -- STEP: 60/1935 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.04654217138886452 (0.04606936424970627) + | > loss_mel_ce: 3.0527563095092773 (3.0085548996925353) + | > loss: 0.38741230964660645 (0.38182803392410275) + | > current_lr: 5e-06 + | > step_time: 0.307 (0.32477421363194797) + | > loader_time: 0.0336 (0.03520751396814982) + + + --> TIME: 2024-12-23 14:59:58 -- STEP: 110/1935 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.044836848974227905 (0.04617286520925435) + | > loss_mel_ce: 3.1157023906707764 (3.012208166989413) + | > loss: 0.39506739377975464 (0.3822976293888959) + | > current_lr: 5e-06 + | > step_time: 0.3476 (0.3236206249757247) + | > loader_time: 0.0349 (0.0347048044204712) + + + --> TIME: 2024-12-23 15:00:24 -- STEP: 160/1935 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.04655368626117706 (0.04616065991576761) + | > loss_mel_ce: 3.031777858734131 (3.0114499062299727) + | > loss: 0.38479143381118774 (0.3822013208642602) + | > current_lr: 5e-06 + | > step_time: 0.2555 (0.324177072942257) + | > loader_time: 0.0307 (0.03263755589723587) + + + --> TIME: 2024-12-23 15:00:49 -- STEP: 210/1935 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.04623880609869957 (0.046240990360577904) + | > loss_mel_ce: 2.946484327316284 (3.0146353903270904) + | > loss: 0.37409040331840515 (0.38260954760369803) + | > current_lr: 5e-06 + | > step_time: 0.3198 (0.3240877514793758) + | > loader_time: 0.0179 (0.03088756969996863) + + + --> TIME: 2024-12-23 15:01:15 -- STEP: 260/1935 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.0470258928835392 (0.04618198176702628) + | > loss_mel_ce: 2.9544878005981445 (3.013089469762949) + | > loss: 0.3751892149448395 (0.3824089316221384) + | > current_lr: 5e-06 + | > step_time: 0.3372 (0.3242174102709843) + | > loader_time: 0.0206 (0.02878639881427473) + + + --> TIME: 2024-12-23 15:01:40 -- STEP: 310/1935 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.04577096179127693 (0.046190133979243625) + | > loss_mel_ce: 3.0320796966552734 (3.014350077413743) + | > loss: 0.38473132252693176 (0.3825675267365671) + | > current_lr: 5e-06 + | > step_time: 0.3191 (0.32414429726139193) + | > loader_time: 0.0327 (0.02790440974696991) + + + --> TIME: 2024-12-23 15:02:06 -- STEP: 360/1935 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.047013457864522934 (0.046214181888434604) + | > loss_mel_ce: 3.051290988922119 (3.0143478466404807) + | > loss: 0.38728806376457214 (0.38257025405764566) + | > current_lr: 5e-06 + | > step_time: 0.2718 (0.323906444840961) + | > loader_time: 0.0228 (0.028268723355399247) + + + --> TIME: 2024-12-23 15:02:32 -- STEP: 410/1935 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.04614213854074478 (0.0462138446911079) + | > loss_mel_ce: 2.998441696166992 (3.0125352120980984) + | > loss: 0.38057297468185425 (0.38234363249162334) + | > current_lr: 5e-06 + | > step_time: 0.3423 (0.32423416812245426) + | > loader_time: 0.0349 (0.028693698673713505) + + + --> TIME: 2024-12-23 15:02:58 -- STEP: 460/1935 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.04514291509985924 (0.046228008393360207) + | > loss_mel_ce: 2.927574396133423 (3.0106023643327795) + | > loss: 0.37158966064453125 (0.382103797080724) + | > current_lr: 5e-06 + | > step_time: 0.2976 (0.3243098134579865) + | > loader_time: 0.0336 (0.029315516223078198) + + + --> TIME: 2024-12-23 15:03:24 -- STEP: 510/1935 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.04683288559317589 (0.04624869401256242) + | > loss_mel_ce: 3.062464475631714 (3.0109340218936698) + | > loss: 0.3886621594429016 (0.38214783978228467) + | > current_lr: 5e-06 + | > step_time: 0.348 (0.32476666207407023) + | > loader_time: 0.0349 (0.02968348334817326) + + + --> TIME: 2024-12-23 15:03:51 -- STEP: 560/1935 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.04664469137787819 (0.04624629975296555) + | > loss_mel_ce: 3.134509801864624 (3.01151411320482) + | > loss: 0.3976443111896515 (0.382220052023019) + | > current_lr: 5e-06 + | > step_time: 0.328 (0.32517951045717497) + | > loader_time: 0.0359 (0.030091417687279843) + + + --> TIME: 2024-12-23 15:04:17 -- STEP: 610/1935 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.047402143478393555 (0.046240215174487356) + | > loss_mel_ce: 3.036585569381714 (3.0114947580900373) + | > loss: 0.3854984641075134 (0.3822168718595976) + | > current_lr: 5e-06 + | > step_time: 0.3385 (0.3252952208284471) + | > loader_time: 0.0412 (0.030363254859799252) + + + --> TIME: 2024-12-23 15:04:43 -- STEP: 660/1935 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.04422467574477196 (0.04622767041578434) + | > loss_mel_ce: 2.94876766204834 (3.0112368507818754) + | > loss: 0.37412405014038086 (0.38218306524283985) + | > current_lr: 5e-06 + | > step_time: 0.3212 (0.32516483393582424) + | > loader_time: 0.0333 (0.03072237245964282) + + + --> TIME: 2024-12-23 15:05:09 -- STEP: 710/1935 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.04540972784161568 (0.046206386870061816) + | > loss_mel_ce: 3.0477781295776367 (3.0108330803857735) + | > loss: 0.3866484761238098 (0.3821299333387699) + | > current_lr: 5e-06 + | > step_time: 0.3145 (0.32509157993424076) + | > loader_time: 0.0383 (0.031001881142737164) + + + --> TIME: 2024-12-23 15:05:35 -- STEP: 760/1935 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.04561389982700348 (0.046190360575718285) + | > loss_mel_ce: 3.0183982849121094 (3.0100511340718534) + | > loss: 0.38300153613090515 (0.38203018684136253) + | > current_lr: 5e-06 + | > step_time: 0.3179 (0.32539699893248664) + | > loader_time: 0.0334 (0.03125724321917484) + + + --> TIME: 2024-12-23 15:06:02 -- STEP: 810/1935 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.04768729954957962 (0.04619884623881473) + | > loss_mel_ce: 2.946911573410034 (3.009731291841579) + | > loss: 0.37432485818862915 (0.38199126720428483) + | > current_lr: 5e-06 + | > step_time: 0.3107 (0.3257825044938073) + | > loader_time: 0.0334 (0.031496868604495194) + + + --> TIME: 2024-12-23 15:06:28 -- STEP: 860/1935 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.04629354923963547 (0.04619476510601679) + | > loss_mel_ce: 3.100855827331543 (3.0099875902020665) + | > loss: 0.3933936655521393 (0.3820227942840999) + | > current_lr: 5e-06 + | > step_time: 0.335 (0.3255158635072927) + | > loader_time: 0.0379 (0.031704853301824536) + + + --> TIME: 2024-12-23 15:06:54 -- STEP: 910/1935 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.047129787504673004 (0.04618232736548222) + | > loss_mel_ce: 3.0441384315490723 (3.0101135120287057) + | > loss: 0.3864085376262665 (0.3820369798731019) + | > current_lr: 5e-06 + | > step_time: 0.3386 (0.32553356322613364) + | > loader_time: 0.0353 (0.03190826609894469) + + + --> TIME: 2024-12-23 15:07:20 -- STEP: 960/1935 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.0470181442797184 (0.046192822403584896) + | > loss_mel_ce: 3.0648550987243652 (3.0094457211593797) + | > loss: 0.38898414373397827 (0.38195481793954983) + | > current_lr: 5e-06 + | > step_time: 0.2946 (0.3254952212174731) + | > loader_time: 0.0339 (0.03205760891238848) + + + --> TIME: 2024-12-23 15:07:46 -- STEP: 1010/1935 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.04465581476688385 (0.04620277505404876) + | > loss_mel_ce: 2.952159881591797 (3.0092961061118864) + | > loss: 0.37460196018218994 (0.38193736005537593) + | > current_lr: 5e-06 + | > step_time: 0.3338 (0.32539885563425475) + | > loader_time: 0.0354 (0.03214371652886422) + + + --> TIME: 2024-12-23 15:08:13 -- STEP: 1060/1935 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.04511341452598572 (0.04619432897922001) + | > loss_mel_ce: 2.922577381134033 (3.0094176317161) + | > loss: 0.37096133828163147 (0.38195149496478864) + | > current_lr: 5e-06 + | > step_time: 0.3391 (0.3254958166266385) + | > loader_time: 0.0339 (0.03217228763508348) + + + --> TIME: 2024-12-23 15:08:38 -- STEP: 1110/1935 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.04512495547533035 (0.04618328557693738) + | > loss_mel_ce: 3.0128173828125 (3.0090471924962228) + | > loss: 0.3822427988052368 (0.3819038096311932) + | > current_lr: 5e-06 + | > step_time: 0.3011 (0.325310952598984) + | > loader_time: 0.0339 (0.03227175098281723) + + + --> TIME: 2024-12-23 15:09:04 -- STEP: 1160/1935 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.047611355781555176 (0.04617210755060452) + | > loss_mel_ce: 3.0410501956939697 (3.009285847277476) + | > loss: 0.3860827088356018 (0.38193224429570427) + | > current_lr: 5e-06 + | > step_time: 0.3128 (0.32523946001611886) + | > loader_time: 0.0347 (0.03234854176126679) + + + --> TIME: 2024-12-23 15:09:31 -- STEP: 1210/1935 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.04522492736577988 (0.04616499443746295) + | > loss_mel_ce: 2.9889097213745117 (3.008628298034352) + | > loss: 0.3792668282985687 (0.3818491614800844) + | > current_lr: 5e-06 + | > step_time: 0.3136 (0.3253037267480014) + | > loader_time: 0.0348 (0.032433985284537344) + + + --> TIME: 2024-12-23 15:09:57 -- STEP: 1260/1935 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.04434511438012123 (0.04616155397915642) + | > loss_mel_ce: 3.0079939365386963 (3.008413060127743) + | > loss: 0.3815423846244812 (0.38182182669166587) + | > current_lr: 5e-06 + | > step_time: 0.3135 (0.3255126037294901) + | > loader_time: 0.0329 (0.03251586206375608) + + + --> TIME: 2024-12-23 15:10:23 -- STEP: 1310/1935 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.044008925557136536 (0.046150249168845456) + | > loss_mel_ce: 2.966014862060547 (3.0085091550841585) + | > loss: 0.3762529790401459 (0.38183242546693097) + | > current_lr: 5e-06 + | > step_time: 0.2721 (0.32522078452219455) + | > loader_time: 0.0327 (0.032549205627150204) + + + --> TIME: 2024-12-23 15:10:49 -- STEP: 1360/1935 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.04318307340145111 (0.04614000118556699) + | > loss_mel_ce: 3.087390422821045 (3.0081581548732883) + | > loss: 0.39132168889045715 (0.38178726943100244) + | > current_lr: 5e-06 + | > step_time: 0.3173 (0.3252044937189886) + | > loader_time: 0.0344 (0.03260638222974892) + + + --> TIME: 2024-12-23 15:11:15 -- STEP: 1410/1935 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.046862561255693436 (0.04612603061006843) + | > loss_mel_ce: 3.0312225818634033 (3.00832238231145) + | > loss: 0.38476064801216125 (0.38180605151974606) + | > current_lr: 5e-06 + | > step_time: 0.3375 (0.3250797620056368) + | > loader_time: 0.0355 (0.03266523205642163) + + + --> TIME: 2024-12-23 15:11:41 -- STEP: 1460/1935 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.04553862288594246 (0.04612220047577599) + | > loss_mel_ce: 3.065669059753418 (3.008136524402933) + | > loss: 0.3889009654521942 (0.38178234053392973) + | > current_lr: 5e-06 + | > step_time: 0.336 (0.32511261636263716) + | > loader_time: 0.0347 (0.03275247697960848) + + + --> TIME: 2024-12-23 15:12:07 -- STEP: 1510/1935 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.04664476215839386 (0.04612016254505571) + | > loss_mel_ce: 3.029085159301758 (3.0078965787066543) + | > loss: 0.3844662308692932 (0.3817520925343435) + | > current_lr: 5e-06 + | > step_time: 0.3537 (0.32515569680573947) + | > loader_time: 0.0351 (0.032821121121084526) + + + --> TIME: 2024-12-23 15:12:33 -- STEP: 1560/1935 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.045297425240278244 (0.046114450244185215) + | > loss_mel_ce: 2.9952926635742188 (3.007677036065323) + | > loss: 0.38007375597953796 (0.3817239356919745) + | > current_lr: 5e-06 + | > step_time: 0.3185 (0.32496412549263437) + | > loader_time: 0.0344 (0.03286931147942179) + + + --> TIME: 2024-12-23 15:12:59 -- STEP: 1610/1935 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.048058390617370605 (0.04611602269622106) + | > loss_mel_ce: 2.985349416732788 (3.007557048264499) + | > loss: 0.37917596101760864 (0.3817091338382748) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.32496678162805825) + | > loader_time: 0.0356 (0.03292708870787063) + + + --> TIME: 2024-12-23 15:13:26 -- STEP: 1660/1935 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.044155459851026535 (0.0461072952363146) + | > loss_mel_ce: 2.97674822807312 (3.007235436985293) + | > loss: 0.3776129484176636 (0.38166784155799716) + | > current_lr: 5e-06 + | > step_time: 0.3351 (0.32507875410907283) + | > loader_time: 0.0347 (0.032982946734830584) + + + --> TIME: 2024-12-23 15:13:51 -- STEP: 1710/1935 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.04813963919878006 (0.04610189905509959) + | > loss_mel_ce: 3.084886074066162 (3.007302891301831) + | > loss: 0.3916282057762146 (0.38167559880959395) + | > current_lr: 5e-06 + | > step_time: 0.314 (0.3249428523214244) + | > loader_time: 0.0339 (0.03302442121226888) + + + --> TIME: 2024-12-23 15:14:18 -- STEP: 1760/1935 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.044607628136873245 (0.04609085412442004) + | > loss_mel_ce: 2.949904203414917 (3.0073086884888744) + | > loss: 0.37431398034095764 (0.38167494286529074) + | > current_lr: 5e-06 + | > step_time: 0.3098 (0.3252587239850655) + | > loader_time: 0.0345 (0.03304842114448551) + + + --> TIME: 2024-12-23 15:14:44 -- STEP: 1810/1935 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.0448877327144146 (0.046085064545356075) + | > loss_mel_ce: 3.012542724609375 (3.0074493118412597) + | > loss: 0.38217881321907043 (0.3816917970858888) + | > current_lr: 5e-06 + | > step_time: 0.3478 (0.3251406829001499) + | > loader_time: 0.0339 (0.033082488887217834) + + + --> TIME: 2024-12-23 15:15:10 -- STEP: 1860/1935 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.044114746153354645 (0.046083015610053304) + | > loss_mel_ce: 2.926703929901123 (3.007299901977663) + | > loss: 0.37135234475135803 (0.381672864783836) + | > current_lr: 5e-06 + | > step_time: 0.2942 (0.3250716017138577) + | > loader_time: 0.0345 (0.03310108274541879) + + + --> TIME: 2024-12-23 15:15:36 -- STEP: 1910/1935 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.0473145954310894 (0.04607928468606856) + | > loss_mel_ce: 2.888424873352051 (3.0071899329180503) + | > loss: 0.36696743965148926 (0.3816586523467961) + | > current_lr: 5e-06 + | > step_time: 0.287 (0.32506233896884656) + | > loader_time: 0.0332 (0.033136209892352886) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11755510532494738 (+0.0005607773559261342) + | > avg_loss_text_ce: 0.04411539661161826 (-0.0004991798313579363) + | > avg_loss_mel_ce: 3.019207513693608 (-0.009142849180432844) + | > avg_loss: 3.06332291617538 (-0.009642024232884072) + + + > EPOCH: 5/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 15:16:37)  + + --> TIME: 2024-12-23 15:16:52 -- STEP: 25/1935 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.04684947803616524 (0.045761347115039826) + | > loss_mel_ce: 3.0252816677093506 (2.9810088157653807) + | > loss: 0.38401639461517334 (0.3783462715148926) + | > current_lr: 5e-06 + | > step_time: 0.578 (0.3294730377197266) + | > loader_time: 0.0354 (0.033246250152587885) + + + --> TIME: 2024-12-23 15:17:17 -- STEP: 75/1935 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.04577083885669708 (0.04574925303459167) + | > loss_mel_ce: 3.0281858444213867 (2.9989836597442627) + | > loss: 0.3842445909976959 (0.3805916162331899) + | > current_lr: 5e-06 + | > step_time: 0.3432 (0.3231792608896891) + | > loader_time: 0.0343 (0.03383216540018716) + + + --> TIME: 2024-12-23 15:17:44 -- STEP: 125/1935 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.04652072861790657 (0.045714041382074355) + | > loss_mel_ce: 2.9623630046844482 (2.9993136768341064) + | > loss: 0.3761104643344879 (0.3806284658908844) + | > current_lr: 5e-06 + | > step_time: 0.2741 (0.3249165325164794) + | > loader_time: 0.0328 (0.0339723606109619) + + + --> TIME: 2024-12-23 15:18:10 -- STEP: 175/1935 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.04576065391302109 (0.045777042806148506) + | > loss_mel_ce: 3.0459394454956055 (2.9966372571672717) + | > loss: 0.3864625096321106 (0.380301788364138) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3791 (0.3262584686279296) + | > loader_time: 0.0341 (0.033225270680018855) + + + --> TIME: 2024-12-23 15:18:35 -- STEP: 225/1935 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.04620489478111267 (0.04577099791831439) + | > loss_mel_ce: 3.0088086128234863 (3.0001310327317983) + | > loss: 0.381876677274704 (0.3807377543714312) + | > current_lr: 5e-06 + | > step_time: 0.3189 (0.32407522519429516) + | > loader_time: 0.0347 (0.03342245843675403) + + + --> TIME: 2024-12-23 15:19:02 -- STEP: 275/1935 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.0448295883834362 (0.04576478471810166) + | > loss_mel_ce: 2.9738287925720215 (3.000358570272273) + | > loss: 0.37733229994773865 (0.38076541954820814) + | > current_lr: 5e-06 + | > step_time: 0.296 (0.32467958276922054) + | > loader_time: 0.0344 (0.03351290616122161) + + + --> TIME: 2024-12-23 15:19:28 -- STEP: 325/1935 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.044253457337617874 (0.04582393069679918) + | > loss_mel_ce: 2.9612820148468018 (3.0012872938009414) + | > loss: 0.3756919205188751 (0.3808889035078198) + | > current_lr: 5e-06 + | > step_time: 0.2991 (0.32467346484844495) + | > loader_time: 0.0405 (0.03349445489736705) + + + --> TIME: 2024-12-23 15:19:54 -- STEP: 375/1935 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.04591157287359238 (0.04583023664355277) + | > loss_mel_ce: 2.899650812149048 (3.0018063621521) + | > loss: 0.3681952953338623 (0.3809545749823255) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3735 (0.3247335618336995) + | > loader_time: 0.0346 (0.03361160977681479) + + + --> TIME: 2024-12-23 15:20:20 -- STEP: 425/1935 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.046621937304735184 (0.045824461114757194) + | > loss_mel_ce: 3.1325695514678955 (3.003184932259953) + | > loss: 0.3973989486694336 (0.38112617443589625) + | > current_lr: 5e-06 + | > step_time: 0.3459 (0.32512670404770794) + | > loader_time: 0.0321 (0.03362470626831055) + + + --> TIME: 2024-12-23 15:20:46 -- STEP: 475/1935 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.043709345161914825 (0.0458225645517048) + | > loss_mel_ce: 2.9770328998565674 (3.0013775253295902) + | > loss: 0.3775927722454071 (0.3809000116272978) + | > current_lr: 5e-06 + | > step_time: 0.3142 (0.3250770076952483) + | > loader_time: 0.0336 (0.03361963623448424) + + + --> TIME: 2024-12-23 15:21:12 -- STEP: 525/1935 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.046097997575998306 (0.04582199890698705) + | > loss_mel_ce: 2.878180503845215 (3.0012389637175065) + | > loss: 0.36553481221199036 (0.38088262081146257) + | > current_lr: 5e-06 + | > step_time: 0.3122 (0.325159854888916) + | > loader_time: 0.0333 (0.03361667042686827) + + + --> TIME: 2024-12-23 15:21:38 -- STEP: 575/1935 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.04540036618709564 (0.04581607439595719) + | > loss_mel_ce: 2.924616575241089 (3.0012174058997116) + | > loss: 0.3712521195411682 (0.3808791856662087) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3395 (0.3249192664934241) + | > loader_time: 0.0336 (0.033555692175160315) + + + --> TIME: 2024-12-23 15:22:04 -- STEP: 625/1935 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.04647592082619667 (0.045816770446300496) + | > loss_mel_ce: 2.916142463684082 (3.0007797016143796) + | > loss: 0.37032729387283325 (0.380824559545517) + | > current_lr: 5e-06 + | > step_time: 0.3366 (0.3244687549591064) + | > loader_time: 0.0342 (0.03348672142028811) + + + --> TIME: 2024-12-23 15:22:30 -- STEP: 675/1935 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.04409250244498253 (0.045812932942752474) + | > loss_mel_ce: 2.978337287902832 (3.000835029460766) + | > loss: 0.3778037130832672 (0.3808309958157716) + | > current_lr: 5e-06 + | > step_time: 0.2989 (0.3243854914771184) + | > loader_time: 0.0343 (0.033534260855780734) + + + --> TIME: 2024-12-23 15:22:56 -- STEP: 725/1935 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.045723073184490204 (0.04579500946505315) + | > loss_mel_ce: 2.990461587905884 (2.9998697507792507) + | > loss: 0.37952306866645813 (0.3807080954929878) + | > current_lr: 5e-06 + | > step_time: 0.3364 (0.3244455465777167) + | > loader_time: 0.0336 (0.033539357349790376) + + + --> TIME: 2024-12-23 15:23:22 -- STEP: 775/1935 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.04589154198765755 (0.045803589854509594) + | > loss_mel_ce: 3.040916919708252 (3.00037176839767) + | > loss: 0.38585105538368225 (0.380771920065726) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3279 (0.3245708422507007) + | > loader_time: 0.0339 (0.033609242900725325) + + + --> TIME: 2024-12-23 15:23:49 -- STEP: 825/1935 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.04502232000231743 (0.04580148752440105) + | > loss_mel_ce: 3.055539131164551 (3.0000519333463727) + | > loss: 0.38757017254829407 (0.38073167793678514) + | > current_lr: 5e-06 + | > step_time: 0.3093 (0.32466064886613294) + | > loader_time: 0.0336 (0.03366220676537716) + + + --> TIME: 2024-12-23 15:24:15 -- STEP: 875/1935 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.04529384523630142 (0.04577135608451706) + | > loss_mel_ce: 3.0173847675323486 (2.999636504854475) + | > loss: 0.3828348219394684 (0.3806759828158787) + | > current_lr: 5e-06 + | > step_time: 0.318 (0.3246293277740475) + | > loader_time: 0.0342 (0.03367597906930105) + + + --> TIME: 2024-12-23 15:24:41 -- STEP: 925/1935 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.04499606043100357 (0.04575706018789394) + | > loss_mel_ce: 3.0796773433685303 (2.9994826244663546) + | > loss: 0.39058417081832886 (0.38065496070964916) + | > current_lr: 5e-06 + | > step_time: 0.2963 (0.32470502853393524) + | > loader_time: 0.0324 (0.03369026158307051) + + + --> TIME: 2024-12-23 15:25:07 -- STEP: 975/1935 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.04485870897769928 (0.045740944746977236) + | > loss_mel_ce: 2.9693617820739746 (2.999879741668701) + | > loss: 0.3767775595188141 (0.3807025858072134) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.3247713172130093) + | > loader_time: 0.0357 (0.033732840220133474) + + + --> TIME: 2024-12-23 15:25:33 -- STEP: 1025/1935 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.046511199325323105 (0.045743364295581485) + | > loss_mel_ce: 3.0610485076904297 (2.9991903458572016) + | > loss: 0.3884449601173401 (0.38061671379135875) + | > current_lr: 5e-06 + | > step_time: 0.3835 (0.3245309973926076) + | > loader_time: 0.0373 (0.03372033235503407) + + + --> TIME: 2024-12-23 15:25:59 -- STEP: 1075/1935 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.04456232115626335 (0.04573830408758895) + | > loss_mel_ce: 3.115910053253174 (2.999664983971175) + | > loss: 0.3950590491294861 (0.38067541097485746) + | > current_lr: 5e-06 + | > step_time: 0.3375 (0.3243537443737647) + | > loader_time: 0.0247 (0.03368041060691658) + + + --> TIME: 2024-12-23 15:26:24 -- STEP: 1125/1935 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.04765917733311653 (0.04573076530628735) + | > loss_mel_ce: 2.9697201251983643 (3.0004410046471497) + | > loss: 0.37717241048812866 (0.3807714713149601) + | > current_lr: 5e-06 + | > step_time: 0.2972 (0.32415352100796135) + | > loader_time: 0.0328 (0.03362050713433162) + + + --> TIME: 2024-12-23 15:26:50 -- STEP: 1175/1935 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.04470236971974373 (0.0457183977359153) + | > loss_mel_ce: 2.989981174468994 (3.000860377778397) + | > loss: 0.3793354332447052 (0.38082234697138995) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3173 (0.32426338804529053) + | > loader_time: 0.033 (0.03355539626263558) + + + --> TIME: 2024-12-23 15:27:16 -- STEP: 1225/1935 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.04540817812085152 (0.045709079081306676) + | > loss_mel_ce: 3.033310651779175 (3.0005154317739047) + | > loss: 0.38483986258506775 (0.38077806390061636) + | > current_lr: 5e-06 + | > step_time: 0.3174 (0.32428814187341776) + | > loader_time: 0.0267 (0.033495789936610634) + + + --> TIME: 2024-12-23 15:27:42 -- STEP: 1275/1935 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.046883486211299896 (0.04570079360230297) + | > loss_mel_ce: 2.936363935470581 (3.000546888463637) + | > loss: 0.3729059398174286 (0.3807809602279286) + | > current_lr: 5e-06 + | > step_time: 0.3048 (0.3243164617875039) + | > loader_time: 0.0208 (0.033239735061047146) + + + --> TIME: 2024-12-23 15:28:08 -- STEP: 1325/1935 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.04458845034241676 (0.04569948825914904) + | > loss_mel_ce: 2.965867042541504 (3.0001342786033174) + | > loss: 0.37630695104599 (0.3807292208806519) + | > current_lr: 5e-06 + | > step_time: 0.2527 (0.32436941218825965) + | > loader_time: 0.0283 (0.03321126128142735) + + + --> TIME: 2024-12-23 15:28:35 -- STEP: 1375/1935 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.04659080505371094 (0.045690385458144286) + | > loss_mel_ce: 2.9846160411834717 (3.000084211176092) + | > loss: 0.3789008557796478 (0.38072182458097237) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3298 (0.3245530024441802) + | > loader_time: 0.0431 (0.03319784095070581) + + + --> TIME: 2024-12-23 15:29:01 -- STEP: 1425/1935 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.04610831290483475 (0.04567733092004793) + | > loss_mel_ce: 3.0661814212799072 (2.999809697134453) + | > loss: 0.3890362083911896 (0.38068587853197416) + | > current_lr: 5e-06 + | > step_time: 0.3189 (0.32455165695725785) + | > loader_time: 0.0359 (0.03316003448084783) + + + --> TIME: 2024-12-23 15:29:27 -- STEP: 1475/1935 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.04328638315200806 (0.04566585257649422) + | > loss_mel_ce: 2.9543843269348145 (2.9995686139898785) + | > loss: 0.3747088313102722 (0.38065430837162423) + | > current_lr: 5e-06 + | > step_time: 0.3125 (0.32491680630182806) + | > loader_time: 0.0248 (0.03314905134298036) + + + --> TIME: 2024-12-23 15:29:53 -- STEP: 1525/1935 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.044617824256420135 (0.04565663785963762) + | > loss_mel_ce: 2.9921412467956543 (2.999836995640739) + | > loss: 0.3795948922634125 (0.3806867041744165) + | > current_lr: 5e-06 + | > step_time: 0.3457 (0.3249039521764539) + | > loader_time: 0.0364 (0.03315519473591792) + + + --> TIME: 2024-12-23 15:30:19 -- STEP: 1575/1935 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.04442613944411278 (0.04566059698188116) + | > loss_mel_ce: 3.002336025238037 (3.000009483307127) + | > loss: 0.3808452785015106 (0.38070876002311665) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.32502967622545054) + | > loader_time: 0.0345 (0.032880412964593826) + + + --> TIME: 2024-12-23 15:30:46 -- STEP: 1625/1935 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.045422110706567764 (0.04563973981142044) + | > loss_mel_ce: 3.0365612506866455 (2.999875743719248) + | > loss: 0.3852479159832001 (0.3806894353903253) + | > current_lr: 5e-06 + | > step_time: 0.3168 (0.32513524392934967) + | > loader_time: 0.0347 (0.03293061769925633) + + + --> TIME: 2024-12-23 15:31:12 -- STEP: 1675/1935 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.04656560346484184 (0.045634213505841015) + | > loss_mel_ce: 2.9403483867645264 (2.9995737158362545) + | > loss: 0.37336423993110657 (0.3806509911302306) + | > current_lr: 5e-06 + | > step_time: 0.3461 (0.32514804825853966) + | > loader_time: 0.0355 (0.03298234654896298) + + + --> TIME: 2024-12-23 15:31:38 -- STEP: 1725/1935 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.04550325870513916 (0.04563967010897139) + | > loss_mel_ce: 2.93410325050354 (2.999436964366747) + | > loss: 0.3724508285522461 (0.3806345792611436) + | > current_lr: 5e-06 + | > step_time: 0.3416 (0.32503693870876166) + | > loader_time: 0.0354 (0.03302513924197877) + + + --> TIME: 2024-12-23 15:32:03 -- STEP: 1775/1935 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.04514764994382858 (0.04562886934465086) + | > loss_mel_ce: 2.944298505783081 (2.9995590303985167) + | > loss: 0.3736807703971863 (0.3806484874033588) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3154 (0.3249474137265919) + | > loader_time: 0.0337 (0.03304709031548304) + + + --> TIME: 2024-12-23 15:32:30 -- STEP: 1825/1935 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.04574848711490631 (0.04562291781379752) + | > loss_mel_ce: 3.0286598205566406 (2.999430706076426) + | > loss: 0.3843010365962982 (0.3806317028934005) + | > current_lr: 5e-06 + | > step_time: 0.3336 (0.3249552583041258) + | > loader_time: 0.0351 (0.033081500432262725) + + + --> TIME: 2024-12-23 15:32:56 -- STEP: 1875/1935 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.04346506670117378 (0.04561077460050583) + | > loss_mel_ce: 2.9997806549072266 (2.999074052556356) + | > loss: 0.3804057240486145 (0.3805856033166246) + | > current_lr: 5e-06 + | > step_time: 0.3376 (0.32500104064941426) + | > loader_time: 0.0335 (0.03311871795654304) + + + --> TIME: 2024-12-23 15:33:22 -- STEP: 1925/1935 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.04589160904288292 (0.045607612862602454) + | > loss_mel_ce: 3.014502763748169 (2.9990025199543346) + | > loss: 0.3825492858886719 (0.38057626646834497) + | > current_lr: 5e-06 + | > step_time: 0.3495 (0.32486363448105865) + | > loader_time: 0.0355 (0.033153665592144145) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.1162349792441936 (-0.0013201260807537835) + | > avg_loss_text_ce: 0.043673324991356226 (-0.00044207162026203145) + | > avg_loss_mel_ce: 3.012298130025766 (-0.006909383667841773) + | > avg_loss: 3.0559714611130544 (-0.007351455062325485) + + > BEST MODEL : /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000/best_model_11610.pth + + > EPOCH: 6/7 + --> /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000 + + > TRAINING (2024-12-23 15:34:20)  + + --> TIME: 2024-12-23 15:34:41 -- STEP: 40/1935 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.044062357395887375 (0.045226150378584865) + | > loss_mel_ce: 3.0430827140808105 (2.9801291048526766) + | > loss: 0.3858931362628937 (0.3781694084405899) + | > current_lr: 5e-06 + | > step_time: 0.3399 (0.3259426176548004) + | > loader_time: 0.0163 (0.018322527408599854) + + + --> TIME: 2024-12-23 15:35:07 -- STEP: 90/1935 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.0438748337328434 (0.04540510202447574) + | > loss_mel_ce: 2.9227664470672607 (2.983218095037671) + | > loss: 0.37083014845848083 (0.37857790026399835) + | > current_lr: 5e-06 + | > step_time: 0.3108 (0.32397663858201775) + | > loader_time: 0.0329 (0.02470499939388699) + + + --> TIME: 2024-12-23 15:35:33 -- STEP: 140/1935 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.047171883285045624 (0.04541670996695757) + | > loss_mel_ce: 3.0870940685272217 (2.987199400152478) + | > loss: 0.3917832374572754 (0.3790770141141756) + | > current_lr: 5e-06 + | > step_time: 0.2767 (0.32673670564379026) + | > loader_time: 0.0317 (0.02768256323678153) + + + --> TIME: 2024-12-23 15:35:59 -- STEP: 190/1935 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.04530227556824684 (0.045502357949551764) + | > loss_mel_ce: 2.9632761478424072 (2.99070569214068) + | > loss: 0.37607231736183167 (0.3795260063911739) + | > current_lr: 5e-06 + | > step_time: 0.3091 (0.3245510427575363) + | > loader_time: 0.0328 (0.02901607438137657) + + + --> TIME: 2024-12-23 15:36:25 -- STEP: 240/1935 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.04403872415423393 (0.04549616305157541) + | > loss_mel_ce: 3.048917055130005 (2.9922143995761874) + | > loss: 0.3866194784641266 (0.3797138202935457) + | > current_lr: 5e-06 + | > step_time: 0.3314 (0.3251861840486527) + | > loader_time: 0.0334 (0.029900618394215903) + + + --> TIME: 2024-12-23 15:36:51 -- STEP: 290/1935 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.04322359338402748 (0.04547273091458041) + | > loss_mel_ce: 3.0447990894317627 (2.993211592476944) + | > loss: 0.3860028386116028 (0.3798355405700618) + | > current_lr: 5e-06 + | > step_time: 0.3257 (0.3244499453182879) + | > loader_time: 0.0344 (0.03038669536853659) + + + --> TIME: 2024-12-23 15:37:17 -- STEP: 340/1935 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.04534347355365753 (0.04544366820551019) + | > loss_mel_ce: 2.9420387744903564 (2.993645957638236) + | > loss: 0.373422771692276 (0.37988620339071066) + | > current_lr: 5e-06 + | > step_time: 0.2909 (0.32448954652337486) + | > loader_time: 0.0311 (0.030799564894507914) + + + --> TIME: 2024-12-23 15:37:43 -- STEP: 390/1935 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.04359586536884308 (0.0453856751513787) + | > loss_mel_ce: 2.9551968574523926 (2.993248087320573) + | > loss: 0.3748490810394287 (0.3798292205119746) + | > current_lr: 5e-06 + | > step_time: 0.3195 (0.3245905747780435) + | > loader_time: 0.0336 (0.03110818129319411) + + + > CHECKPOINT : /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000/checkpoint_12000.pth + + --> TIME: 2024-12-23 15:38:14 -- STEP: 440/1935 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.043695930391550064 (0.04534592324529183) + | > loss_mel_ce: 3.0493128299713135 (2.993870320645246) + | > loss: 0.38662609457969666 (0.37990203066305694) + | > current_lr: 5e-06 + | > step_time: 0.2869 (0.3239094967191872) + | > loader_time: 0.0335 (0.031443604555996964) + + + --> TIME: 2024-12-23 15:38:40 -- STEP: 490/1935 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.044572170823812485 (0.04535696489014189) + | > loss_mel_ce: 2.95894455909729 (2.9925743394968465) + | > loss: 0.3754395842552185 (0.3797414130094101) + | > current_lr: 5e-06 + | > step_time: 0.3429 (0.32365477912280044) + | > loader_time: 0.0352 (0.03173394349156592) + + + --> TIME: 2024-12-23 15:39:06 -- STEP: 540/1935 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.044120095670223236 (0.04536574306311432) + | > loss_mel_ce: 2.9220428466796875 (2.9906814712065244) + | > loss: 0.3707703649997711 (0.37950590142497326) + | > current_lr: 5e-06 + | > step_time: 0.2955 (0.323286239747648) + | > loader_time: 0.0339 (0.03196749201527345) + + + --> TIME: 2024-12-23 15:39:32 -- STEP: 590/1935 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.04432106763124466 (0.04536422578838922) + | > loss_mel_ce: 3.0402724742889404 (2.991418972661941) + | > loss: 0.38557419180870056 (0.3795978993177415) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.3243210202556545) + | > loader_time: 0.036 (0.032146025512178045) + + + --> TIME: 2024-12-23 15:39:58 -- STEP: 640/1935 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.04701656103134155 (0.04536673073889689) + | > loss_mel_ce: 2.9804141521453857 (2.990498902648691) + | > loss: 0.3784288465976715 (0.3794832037296147) + | > current_lr: 5e-06 + | > step_time: 0.3257 (0.3240126542747021) + | > loader_time: 0.0365 (0.03232643045485018) + + + --> TIME: 2024-12-23 15:40:24 -- STEP: 690/1935 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.048134516924619675 (0.045346080949125074) + | > loss_mel_ce: 3.0584042072296143 (2.9912591643955415) + | > loss: 0.388317346572876 (0.3795756551666536) + | > current_lr: 5e-06 + | > step_time: 0.3168 (0.3240569767744647) + | > loader_time: 0.0345 (0.03249713337939717) + + + --> TIME: 2024-12-23 15:40:50 -- STEP: 740/1935 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.0440949946641922 (0.04533663909076839) + | > loss_mel_ce: 3.039722442626953 (2.9911975029352567) + | > loss: 0.3854771852493286 (0.3795667673284943) + | > current_lr: 5e-06 + | > step_time: 0.3382 (0.3242373930441367) + | > loader_time: 0.0358 (0.03264659127673584) + + + --> TIME: 2024-12-23 15:41:17 -- STEP: 790/1935 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.04436204209923744 (0.0453298593482262) + | > loss_mel_ce: 2.957324981689453 (2.991199469566347) + | > loss: 0.37521088123321533 (0.37956616580486296) + | > current_lr: 5e-06 + | > step_time: 0.2925 (0.32427256831639917) + | > loader_time: 0.0338 (0.032665876195400546) + + + --> TIME: 2024-12-23 15:41:43 -- STEP: 840/1935 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.04652686417102814 (0.0453273521337126) + | > loss_mel_ce: 3.0382423400878906 (2.9910512067022794) + | > loss: 0.3855961561203003 (0.37954731963220095) + | > current_lr: 5e-06 + | > step_time: 0.3188 (0.3245062104293277) + | > loader_time: 0.0335 (0.03274617876325334) + + + --> TIME: 2024-12-23 15:42:09 -- STEP: 890/1935 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.04441365227103233 (0.04531904866688707) + | > loss_mel_ce: 2.9631528854370117 (2.990829706192018) + | > loss: 0.3759458065032959 (0.3795185940319233) + | > current_lr: 5e-06 + | > step_time: 0.3424 (0.32456856684738317) + | > loader_time: 0.0353 (0.03278186080161105) + + + --> TIME: 2024-12-23 15:42:35 -- STEP: 940/1935 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.044940605759620667 (0.04530925080972783) + | > loss_mel_ce: 2.9322118759155273 (2.9899115209883846) + | > loss: 0.37214407324790955 (0.3794025962023025) + | > current_lr: 5e-06 + | > step_time: 0.2991 (0.32445583723961025) + | > loader_time: 0.041 (0.03284077948712289) + + + --> TIME: 2024-12-23 15:43:01 -- STEP: 990/1935 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.04368729516863823 (0.045306935936513575) + | > loss_mel_ce: 3.0315964221954346 (2.989615742847174) + | > loss: 0.38441047072410583 (0.3793653346673407) + | > current_lr: 5e-06 + | > step_time: 0.3474 (0.32416650237459116) + | > loader_time: 0.0344 (0.032906993711837626) + + + --> TIME: 2024-12-23 15:43:27 -- STEP: 1040/1935 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.043960876762866974 (0.04528512266249611) + | > loss_mel_ce: 3.0140902996063232 (2.990207819984511) + | > loss: 0.3822563886642456 (0.37943661760252256) + | > current_lr: 5e-06 + | > step_time: 0.2856 (0.32419740374271677) + | > loader_time: 0.0344 (0.03296841062032261) + + + --> TIME: 2024-12-23 15:43:53 -- STEP: 1090/1935 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.04566548392176628 (0.045269195602164365) + | > loss_mel_ce: 3.0605623722076416 (2.9901533188076193) + | > loss: 0.3882784843444824 (0.37942781410086046) + | > current_lr: 5e-06 + | > step_time: 0.336 (0.3243873082169699) + | > loader_time: 0.029 (0.033014968994560585) + + + --> TIME: 2024-12-23 15:44:19 -- STEP: 1140/1935 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.04609273374080658 (0.04526678515892281) + | > loss_mel_ce: 2.9851951599121094 (2.990150252559729) + | > loss: 0.37891098856925964 (0.3794271295530754) + | > current_lr: 5e-06 + | > step_time: 0.2763 (0.3245446458197477) + | > loader_time: 0.0331 (0.03305681504701315) + + + --> TIME: 2024-12-23 15:44:45 -- STEP: 1190/1935 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.04449916258454323 (0.045254339919495984) + | > loss_mel_ce: 3.0358643531799316 (2.9900364154527175) + | > loss: 0.38504543900489807 (0.3794113442677408) + | > current_lr: 5e-06 + | > step_time: 0.272 (0.3244971101023571) + | > loader_time: 0.0327 (0.03309741380835782) + + + --> TIME: 2024-12-23 15:45:11 -- STEP: 1240/1935 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.04526558145880699 (0.04525307549704468) + | > loss_mel_ce: 3.0638349056243896 (2.9887796152022585) + | > loss: 0.38863757252693176 (0.37925408619546114) + | > current_lr: 5e-06 + | > step_time: 0.3078 (0.32444243238818254) + | > loader_time: 0.0354 (0.033143641102698525) + + + --> TIME: 2024-12-23 15:45:37 -- STEP: 1290/1935 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.045943040400743484 (0.04524187134622141) + | > loss_mel_ce: 2.9567666053771973 (2.988278123205021) + | > loss: 0.37533870339393616 (0.37918999920519736) + | > current_lr: 5e-06 + | > step_time: 0.3 (0.3244469088177348) + | > loader_time: 0.0407 (0.03321341802907543) + + + --> TIME: 2024-12-23 15:46:04 -- STEP: 1340/1935 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.04499118775129318 (0.04524290569182207) + | > loss_mel_ce: 2.9821531772613525 (2.987714717103473) + | > loss: 0.3783930540084839 (0.37911970277330753) + | > current_lr: 5e-06 + | > step_time: 0.3331 (0.32455295075231544) + | > loader_time: 0.0343 (0.0332728919698231) + + + --> TIME: 2024-12-23 15:46:30 -- STEP: 1390/1935 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.044992826879024506 (0.04523524151270766) + | > loss_mel_ce: 2.9840919971466064 (2.987752754568196) + | > loss: 0.37863561511039734 (0.37912349945349627) + | > current_lr: 5e-06 + | > step_time: 0.3076 (0.3247653671305818) + | > loader_time: 0.032 (0.03328144721847641) + + + --> TIME: 2024-12-23 15:46:56 -- STEP: 1440/1935 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.045743536204099655 (0.045217272121873174) + | > loss_mel_ce: 2.927184581756592 (2.9873936598499617) + | > loss: 0.371616005897522 (0.37907636647836995) + | > current_lr: 5e-06 + | > step_time: 0.294 (0.3248187452554707) + | > loader_time: 0.0347 (0.03326343811220588) + + + --> TIME: 2024-12-23 15:47:23 -- STEP: 1490/1935 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.04170152172446251 (0.04520021741772259) + | > loss_mel_ce: 2.96610164642334 (2.9873379985758124) + | > loss: 0.3759754002094269 (0.3790672770482579) + | > current_lr: 5e-06 + | > step_time: 0.3346 (0.32489039465885106) + | > loader_time: 0.0343 (0.03331739518466411) + + + --> TIME: 2024-12-23 15:47:48 -- STEP: 1540/1935 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.046875789761543274 (0.045184280566097086) + | > loss_mel_ce: 2.971383571624756 (2.987454295003569) + | > loss: 0.37728241086006165 (0.37907982199610024) + | > current_lr: 5e-06 + | > step_time: 0.3183 (0.32470225080267223) + | > loader_time: 0.0348 (0.03329452245266403) + + + --> TIME: 2024-12-23 15:48:14 -- STEP: 1590/1935 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.04706810042262077 (0.04518248029053212) + | > loss_mel_ce: 3.046093463897705 (2.9875605907080307) + | > loss: 0.38664519786834717 (0.3790928839325152) + | > current_lr: 5e-06 + | > step_time: 0.3177 (0.3246653619802229) + | > loader_time: 0.035 (0.033290178071027986) + + + --> TIME: 2024-12-23 15:48:40 -- STEP: 1640/1935 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.04711534082889557 (0.04517411102381785) + | > loss_mel_ce: 2.946345806121826 (2.986931311357312) + | > loss: 0.37418264150619507 (0.37901317784335525) + | > current_lr: 5e-06 + | > step_time: 0.2549 (0.32470059627439973) + | > loader_time: 0.0334 (0.03332732479746743) + + + --> TIME: 2024-12-23 15:49:06 -- STEP: 1690/1935 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.04595552012324333 (0.04517220776506078) + | > loss_mel_ce: 3.0156567096710205 (2.986784256844831) + | > loss: 0.3827015161514282 (0.3789945581079232) + | > current_lr: 5e-06 + | > step_time: 0.275 (0.3246354579925542) + | > loader_time: 0.0328 (0.03334352716186337) + + + --> TIME: 2024-12-23 15:49:32 -- STEP: 1740/1935 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.045698150992393494 (0.045164559738731944) + | > loss_mel_ce: 2.953768491744995 (2.9866194256420795) + | > loss: 0.3749333322048187 (0.37897299815868485) + | > current_lr: 5e-06 + | > step_time: 0.3857 (0.3246630157547441) + | > loader_time: 0.0356 (0.03336951595613318) + + + --> TIME: 2024-12-23 15:49:59 -- STEP: 1790/1935 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.04614006355404854 (0.04516546140169965) + | > loss_mel_ce: 2.9911601543426514 (2.9866576989935765) + | > loss: 0.37966251373291016 (0.3789778950327599) + | > current_lr: 5e-06 + | > step_time: 0.3381 (0.32471660768519617) + | > loader_time: 0.0255 (0.033373557122725925) + + + --> TIME: 2024-12-23 15:50:25 -- STEP: 1840/1935 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.04269757494330406 (0.04516081309917829) + | > loss_mel_ce: 2.778010845184326 (2.98650021954723) + | > loss: 0.35258856415748596 (0.37895762909041775) + | > current_lr: 5e-06 + | > step_time: 0.2887 (0.3247720732637079) + | > loader_time: 0.0352 (0.033375904093617896) + + + --> TIME: 2024-12-23 15:50:51 -- STEP: 1890/1935 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.0452643446624279 (0.04516420783425766) + | > loss_mel_ce: 2.947503089904785 (2.986167579983908) + | > loss: 0.3740959167480469 (0.3789164735211264) + | > current_lr: 5e-06 + | > step_time: 0.3381 (0.32465864678539347) + | > loader_time: 0.0353 (0.03339664507164521) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.1170845103986335 (+0.0008495311544399059) + | > avg_loss_text_ce: 0.04324476138660401 (-0.00042856360475221345) + | > avg_loss_mel_ce: 3.0061961978372906 (-0.006101932188475434) + | > avg_loss: 3.0494409691203734 (-0.006530491992680965) + + > BEST MODEL : /checkpoints/GPT_XTTS_FT-December-23-2024_01+48PM-0000000/best_model_13545.pth