source/train/XLearningRate.cpp
float XLearningRate::MakeLRTransformer(const float lrate, const int nstep, const int nwarmup, const float warmupInitLR)
{
    float lr = 0.0F;
    float warmupEndLR = lrate;
    float lrStep = (warmupEndLR - warmupInitLR) / nwarmup;
    float decayFactor = warmupEndLR * pow(float(nwarmup), 0.5F);

    /* learning rate, scheduled by inverse square root */
    if (nstep < nwarmup)
        lr = warmupInitLR + nstep * lrStep;
    else
        lr = decayFactor * pow((float)nstep, -0.5F);

    return lr;
}
/* a Transformer-style scheduler. For more details, see
"Attention is all need" by Vaswani at al. 
>> lrate - the learning rate
>> nstep - the update step number 
>> nwarmup - the warmup step number 
*/
a Transformer-style scheduler. For more details, see
"Attention is all need" by Vaswani at al.
a Transformer - style scheduler . For more details see " Attention is all need " by Vaswani at al .
float XLearningRate::MakeLRTransformer(const float lrate, const int nstep, const int nwarmup, const float warmupInitLR)
{
    float lr = 0.0F;
    float warmupEndLR = lrate;
    float lrStep = (warmupEndLR - warmupInitLR) / nwarmup;
    float decayFactor = warmupEndLR * pow(float(nwarmup), 0.5F);
    if (nstep < nwarmup)
        lr = warmupInitLR + nstep * lrStep;
    else
        lr = decayFactor * pow((float)nstep, -0.5F);
    return lr;
}
float XLearningRate :: MakeLRTransformer ( const float lrate , const int nstep , const int nwarmup , const float warmupInitLR ) { float lr = 0.0F ; float warmupEndLR = lrate ; float lrStep = ( warmupEndLR - warmupInitLR ) / nwarmup ; float decayFactor = warmupEndLR * pow ( float ( nwarmup ) , 0.5F ) ; if ( nstep < nwarmup ) lr = warmupInitLR + nstep * lrStep ; else lr = decayFactor * pow ( ( float ) nstep , -0.5F ) ; return lr ; }
a Transformer-style scheduler.
a Transformer - style scheduler .
/* learning rate, scheduled by inverse square root */