source/sample/transformer/submodel/LayerNorm.cpp
XTensor LayerNorm::RunL2Norm(XTensor& input)
{
    XTensor& x = input;
    XTensor xn;
    XTensor mean;
    XTensor variance;
    XTensor standard;
    XTensor meanFilled;
    XTensor standardFilled;

    TENSOR_DATA_TYPE dataType = input.dataType;

    if (dataType == X_FLOAT16) {
        /* reduce functions can only run with FP32 */
        x = ConvertDataType(input, X_FLOAT);
    }

    /* \mu = (sum_i x_i)/m */
    mean = ReduceMean(x, x.order - 1);

    /* \sigma = (sum_i (x_i - \mu)^2)/m */
    variance = ReduceVariance(x, x.order - 1, mean, false);

    if (!weight.enableGrad)
        return Normalize(x, x.order - 1, mean, variance, weight, bias, 0.0F);

    /* standard = sqrt(variance) */
    standard = Power(variance, 0.5F);

    /* unsqueeze mean and standard deviation to fit them into
       the same shape of x */
    meanFilled = Unsqueeze(mean, x.order - 1, x.GetDim(-1));
    standardFilled = Unsqueeze(standard, x.order - 1, x.GetDim(-1));

    /* x' = (x - \mu)/standard */
    xn = (x - meanFilled) / standardFilled;

    if (dataType != mean.dataType) {
        x = ConvertDataType(x, dataType);
        xn = ConvertDataType(xn, dataType);
    }

    /* result = x' * w + b   */
    xn = xn * weight;

    xn = Sum(xn, bias, /*inplace=*/true);

    return xn;
}
/*
run layernorm for inference
>> input - the input tensor
>> return - layer normalization output
*/
run layernorm for inference
>> input - the input tensor
>> return - layer normalization output
run layernorm for inference >> input - the input tensor >> return - layer normalization output
XTensor LayerNorm::RunL2Norm(XTensor& input)
{
    XTensor& x = input;
    XTensor xn;
    XTensor mean;
    XTensor variance;
    XTensor standard;
    XTensor meanFilled;
    XTensor standardFilled;
    TENSOR_DATA_TYPE dataType = input.dataType;
    if (dataType == X_FLOAT16) {
        x = ConvertDataType(input, X_FLOAT);
    }
    mean = ReduceMean(x, x.order - 1);
    variance = ReduceVariance(x, x.order - 1, mean, false);
    if (!weight.enableGrad)
        return Normalize(x, x.order - 1, mean, variance, weight, bias, 0.0F);
    standard = Power(variance, 0.5F);
    meanFilled = Unsqueeze(mean, x.order - 1, x.GetDim(-1));
    standardFilled = Unsqueeze(standard, x.order - 1, x.GetDim(-1));
    xn = (x - meanFilled) / standardFilled;
    if (dataType != mean.dataType) {
        x = ConvertDataType(x, dataType);
        xn = ConvertDataType(xn, dataType);
    }
    xn = xn * weight;
    xn = Sum(xn, bias, true);
    return xn;
}
XTensor LayerNorm :: RunL2Norm ( XTensor & input ) { XTensor & x = input ; XTensor xn ; XTensor mean ; XTensor variance ; XTensor standard ; XTensor meanFilled ; XTensor standardFilled ; TENSOR_DATA_TYPE dataType = input . dataType ; if ( dataType == X_FLOAT16 ) { x = ConvertDataType ( input , X_FLOAT ) ; } mean = ReduceMean ( x , x . order - 1 ) ; variance = ReduceVariance ( x , x . order - 1 , mean , false ) ; if ( ! weight . enableGrad ) return Normalize ( x , x . order - 1 , mean , variance , weight , bias , 0.0F ) ; standard = Power ( variance , 0.5F ) ; meanFilled = Unsqueeze ( mean , x . order - 1 , x . GetDim ( -1 ) ) ; standardFilled = Unsqueeze ( standard , x . order - 1 , x . GetDim ( -1 ) ) ; xn = ( x - meanFilled ) / standardFilled ; if ( dataType != mean . dataType ) { x = ConvertDataType ( x , dataType ) ; xn = ConvertDataType ( xn , dataType ) ; } xn = xn * weight ; xn = Sum ( xn , bias , true ) ; return xn ; }
run layernorm for inference
>> input - the input tensor
>> return - layer normalization output
run layernorm for inference >> input - the input tensor >> return - layer normalization output
/* reduce functions can only run with FP32 */ /* \mu = (sum_i x_i)/m */ /* \sigma = (sum_i (x_i - \mu)^2)/m */ /* standard = sqrt(variance) */ /* unsqueeze mean and standard deviation to fit them into
       the same shape of x */ /* x' = (x - \mu)/standard */ /* result = x' * w + b   */ /*inplace=*/