Fixed more formating and tidy issues.

Fixed more formating and tidy issues.

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#ifndef MIO_BN_LDS_SIZE
#define MIO_BN_LDS_SIZE 1
#endif

#ifndef MIO_BN_N
#define MIO_BN_N 1
#endif

#ifndef MIO_BN_NHW
#define MIO_BN_NHW 1
#endif

#ifndef MIO_BN_HW
#define MIO_BN_HW 1
#endif

#ifndef MIO_BN_GRP0
#define MIO_BN_GRP0 1
#endif

#ifndef MIO_BN_GRP1
#define MIO_BN_GRP1 1
#endif

#ifndef MIO_BN_GRP2
#define MIO_BN_GRP2 1
#endif

#ifndef MIO_BN_NGRPS
#define MIO_BN_NGRPS 1
#endif

#ifndef MIO_BN_VARIANT
#define MIO_BN_VARIANT 3
#endif

#define UNUSED __attribute__((__unused__))

//#ifdef __AMDGCN__
//#undef __AMDGCN__
//#endif

// Disable specific warnings
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wconditional-uninitialized"
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wsometimes-uninitialized"
#endif

static inline void ReduceKernel(__local _FLOAT* lcl_mem,
                                unsigned int sum_stride,
                                unsigned int unit_id,
                                unsigned int unit_len)
{
    _FLOAT sum              = 0;
    unsigned int lcl_offset = unit_id * unit_len;

#pragma unroll
    for(unsigned int i = 0; i < unit_len; i += sum_stride)
    {
        sum += lcl_mem[lcl_offset + i];
    }
    lcl_mem[lcl_offset] = sum;
}

static inline void
regLDSreduce(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{
    data[localID] = *value;
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(data, 1, localID, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(data, 4, localID, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID == 0)
        ReduceKernel(data, 16, localID, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0] * scale;
}

static inline void
regLDSreduceNoSet(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{

    if(localID < 128)
        ReduceKernel(data, 1, localID, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID < 16)
        ReduceKernel(data, 4, localID, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID == 0)
        ReduceKernel(data, 16, localID, 256);
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0] * scale;
}

static inline void dppRegReduce64(_FLOAT* value, _FLOAT scale)
{
    _FLOAT tmp = 0.;
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x111, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x112, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x114, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x142, 0xF, 0xF, 0));
    *value += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x143, 0xF, 0xF, 0));
    *value += tmp;
    *value = as_float(__builtin_amdgcn_readlane(as_int(*value), 63));
    *value *= scale;
}

static inline void dppRegReduce16(_FLOAT* value, _FLOAT scale)
{

    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x101, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x102, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x104, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x108, 0xF, 0xF, 0));
    *value = as_float(__builtin_amdgcn_readlane(as_int(*value), 0));
    *value *= scale;
}

static inline void
dppLDSReduce64(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{
    _FLOAT tmp = 0.;
    *value     = data[localID];
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x111, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x112, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x114, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x142, 0xF, 0xF, 0));
    *value += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x143, 0xF, 0xF, 0));
    *value += tmp;
    if(localID == 63)
        data[0] = *value * scale;
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0];
}

static inline void
dppLDSReduce16(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{

    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x101, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x102, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x104, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x108, 0xF, 0xF, 0));
    if(localID == 0)
        data[0] = *value * scale;
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0];
}

#if(MIO_BN_VARIANT == 0)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialEst(const __global _FLOAT* __restrict in, /* x input */
                            __global _FLOAT* __restrict out,      /* y output */
                            const __global _FLOAT* __restrict estimatedMean,
                            const __global _FLOAT* __restrict estimatedVariance,
                            const __global _FLOAT* __restrict scale,
                            const __global _FLOAT* __restrict bias,
                            double epsilon)
{

    int xgid = get_global_id(0);
    int ygid = get_global_id(1);

    local _FLOAT lmean;
    local _FLOAT lvar;
    local _FLOAT lscale;
    local _FLOAT lbias;

    unsigned int cidx = xgid * MIO_BN_HW;
    unsigned int index;

    _FLOAT mean, variance, invVariance;
    _FLOAT inhat;
    _FLOAT pscale, pbias;

    if(get_local_id(1) == 0)
    {
        lmean  = estimatedMean[xgid];
        lvar   = estimatedVariance[xgid];
        lscale = scale[xgid]; // dims 1xCx1x1
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // move across the sections of the image mini_batch stack
    if(ygid < MIO_BN_HW)
    {

        mean        = lmean;
        variance    = lvar;
        pscale      = lscale;
        pbias       = lbias;
        invVariance = rsqrt(fabs(variance + epsilon));

#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index      = n * MIO_BN_CHW + cidx + ygid;
            inhat      = (in[index] - mean) * invVariance;
            out[index] = mad(pscale, inhat, pbias); // y_i = gamma*x_hat + beta
        }                                           // end for(img_offset)
    }
} // end spatial norm

//=========================================================

//=== SPATIAL NO SAVED DATA ===============================

#elif(MIO_BN_VARIANT == 1)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialSingleNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias,
                                   double epsilon,
                                   double INHW)
{

    // SPATIAL
    _FLOAT mean        = 0.;
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;
    _FLOAT inhat       = 0.;
    _FLOAT pvt_scale   = 0.;
    _FLOAT pvt_bias    = 0.;
    _FLOAT elemStd     = 0.;
    _FLOAT minibatch[MIO_BN_HW];

#ifndef __AMDGCN__
#if (MIO_BN_N > 1)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
#endif
#endif

    local _FLOAT lscale;
    local _FLOAT lbias;

    unsigned int index;
    unsigned int ylid = get_local_id(1);
    unsigned int xgid = get_global_id(0);
    unsigned int ygid = get_global_id(1);
    unsigned int cidx = xgid * MIO_BN_HW;

    if(ylid == 0)
    {
        lscale = scale[xgid];
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ylid < MIO_BN_N)
    {
#pragma unroll
        for(unsigned int hw = 0; hw < MIO_BN_HW; hw++)
        {
            index                 = ylid * MIO_BN_CHW + cidx + hw;
            mean += minibatch[hw] = in[index];
        }
    }

#ifdef __AMDGCN__
#if(MIO_BN_N > 16)
    dppRegReduce64(&mean, INHW);
#elif(MIO_BN_N > 1)
    dppRegReduce16(&mean, INHW);
#else
    mean *= INHW;
#endif // N
#else  // if not GCN

#if(MIO_BN_N > 16)
    regLDSreduce(&mean, lcl_data, ylid, INHW);
#elif(MIO_BN_N > 1)
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = 0.;
#pragma unroll
    for(int n = 0; n < MIO_BN_N; n++)
    {
        mean += lcl_data[n];
    }
    mean *= INHW;
#else
    mean *= INHW;
#endif // N
#endif // GCN

    if(ylid < MIO_BN_N)
    {
#pragma unroll
        for(int hw = 0; hw < MIO_BN_HW; hw++)
        {
            index    = ylid * MIO_BN_CHW + cidx + hw;
            elemStd  = (minibatch[hw] - mean);
            variance = mad(elemStd, elemStd, variance);
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_N > 16)
    dppRegReduce64(&variance, INHW);
#elif(MIO_BN_N > 1)
    dppRegReduce16(&variance, INHW);
#else
    variance *= INHW;
#endif // N

#else // if not GCN

#if(MIO_BN_N > 16)
    regLDSreduce(&variance, lcl_data, ylid, INHW);
#elif(MIO_BN_N > 1)
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    variance = 0.;
#pragma unroll
    for(unsigned int n = 0; n < MIO_BN_N; n++)
    {
        variance += lcl_data[n];
    }
    variance *= INHW;
#else
    variance *= INHW;
#endif // N

#endif // GCN

    // #3 add epsilon for numeric stability, sq_root, and invert
    invVariance = rsqrt(variance + epsilon);

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid < MIO_BN_N)
    {

        pvt_scale = lscale;
        pvt_bias  = lbias;

#pragma unroll
        for(unsigned int hw = 0; hw < MIO_BN_HW; hw++)
        { // apply normalization
            index = ylid * MIO_BN_CHW + cidx + hw;
            inhat = (minibatch[hw] - mean) * invVariance;
            // #5 Gamma and Beta adjust: y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for
    }     // end if

} // end spatial norm

#elif(MIO_BN_VARIANT == 2)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialSingleNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias,
                                   double epsilon,
                                   double INHW)
{

    // SPATIAL
    _FLOAT mean        = 0.;
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;
    _FLOAT inhat       = 0.;
    _FLOAT pvt_scale   = 0.;
    _FLOAT pvt_bias    = 0.;
    _FLOAT elemStd     = 0.;
    _FLOAT minibatch[MIO_BN_N];

    local _FLOAT lscale;
    local _FLOAT lbias;

#ifndef __AMDGCN__
#if (MIO_BN_HW > 1)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
#endif
#endif

    unsigned int index;
    unsigned int ylid = get_local_id(1);
    unsigned int xgid = get_global_id(0);
    unsigned int cidx = xgid * MIO_BN_HW;

    if(ylid == 0)
    {
        lscale = scale[xgid];
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            index                = n * MIO_BN_CHW + cidx + ylid;
            mean += minibatch[n] = in[index];
        }
    }

#ifdef __AMDGCN__
#if(MIO_BN_HW > 16)
    dppRegReduce64(&mean, INHW);
#elif(MIO_BN_HW > 1)
    dppRegReduce16(&mean, INHW);
#else
    mean *= INHW;
#endif // HW
#else  // if not GCN

#if(MIO_BN_HW > 16)
    regLDSreduce(&mean, lcl_data, ylid, INHW);
#elif(MIO_BN_HW > 1)
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = 0.;
#pragma unroll
    for(int hw = 0; hw < MIO_BN_HW; hw++)
    {
        mean += lcl_data[hw];
    }
    mean *= INHW;
#else
    mean *= INHW;
#endif // HW
#endif // GCN

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index    = n * MIO_BN_CHW + cidx + ylid;
            elemStd  = (minibatch[n] - mean);
            variance = mad(elemStd, elemStd, variance);
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_HW > 16)
    dppRegReduce64(&variance, INHW);
#elif(MIO_BN_HW > 1)
    dppRegReduce16(&variance, INHW);
#else
    variance *= INHW;
#endif // HW
#else  // if not GCN

#if(MIO_BN_HW > 16)
    regLDSreduce(&variance, lcl_data, ylid, INHW);
#elif(MIO_BN_HW > 1)
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    variance = 0.;
#pragma unroll
    for(unsigned int hw = 0; hw < MIO_BN_HW; hw++)
    {
        variance += lcl_data[hw];
    }
    variance *= INHW;
#else
    variance *= INHW;
#endif // HW
#endif // GCN

    // #3 add epsilon for numeric stability, sq_root, and invert
    invVariance = rsqrt(variance + epsilon);

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid < MIO_BN_HW)
    {
        pvt_scale = lscale;
        pvt_bias  = lbias;

#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ylid;
            inhat = (minibatch[n] - mean) * invVariance;
            // #5 Gamma and Beta adjust: y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for
    }     // end if

} // end spatial norm

#elif(MIO_BN_VARIANT == 3)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialSingleNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias,
                                   double epsilon,
                                   double INHW)
{

    // SPATIAL
    _FLOAT mean        = 0.;
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;
    _FLOAT inhat       = 0.;
    _FLOAT pvt_scale   = 0.;
    _FLOAT pvt_bias    = 0.;
    _FLOAT elemStd     = 0.;

    local _FLOAT lscale;
    local _FLOAT lbias;

    unsigned int index;
    unsigned int ylid = get_local_id(1);
    unsigned int xgid = get_global_id(0);
    unsigned int cidx = xgid * MIO_BN_HW;

    if(ylid == 0)
    {
        lscale = scale[xgid];
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index = n * MIO_BN_CHW + cidx + ylid;
            mean += in[index];
        }
    }

#ifdef __AMDGCN__
#if(MIO_BN_HW > 64)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 32; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    dppLDSReduce64(&mean, lcl_data, ylid, INHW);
#elif(MIO_BN_HW > 16)
    dppRegReduce64(&mean, INHW);
#else
    dppRegReduce16(&mean, INHW);
#endif // GRP1
#else  // no GCN
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = 0.;
#if(MIO_BN_HW > 16)
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 128; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    regLDSreduceNoSet(&mean, lcl_data, ylid, INHW);
#else
#pragma unroll
    for(int i = 0; i < MIO_BN_HW; i++)
    {
        mean += lcl_data[i];
    }
    mean *= INHW;
#endif // GRP1
#endif // endif GCN
// Done with mean

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index   = n * MIO_BN_CHW + cidx + ylid;
            elemStd = (in[index] - mean);
            variance = mad(elemStd, elemStd, variance);
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_GRP1 > 64)
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 32; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    dppLDSReduce64(&variance, lcl_data, ylid, INHW);
#elif(MIO_BN_GRP1 > 16)
    dppRegReduce64(&variance, INHW);
#else
    dppRegReduce16(&variance, INHW);

#endif // GRP1
#else  // GCN
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    variance = 0.;
#if(MIO_BN_GRP1 > 16)
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 128; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    regLDSreduceNoSet(&variance, lcl_data, ylid, INHW);
#else
#pragma unroll
    for(int i = 0; i < MIO_BN_GRP1; i++)
    {
        variance += lcl_data[i];
    }
    variance *= INHW;

#endif // GRP1
#endif // GCN

    // #3 add epsilon for numeric stability, sq_root, and invert
    invVariance = rsqrt(variance + epsilon);

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid < MIO_BN_HW)
    {
        pvt_scale = lscale;
        pvt_bias  = lbias;

#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ylid;
            inhat = (in[index] - mean) * invVariance;
            // #5 Gamma and Beta adjust
            // y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for
    }     // end if

} // end spatial norm

#elif(MIO_BN_VARIANT == 4)

__kernel void BatchNormFwdInferSpatialNorm(const __global _FLOAT* __restrict in,
                                           __global _FLOAT* __restrict out,
                                           const __global _FLOAT* __restrict scale,
                                           const __global _FLOAT* __restrict bias)
{

    // SPATIAL
    _FLOAT mean   = 0.;
    _FLOAT invVar = 0.;
    _FLOAT inhat  = 0.;
    _FLOAT pscale = 0.;
    _FLOAT pbias  = 0.;
    __local _FLOAT lb, ls, lm, liv;
    unsigned int ygrp_id = get_group_id(1);
    unsigned int xgid    = get_global_id(0);
    unsigned int ygid    = get_global_id(1);
    unsigned int ygrp_sz = get_local_size(1);
    unsigned int ylid    = get_local_id(1);
    unsigned int index;
    unsigned int cidx = xgid * MIO_BN_HW;

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid == 0)
    {
        unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
        unsigned int varstashindex  = cidx + ygrp_sz * ygrp_id + 3;
        lm                          = out[meanstashindex]; // load stashed mean
        liv                         = out[varstashindex];
        ls                          = scale[xgid];
        lb                          = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ygid < MIO_BN_HW)
    {
        pscale = ls;
        pbias  = lb;
        mean   = lm;
        invVar = liv;
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ygid;
            inhat = (in[index] - mean) * invVar;
            out[index] =
                mad(pscale, inhat, pbias); // #5 Gamma and Beta adjust :: y_i = gamma*x_hat + beta
        }                                  // end for(n)
    }                                      // end if(inImgIndex)
} // end spatial norm

__kernel void BatchNormFwdInferSpatialFinalVariance(__global _FLOAT* __restrict varbuff,
                                                    double epsilon)
{

    // SPATIAL
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;

    unsigned int ylid          = get_local_id(1);
    unsigned int ygrp_id       = get_group_id(1);
    unsigned int xgid          = get_global_id(0);
    unsigned int ygrp_sz       = get_local_size(1);
    unsigned int yngrps        = get_num_groups(1);
    unsigned int cidx          = xgid * MIO_BN_HW;
    unsigned int varstashindex = cidx + ygrp_sz * ygrp_id + 3;
    _FLOAT NHW                 = (_FLOAT)MIO_BN_NHW;

    for(int gn = 0; gn < yngrps; gn++)
    {
        unsigned int offset   = gn * ygrp_sz + ylid;
        unsigned int varindex = cidx + ygrp_sz * offset + 2;
        if(offset < yngrps)
        {                                  // modified to span larger number of groups
            variance += varbuff[varindex]; // load per group variance
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_NGRPS > 64)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];

    _FLOAT tmp     = 0.;
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    variance = lcl_data[ylid];
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    variance /= NHW;
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 63)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }

#elif(MIO_BN_NGRPS > 16)

    _FLOAT tmp = 0.;
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    variance = as_float(__builtin_amdgcn_readlane(as_int(variance), 63));
    variance /= NHW;
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 63)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }

#else
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x101, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x102, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x104, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x108, 0xF, 0xF, 0));
    variance /= NHW;
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 0)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }

#endif

#else
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
#if(MIO_BN_NGRPS > 16)
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    variance    = lcl_data[0] / NHW;

#else

    variance = 0.;
#pragma unroll
    for(int i = 0; i < MIO_BN_NGRPS; i++)
    {
        variance += lcl_data[i];
    }
    variance /= NHW;
#endif
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 0)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }
#endif
}

__kernel void BatchNormFwdInferSpatialVariance(const __global _FLOAT* __restrict in,
                                               __global _FLOAT* __restrict meanvarbuff)
{

    // SPATIAL
    _FLOAT mean     = 0.;
    _FLOAT elemStd  = 0.;
    _FLOAT variance = 0.;

    __local _FLOAT lm;

    unsigned int ylid    = get_local_id(1);
    unsigned int ygrp_id = get_group_id(1);
    unsigned int xgid    = get_global_id(0);
    unsigned int ygid    = get_global_id(1);
    unsigned int ygrp_sz = get_local_size(1);
    unsigned int index, ncIdx;
    unsigned int cidx     = xgid * MIO_BN_HW;
    unsigned int varindex = cidx + ygrp_sz * ygrp_id + 2;

    if(ylid == 0)
    {
        unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
        lm                          = meanvarbuff[meanstashindex]; // load stashed mean
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ygid < MIO_BN_HW)
    {
        mean = lm;
#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            ncIdx   = n * MIO_BN_CHW + cidx;
            index   = ncIdx + ygid;
            elemStd = (in[index] - mean);
            variance += elemStd * elemStd;
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_GRP1 > 64)

    _FLOAT tmp = 0.;
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    variance = lcl_data[ylid];
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    if(ylid == 63)
    {
        meanvarbuff[varindex] = variance; // pre-stage for group reduction
    }

#elif(MIO_BN_GRP1 > 16)

    _FLOAT tmp = 0.;
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    if(ylid == 63)
    {
        meanvarbuff[varindex] = variance; // pre-stage for group reduction
    }
#else
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x101, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x102, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x104, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x108, 0xF, 0xF, 0));
    if(ylid == 0)
    {
        meanvarbuff[varindex] = variance; // pre-stage for group reduction
    }

#endif

#else

    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    // Reduction over a work-grp: 256 -> 64 -> 16 -> 1
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
    {
        meanvarbuff[varindex] = lcl_data[0];
    }
#endif

} // end spatial variance

__kernel void BatchNormFwdInferSpatialFinalMean(__global _FLOAT* __restrict meanvarbuff)
{

    _FLOAT mean = 0.;

    unsigned int ylid           = get_local_id(1);
    unsigned int ygrp_id        = get_group_id(1);
    unsigned int xgid           = get_global_id(0);
    unsigned int ygrp_sz        = get_local_size(1);
    unsigned int yngrps         = get_num_groups(1);
    unsigned int cidx           = xgid * MIO_BN_HW;
    unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
    _FLOAT NHW                  = (_FLOAT)MIO_BN_NHW;

    for(int gn = 0; gn < yngrps; gn++)
    {
        unsigned int offset    = gn * ygrp_sz + ylid;
        unsigned int meanindex = cidx + ygrp_sz * offset;
        if(offset < yngrps)
        { // modify to span larger number of groups
            mean += meanvarbuff[meanindex];
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_NGRPS > 64)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];

    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    _FLOAT tmp = 0.;
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    mean = lcl_data[ylid];
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x111, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x112, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x114, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x142, 0xF, 0xF, 0));
    mean += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x143, 0xF, 0xF, 0));
    mean += tmp;
    if(ylid == 63)
    {
        meanvarbuff[meanstashindex] = mean / NHW; // stash mean
    }
#elif(MIO_BN_NGRPS > 16)
    _FLOAT tmp = 0.;
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x111, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x112, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x114, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x142, 0xF, 0xF, 0));
    mean += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x143, 0xF, 0xF, 0));
    mean += tmp;
    if(ylid == 63)
    {
        meanvarbuff[meanstashindex] = mean / NHW; // stash mean
    }
#else
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x101, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x102, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x104, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x108, 0xF, 0xF, 0));
    if(ylid == 0)
    {
        meanvarbuff[meanstashindex] = mean / NHW; // stash mean
    }
#endif

#else

    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
#if(MIO_BN_NGRPS > 16)
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = lcl_data[0] / NHW;
#else

    mean = 0.;
#pragma unroll
    for(int i = 0; i < MIO_BN_NGRPS; i++)
    {
        mean += lcl_data[i];
    }
    mean /= NHW;
#endif
    if(ylid == 0)
    {
        meanvarbuff[meanstashindex] = mean; // stash mean
    }

#endif
}

__kernel void BatchNormFwdInferSpatialMean(const __global _FLOAT* __restrict in,
                                           __global _FLOAT* __restrict meanbuff)
{

    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];

    _FLOAT mean            = 0.;
    unsigned int ylid      = get_local_id(1);
    unsigned int ygrp_id   = get_group_id(1);
    unsigned int xgid      = get_global_id(0);
    unsigned int ygid      = get_global_id(1);
    unsigned int ygrp_sz   = get_local_size(1);
    unsigned int index     = 0;
    unsigned int cidx      = xgid * MIO_BN_HW;
    unsigned int meanindex = cidx + ygrp_sz * ygrp_id; // making assumption of n=0 here

    // move across the sections of the image mini_batch stack
    if(ygid < MIO_BN_HW)
    {

#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            index = n * MIO_BN_CHW + cidx + ygid;
            mean += in[index];
        }
    }

    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);

#ifdef __AMDGCN__

    _FLOAT tmp = 0.;
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    mean = lcl_data[ylid];
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x111, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x112, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x114, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x142, 0xF, 0xF, 0));
    mean += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x143, 0xF, 0xF, 0));
    mean += tmp;

    if(ylid == 63)
    {
        meanbuff[meanindex] = mean; // pre-stage for group reduction
    }
#else

#if(MIO_BN_GRP1 > 16)
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = lcl_data[0];
#else

    mean = 0.;
#pragma unroll
    for(int i = 0; i < MIO_BN_GRP1; i++)
    {
        mean += lcl_data[i];
    }
#endif
    if(ylid == 0)
    {
        meanbuff[meanindex] = mean; // pre-stage for group reduction
    }
#endif

} // end spatial mean kernel

//====================================================

#endif

#ifdef __clang__
#pragma clang diagnostic pop
#pragma clang diagnostic pop
#endif

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#ifndef MIO_BN_LDS_SIZE
#define MIO_BN_LDS_SIZE 1
#endif

#ifndef MIO_BN_N
#define MIO_BN_N 1
#endif

#ifndef MIO_BN_NHW
#define MIO_BN_NHW 1
#endif

#ifndef MIO_BN_HW
#define MIO_BN_HW 1
#endif

#ifndef MIO_BN_GRP0
#define MIO_BN_GRP0 1
#endif

#ifndef MIO_BN_GRP1
#define MIO_BN_GRP1 1
#endif

#ifndef MIO_BN_GRP2
#define MIO_BN_GRP2 1
#endif

#ifndef MIO_BN_NGRPS
#define MIO_BN_NGRPS 1
#endif

#ifndef MIO_BN_VARIANT
#define MIO_BN_VARIANT 3
#endif

#define UNUSED __attribute__((__unused__))

//#ifdef __AMDGCN__
//#undef __AMDGCN__
//#endif

// Disable specific warnings
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wconditional-uninitialized"
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wsometimes-uninitialized"
#endif

static inline void ReduceKernel(__local _FLOAT* lcl_mem,
                                unsigned int sum_stride,
                                unsigned int unit_id,
                                unsigned int unit_len)
{
    _FLOAT sum              = 0;
    unsigned int lcl_offset = unit_id * unit_len;

#pragma unroll
    for(unsigned int i = 0; i < unit_len; i += sum_stride)
    {
        sum += lcl_mem[lcl_offset + i];
    }
    lcl_mem[lcl_offset] = sum;
}

static inline void
regLDSreduce(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{
    data[localID] = *value;
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(data, 1, localID, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(data, 4, localID, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID == 0)
        ReduceKernel(data, 16, localID, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0] * scale;
}

static inline void
regLDSreduceNoSet(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{

    if(localID < 128)
        ReduceKernel(data, 1, localID, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID < 16)
        ReduceKernel(data, 4, localID, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(localID == 0)
        ReduceKernel(data, 16, localID, 256);
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0] * scale;
}

static inline void dppRegReduce64(_FLOAT* value, _FLOAT scale)
{
    _FLOAT tmp = 0.;
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x111, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x112, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x114, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x142, 0xF, 0xF, 0));
    *value += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x143, 0xF, 0xF, 0));
    *value += tmp;
    *value = as_float(__builtin_amdgcn_readlane(as_int(*value), 63));
    *value *= scale;
}

static inline void dppRegReduce16(_FLOAT* value, _FLOAT scale)
{

    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x101, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x102, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x104, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x108, 0xF, 0xF, 0));
    *value = as_float(__builtin_amdgcn_readlane(as_int(*value), 0));
    *value *= scale;
}

static inline void
dppLDSReduce64(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{
    _FLOAT tmp = 0.;
    *value     = data[localID];
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x111, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x112, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x114, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x142, 0xF, 0xF, 0));
    *value += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x143, 0xF, 0xF, 0));
    *value += tmp;
    if(localID == 63)
        data[0] = *value * scale;
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0];
}

static inline void
dppLDSReduce16(_FLOAT* value, __local _FLOAT* data, unsigned int localID, _FLOAT scale)
{

    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x101, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x102, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x104, 0xF, 0xF, 0));
    *value += as_float(__builtin_amdgcn_mov_dpp(as_int(*value), 0x108, 0xF, 0xF, 0));
    if(localID == 0)
        data[0] = *value * scale;
    barrier(CLK_LOCAL_MEM_FENCE);
    *value = data[0];
}

#if(MIO_BN_VARIANT == 0)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialEst(const __global _FLOAT* __restrict in, /* x input */
                            __global _FLOAT* __restrict out,      /* y output */
                            const __global _FLOAT* __restrict estimatedMean,
                            const __global _FLOAT* __restrict estimatedVariance,
                            const __global _FLOAT* __restrict scale,
                            const __global _FLOAT* __restrict bias,
                            double epsilon)
{

    int xgid = get_global_id(0);
    int ygid = get_global_id(1);

    local _FLOAT lmean;
    local _FLOAT lvar;
    local _FLOAT lscale;
    local _FLOAT lbias;

    unsigned int cidx = xgid * MIO_BN_HW;
    unsigned int index;

    _FLOAT mean, variance, invVariance;
    _FLOAT inhat;
    _FLOAT pscale, pbias;

    if(get_local_id(1) == 0)
    {
        lmean  = estimatedMean[xgid];
        lvar   = estimatedVariance[xgid];
        lscale = scale[xgid]; // dims 1xCx1x1
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // move across the sections of the image mini_batch stack
    if(ygid < MIO_BN_HW)
    {

        mean        = lmean;
        variance    = lvar;
        pscale      = lscale;
        pbias       = lbias;
        invVariance = rsqrt(fabs(variance + epsilon));

#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index      = n * MIO_BN_CHW + cidx + ygid;
            inhat      = (in[index] - mean) * invVariance;
            out[index] = mad(pscale, inhat, pbias); // y_i = gamma*x_hat + beta
        }                                           // end for(img_offset)
    }
} // end spatial norm

//=========================================================

//=== SPATIAL NO SAVED DATA ===============================

#elif(MIO_BN_VARIANT == 1)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialSingleNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias,
                                   double epsilon,
                                   double INHW)
{

    // SPATIAL
    _FLOAT mean        = 0.;
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;
    _FLOAT inhat       = 0.;
    _FLOAT pvt_scale   = 0.;
    _FLOAT pvt_bias    = 0.;
    _FLOAT elemStd     = 0.;
    _FLOAT minibatch[MIO_BN_HW];

#ifndef __AMDGCN__
#if (MIO_BN_N > 1)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
#endif
#endif

    local _FLOAT lscale;
    local _FLOAT lbias;

    unsigned int index;
    unsigned int ylid = get_local_id(1);
    unsigned int xgid = get_global_id(0);
    unsigned int ygid = get_global_id(1);
    unsigned int cidx = xgid * MIO_BN_HW;

    if(ylid == 0)
    {
        lscale = scale[xgid];
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ylid < MIO_BN_N)
    {
#pragma unroll
        for(unsigned int hw = 0; hw < MIO_BN_HW; hw++)
        {
            index                 = ylid * MIO_BN_CHW + cidx + hw;
            mean += minibatch[hw] = in[index];
        }
    }

#ifdef __AMDGCN__
#if(MIO_BN_N > 16)
    dppRegReduce64(&mean, INHW);
#elif(MIO_BN_N > 1)
    dppRegReduce16(&mean, INHW);
#else
    mean *= INHW;
#endif // N
#else  // if not GCN

#if(MIO_BN_N > 16)
    regLDSreduce(&mean, lcl_data, ylid, INHW);
#elif(MIO_BN_N > 1)
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = 0.;
#pragma unroll
    for(int n = 0; n < MIO_BN_N; n++)
    {
        mean += lcl_data[n];
    }
    mean *= INHW;
#else
    mean *= INHW;
#endif // N
#endif // GCN

    if(ylid < MIO_BN_N)
    {
#pragma unroll
        for(int hw = 0; hw < MIO_BN_HW; hw++)
        {
            index    = ylid * MIO_BN_CHW + cidx + hw;
            elemStd  = (minibatch[hw] - mean);
            variance = mad(elemStd, elemStd, variance);
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_N > 16)
    dppRegReduce64(&variance, INHW);
#elif(MIO_BN_N > 1)
    dppRegReduce16(&variance, INHW);
#else
    variance *= INHW;
#endif // N

#else // if not GCN

#if(MIO_BN_N > 16)
    regLDSreduce(&variance, lcl_data, ylid, INHW);
#elif(MIO_BN_N > 1)
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    variance = 0.;
#pragma unroll
    for(unsigned int n = 0; n < MIO_BN_N; n++)
    {
        variance += lcl_data[n];
    }
    variance *= INHW;
#else
    variance *= INHW;
#endif // N

#endif // GCN

    // #3 add epsilon for numeric stability, sq_root, and invert
    invVariance = rsqrt(variance + epsilon);

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid < MIO_BN_N)
    {

        pvt_scale = lscale;
        pvt_bias  = lbias;

#pragma unroll
        for(unsigned int hw = 0; hw < MIO_BN_HW; hw++)
        { // apply normalization
            index = ylid * MIO_BN_CHW + cidx + hw;
            inhat = (minibatch[hw] - mean) * invVariance;
            // #5 Gamma and Beta adjust: y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for
    }     // end if

} // end spatial norm

#elif(MIO_BN_VARIANT == 2)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialSingleNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias,
                                   double epsilon,
                                   double INHW)
{

    // SPATIAL
    _FLOAT mean        = 0.;
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;
    _FLOAT inhat       = 0.;
    _FLOAT pvt_scale   = 0.;
    _FLOAT pvt_bias    = 0.;
    _FLOAT elemStd     = 0.;
    _FLOAT minibatch[MIO_BN_N];

    local _FLOAT lscale;
    local _FLOAT lbias;

#ifndef __AMDGCN__
#if (MIO_BN_HW > 1)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
#endif
#endif

    unsigned int index;
    unsigned int ylid = get_local_id(1);
    unsigned int xgid = get_global_id(0);
    unsigned int ygid = get_global_id(1);
    unsigned int cidx = xgid * MIO_BN_HW;

    if(ylid == 0)
    {
        lscale = scale[xgid];
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            index                = n * MIO_BN_CHW + cidx + ylid;
            mean += minibatch[n] = in[index];
        }
    }

#ifdef __AMDGCN__
#if(MIO_BN_HW > 16)
    dppRegReduce64(&mean, INHW);
#elif(MIO_BN_HW > 1)
    dppRegReduce16(&mean, INHW);
#else
    mean *= INHW;
#endif // HW
#else  // if not GCN

#if(MIO_BN_HW > 16)
    regLDSreduce(&mean, lcl_data, ylid, INHW);
#elif(MIO_BN_HW > 1)
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = 0.;
#pragma unroll
    for(int hw = 0; hw < MIO_BN_HW; hw++)
    {
        mean += lcl_data[hw];
    }
    mean *= INHW;
#else
    mean *= INHW:
#endif // HW
#endif // GCN

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index    = n * MIO_BN_CHW + cidx + ylid;
            elemStd  = (minibatch[n] - mean);
            variance = mad(elemStd, elemStd, variance);
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_HW > 16)
    dppRegReduce64(&variance, INHW);
#elif(MIO_BN_HW > 1)
    dppRegReduce16(&variance, INHW);
#else
    variance *= INHW;
#endif // HW
#else  // if not GCN

#if(MIO_BN_HW > 16)
    regLDSreduce(&variance, lcl_data, ylid, INHW);
#elif(MIO_BN_HW > 1)
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    variance = 0.;
#pragma unroll
    for(unsigned int hw = 0; hw < MIO_BN_HW; hw++)
    {
        variance += lcl_data[hw];
    }
    variance *= INHW;
#else
    variance *= INHW:
#endif // HW
#endif // GCN

    // #3 add epsilon for numeric stability, sq_root, and invert
    invVariance = rsqrt(variance + epsilon);

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid < MIO_BN_HW)
    {
        pvt_scale = lscale;
        pvt_bias  = lbias;

#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ylid;
            inhat = (minibatch[n] - mean) * invVariance;
            // #5 Gamma and Beta adjust: y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for
    }     // end if

} // end spatial norm

#elif(MIO_BN_VARIANT == 3)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
BatchNormFwdInferSpatialSingleNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias,
                                   double epsilon,
                                   double INHW)
{

    // SPATIAL
    _FLOAT mean        = 0.;
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;
    _FLOAT inhat       = 0.;
    _FLOAT pvt_scale   = 0.;
    _FLOAT pvt_bias    = 0.;
    _FLOAT elemStd     = 0.;

    local _FLOAT lscale;
    local _FLOAT lbias;

    unsigned int index;
    unsigned int ylid = get_local_id(1);
    unsigned int xgid = get_global_id(0);
    unsigned int cidx = xgid * MIO_BN_HW;

    if(ylid == 0)
    {
        lscale = scale[xgid];
        lbias  = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index = n * MIO_BN_CHW + cidx + ylid;
            mean += in[index];
        }
    }

#ifdef __AMDGCN__
#if(MIO_BN_HW > 64)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 32; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    dppLDSReduce64(&mean, lcl_data, ylid, INHW);
#elif(MIO_BN_HW > 16)
    dppRegReduce64(&mean, INHW);
#else
    dppRegReduce16(&mean, INHW);
#endif // GRP1
#else  // no GCN
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = 0.;
#if(MIO_BN_HW > 16)
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 128; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    regLDSreduceNoSet(&mean, lcl_data, ylid, INHW);
#else
#pragma unroll
    for(int i = 0; i < MIO_BN_HW; i++)
    {
        mean += lcl_data[i];
    }
    mean *= INHW;
#endif // GRP1
#endif // endif GCN
// Done with mean

    if(ylid < MIO_BN_HW)
    {
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        {
            index   = n * MIO_BN_CHW + cidx + ylid;
            elemStd = (in[index] - mean);
            variance = mad(elemStd, elemStd, variance);
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_GRP1 > 64)
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 32; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    dppLDSReduce64(&variance, lcl_data, ylid, INHW);
#elif(MIO_BN_GRP1 > 16)
    dppRegReduce64(&variance, INHW);
#else
    dppRegReduce16(&variance, INHW);

#endif // GRP1
#else  // GCN
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    variance = 0.;
#if(MIO_BN_GRP1 > 16)
    for(unsigned int red = (MIO_BN_GRP1 >> 1); red > 128; red >>= 1)
    {
        if(ylid < red)
            lcl_data[ylid] += lcl_data[ylid + red];
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    regLDSreduceNoSet(&variance, lcl_data, ylid, INHW);
#else
#pragma unroll
    for(int i = 0; i < MIO_BN_GRP1; i++)
    {
        variance += lcl_data[i];
    }
    variance *= INHW;

#endif // GRP1
#endif // GCN

    // #3 add epsilon for numeric stability, sq_root, and invert
    invVariance = rsqrt(variance + epsilon);

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid < MIO_BN_HW)
    {
        pvt_scale = lscale;
        pvt_bias  = lbias;

#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ylid;
            inhat = (in[index] - mean) * invVariance;
            // #5 Gamma and Beta adjust
            // y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for
    }     // end if

} // end spatial norm

#elif(MIO_BN_VARIANT == 4)

__kernel void BatchNormFwdInferSpatialNorm(const __global _FLOAT* __restrict in,
                                           __global _FLOAT* __restrict out,
                                           const __global _FLOAT* __restrict scale,
                                           const __global _FLOAT* __restrict bias)
{

    // SPATIAL
    _FLOAT mean   = 0.;
    _FLOAT invVar = 0.;
    _FLOAT inhat  = 0.;
    _FLOAT pscale = 0.;
    _FLOAT pbias  = 0.;
    __local _FLOAT lb, ls, lm, liv;
    unsigned int ygrp_id = get_group_id(1);
    unsigned int xgid    = get_global_id(0);
    unsigned int ygid    = get_global_id(1);
    unsigned int ygrp_sz = get_local_size(1);
    unsigned int ylid    = get_local_id(1);
    unsigned int index;
    unsigned int cidx = xgid * MIO_BN_HW;

    // #4 apply the normalization
    // x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(ylid == 0)
    {
        unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
        unsigned int varstashindex  = cidx + ygrp_sz * ygrp_id + 3;
        lm                          = out[meanstashindex]; // load stashed mean
        liv                         = out[varstashindex];
        ls                          = scale[xgid];
        lb                          = bias[xgid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ygid < MIO_BN_HW)
    {
        pscale = ls;
        pbias  = lb;
        mean   = lm;
        invVar = liv;
#pragma unroll
        for(int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ygid;
            inhat = (in[index] - mean) * invVar;
            out[index] =
                mad(pscale, inhat, pbias); // #5 Gamma and Beta adjust :: y_i = gamma*x_hat + beta
        }                                  // end for(n)
    }                                      // end if(inImgIndex)
} // end spatial norm

__kernel void BatchNormFwdInferSpatialFinalVariance(__global _FLOAT* __restrict varbuff,
                                                    double epsilon)
{

    // SPATIAL
    _FLOAT variance    = 0.;
    _FLOAT invVariance = 0.;

    unsigned int ylid          = get_local_id(1);
    unsigned int ygrp_id       = get_group_id(1);
    unsigned int xgid          = get_global_id(0);
    unsigned int ygrp_sz       = get_local_size(1);
    unsigned int yngrps        = get_num_groups(1);
    unsigned int cidx          = xgid * MIO_BN_HW;
    unsigned int varstashindex = cidx + ygrp_sz * ygrp_id + 3;
    _FLOAT NHW                 = (_FLOAT)MIO_BN_NHW;

    for(int gn = 0; gn < yngrps; gn++)
    {
        unsigned int offset   = gn * ygrp_sz + ylid;
        unsigned int varindex = cidx + ygrp_sz * offset + 2;
        if(offset < yngrps)
        {                                  // modified to span larger number of groups
            variance += varbuff[varindex]; // load per group variance
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_NGRPS > 64)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];

    _FLOAT tmp     = 0.;
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    variance = lcl_data[ylid];
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    variance /= NHW;
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 63)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }

#elif(MIO_BN_NGRPS > 16)

    _FLOAT tmp = 0.;
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    variance = as_float(__builtin_amdgcn_readlane(as_int(variance), 63));
    variance /= NHW;
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 63)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }

#else
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x101, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x102, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x104, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x108, 0xF, 0xF, 0));
    variance /= NHW;
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 0)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }

#endif

#else
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
#if(MIO_BN_NGRPS > 16)
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    variance    = lcl_data[0] / NHW;

#else

    variance = 0.;
#pragma unroll
    for(int i = 0; i < MIO_BN_NGRPS; i++)
    {
        variance += lcl_data[i];
    }
    variance /= NHW;
#endif
    invVariance = rsqrt(variance + epsilon);
    if(ylid == 0)
    {
        varbuff[varstashindex] = invVariance; // stash mean
    }
#endif
}

__kernel void BatchNormFwdInferSpatialVariance(const __global _FLOAT* __restrict in,
                                               __global _FLOAT* __restrict meanvarbuff)
{

    // SPATIAL
    _FLOAT mean     = 0.;
    _FLOAT elemStd  = 0.;
    _FLOAT variance = 0.;

    __local _FLOAT lm;

    unsigned int ylid    = get_local_id(1);
    unsigned int ygrp_id = get_group_id(1);
    unsigned int xgid    = get_global_id(0);
    unsigned int ygid    = get_global_id(1);
    unsigned int ygrp_sz = get_local_size(1);
    unsigned int index, ncIdx;
    unsigned int cidx     = xgid * MIO_BN_HW;
    unsigned int varindex = cidx + ygrp_sz * ygrp_id + 2;

    if(ylid == 0)
    {
        unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
        lm                          = meanvarbuff[meanstashindex]; // load stashed mean
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ygid < MIO_BN_HW)
    {
        mean = lm;
#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            ncIdx   = n * MIO_BN_CHW + cidx;
            index   = ncIdx + ygid;
            elemStd = (in[index] - mean);
            variance += elemStd * elemStd;
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_GRP1 > 64)

    _FLOAT tmp = 0.;
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    variance = lcl_data[ylid];
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    if(ylid == 63)
    {
        meanvarbuff[varindex] = variance; // pre-stage for group reduction
    }

#elif(MIO_BN_GRP1 > 16)

    _FLOAT tmp = 0.;
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x111, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x112, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x114, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x142, 0xF, 0xF, 0));
    variance += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x143, 0xF, 0xF, 0));
    variance += tmp;
    if(ylid == 63)
    {
        meanvarbuff[varindex] = variance; // pre-stage for group reduction
    }
#else
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x101, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x102, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x104, 0xF, 0xF, 0));
    variance += as_float(__builtin_amdgcn_mov_dpp(as_int(variance), 0x108, 0xF, 0xF, 0));
    if(ylid == 0)
    {
        meanvarbuff[varindex] = variance; // pre-stage for group reduction
    }

#endif

#else

    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = variance;
    barrier(CLK_LOCAL_MEM_FENCE);
    // Reduction over a work-grp: 256 -> 64 -> 16 -> 1
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
    {
        meanvarbuff[varindex] = lcl_data[0];
    }
#endif

} // end spatial variance

__kernel void BatchNormFwdInferSpatialFinalMean(__global _FLOAT* __restrict meanvarbuff)
{

    _FLOAT mean = 0.;

    unsigned int ylid           = get_local_id(1);
    unsigned int ygrp_id        = get_group_id(1);
    unsigned int xgid           = get_global_id(0);
    unsigned int ygrp_sz        = get_local_size(1);
    unsigned int yngrps         = get_num_groups(1);
    unsigned int cidx           = xgid * MIO_BN_HW;
    unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
    _FLOAT NHW                  = (_FLOAT)MIO_BN_NHW;

    for(int gn = 0; gn < yngrps; gn++)
    {
        unsigned int offset    = gn * ygrp_sz + ylid;
        unsigned int meanindex = cidx + ygrp_sz * offset;
        if(offset < yngrps)
        { // modify to span larger number of groups
            mean += meanvarbuff[meanindex];
        }
    }

#ifdef __AMDGCN__

#if(MIO_BN_NGRPS > 64)
    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];

    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
    _FLOAT tmp = 0.;
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    mean = lcl_data[ylid];
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x111, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x112, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x114, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x142, 0xF, 0xF, 0));
    mean += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x143, 0xF, 0xF, 0));
    mean += tmp;
    if(ylid == 63)
    {
        meanvarbuff[meanstashindex] = mean / NHW; // stash mean
    }
#elif(MIO_BN_NGRPS > 16)
    _FLOAT tmp = 0.;
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x111, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x112, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x114, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x142, 0xF, 0xF, 0));
    mean += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x143, 0xF, 0xF, 0));
    mean += tmp;
    if(ylid == 63)
    {
        meanvarbuff[meanstashindex] = mean / NHW; // stash mean
    }
#else
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x101, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x102, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x104, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x108, 0xF, 0xF, 0));
    if(ylid == 0)
    {
        meanvarbuff[meanstashindex] = mean / NHW; // stash mean
    }
#endif

#else

    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];
    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);
#if(MIO_BN_NGRPS > 16)
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = lcl_data[0] / NHW;
#else

    mean = 0.;
#pragma unroll
    for(int i = 0; i < MIO_BN_NGRPS; i++)
    {
        mean += lcl_data[i];
    }
    mean /= NHW;
#endif
    if(ylid == 0)
    {
        meanvarbuff[meanstashindex] = mean; // stash mean
    }

#endif
}

__kernel void BatchNormFwdInferSpatialMean(const __global _FLOAT* __restrict in,
                                           __global _FLOAT* __restrict meanbuff)
{

    __local _FLOAT lcl_data[MIO_BN_LDS_SIZE];

    _FLOAT mean            = 0.;
    unsigned int ylid      = get_local_id(1);
    unsigned int ygrp_id   = get_group_id(1);
    unsigned int xgid      = get_global_id(0);
    unsigned int ygid      = get_global_id(1);
    unsigned int ygrp_sz   = get_local_size(1);
    unsigned int index     = 0;
    unsigned int cidx      = xgid * MIO_BN_HW;
    unsigned int meanindex = cidx + ygrp_sz * ygrp_id; // making assumption of n=0 here

    // move across the sections of the image mini_batch stack
    if(ygid < MIO_BN_HW)
    {

#pragma unroll
        for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            index = n * MIO_BN_CHW + cidx + ygid;
            mean += in[index];
        }
    }

    lcl_data[ylid] = mean;
    barrier(CLK_LOCAL_MEM_FENCE);

#ifdef __AMDGCN__

    _FLOAT tmp = 0.;
    if(ylid < 128)
        lcl_data[ylid] += lcl_data[ylid + 128];
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < 64)
        lcl_data[ylid] += lcl_data[ylid + 64];
    barrier(CLK_LOCAL_MEM_FENCE);

    mean = lcl_data[ylid];
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x111, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x112, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x114, 0xF, 0xF, 0));
    mean += as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x118, 0xF, 0xF, 0));
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x142, 0xF, 0xF, 0));
    mean += tmp;
    tmp = as_float(__builtin_amdgcn_mov_dpp(as_int(mean), 0x143, 0xF, 0xF, 0));
    mean += tmp;

    if(ylid == 63)
    {
        meanbuff[meanindex] = mean; // pre-stage for group reduction
    }
#else

#if(MIO_BN_GRP1 > 16)
    if(ylid < (MIO_BN_LDS_SIZE >> 2))
        ReduceKernel(lcl_data, 1, ylid, 4);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid < (MIO_BN_LDS_SIZE >> 4))
        ReduceKernel(lcl_data, 4, ylid, 16);
    barrier(CLK_LOCAL_MEM_FENCE);
    if(ylid == 0)
        ReduceKernel(lcl_data, 16, ylid, MIO_BN_LDS_SIZE);
    barrier(CLK_LOCAL_MEM_FENCE);
    mean = lcl_data[0];
#else

    mean = 0.;
#pragma unroll
    for(int i = 0; i < MIO_BN_GRP1; i++)
    {
        mean += lcl_data[i];
    }
#endif
    if(ylid == 0)
    {
        meanbuff[meanindex] = mean; // pre-stage for group reduction
    }
#endif

} // end spatial mean kernel

//====================================================

#endif

#ifdef __clang__
#pragma clang diagnostic pop
#pragma clang diagnostic pop
#endif


removed shared memory usage

removed shared memory usage

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVF2DInit(__read_only image2d_t volume, __write_only image2d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int2 pos = {get_global_id(0), get_global_id(1)};

    float f10 = read_imagef(volume, sampler, pos + (int2)(1,0)).x;
    float f_10 = read_imagef(volume, sampler, pos - (int2)(1,0)).x;
    float f01 = read_imagef(volume, sampler, pos + (int2)(0,1)).x;
    float f0_1 = read_imagef(volume, sampler, pos - (int2)(0,1)).x;

    float4 gradient = {
        0.5f*(f10-f_10),
        0.5f*(f01-f0_1),
        0,
        0
    };

    write_imagef(vector_field, pos, gradient);
}

__kernel __attribute__((reqd_work_group_size(16,16,1))) void GVF2DIteration(__read_only image2d_t init_vector_field, __read_only image2d_t read_vector_field, __write_only image2d_t write_vector_field, __private float mu) {

    int2 writePos = {
        get_global_id(0)
        get_global_id(1)
    };
    // Enforce mirror boundary conditions
    int2 size = {get_image_width(init_vector_field), get_image_height(init_vector_field)};
    int2 pos = writePos;
    pos = select(pos, (int2)(2,2), pos == (int2)(0,0));
    pos = select(pos, size-3, pos >= size-1);

    float2 v = read_imagef(read_vector_field, sampler, pos).xy;

        // Load data from shared memory and do calculations
        float2 init_vector = read_imagef(init_vector_field, sampler, pos).xy;

        float2 fx1, fx_1, fy1, fy_1;
        fx1 = read_imagef(read_vector_field, sampler, pos + (int2)(1,0)).xy;
        fy1 = read_imagef(read_vector_field, sampler, pos + (int2)(0,1)).xy;
        fx_1 = read_imagef(read_vector_field, sampler, pos - (int2)(1,0)).xy;
        fy_1 = read_imagef(read_vector_field, sampler, pos - (int2)(0,1)).xy;

        // Update the vector field: Calculate Laplacian using a 3D central difference scheme
        float2 laplacian = -4*v + fx1 + fx_1 + fy1 + fy_1;

        v += mu * laplacian - (v - init_vector)*(init_vector.x*init_vector.x+init_vector.y*init_vector.y);

        write_imagef(write_vector_field, writePos, (float4)(v.x,v.y,0,0));
}

__kernel void GVF2DResult(__write_only image2d_t result, __read_only image2d_t vectorField) {
    int2 pos = {get_global_id(0), get_global_id(1)};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.z = 0;
    vector.w = 0;
    write_imagef(result, pos, length(vector));
}

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVF2DInit(__read_only image2d_t volume, __write_only image2d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int2 pos = {get_global_id(0), get_global_id(1)};

    float f10 = read_imagef(volume, sampler, pos + (int2)(1,0)).x;
    float f_10 = read_imagef(volume, sampler, pos - (int2)(1,0)).x;
    float f01 = read_imagef(volume, sampler, pos + (int2)(0,1)).x;
    float f0_1 = read_imagef(volume, sampler, pos - (int2)(0,1)).x;

    float4 gradient = {
        0.5f*(f10-f_10),
        0.5f*(f01-f0_1),
        0,
        0
    };

    write_imagef(vector_field, pos, gradient);
}

#define LA2D(x,y) (x + (y<<4))
__kernel __attribute__((reqd_work_group_size(16,16,1))) void GVF2DIteration(__read_only image2d_t init_vector_field, __read_only image2d_t read_vector_field, __write_only image2d_t write_vector_field, __private float mu) {

    int2 writePos = {
        get_global_id(0)-(get_group_id(0)*2+1),
        get_global_id(1)-(get_group_id(1)*2+1)
    };
    int2 localPos = {get_local_id(0), get_local_id(1)};

    // TODO: Enforce mirror boundary conditions
    int2 size = {get_image_width(init_vector_field), get_image_height(init_vector_field)};
    int2 pos = writePos;
    pos = select(pos, (int2)(2,2), pos == (int2)(0,0));
    pos = select(pos, size-3, pos >= size-1);

    // Allocate shared memory
    __local float2 sharedMemory[256];

    // Read into shared memory
    float2 v = read_imagef(read_vector_field, sampler, pos).xy;
    sharedMemory[LA2D(localPos.x,localPos.y)]= v;

    // Ensure that it don't write outside of the image
    int2 comp = (localPos == (int2)(0,0)) +
        (localPos == (int2)(15,15)) + (writePos > size-1);

    // Synchronize the threads in the group
    barrier(CLK_LOCAL_MEM_FENCE);

    if(comp.x+comp.y ==  0) {
        // Load data from shared memory and do calculations
        float2 init_vector = read_imagef(init_vector_field, sampler, pos).xy;

        float2 fx1, fx_1, fy1, fy_1;
        fx1 = sharedMemory[LA2D(localPos.x+1,localPos.y)];
        fy1 = sharedMemory[LA2D(localPos.x,localPos.y+1)];
        fx_1 = sharedMemory[LA2D(localPos.x-1,localPos.y)];
        fy_1 = sharedMemory[LA2D(localPos.x,localPos.y-1)];

        // Update the vector field: Calculate Laplacian using a 3D central difference scheme
        float2 laplacian = -4*v + fx1 + fx_1 + fy1 + fy_1;

        v += mu * laplacian - (v - init_vector)*(init_vector.x*init_vector.x+init_vector.y*init_vector.y);

        write_imagef(write_vector_field, writePos, (float4)(v.x,v.y,0,0));
    }
}

__kernel void GVF2DResult(__write_only image2d_t result, __read_only image2d_t vectorField) {
    int2 pos = {get_global_id(0), get_global_id(1)};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.z = 0;
    vector.w = 0;
    write_imagef(result, pos, length(vector));
}