formatted WrW1x1

formatted WrW1x1

#if 0 // nef ML_OPEN_RUNNING
// W 7 x H 7 x C 2048 x K 2048
//#define MLO_GRP_SZ
#define MLO_GRP_SZ0 64
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_FILTER_SIZE0 1
#define MLO_FILTER_SIZE1 1
#define MLO_FILTER_PAD0 3
#define MLO_FILTER_PAD1 3
#define MLO_FILTER_STRIDE0 2
#define MLO_FILTER_STRIDE1 2
#define STRIDE_W 1
#define STRIDE_H 1
#define MLO_N_OUTPUTS 2048
#define MLO_N_INPUTS 1024
#define MLO_BATCH_SZ 16
//MLO_N_BATCH_LOOPS
#define MLO_IN_WIDTH 7
#define MLO_IN_HEIGHT 7
#define MLO_OUT_WIDTH 7
#define MLO_OUT_HEIGHT 7
#endif

#if 0 // nef ML_OPEN_RUNNING
// W 7 x H 7 x C 2048 x K 2048
//#define MLO_GRP_SZ
#define MLO_GRP_SZ0 256
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_FILTER_SIZE0 1
#define MLO_FILTER_SIZE1 1
#define MLO_FILTER_PAD0 0
#define MLO_FILTER_PAD1 0
#define MLO_FILTER_STRIDE0 2
#define MLO_FILTER_STRIDE1 2
#define STRIDE_W 1
#define STRIDE_H 1
#define MLO_N_OUTPUTS 2048
#define MLO_N_INPUTS 1024
#define MLO_BATCH_SZ 16
//MLO_N_BATCH_LOOPS
#define MLO_IN_WIDTH 14
#define MLO_IN_HEIGHT 14
#define MLO_OUT_WIDTH 7
#define MLO_OUT_HEIGHT 7
#endif

#if 0 // nef ML_OPEN_RUNNING
// W 14 x H 14 x C 1024 x K 512
//#define MLO_GRP_SZ
#define MLO_GRP_SZ0 256
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_FILTER_SIZE0 1
#define MLO_FILTER_SIZE1 1
#define MLO_FILTER_PAD0 0
#define MLO_FILTER_PAD1 0
#define MLO_FILTER_STRIDE0 2
#define MLO_FILTER_STRIDE1 2
#define STRIDE_W 1
#define STRIDE_H 1
#define MLO_N_OUTPUTS 512
#define MLO_N_INPUTS 1024
#define MLO_BATCH_SZ 16
//MLO_N_BATCH_LOOPS
#define MLO_IN_WIDTH 14
#define MLO_IN_HEIGHT 14
#define MLO_OUT_WIDTH 7
#define MLO_OUT_HEIGHT 7

#define MLO_LDS_REDUCTOIN 1
#define MLO_GLOBAL_ATOMIC 0
#define MLO_N_LDS_REDUCTION_ONCE 8
#define MLO_N_LDS_SIZE_PER_THREAD 8

#endif

#if 0 // nef ML_OPEN_RUNNING
// W 28 x H 28 x C 192 x K 64 X N 16
//#define MLO_GRP_SZ
#define MLO_GRP_SZ0 64
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_FILTER_SIZE0 1
#define MLO_FILTER_SIZE1 1
#define MLO_FILTER_PAD0 0
#define MLO_FILTER_PAD1 0
#define MLO_FILTER_STRIDE0 1
#define MLO_FILTER_STRIDE1 1
#define STRIDE_W 1
#define STRIDE_H 1
#define MLO_N_OUTPUTS 64
#define MLO_N_INPUTS 192
#define MLO_BATCH_SZ 16
#define MLO_IN_WIDTH 28
#define MLO_IN_HEIGHT 28
#define MLO_OUT_WIDTH 28
#define MLO_OUT_HEIGHT 28
#define MLO_LDS_REDUCTOIN 1
#define MLO_GLOBAL_ATOMIC 0

#define MLO_N_LOAD_DWORDS_PER_MAP_ONCE 64
#define MLO_N_LCL_IN_MAPS 8
#define MLO_N_LCL_OUT_MAPS 8

#define MLO_N_LCL_IN_MAPS_ONCE 8
#define MLO_N_LCL_OUT_MAPS_ONCE 8

#define MLO_READ_UNIT 2

//READ_UNIT == 1 for STRIDE and PAD mode

#define MLO_OUT_BATCH_STRIDE (MLO_OUT_WIDTH * MLO_OUT_HEIGHT * MLO_N_OUTPUTS)
#define MLO_OUT_CHANNEL_STRIDE (MLO_OUT_WIDTH * MLO_OUT_WIDTH)

#define MLO_IN_BATCH_STRIDE (MLO_IN_WIDTH * MLO_IN_HEIGHT * MLO_N_INPUTS)
#define MLO_IN_CHANNEL_STRIDE (MLO_IN_WIDTH * MLO_IN_HEIGHT)
#define MLO_WEI_BATCH_STRIDE (MLO_N_INPUTS * MLO_N_OUTPUTS)
#define MLO_WEI_CHANNEL_STRIDE (1 * 1 * MLO_N_INPUTS)
#define MLO_MAX_LOADS ((MLO_OUT_CHANNEL_STRIDE / MLO_READ_UNIT) * MLO_BATCH_SZ)

#define MLO_ACCUM_SZ (MLO_N_LCL_IN_MAPS * MLO_N_LCL_OUT_MAPS)
#define MLO_OUT_READ_SZ (N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_IN_READ_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

#define MLO_OUT_CHANNEL_READ_SZ (MLO_OUT_CHANNEL_STRIDE / MLO_READ_UNIT)

#define MLO_N_IN_TILE_BLOCK 4
#endif

#if 0
#define MLO_READ_UNIT 4

#define MLO_OUT_BATCH_STRIDE (MLO_OUT_WIDTH * MLO_OUT_HEIGHT * MLO_N_OUTPUTS)
#define MLO_OUT_CHANNEL_STRIDE (MLO_OUT_WIDTH * MLO_OUT_WIDTH)
#define MLO_OUT_STRIDE (1)
#define MLO_IN_BATCH_STRIDE (MLO_IN_WIDTH * MLO_IN_HEIGHT * MLO_N_INPUTS)
#define MLO_IN_CHANNEL_STRIDE (MLO_IN_WIDTH * MLO_IN_HEIGHT)
#define MLO_IN_STRIDE (2)
#define MLO_WEI_BATCH_STRIDE (MLO_N_INPUTS * MLO_N_OUTPUTS)
#define MLO_WEI_CHANNEL_STRIDE (1 * 1 * MLO_N_INPUTS)

//#define MLO_N_LCL_IN_MAPS       8
//#define MLO_N_LCL_OUT_MAPS      8
#define MLO_CACHELINE_DWORD_SZ 64

#endif

#if 0
#if(MLO_FILTER_PAD0 > 0 || MLO_FILTER_PAD1 > 0)
#define MLO_IN_PAD_MIN_X0 (MLO_FILTER_STRIDE0 - (MLO_FILTER_PAD0 % MLO_FILTER_STRIDE0))
#define MLO_IN_PAD_MIN_Y0 (MLO_FILTER_STRIDE1 - (MLO_FILTER_PAD1 % MLO_FILTER_STRIDE1))

#define MLO_IN_PAD_MIN_X (MLO_IN_PAD_MIN_X0 % MLO_FILTER_STRIDE0)
#define MLO_IN_PAD_MIN_Y (MLO_IN_PAD_MIN_Y0 % MLO_FILTER_STRIDE1)

#define MLO_OUT_PAD_MIN_X ((MLO_FILTER_PAD0 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0)
#define MLO_OUT_PAD_MIN_Y ((MLO_FILTER_PAD1 + MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)

#define MLO_OUT_PAD_WIDTH \
    (((MLO_IN_WIDTH - MLO_IN_PAD_MIN_X + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0))
#define MLO_OUT_PAD_HEIGHT \
    (((MLO_IN_HEIGHT - MLO_IN_PAD_MIN_Y + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE1))

#else
#define MLO_IN_PAD_MIN_X 0
#define MLO_IN_PAD_MIN_Y 0

#define MLO_OUT_PAD_MIN_X 0
#define MLO_OUT_PAD_MIN_Y 0

#define MLO_OUT_PAD_WIDTH MLO_OUT_WIDTH
#define MLO_OUT_PAD_HEIGHT MLO_OUT_HEIGHT
#endif
#endif

#if 0

#define MLO_N_BATCH_PER_WAVE ((MLO_BATCH_SZ + (MLO_GRP_SZ0 / 64) - 1) / (MLO_GRP_SZ0 / 64))
#define MLO_N_LOOPS_PER_MAP                                                          \
    ((MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT + MLO_N_LOAD_DWORDS_PER_MAP_ONCE - 1) / \
     (MLO_N_LOAD_DWORDS_PER_MAP_ONCE))
#define MLO_N_DWORDS_LAST_LOAD_PER_MAP \
    ((MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT) % (MLO_N_LOAD_DWORDS_PER_MAP_ONCE))

#define MLO_IN_NON_ALIGN (MLO_N_INPUTS & 0x7)
#define MLO_OUT_NON_ALIGN (MLO_N_OUTPUTS & 0x7)
#define MLO_IN_LAST_GROUP (MLO_N_INPUTS & (~0x7))
#define MLO_OUT_LAST_GROUP (MLO_N_OUTPUTS & (~0x7))
#define MLO_MAX_LOAD_DWORD ((MLO_OUT_CHANNEL_STRIDE / MLO_READ_UNIT) * MLO_BATCH_SZ)
#endif

// FLAT_LOAD_DWORDx4 is max
// MIN(MLO_OUT_PAD_WIDTH , 8)

#if 0

//64x64 only works MLO_OUT_PAD_WIDTH<=8
//following are trick to speed-up shader compiler for unused MIOpenCvBwdWrW_64x64
#if MLO_OUT_PAD_WIDTH > 8
#define MLO_N_LOAD_DWORD_ONCE_PER_THREAD 1
#else
#define MLO_N_DWORD_PER_LOOP (MLO_OUT_PAD_WIDTH)
#endif

#endif

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

// top_df        ==> out        in [Batch][output][out_H][out_W]
// bot           ==> gard_input in [Batch][inputs][IN_H][IN_W]
// weights_df    ==> weights    in [output][input][filter][filter]

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenCvBwdWrW_8x8map(const __global _FLOAT* __restrict top_df,
                      const __global _FLOAT* __restrict bot,
                      __global _FLOAT* __restrict weights_df,
                      UNUSED _FLOAT padding_val)
{
    __local _FLOAT sdata[MLO_GRP_SZ0 * 8];
    // 8x8 MUL_ADD per thread
    // Every thread 1DWORDs per MAP * 8 Maps
    // every 64 threads Load continous 64 DWORDs per MAP * 8 Maps
    // LDS reduction of every 64 threads for 8x8 results
    // NO LDS data exchange

    // Global_group_Id0:  [TILE_BLOCK]
    // Global_Id0:  [C/8] * 64 or 256
    // Global_group_Id1:  [K/8]
    // Global_group_Id2:  [C/8/ (MLO_N_IN_TILE_BLOCK= 8 or 4) ]

    // uint group_id0 = get_group_id(0);
    // uint group_id1 = get_group_id(1);

    uint local_Id0 = get_local_id(0);

// traverse small batch size to have better performance
#if MLO_IN_BATCH_STRIDE < MLO_OUT_BATCH_STRIDE
    uint C_OFFSET = get_group_id(0) * MLO_N_LCL_IN_MAPS;
    uint K_OFFSET = get_group_id(1) * MLO_N_LCL_OUT_MAPS;

#else
    uint K_OFFSET          = get_group_id(0) * MLO_N_LCL_OUT_MAPS;
    uint C_OFFSET          = get_group_id(1) * MLO_N_LCL_IN_MAPS;

#endif

    uint glb_out_off0 = K_OFFSET * MLO_OUT_CHANNEL_STRIDE;
    uint glb_in_off0  = C_OFFSET * MLO_IN_CHANNEL_STRIDE;

    // NO preload
    __private _FLOAT load_buf_top[MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT];
    __private _FLOAT load_buf_bot[MLO_N_LCL_IN_MAPS * MLO_READ_UNIT];

    __private _FLOAT accum[MLO_ACCUM_SZ];

    // CNHW will be continous address to utlize X4 load;
    // NCHW will be hard mode till now

    for(uint i = 0; i < MLO_ACCUM_SZ; i++)
    {
        accum[i] = 0;
    }

    for(uint i = 0; i < MLO_N_LCL_IN_MAPS; i++)
    {
        sdata[local_Id0 + i * MLO_GRP_SZ0] = 0;
    }

    for(uint faked_off = local_Id0; faked_off < MLO_MAX_LOADS; faked_off += MLO_GRP_SZ0)
    {
#if MLO_FILTER_STRIDE0 > 1 || MLO_FILTER_STRIDE1 > 1 || MLO_FILTER_STRIDE0 > 1 || \
    MLO_FILTER_STRIDE1 > 1
        // MLO_READ_UNIT==1

        uint batch_id   = iDiv(faked_off, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));
        uint faked_off2 = iMod(faked_off, batch_id, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));

        uint out_x_off = iDiv(faked_off2, MLO_OUT_PAD_WIDTH);
        uint out_y_off = iMod(faked_off2, out_x_off, MLO_OUT_PAD_WIDTH);

        uint out_image_off =
            (out_y_off + MLO_OUT_PAD_MIN_Y) * MLO_OUT_WIDTH + (out_x_off + MLO_OUT_PAD_MIN_X);

        uint in_x_off = out_x_off * MLO_FILTER_STRIDE0 + MLO_IN_PAD_MIN_X;
        uint in_y_off = out_y_off * MLO_FILTER_STRIDE1 + MLO_IN_PAD_MIN_Y;

        uint in_image_off = in_y_off * MLO_IN_WIDTH + in_x_off;

#else
        uint batch_id      = iDiv(faked_off, (MLO_OUT_CHANNEL_READ_SZ));           // batch
        uint image_off     = iMod(faked_off, batch_id, (MLO_OUT_CHANNEL_READ_SZ)); // pixel offset
        uint in_image_off  = image_off * MLO_READ_UNIT;
        uint out_image_off = image_off * MLO_READ_UNIT;
#endif
        uint glb_in_off = glb_in_off0 + batch_id * MLO_IN_BATCH_STRIDE + in_image_off;

        // Address offset mode is not used in shader compiler
        // Most VGPRs are used up by Address
        for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
        {
            uint bot_off                = glb_in_off + c * MLO_IN_CHANNEL_STRIDE;
            const __global _FLOAT* bot1 = &bot[bot_off];

            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                load_buf_bot[c * MLO_READ_UNIT + i] = bot1[i];
            }
        }

        uint glb_out_off = glb_out_off0 + batch_id * MLO_OUT_BATCH_STRIDE + out_image_off;
        for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
        {
            uint top_off = glb_out_off + k * MLO_OUT_CHANNEL_STRIDE;

            const __global _FLOAT* top1 = &top_df[top_off];

            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                load_buf_top[k * MLO_READ_UNIT + i] = top1[i];
            }
        }

        // processing
        // outside loop-i save 1 VGPR than loop-i inside
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
            {
                for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
                {
                    {
                        accum[k * MLO_N_LCL_IN_MAPS + c] += load_buf_bot[c * MLO_READ_UNIT + i] *
                                                            load_buf_top[k * MLO_READ_UNIT + i];
                    }
                }
            }
        }
    }

#define LAST_PIXELS (MLO_OUT_CHANNEL_STRIDE % MLO_READ_UNIT)

// PAD/STRIDE never goes to following since MLO_READ_UNIT == 1
#if LAST_PIXELS > 0
#define MLO_MAX_LOADS2 (MLO_BATCH_SZ * LAST_PIXELS)
#define MLO_LAST_PIXEL_OFFSET (MLO_OUT_CHANNEL_STRIDE - LAST_PIXELS)

    for(uint faked_off = local_Id0; faked_off < MLO_MAX_LOADS2; faked_off += MLO_GRP_SZ0)
    {
#if MLO_FILTER_STRIDE0 > 1 || MLO_FILTER_STRIDE1 > 1 || MLO_FILTER_STRIDE0 > 1 || \
    MLO_FILTER_STRIDE1 > 1
        uint batch_id   = iDiv(faked_off, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));
        uint faked_off2 = iMod(faked_off, batch_id, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));

        uint out_x_off = iDiv(faked_off2, MLO_OUT_PAD_WIDTH);
        uint out_y_off = iMod(faked_off2, out_x_off, MLO_OUT_PAD_WIDTH);

        uint out_image_off =
            (out_y_off + MLO_OUT_PAD_MIN_Y) * MLO_OUT_WIDTH + (out_x_off + MLO_OUT_PAD_MIN_X);

        uint in_x_off = out_x_off * MLO_FILTER_STRIDE0 + MLO_IN_PAD_MIN_X;
        uint in_y_off = out_y_off * MLO_FILTER_STRIDE1 + MLO_IN_PAD_MIN_Y;

        uint in_image_off = in_y_off * MLO_IN_WIDTH + in_x_off;

#else
        uint batch_id = iDiv(faked_off, (LAST_PIXELS)); // batch
        uint image_off =
            iMod(faked_off, batch_id, (LAST_PIXELS)) + MLO_LAST_PIXEL_OFFSET; // pixel offset
        uint in_image_off  = image_off * 1;
        uint out_image_off = image_off * 1;
#endif
        uint glb_in_off = glb_in_off0 + batch_id * MLO_IN_BATCH_STRIDE + in_image_off;

        // Address offset mode is not used in shader compiler
        // Most VGPRs are used up by Address
        for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
        {
            uint bot_off                = glb_in_off + c * MLO_IN_CHANNEL_STRIDE;
            const __global _FLOAT* bot1 = &bot[bot_off];

            for(uint i = 0; i < 1; ++i)
            {
                load_buf_bot[c * MLO_READ_UNIT + i] = bot1[i];
            }
        }

        uint glb_out_off = glb_out_off0 + batch_id * MLO_OUT_BATCH_STRIDE + out_image_off;
        for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
        {
            uint top_off = glb_out_off + k * MLO_OUT_CHANNEL_STRIDE;

            const __global _FLOAT* top1 = &top_df[top_off];

            for(uint i = 0; i < 1; ++i)
            {
                load_buf_top[k * MLO_READ_UNIT + i] = top1[i];
            }
        }

        // processing
        // outside loop-i save 1 VGPR than loop-i inside
        for(uint i = 0; i < 1; ++i)
        {
            for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
            {
                for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
                {
                    {
                        accum[k * MLO_N_LCL_IN_MAPS + c] += load_buf_bot[c * MLO_READ_UNIT + i] *
                                                            load_buf_top[k * MLO_READ_UNIT + i];
                    }
                }
            }
        }
    }
#endif

    __private _FLOAT accum_to_store = 0;

    for(uint K = 0; K < MLO_N_LCL_OUT_MAPS; K++)
    {
        barrier(CLK_LOCAL_MEM_FENCE);

        for(uint C = 0; C < MLO_N_LCL_IN_MAPS; C++)
        {
            sdata[local_Id0 + MLO_GRP_SZ0 * C] = accum[K * MLO_N_LCL_IN_MAPS + C];
        }

        barrier(CLK_LOCAL_MEM_FENCE);

        // Reduction every MLO_GRP_SZ0* MLO_GRP_SZ1 to trhead 0
        for(uint s = ((MLO_GRP_SZ0) >> 2); s > 0; s = (s >> 2))
        {
            if(local_Id0 < s)
            {
                for(uint C = 0; C < MLO_N_LCL_IN_MAPS; C++)
                {
                    sdata[local_Id0 + MLO_GRP_SZ0 * C] +=
                        sdata[MLO_GRP_SZ0 * C + local_Id0 + s] +
                        sdata[MLO_GRP_SZ0 * C + local_Id0 + 2 * s] +
                        sdata[MLO_GRP_SZ0 * C + local_Id0 + 3 * s];
                }
            }
            // NO need inside 1 wave: barrier(CLK_LOCAL_MEM_FENCE);
        }

        barrier(CLK_LOCAL_MEM_FENCE);

        // MLO_N_LCL_IN_MAPS store
        if((local_Id0 & ~0x7) == (K * MLO_N_LCL_IN_MAPS))
        {
            accum_to_store = sdata[0 + (local_Id0 & 0x7) * MLO_GRP_SZ0];
        }

        // only 1st wave need to barrier: it will remove all scratch registers
    }

    if(local_Id0 < (MLO_ACCUM_SZ))
    {

        // Store to Memory
        __global _FLOAT* __restrict weights_ptr =
            weights_df + (K_OFFSET + (local_Id0 / MLO_N_LCL_IN_MAPS)) * MLO_WEI_CHANNEL_STRIDE +
            C_OFFSET + (local_Id0 % MLO_N_LCL_IN_MAPS);

        {
            __global _FLOAT* weights_ptr2 = weights_ptr;
            *weights_ptr2                 = accum_to_store;
        }
    }
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenCvBwdWrW_16x16map(const __global _FLOAT* __restrict top_df,
                        const __global _FLOAT* __restrict bot,
                        __global _FLOAT* __restrict weights_df,
                        UNUSED _FLOAT padding_val)
{
    __local _FLOAT sdata[MLO_GRP_SZ0 * 8];
    // 64 threds split into 4 grpoups: every 16 threads accumulate 8x8
    //
    // 256 threads split inot 4 groups : every 64 threads accumulate 8x8
    //

    // Global_group_Id0:  [TILE_BLOCK]
    // Global_Id0:  [C/16] * 64 or 256
    // Global_group_Id1:  [K/16]
    // Global_group_Id2:  [C/16/ (MLO_N_IN_TILE_BLOCK= 8 or 4) ]

    // uint group_id0 = get_group_id(0);
    // uint group_id1 = get_group_id(1);

    uint local_Id0 = get_local_id(0);

// traverse small batch size to have better performance
#if MLO_IN_BATCH_STRIDE < MLO_OUT_BATCH_STRIDE
    uint C_OFFSET = get_group_id(0) * MLO_N_LCL_IN_MAPS;
    uint K_OFFSET = get_group_id(1) * MLO_N_LCL_OUT_MAPS;

#else
    uint K_OFFSET          = get_group_id(0) * MLO_N_LCL_OUT_MAPS;
    uint C_OFFSET          = get_group_id(1) * MLO_N_LCL_IN_MAPS;

#endif

    // Split into 4 groups for C[0,1,0,1], K [0,0,1,1]
    uint k_offset2 = ((local_Id0 / (MLO_GRP_SZ0 / 4)) / 2) * MLO_N_LCL_OUT_MAPS_ONCE;
    uint c_offset2 = ((local_Id0 / (MLO_GRP_SZ0 / 4)) % 2) * MLO_N_LCL_IN_MAPS_ONCE;

    uint glb_out_off0 = (K_OFFSET + k_offset2) * MLO_OUT_CHANNEL_STRIDE;
    uint glb_in_off0  = (C_OFFSET + c_offset2) * MLO_IN_CHANNEL_STRIDE;

    // NO preload
    __private _FLOAT load_buf_top[MLO_N_LCL_OUT_MAPS_ONCE * MLO_READ_UNIT];
    __private _FLOAT load_buf_bot[MLO_N_LCL_IN_MAPS_ONCE * MLO_READ_UNIT];

    __private _FLOAT accum[MLO_ACCUM_SZ];

    // CNHW will be continous address to utlize X4 load;
    // NCHW will be hard mode till now

    for(uint i = 0; i < MLO_ACCUM_SZ; i++)
    {
        accum[i] = 0;
    }

    for(uint i = 0; i < MLO_N_LCL_OUT_MAPS_ONCE; i++)
    {
        sdata[local_Id0 + i * MLO_GRP_SZ0] = 0;
    }

    for(uint faked_off = (local_Id0 % (MLO_GRP_SZ0 / 4)); faked_off < MLO_MAX_LOADS;
        faked_off += (MLO_GRP_SZ0 / 4))
    {
#if MLO_FILTER_STRIDE0 > 1 || MLO_FILTER_STRIDE1 > 1 || MLO_FILTER_STRIDE0 > 1 || \
    MLO_FILTER_STRIDE1 > 1

#if MLO_READ_UNIT == 1
        uint batch_id   = iDiv(faked_off, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));
        uint faked_off2 = iMod(faked_off, batch_id, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));

        uint out_x_off = iDiv(faked_off2, MLO_OUT_PAD_WIDTH);
        uint out_y_off = iMod(faked_off2, out_x_off, MLO_OUT_PAD_WIDTH);

        uint out_image_off =
            (out_y_off + MLO_OUT_PAD_MIN_Y) * MLO_OUT_WIDTH + (out_x_off + MLO_OUT_PAD_MIN_X);

        uint in_x_off = out_x_off * MLO_FILTER_STRIDE0 + MLO_IN_PAD_MIN_X;
        uint in_y_off = out_y_off * MLO_FILTER_STRIDE1 + MLO_IN_PAD_MIN_Y;

        uint in_image_off = in_y_off * MLO_IN_WIDTH + in_x_off;
#endif
#if 0 // PER_ROW which will be enabled after SGPR offset is enabled.
        uint batch_id   = iDiv( faked_off,  (MLO_OUT_PAD_WIDTH ));
        uint faked_off2 = iMod( faked_off,  batch_id, (MLO_OUT_PAD_WIDTH ));

        uint out_x_off = 0;
        uint out_y_off = faked_off2;

        uint out_image_off = (out_y_off + MLO_OUT_PAD_MIN_Y) * MLO_OUT_WIDTH + (out_x_off +MLO_OUT_PAD_MIN_X);

        uint in_x_off = out_x_off * MLO_FILTER_STRIDE0 + MLO_IN_PAD_MIN_X;
        uint in_y_off = out_y_off * MLO_FILTER_STRIDE1 + MLO_IN_PAD_MIN_Y;

        uint in_image_off  = in_y_off * MLO_IN_WIDTH + in_x_off;

        //uint glb_in_off  = glb_in_off0  + batch_id * MLO_IN_BATCH_STRIDE   + in_image_off ;
        //uint glb_out_off = glb_out_off0 + batch_id * MLO_OUT_BATCH_STRIDE  + out_image_off;

#endif
#else
        uint batch_id      = iDiv(faked_off, (MLO_OUT_CHANNEL_READ_SZ));           // batch
        uint image_off     = iMod(faked_off, batch_id, (MLO_OUT_CHANNEL_READ_SZ)); // pixel offset
        uint in_image_off  = image_off * MLO_READ_UNIT;
        uint out_image_off = image_off * MLO_READ_UNIT;
#endif
        uint glb_in_off = glb_in_off0 + batch_id * MLO_IN_BATCH_STRIDE + in_image_off;

        // Address offset mode is not used in shader compiler
        // Most VGPRs are used up by Address
        for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
        {
            uint bot_off                = glb_in_off + c * MLO_IN_CHANNEL_STRIDE;
            const __global _FLOAT* bot1 = &bot[bot_off];

            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                load_buf_bot[c * MLO_READ_UNIT + i] = bot1[i * MLO_FILTER_STRIDE0];
            }
        }

        uint glb_out_off = glb_out_off0 + batch_id * MLO_OUT_BATCH_STRIDE + out_image_off;
        for(uint k = 0; k < MLO_N_LCL_OUT_MAPS_ONCE; ++k)
        {
            uint top_off = glb_out_off + k * MLO_OUT_CHANNEL_STRIDE;

            const __global _FLOAT* top1 = &top_df[top_off];

            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                load_buf_top[k * MLO_READ_UNIT + i] = top1[i];
            }
        }

        // processing
        // outside loop-i save 1 VGPR than loop-i inside
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            for(uint k = 0; k < MLO_N_LCL_OUT_MAPS_ONCE; ++k)
            {
                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    // for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        accum[k * MLO_N_LCL_IN_MAPS_ONCE + c] +=
                            load_buf_bot[c * MLO_READ_UNIT + i] *
                            load_buf_top[k * MLO_READ_UNIT + i];
                    }
                }
            }
        }
    }

#undef LAST_PIXELS
#undef MLO_MAX_LOADS2
#undef MLO_LAST_PIXEL_OFFSET

#define LAST_PIXELS (MLO_OUT_CHANNEL_STRIDE % MLO_READ_UNIT)

// PAD/STRIDE never goes to LAST_PIXELS
#if LAST_PIXELS > 0
#define MLO_MAX_LOADS2 (MLO_BATCH_SZ * LAST_PIXELS)
#define MLO_LAST_PIXEL_OFFSET (MLO_OUT_CHANNEL_STRIDE - LAST_PIXELS)

    for(uint faked_off = (local_Id0 % (MLO_GRP_SZ0 / 4)); faked_off < MLO_MAX_LOADS2;
        faked_off += (MLO_GRP_SZ0 / 4))
    {
#if MLO_FILTER_STRIDE0 > 1 || MLO_FILTER_STRIDE1 > 1 || MLO_FILTER_STRIDE0 > 1 || \
    MLO_FILTER_STRIDE1 > 1
        uint batch_id   = iDiv(faked_off, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));
        uint faked_off2 = iMod(faked_off, batch_id, (MLO_OUT_PAD_WIDTH * MLO_OUT_PAD_HEIGHT));

        uint out_x_off = iDiv(faked_off2, MLO_OUT_PAD_WIDTH);
        uint out_y_off = iMod(faked_off2, x_off, MLO_OUT_PAD_WIDTH) + MLO_OUT_PAD_MIN_X;

        uint out_image_off =
            (out_y_off + MLO_OUT_PAD_MIN_X) * MLO_OUT_WIDTH + out_x_off + MLO_OUT_PAD_MIN_X;

        uint in_x_off = out_x_off * MLO_FILTER_STRIDE0 + MLO_IN_PAD_MIN_X;
        uint in_y_off = out_y_off * MLO_FILTER_STRIDE1 + MLO_IN_PAD_MIN_Y;

        uint in_image_off = in_y_off * MLO_IN_WIDTH + in_x_off;

#else
        uint batch_id      = iDiv(faked_off, (LAST_PIXELS));           // batch
        uint image_off     = iMod(faked_off, batch_id, (LAST_PIXELS)); // pixel offset
        image_off += MLO_LAST_PIXEL_OFFSET;
#endif
        uint glb_in_off = glb_in_off0 + batch_id * MLO_IN_BATCH_STRIDE + image_off * 1;

        // Address offset mode is not used in shader compiler
        // Most VGPRs are used up by Address
        for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
        {
            uint bot_off                = glb_in_off + c * MLO_IN_CHANNEL_STRIDE;
            const __global _FLOAT* bot1 = &bot[bot_off];

            for(uint i = 0; i < 1; ++i)
            {
                load_buf_bot[c * MLO_READ_UNIT + i] = bot1[i];
            }
        }

        uint glb_out_off = glb_out_off0 + batch_id * MLO_OUT_BATCH_STRIDE + image_off * 1;
        for(uint k = 0; k < MLO_N_LCL_OUT_MAPS_ONCE; ++k)
        {
            uint top_off = glb_out_off + k * MLO_OUT_CHANNEL_STRIDE;

            const __global _FLOAT* top1 = &top_df[top_off];

            for(uint i = 0; i < 1; ++i)
            {
                load_buf_top[k * MLO_READ_UNIT + i] = top1[i];
            }
        }

        // processing
        // outside loop-i save 1 VGPR than loop-i inside
        for(uint i = 0; i < 1; ++i)
        {
            for(uint k = 0; k < MLO_N_LCL_OUT_MAPS_ONCE; ++k)
            {
                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    {
                        accum[k * MLO_N_LCL_IN_MAPS_ONCE + c] +=
                            load_buf_bot[c * MLO_READ_UNIT + i] *
                            load_buf_top[k * MLO_READ_UNIT + i];
                    }
                }
            }
        }
    }
#endif

    __private _FLOAT accum_to_store[4];
    accum_to_store[0] = 0;
    accum_to_store[1] = 0;
    accum_to_store[2] = 0;
    accum_to_store[3] = 0;

    // 1 loop reduction 8xC per thread, 32 result per workgroup

    for(uint K = 0; K < MLO_N_LCL_OUT_MAPS_ONCE; K++)
    {
        barrier(CLK_LOCAL_MEM_FENCE);

        for(uint C = 0; C < MLO_N_LCL_IN_MAPS_ONCE; C++)
        {
            sdata[local_Id0 + MLO_GRP_SZ0 * C] = accum[K * MLO_N_LCL_IN_MAPS_ONCE + C];
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        // Reduction every MLO_GRP_SZ0* MLO_GRP_SZ1 to trhead 0 inside 1 wave
        for(uint s = (MLO_GRP_SZ0 >> 2); s >= 4; s = (s >> 2))
        {
            // Every time reduce to 1/4
            // Final: offset 0, 1, 2, 3 has the accumualte value
            if(local_Id0 < s)
            {
                for(uint C = 0; C < MLO_N_LCL_IN_MAPS_ONCE; C++)
                {
                    // reduce to final 4x
                    sdata[local_Id0 + MLO_GRP_SZ0 * C] =
                        sdata[local_Id0 * 4 + 0 + MLO_GRP_SZ0 * C] +
                        sdata[local_Id0 * 4 + 1 + MLO_GRP_SZ0 * C] +
                        sdata[local_Id0 * 4 + 2 + MLO_GRP_SZ0 * C] +
                        sdata[local_Id0 * 4 + 3 + MLO_GRP_SZ0 * C];
                }
            }
            // NO need inside 1 wave: barrier(CLK_LOCAL_MEM_FENCE);
        }

        barrier(CLK_LOCAL_MEM_FENCE);

        // MLO_N_LCL_IN_MAPS store 32 DWORD once
        if((local_Id0 & ~0x7) == (K * (MLO_N_LCL_IN_MAPS_ONCE)))
        {
            accum_to_store[0] = sdata[0 + (local_Id0 & 0x7) * MLO_GRP_SZ0];
            accum_to_store[1] = sdata[1 + (local_Id0 & 0x7) * MLO_GRP_SZ0];
            accum_to_store[2] = sdata[2 + (local_Id0 & 0x7) * MLO_GRP_SZ0];
            accum_to_store[3] = sdata[3 + (local_Id0 & 0x7) * MLO_GRP_SZ0];
        }

        // only 1st wave need to barrier: it will remove all scratch registers
    }

    if(local_Id0 < (MLO_ACCUM_SZ))
    {

        // Store to Memory
        __global _FLOAT* __restrict weights_ptr =
            weights_df +
            (K_OFFSET + (local_Id0 / MLO_N_LCL_IN_MAPS_ONCE)) * MLO_WEI_CHANNEL_STRIDE + C_OFFSET +
            (local_Id0 % MLO_N_LCL_IN_MAPS_ONCE);

        {
            __global _FLOAT* weights_ptr1 = weights_ptr;
            for(uint k = 0; k < 2; k++)
                for(uint c = 0; c < 2; c++)
                {
                    __global _FLOAT* weights_ptr2 =
                        weights_ptr1 + k * MLO_N_LCL_IN_MAPS_ONCE * MLO_WEI_CHANNEL_STRIDE +
                        c * MLO_N_LCL_IN_MAPS_ONCE;
                    *weights_ptr2 = accum_to_store[k * 2 + c];
                }
        }
    }
}


Fix unused variables in lrn

Fix unused variables in lrn

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

/*

This is a naive implementation.
The "sliding window" -based implementation is in MIOpenLRNFwd.cl file

*/

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannelBwd(const __global _FLOAT* top,
                          const __global _FLOAT* bot,
                          const __global _FLOAT* top_df,
                          const __global _FLOAT* scale,
                          __global _FLOAT* bot_df,
                          UNUSED _FLOAT ratio,
                          _FLOAT alpha,
                          _FLOAT beta)
{
    __local _FLOAT top_df_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    __local _FLOAT ratio_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x          = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y          = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0    = get_local_id(0);
    int lcl_id1    = get_local_id(1);
    int ob         = get_global_id(2); // output * batch_sz
    int o          = ob / MLO_LRN_BATCH_SZ;
    int b          = ob - o * MLO_LRN_BATCH_SZ;
    int top_x      = x;
    int top_y      = y;
    int top_df_off = b * MLO_LRN_TOPDF_BATCH_STRIDE + o * MLO_LRN_TOPDF_CHANNEL_STRIDE;
    int scale_off  = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE;
    int bot_x      = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int bot_y      = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX;

    _FLOAT prv_exp_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    //		_FLOAT prv_top_df[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    // load top_diff and scale tiles
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_df_y_off = top_y_act * MLO_LRN_TOPDF_STRIDE;
        int scale_y_off  = top_y_act * MLO_LRN_SCALE_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act];
            _FLOAT scale_val  = scale[scale_off + scale_y_off + top_x_act];

            top_df_val = (invisibleX || invisibleY) ? 0 : top_df_val;
            scale_val  = (invisibleX || invisibleY) ? (_FLOAT)1.f : scale_val;

            top_df_data[lcl_off_v + b_i] = top_df_val;
            ratio_data[lcl_off_v + b_i]  = scale_val;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // actual top_diffs and scales
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int lcl_off_v =
            (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_PAD + j) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT scale_ratio =
                ratio_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i];
            prv_exp_scale[j][i] = exp(-beta * log(scale_ratio));
            //				prv_exp_scale[j][i]= pow(scale, -beta);

            //				prv_top_df[j][i] = top_df_val;
        }
    }

    // read top and load ratio tile
    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE;
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_y_off = top_y_act * MLO_LRN_TOP_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_val = top[top_off + top_y_off + top_x_act];

            top_val = (invisibleX || invisibleY) ? 0 : top_val;

            _FLOAT top_df_val = top_df_data[lcl_off_v + b_i];

            _FLOAT scale_val = ratio_data[lcl_off_v + b_i];

            // scale val is not 0
            _FLOAT ratio_dta = (top_df_val * top_val) / scale_val;
            // replacing scale with ratio
            ratio_data[lcl_off_v + b_i] = ratio_dta;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // caculate bot diff
    _FLOAT prv_bot_diff[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int v_off_v = (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + j);
        int hstart  = y + v_off_v - MLO_LRN_PAD;
        int hend    = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_HEIGHT + MLO_LRN_PAD);

        // accum offset, vertical
        //			int lcl_a_off_v = v_off_v *  MLO_LRN_LCL_DATA_WIDTH;
        // value offset, vertical
        int lcl_v_off_v = (v_off_v + MLO_LRN_PAD) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT prv_ratio_accum = 0;
            int v_off_h            = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + i;

            int wstart = x + v_off_h - MLO_LRN_PAD;
            int wend   = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_WIDTH + MLO_LRN_PAD);

            int adj_area_size = (hend - hstart) * (wend - wstart);

            // accum offset, horiz
            int lcl_a_off_h = v_off_h;
            //	value offset, horiz
            int lcl_v_off_h = lcl_a_off_h + MLO_LRN_PAD;

            for(int k = 0; k < MLO_LRN_KERNEL_SZ; k++)
            {
                for(int l = 0; l < MLO_LRN_KERNEL_SZ; l++)
                {
                    prv_ratio_accum +=
                        ratio_data[(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l];
                }
            }

            _FLOAT top_df_val = top_df_data[lcl_v_off_v + lcl_v_off_h];
            _FLOAT bot_dta    = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * o +
                                 MLO_LRN_BOT_STRIDE * (y + v_off_v) + x + v_off_h];
            _FLOAT adj_ratio       = (_FLOAT)2.f * alpha * beta / adj_area_size;
            _FLOAT prv_accum_ratio = adj_ratio * bot_dta * prv_ratio_accum;
            prv_bot_diff[j][i]     = prv_exp_scale[j][i] * top_df_val - prv_accum_ratio;
        }
    }

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; j++)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            if(bot_y + j < MLO_LRN_BOT_HEIGHT && bot_x + i < MLO_LRN_BOT_WIDTH)
            {

                bot_df[MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * o +
                       MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i] = prv_bot_diff[j][i];
            }
        }
    }
}

#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannelsBwd1(const __global _FLOAT* top,
                            const __global _FLOAT* bot,
                            const __global _FLOAT* top_df,
                            const __global _FLOAT* scale,
                            __global _FLOAT* bot_df,
                            _FLOAT ratio,
                            UNUSED _FLOAT alpha,
                            _FLOAT beta)
{
    int x              = get_global_id(0); // channel x
    int y              = get_global_id(1); // channel y
    int b              = get_global_id(2); // batch
    _FLOAT accum_ratio = 0;
    _FLOAT top_df_in[MLO_LRN_KERNEL_SZ];
    _FLOAT scale_in[MLO_LRN_KERNEL_SZ];
    _FLOAT ratio_dta[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    int bot_df_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD; c_i++)
    {

        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];

        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];

#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
    }

    for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
    {
        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[c_o];

            _FLOAT exp_scale = exp(-beta * log(prv_scale));
            //					pow(prv_scale, -beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[c_o] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++)
    {

        _FLOAT prv_top_df_in =
            top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i +
                   MLO_LRN_TOPDF_STRIDE * y + x];
        _FLOAT prv_scale_in =
            scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                  MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        _FLOAT prv_ratio_dta = prv_top_df_in * top_dta / prv_scale_in;
#if MLO_LOW_CHNL_COUNT == 1
        prv_ratio_dta = (c_i < MLO_LRN_N_OUTPUTS) ? prv_ratio_dta : 0;
#endif

        accum_ratio = accum_ratio + prv_ratio_dta;

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

        top_df_in[MLO_LRN_KERNEL_SZ - 1] = prv_top_df_in;
        scale_in[MLO_LRN_KERNEL_SZ - 1]  = prv_scale_in;
        ratio_dta[MLO_LRN_KERNEL_SZ - 1] = prv_ratio_dta;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = exp(-beta * log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = exp(-beta * log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

/*

This is a naive implementation.
The "sliding window" -based implementation is in MIOpenLRNFwd.cl file

*/

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannelBwd(const __global _FLOAT* top,
                          const __global _FLOAT* bot,
                          const __global _FLOAT* top_df,
                          const __global _FLOAT* scale,
                          __global _FLOAT* bot_df,
                          UNUSED _FLOAT ratio,
                          _FLOAT alpha,
                          _FLOAT beta)
{
    __local _FLOAT top_df_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    __local _FLOAT ratio_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x          = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y          = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0    = get_local_id(0);
    int lcl_id1    = get_local_id(1);
    int ob         = get_global_id(2); // output * batch_sz
    int o          = ob / MLO_LRN_BATCH_SZ;
    int b          = ob - o * MLO_LRN_BATCH_SZ;
    int top_x      = x;
    int top_y      = y;
    int top_df_off = b * MLO_LRN_TOPDF_BATCH_STRIDE + o * MLO_LRN_TOPDF_CHANNEL_STRIDE;
    int scale_off  = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE;
    int bot_x      = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int bot_y      = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX;

    _FLOAT prv_exp_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    //		_FLOAT prv_top_df[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    // load top_diff and scale tiles
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_df_y_off = top_y_act * MLO_LRN_TOPDF_STRIDE;
        int scale_y_off  = top_y_act * MLO_LRN_SCALE_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act];
            _FLOAT scale_val  = scale[scale_off + scale_y_off + top_x_act];

            top_df_val = (invisibleX || invisibleY) ? 0 : top_df_val;
            scale_val  = (invisibleX || invisibleY) ? (_FLOAT)1.f : scale_val;

            top_df_data[lcl_off_v + b_i] = top_df_val;
            ratio_data[lcl_off_v + b_i]  = scale_val;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // actual top_diffs and scales
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int lcl_off_v =
            (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_PAD + j) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT scale_ratio =
                ratio_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i];
            prv_exp_scale[j][i] = exp(-beta * log(scale_ratio));
            //				prv_exp_scale[j][i]= pow(scale, -beta);

            //				prv_top_df[j][i] = top_df_val;
        }
    }

    // read top and load ratio tile
    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE;
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_y_off = top_y_act * MLO_LRN_TOP_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_val = top[top_off + top_y_off + top_x_act];

            top_val = (invisibleX || invisibleY) ? 0 : top_val;

            _FLOAT top_df_val = top_df_data[lcl_off_v + b_i];

            _FLOAT scale_val = ratio_data[lcl_off_v + b_i];

            // scale val is not 0
            _FLOAT ratio_dta = (top_df_val * top_val) / scale_val;
            // replacing scale with ratio
            ratio_data[lcl_off_v + b_i] = ratio_dta;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // caculate bot diff
    _FLOAT prv_bot_diff[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int v_off_v = (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + j);
        int hstart  = y + v_off_v - MLO_LRN_PAD;
        int hend    = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_HEIGHT + MLO_LRN_PAD);

        // accum offset, vertical
        //			int lcl_a_off_v = v_off_v *  MLO_LRN_LCL_DATA_WIDTH;
        // value offset, vertical
        int lcl_v_off_v = (v_off_v + MLO_LRN_PAD) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT prv_ratio_accum = 0;
            int v_off_h            = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + i;

            int wstart = x + v_off_h - MLO_LRN_PAD;
            int wend   = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_WIDTH + MLO_LRN_PAD);

            int adj_area_size = (hend - hstart) * (wend - wstart);

            // accum offset, horiz
            int lcl_a_off_h = v_off_h;
            //	value offset, horiz
            int lcl_v_off_h = lcl_a_off_h + MLO_LRN_PAD;

            for(int k = 0; k < MLO_LRN_KERNEL_SZ; k++)
            {
                for(int l = 0; l < MLO_LRN_KERNEL_SZ; l++)
                {
                    prv_ratio_accum +=
                        ratio_data[(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l];
                }
            }

            _FLOAT top_df_val = top_df_data[lcl_v_off_v + lcl_v_off_h];
            _FLOAT bot_dta    = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * o +
                                 MLO_LRN_BOT_STRIDE * (y + v_off_v) + x + v_off_h];
            _FLOAT adj_ratio       = (_FLOAT)2.f * alpha * beta / adj_area_size;
            _FLOAT prv_accum_ratio = adj_ratio * bot_dta * prv_ratio_accum;
            prv_bot_diff[j][i]     = prv_exp_scale[j][i] * top_df_val - prv_accum_ratio;
        }
    }

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; j++)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            if(bot_y + j < MLO_LRN_BOT_HEIGHT && bot_x + i < MLO_LRN_BOT_WIDTH)
            {

                bot_df[MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * o +
                       MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i] = prv_bot_diff[j][i];
            }
        }
    }
}

#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannelsBwd1(const __global _FLOAT* top,
                            const __global _FLOAT* bot,
                            const __global _FLOAT* top_df,
                            const __global _FLOAT* scale,
                            __global _FLOAT* bot_df,
                            UNUSED _FLOAT ratio,
                            _FLOAT alpha,
                            _FLOAT beta)
{
    int x              = get_global_id(0); // channel x
    int y              = get_global_id(1); // channel y
    int b              = get_global_id(2); // batch
    _FLOAT accum_ratio = 0;
    _FLOAT top_df_in[MLO_LRN_KERNEL_SZ];
    _FLOAT scale_in[MLO_LRN_KERNEL_SZ];
    _FLOAT ratio_dta[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    int bot_df_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD; c_i++)
    {

        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];

        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];

#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
    }

    for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
    {
        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[c_o];

            _FLOAT exp_scale = exp(-beta * log(prv_scale));
            //					pow(prv_scale, -beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[c_o] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++)
    {

        _FLOAT prv_top_df_in =
            top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i +
                   MLO_LRN_TOPDF_STRIDE * y + x];
        _FLOAT prv_scale_in =
            scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                  MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        _FLOAT prv_ratio_dta = prv_top_df_in * top_dta / prv_scale_in;
#if MLO_LOW_CHNL_COUNT == 1
        prv_ratio_dta = (c_i < MLO_LRN_N_OUTPUTS) ? prv_ratio_dta : 0;
#endif

        accum_ratio = accum_ratio + prv_ratio_dta;

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

        top_df_in[MLO_LRN_KERNEL_SZ - 1] = prv_top_df_in;
        scale_in[MLO_LRN_KERNEL_SZ - 1]  = prv_scale_in;
        ratio_dta[MLO_LRN_KERNEL_SZ - 1] = prv_ratio_dta;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = exp(-beta * log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = exp(-beta * log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }
}


[OpenCL] Added a LIT test for ensuring address space mangling is done the same both in OpenCL1.2 and OpenCL2.0.

Patch by Egor Churaev (echuraev).

Reviewers: Anastasia

Subscribers: yaxunl, cfe-commits, bader

Differential Revision: https://reviews.llvm.org/D27403

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@288891 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -ffake-address-space-map -faddress-space-map-mangling=yes -triple %itanium_abi_triple -emit-llvm -o - | FileCheck -check-prefix=ASMANG %s
// RUN: %clang_cc1 %s -ffake-address-space-map -faddress-space-map-mangling=no -triple %itanium_abi_triple -emit-llvm -o - | FileCheck -check-prefix=NOASMANG %s

// We check that the address spaces are mangled the same in both version of OpenCL
// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=CL2.0 -emit-llvm -o - | FileCheck -check-prefix=OCL-20 %s
// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=CL1.2 -emit-llvm -o - | FileCheck -check-prefix=OCL-12 %s

// We can't name this f as private is equivalent to default
// no specifier given address space so we get multiple definition
// warnings, but we do want it for comparison purposes.
__attribute__((overloadable))
void ff(int *arg) { }
// ASMANG: @_Z2ffPi
// NOASMANG: @_Z2ffPi
// OCL-20-DAG: @_Z2ffPU3AS4i
// OCL-12-DAG: @_Z2ffPi

__attribute__((overloadable))
void f(private int *arg) { }
// ASMANG: @_Z1fPi
// NOASMANG: @_Z1fPi
// OCL-20-DAG: @_Z1fPi
// OCL-12-DAG: @_Z1fPi

__attribute__((overloadable))
void f(global int *arg) { }
// ASMANG: @_Z1fPU3AS1i
// NOASMANG: @_Z1fPU8CLglobali
// OCL-20-DAG: @_Z1fPU3AS1i
// OCL-12-DAG: @_Z1fPU3AS1i

__attribute__((overloadable))
void f(local int *arg) { }
// ASMANG: @_Z1fPU3AS2i
// NOASMANG: @_Z1fPU7CLlocali
// OCL-20-DAG: @_Z1fPU3AS2i
// OCL-12-DAG: @_Z1fPU3AS2i

__attribute__((overloadable))
void f(constant int *arg) { }
// ASMANG: @_Z1fPU3AS3i
// NOASMANG: @_Z1fPU10CLconstanti
// OCL-20-DAG: @_Z1fPU3AS3i
// OCL-12-DAG: @_Z1fPU3AS3i

// RUN: %clang_cc1 %s -ffake-address-space-map -faddress-space-map-mangling=yes -triple %itanium_abi_triple -emit-llvm -o - | FileCheck -check-prefix=ASMANG %s
// RUN: %clang_cc1 %s -ffake-address-space-map -faddress-space-map-mangling=no -triple %itanium_abi_triple -emit-llvm -o - | FileCheck -check-prefix=NOASMANG %s

// We can't name this f as private is equivalent to default
// no specifier given address space so we get multiple definition
// warnings, but we do want it for comparison purposes.
__attribute__((overloadable))
void ff(int *arg) { }
// ASMANG: @_Z2ffPi
// NOASMANG: @_Z2ffPi

__attribute__((overloadable))
void f(private int *arg) { }
// ASMANG: @_Z1fPi
// NOASMANG: @_Z1fPi

__attribute__((overloadable))
void f(global int *arg) { }
// ASMANG: @_Z1fPU3AS1i
// NOASMANG: @_Z1fPU8CLglobali

__attribute__((overloadable))
void f(local int *arg) { }
// ASMANG: @_Z1fPU3AS2i
// NOASMANG: @_Z1fPU7CLlocali

__attribute__((overloadable))
void f(constant int *arg) { }
// ASMANG: @_Z1fPU3AS3i
// NOASMANG: @_Z1fPU10CLconstanti


Fix OpenCL compilation error for ROCm sparse

/*******************************************************
 * Copyright (c) 2016, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

#if IS_CPLX
T __cmul(T lhs, T rhs) {
    T out;
    out.x = lhs.x * rhs.x - lhs.y * rhs.y;
    out.y = lhs.x * rhs.y + lhs.y * rhs.x;
    return out;
}

T __ccmul(T lhs, T rhs) {
    T out;
    out.x = lhs.x * rhs.x + lhs.y * rhs.y;
    out.y = lhs.x * rhs.y - lhs.y * rhs.x;
    return out;
}

#define MUL(a, b) __cmul(a, b)

#if IS_CONJ
#define CMUL(a, b) __ccmul(a, b)
#else
#define CMUL(a, b) __cmul(a, b)
#endif

#else
#define MUL(a, b) (a) * (b)
#define CMUL(a, b) (a) * (b)
#endif

// In this kernel, each thread performs one "dot" operation by reading nonzero
// elements from one row and multiplying with the corresponding elements from
// the dense vector to produce a single output value. This kernel should be used
// when the number of nonzero elements per block is fairly small
__kernel void csrmv_thread(__global T *output, __global const T *values,
                           __global const int *rowidx,
                           __global const int *colidx, const int M,
                           __global const T *rhs, const KParam rinfo,
                           const T alpha, const T beta, __global int *counter) {
    rhs += rinfo.offset;
    int rowNext = get_global_id(0);

    while (true) {
        // Each thread performs multiple "dot" operations
#if USE_GREEDY
        // Considering that the number of non zero elements per row can be
        // uneven a greedy approach may be useful. This acheived by getting the
        // next available row to perform the "dot" operation on.
        int rowId = atomic_inc(counter);
#else
        // Unfortunately atomic operations are costly on some architectures.
        // The fallback is to use same number of rows on all threads.
        int rowId = rowNext;
        rowNext += get_global_size(0);
#endif
        if (rowId >= M) return;

        // Find the columns offsets for the current row
        int colStart = rowidx[rowId];
        int colEnd   = rowidx[rowId + 1];

        T outval = 0;
        // Performing the "dot" operation
        for (int id = colStart; id < colEnd; id++) {
            int cid = colidx[id];
            outval += CMUL(values[id], rhs[cid]);
        }

        // Writing out a single output
#if USE_ALPHA
        outval = MUL(alpha, outval);
#endif

#if USE_BETA
        output[rowId] = outval + MUL(beta, output[rowId]);
#else
        output[rowId] = outval;
#endif
    }
}

// In this kernel, each block performs one "dot" operation by having each thread
// read a nonzero element from a row and multiplying with the corresponding
// elements from dense vector to produce a local output values. Then the block
// performs a reduction operation to produce a single output value. This kernel
// should be used when the number of nonzero elements per block is large
__kernel void csrmv_block(__global T *output, __global const T *values,
                          __global const int *rowidx,
                          __global const int *colidx, const int M,
                          __global const T *rhs, const KParam rinfo,
                          const T alpha, const T beta, __global int *counter) {
    rhs += rinfo.offset;
    int lid     = get_local_id(0);
    int rowNext = get_group_id(0);
    __local int s_rowId;

    // Each thread stores part of the output result
    __local T s_outval[THREADS];

    // Each groups performs multiple "dot" operations
    while (true) {
#if USE_GREEDY
        // Considering that the number of non zero elements per row can be
        // uneven a greedy approach may be useful. This acheived by getting the
        // next available row to perform the "dot" operation on. Since the rowId
        // needs is the same across the block, only one thread needs to
        // increment the counter.
        if (lid == 0) { s_rowId = atomic_inc(counter); }
        barrier(CLK_LOCAL_MEM_FENCE);
        int rowId = s_rowId;
#else
        // Unfortunately atomic operations are costly on some architectures.
        // The fallback is to use same number of rows on all blocks.
        int rowId = rowNext;
        rowNext += get_num_groups(0);
#endif
        if (rowId >= M) return;

        int colStart = rowidx[rowId];
        int colEnd   = rowidx[rowId + 1];
        T outval     = 0;

        // Each thread performs "dot" on num_nonzero_elements / THREADS for a
        // given row
        for (int id = colStart + lid; id < colEnd; id += THREADS) {
            int cid = colidx[id];
            outval += MUL(values[id], rhs[cid]);
        }
        s_outval[lid] = outval;
        barrier(CLK_LOCAL_MEM_FENCE);

        // Perform a block reduce operation to get the single output value
        for (int n = THREADS / 2; n > 0; n /= 2) {
            if (lid < n) s_outval[lid] += s_outval[lid + n];
            barrier(CLK_LOCAL_MEM_FENCE);
        }

        // A single thread writes the output value
        if (lid == 0) {
#if USE_ALPHA
            outval = MUL(alpha, s_outval[0]);
#else
            outval        = s_outval[0];
#endif

#if USE_BETA
            output[rowId] = outval + MUL(beta, output[rowId]);
#else
            output[rowId] = outval;
#endif
        }
    }
}

/*******************************************************
 * Copyright (c) 2016, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

#if IS_CPLX
T __cmul(T lhs, T rhs) {
    T out;
    out.x = lhs.x * rhs.x - lhs.y * rhs.y;
    out.y = lhs.x * rhs.y + lhs.y * rhs.x;
    return out;
}

T __ccmul(T lhs, T rhs) {
    T out;
    out.x = lhs.x * rhs.x + lhs.y * rhs.y;
    out.y = lhs.x * rhs.y - lhs.y * rhs.x;
    return out;
}

#define MUL(a, b) __cmul(a, b)

#if IS_CONJ
#define CMUL(a, b) __ccmul(a, b)
#else
#define CMUL(a, b) __cmul(a, b)
#endif

#else
#define MUL(a, b) (a) * (b)
#define CMUL(a, b) (a) * (b)
#endif

// In this kernel, each thread performs one "dot" operation by reading nonzero
// elements from one row and multiplying with the corresponding elements from
// the dense vector to produce a single output value. This kernel should be used
// when the number of nonzero elements per block is fairly small
__kernel void csrmv_thread(__global T *output, __global const T *values,
                           __global const int *rowidx,
                           __global const int *colidx, const int M,
                           __global const T *rhs, const KParam rinfo,
                           const T alpha, const T beta, __global int *counter) {
    rhs += rinfo.offset;
    int rowNext = get_global_id(0);

    while (true) {
        // Each thread performs multiple "dot" operations
#if USE_GREEDY
        // Considering that the number of non zero elements per row can be
        // uneven a greedy approach may be useful. This acheived by getting the
        // next available row to perform the "dot" operation on.
        int rowId = atomic_inc(counter);
#else
        // Unfortunately atomic operations are costly on some architectures.
        // The fallback is to use same number of rows on all threads.
        int rowId = rowNext;
        rowNext += get_global_size(0);
#endif
        if (rowId >= M) return;

        // Find the columns offsets for the current row
        int colStart = rowidx[rowId];
        int colEnd   = rowidx[rowId + 1];

        T outval = 0;
        // Performing the "dot" operation
        for (int id = colStart; id < colEnd; id++) {
            int cid = colidx[id];
            outval += CMUL(values[id], rhs[cid]);
        }

        // Writing out a single output
#if USE_ALPHA
        outval = MUL(alpha, outval);
#endif

#if USE_BETA
        output[rowId] = outval + MUL(beta, output[rowId]);
#else
        output[rowId] = outval;
#endif
    }
}

// In this kernel, each block performs one "dot" operation by having each thread
// read a nonzero element from a row and multiplying with the corresponding
// elements from dense vector to produce a local output values. Then the block
// performs a reduction operation to produce a single output value. This kernel
// should be used when the number of nonzero elements per block is large
__kernel void csrmv_block(__global T *output, __global const T *values,
                          __global const int *rowidx,
                          __global const int *colidx, const int M,
                          __global const T *rhs, const KParam rinfo,
                          const T alpha, const T beta, __global int *counter) {
    rhs += rinfo.offset;
    int lid     = get_local_id(0);
    int rowNext = get_group_id(0);
    __local int s_rowId;

    // Each groups performs multiple "dot" operations
    while (true) {
#if USE_GREEDY
        // Considering that the number of non zero elements per row can be
        // uneven a greedy approach may be useful. This acheived by getting the
        // next available row to perform the "dot" operation on. Since the rowId
        // needs is the same across the block, only one thread needs to
        // increment the counter.
        if (lid == 0) { s_rowId = atomic_inc(counter); }
        barrier(CLK_LOCAL_MEM_FENCE);
        int rowId = s_rowId;
#else
        // Unfortunately atomic operations are costly on some architectures.
        // The fallback is to use same number of rows on all blocks.
        int rowId = rowNext;
        rowNext += get_num_groups(0);
#endif
        if (rowId >= M) return;

        // Each thread stores part of the output result
        __local T s_outval[THREADS];

        int colStart = rowidx[rowId];
        int colEnd   = rowidx[rowId + 1];
        T outval     = 0;

        // Each thread performs "dot" on num_nonzero_elements / THREADS for a
        // given row
        for (int id = colStart + lid; id < colEnd; id += THREADS) {
            int cid = colidx[id];
            outval += MUL(values[id], rhs[cid]);
        }
        s_outval[lid] = outval;
        barrier(CLK_LOCAL_MEM_FENCE);

        // Perform a block reduce operation to get the single output value
        for (int n = THREADS / 2; n > 0; n /= 2) {
            if (lid < n) s_outval[lid] += s_outval[lid + n];
            barrier(CLK_LOCAL_MEM_FENCE);
        }

        // A single thread writes the output value
        if (lid == 0) {
#if USE_ALPHA
            outval = MUL(alpha, s_outval[0]);
#else
            outval        = s_outval[0];
#endif

#if USE_BETA
            output[rowId] = outval + MUL(beta, output[rowId]);
#else
            output[rowId] = outval;
#endif
        }
    }
}


Updated some kmeans OpenCL code

Updated some kmeans OpenCL code

typedef union
{
   float s[2];
   float2 v2;
} Vector2;

#define vectorInitZero(a) \
   (a)->s[0] = 0; \
   (a)->s[1] = 0;

#define vectorAdd(a, b) \
   (a)->s[0] += (b)->s[0]; \
   (a)->s[1] += (b)->s[1];

#define vectorScale(a, c) \
   (a)->s[0] *= (c); \
   (a)->s[1] *= (c);

#define SQR(x) ((x)*(x))
#define vectorDiffNorm(a, b) \
   sqrt(SQR((a)->s[0] - (b)->s[0]) + SQR((a)->s[1] - (b)->s[1]))

/**
 * Implementation of rand(), taken from POSIX example.
 *
 * @param state
 */
int rand(ulong *state)
{
   *state = (*state) * 1103515245 + 12345;
   return ((unsigned)((*state)/65536) % 32768);
}

/**
 * Fetch and build data matrix X for a pair of genes, skipping any expressions
 * that are missing for either gene.
 *
 * @param expressions
 * @param size
 * @param pair
 * @param minThreshold
 * @param X
 * @param labels
 * @return number of rows in X
 */
int fetchData(
   __global const float *expressions, int size,
   int2 pair,
   int minThreshold,
   __global Vector2 *X,
   __global char *labels)
{
   int numSamples = 0;

   pair.x *= size;
   pair.y *= size;

   // build data matrix from expressions and indices
   for ( int i = 0; i < size; ++i )
   {
      float2 v = (float2) (
         expressions[pair.x + i],
         expressions[pair.y + i]
      );

      if ( isnan(v.x) || isnan(v.y) )
      {
         labels[i] = -9;
      }
      else if ( v.x < minThreshold || v.y < minThreshold )
      {
         labels[i] = -6;
      }
      else
      {
         X[numSamples].v2 = v;
         numSamples++;

         labels[i] = 0;
      }
   }

   // return size of X
   return numSamples;
}

/**
 * Compute the log-likelihood of a K-means model given data X.
 *
 * @param X
 * @param N
 * @param y
 * @param means
 * @param K
 */
float computeLogLikelihood(
   __global const Vector2 *X, int N,
   __global const char *y,
   __global const Vector2 *means, int K)
{
   // compute within-class scatter
   float S = 0;

   for ( int k = 0; k < K; ++k )
   {
      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != k )
         {
            continue;
         }

         float dist = vectorDiffNorm(&X[i], &means[k]);

         S += dist * dist;
      }
   }

   return -S;
}

/**
 * Compute a K-means clustering model from a dataset.
 */
void fit(
   __global const Vector2 *X, int N, int K,
   __global int *y,
   float *logL,
   __global Vector2 *means,
   __global char *y_next)
{
   ulong state = 1;

   // initialize means randomly from X
   for ( int k = 0; k < K; ++k )
   {
      int i = rand(&state) % N;
      means[k] = X[i];
   }

   // iterate K means until convergence
   while ( true )
   {
      // compute new labels
      for ( int i = 0; i < N; ++i )
      {
         // find k that minimizes norm(x_i - mu_k)
         int min_k = -1;
         float min_dist;

         for ( int k = 0; k < K; ++k )
         {
            float dist = vectorDiffNorm(&X[i], &means[k]);

            if ( min_k == -1 || dist < min_dist )
            {
               min_k = k;
               min_dist = dist;
            }
         }

         y_next[i] = min_k;
      }

      // check for convergence
      bool converged = true;

      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != y_next[i] )
         {
            converged = false;
            break;
         }
      }

      if ( converged )
      {
         break;
      }

      // update labels
      for ( int i = 0; i < N; ++i )
      {
         y[i] = y_next[i];
      }

      // update means
      for ( int k = 0; k < K; ++k )
      {
         // compute mu_k = mean of all x_i in cluster k
         int n_k = 0;

         vectorInitZero(&means[k]);

         for ( int i = 0; i < N; ++i )
         {
            if ( y[i] == k )
            {
               vectorAdd(&means[k], &X[i]);
               n_k++;
            }
         }

         vectorScale(&means[k], 1.0f / n_k);
      }
   }

   // save outputs
   *logL = computeLogLikelihood(X, N, y, means, K);
}

/**
 * Compute the Bayes information criterion of a K-means model.
 *
 * @param K
 * @param logL
 * @param N
 * @param D
 */
float computeBIC(int K, float logL, int N, int D)
{
   int p = K * D;

   return log((float) N) * p - 2 * logL;
}

/**
 * Compute a block of K-means models given a block of gene pairs.
 *
 * For each gene pair, several models are computed and the best model
 * is selected according to a criterion (BIC). The selected K and the
 * resulting sample mask for each pair is returned.
 */
__kernel void computeKmeansBlock(
   __global const float *expressions, int size,
   __global const int2 *pairs,
   int minSamples, int minThreshold, int minClusters, int maxClusters,
   __global Vector2 *work_X,
   __global char *work_y,
   __global Vector2 *work_means,
   __global char *work_ynext,
   __global int *result_K,
   __global char *result_labels)
{
   int i = get_global_id(0);

   if ( pairs[i].x == 0 && pairs[i].y == 0 )
   {
      return;
   }

   // initialize workspace variables
   __global Vector2 *X = &work_X[i * size];
   __global char *y = &work_y[i * size];
   __global Vector2 *means = &work_means[i * maxClusters];
   __global char *y_next = &work_ynext[i * size];
   __global int *bestK = &result_K[i];
   __global char *bestLabels = &result_labels[i * size];

   // fetch data matrix X from expression matrix
   int numSamples = fetchData(expressions, size, pairs[i], minThreshold, X, bestLabels);

   // perform clustering only if there are enough samples
   *bestK = 0;

   if ( numSamples >= minSamples )
   {
      float bestValue = INFINITY;

      for ( int K = minClusters; K <= maxClusters; ++K )
      {
         // run each clustering model
         float logL;
         fit(X, numSamples, K, y, &logL, means, y_next);

         // evaluate model
         float value = computeBIC(K, logL, numSamples, 2);

         // save the best model
         if ( value < bestValue )
         {
            *bestK = K;
            bestValue = value;

            for ( int i = 0, j = 0; i < numSamples; ++i )
            {
               if ( bestLabels[i] >= 0 )
               {
                  bestLabels[i] = y[j];
                  ++j;
               }
            }
         }
      }
   }
}

typedef union
{
   float s[2];
   float2 v2;
} Vector2;

#define vectorInitZero(a) \
   (a)->s[0] = 0; \
   (a)->s[1] = 0;

#define vectorAdd(a, b) \
   (a)->s[0] += (b)->s[0]; \
   (a)->s[1] += (b)->s[1];

#define vectorScale(a, c) \
   (a)->s[0] *= (c); \
   (a)->s[1] *= (c);

#define SQR(x) ((x)*(x))
#define vectorDiffNorm(a, b) \
   sqrt(SQR((a)->s[0] - (b)->s[0]) + SQR((a)->s[1] - (b)->s[1]))

/**
 * Implementation of rand(), taken from POSIX example.
 *
 * @param state
 */
int rand(ulong *state)
{
   *state = (*state) * 1103515245 + 12345;
   return ((unsigned)((*state)/65536) % 32768);
}

/**
 * Fetch and build data matrix X for a pair of genes, skipping any expressions
 * that are missing for either gene.
 *
 * @param expressions
 * @param size
 * @param pair
 * @param minThreshold
 * @param X
 * @param labels
 * @return number of rows in X
 */
int fetchData(
   __global const float *expressions, int size,
   int2 pair,
   int minThreshold,
   __global Vector2 *X,
   __global char *labels)
{
   int numSamples = 0;

   pair.x *= size;
   pair.y *= size;

   // build data matrix from expressions and indices
   for ( int i = 0; i < size; ++i )
   {
      float2 v = (float2) (
         expressions[pair.x + i],
         expressions[pair.y + i]
      );

      if ( isnan(v.x) || isnan(v.y) )
      {
         labels[i] = -9;
      }
      else if ( v.x < minThreshold || v.y < minThreshold )
      {
         labels[i] = -6;
      }
      else
      {
         X[numSamples].v2 = v;
         numSamples++;

         labels[i] = 0;
      }
   }

   // return size of X
   return numSamples;
}

/**
 * Compute the log-likelihood of a K-means model given data X.
 *
 * @param X
 * @param N
 * @param y
 * @param means
 * @param K
 */
float computeLogLikelihood(
   __global const Vector2 *X, int N,
   __global const char *y,
   __global const Vector2 *means, int K)
{
   // compute within-class scatter
   float S = 0;

   for ( int k = 0; k < K; ++k )
   {
      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != k )
         {
            continue;
         }

         float dist = vectorDiffNorm(&X[i], &means[k]);

         S += dist * dist;
      }
   }

   return -S;
}

/**
 * Compute a K-means clustering model from a dataset.
 */
void fit(
   __global const Vector2 *X, int N, int K,
   __global int *y,
   float *logL,
   __global Vector2 *means,
   __global char *y_next)
{
   ulong state = 1;

   // initialize means randomly from X
   for ( int k = 0; k < K; ++k )
   {
      int i = rand(&state) % N;
      means[k] = X[i];
   }

   // iterate K means until convergence
   while ( true )
   {
      // compute new labels
      for ( int i = 0; i < N; ++i )
      {
         // find k that minimizes norm(x_i - mu_k)
         int min_k = -1;
         float min_dist;

         for ( int k = 0; k < K; ++k )
         {
            float dist = vectorDiffNorm(&X[i], &means[k]);

            if ( min_k == -1 || dist < min_dist )
            {
               min_k = k;
               min_dist = dist;
            }
         }

         y_next[i] = min_k;
      }

      // check for convergence
      bool converged = true;

      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != y_next[i] )
         {
            converged = false;
            break;
         }
      }

      if ( converged )
      {
         break;
      }

      // update labels
      for ( int i = 0; i < N; ++i )
      {
         y[i] = y_next[i];
      }

      // update means
      for ( int k = 0; k < K; ++k )
      {
         // compute mu_k = mean of all x_i in cluster k
         int n_k = 0;

         vectorInitZero(&means[k]);

         for ( int i = 0; i < N; ++i )
         {
            if ( y[i] == k )
            {
               vectorAdd(&means[k], &X[i]);
               n_k++;
            }
         }

         vectorScale(&means[k], 1.0f / n_k);
      }
   }

   // save outputs
   *logL = computeLogLikelihood(X, N, y, means, K);
}

/**
 * Compute the Bayes information criterion of a K-means model.
 *
 * @param K
 * @param logL
 * @param N
 * @param D
 */
float computeBIC(int K, float logL, int N, int D)
{
   int p = K * D;

   return log((float) N) * p - 2 * logL;
}

/**
 * Compute a block of K-means models given a block of gene pairs.
 *
 * For each gene pair, several models are computed and the best model
 * is selected according to a criterion (BIC). The selected K and the
 * resulting sample mask for each pair is returned.
 */
__kernel void computeKmeansBlock(
   __global const float *expressions, int size,
   __global const int2 *pairs,
   int minSamples, int minThreshold, int minClusters, int maxClusters,
   __global Vector2 *work_X,
   __global char *work_y,
   __global Vector2 *work_means,
   __global char *work_ynext,
   __global int *result_K,
   __global char *result_labels)
{
   // initialize workspace variables
   int i = get_global_id(0);
   __global Vector2 *X = &work_X[i * size];
   __global char *y = &work_y[i * size];
   __global Vector2 *means = &work_means[i * maxClusters];
   __global char *y_next = &work_ynext[i * size];
   __global int *bestK = &result_K[i];
   __global char *bestLabels = &result_labels[i * size];

   // fetch data matrix X from expression matrix
   int numSamples = fetchData(expressions, size, pairs[i], minThreshold, X, bestLabels);

   // initialize output
   *bestK = 0;

   // make sure minimum number of related samples is reached
   if ( numSamples >= minSamples )
   {
      float bestValue = INFINITY;

      for ( int K = minClusters; K <= maxClusters; ++K )
      {
         // run each clustering model
         float logL;
         fit(X, numSamples, K, y, &logL, means, y_next);

         // evaluate model
         float value = computeBIC(K, logL, numSamples, 2);

         // save the best model
         if ( value < bestValue )
         {
            *bestK = K;
            bestValue = value;

            for ( int i = 0, j = 0; i < numSamples; ++i )
            {
               if ( bestLabels[i] >= 0 )
               {
                  bestLabels[i] = y[j];
                  ++j;
               }
            }
         }
      }
   }
}