placeholders for kernel

placeholders for kernel

kernel void copy(read_only image2d_t input, write_only image2d_t output)
{
    int x = get_global_id(0);
    int y = get_global_id(1);
    sampler_t sampler = CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
    float4 colour = read_imagef(input, sampler, (int2)(x, y));
    write_imagef(output, (int2)(x, y), colour);
}

kernel void add_weighted(read_only image2d_t input, read_only image2d_t blur, write_only image2d_t output)
{
    sampler_t sampler = CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
    int2 px = (int2)(get_global_id(0), get_global_id(1));

    float alpha = 1.5, beta = -0.5, gamma = 0.0;
    float4 colour = read_imagef(input, sampler, px);
    float4 blurred = read_imagef(blur, sampler, px);

    float r = colour.x * alpha + blurred.x * beta + gamma;
    float g = colour.y * alpha + blurred.y * beta + gamma;
    float b = colour.z * alpha + blurred.z * beta + gamma;

    write_imagef(output, px, (float4)(r, g, b, colour.w));
}

kernel void gaussian_blur(read_only image2d_t input, write_only image2d_t output, const float* mask, const int radius)
{
    int2 px = (int2)(get_global_id(0), get_global_id(y));

    float r = 0.0, g = 0.0, b = 0.0;

    // TODO: implement blur loop

    write_imagef(output, px, (float4)(r, g, b, input.w));
}

kernel void copy(read_only image2d_t input, write_only image2d_t output)
{
    int x = get_global_id(0);
    int y = get_global_id(1);
    sampler_t sampler = CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
    float4 colour = read_imagef(input, sampler, (int2)(x, y));
    write_imagef(output, (int2)(x, y), colour);
}

kernel void add_weighted(read_only image2d_t input, read_only image2d_t blur, write_only image2d_t output)
{
    sampler_t sampler = CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
    int2 px = (int2)(get_global_id(0), get_global_id(1));

    float alpha = 1.5, beta = -0.5, gamma = 0.0;
    float4 colour = read_imagef(input, sampler, px);
    float4 blurred = read_imagef(blur, sampler, px);

    float r = colour.x * alpha + blurred.x * beta + gamma;
    float g = colour.y * alpha + blurred.y * beta + gamma;
    float b = colour.z * alpha + blurred.z * beta + gamma;

    write_imagef(output, px, (float4)(r, g, b, colour.w));
}

kernel void gaussian_blur(read_only image2d_t input, write_only image2d_t output, int radius)
{
    size_t x = get_global_id(0);
    size_t y = get_global_id(1);

    // TODO: Implement full gaussian blur
}


fixed formatting

fixed formatting

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

// Trying to use float ATOMIC_ADD to increase total waves
// For example,  7x7 : 49
// Global 0:  H * W * N * (K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1)
// N * H * W =
//    [groupId / ((K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1))] * 64 + localId
// For example, H7*W7*N*C256*K64
// FIRST 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD
// Assembly Shader can utlize LDS to Reductiion
// However Shader Compiler of OpenCL will compile any localId[1] to FLAT_BUFFER_LOAD not constant
// load

// WAVE 0  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 0-127
// WAVE 1  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 0-127
// WAVE 2  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 0-127
// WAVE 3  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 0-127
// 2nd 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD

// WAVE 4  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 128-255
// WAVE 5  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 128-255
// WAVE 6  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 128-255
// WAVE 7  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 128-255

// STRIDE Mode: Global 0:  H_out * W_out * N * (K/ 16 outplane per threads ) * (( Inputlanes/ 128 )
// or 1)

// example
#if 0 // ndef MLopen_RUNNING
#define MLO_FILTER_STRIDE0 2
#define MLO_FILTER_STRIDE1 2
#define MLO_N_LCL_IN_MAPS_ONCE 8

#define H 28
#define W 28
#define C 192
#define K 64

#define MLO_IN_HEIGHT H
#define MLO_IN_WIDTH W
#define MLO_N_INPUTS C

//128 or MLO_N_INPUTS
#define MLO_N_LCL_IN_MAPS 192

#define MLO_N_OUTPUTS K

#define H_out 28
#define W_out 28
#define MLO_N_LCL_OUT_MAPS 16

#define MLO_N_IN_GROUPS ((MLO_N_INPUTS + MLO_N_LCL_IN_MAPS - 1) / MLO_N_LCL_IN_MAPS)
#define MLO_CLOOP0 (MLO_N_LCL_IN_MAPS / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CLOOP2 \
    ((MLO_N_INPUTS - MLO_N_LCL_IN_MAPS * (MLO_N_IN_GROUPS - 1)) / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CHEAT_SHADER_COMPILER 1

#endif

#define MLO_IN_CHANNEL_STRIDE (H * W)
#define MLO_IN_BATCH_STRIDE (H * W * C)

#define MLO_WEI_BSTRIDE (1 * 1 * C * K)
#define MLO_WEI_CHANNEL_STRIDE (1 * 1 * C)

#define MLO_OUT_BATCH_STRIDE (H_out * W_out * K)
#define MLO_OUT_CHANNEL_STRIDE (H_out * W_out)

#define FIXED_WORKGROUP_SIZE 64

#define MLO_N_OUT_GROUPS (MLO_N_OUTPUTS / MLO_N_LCL_OUT_MAPS)

#define MLO_GRP_SZ0 64
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1

#define _FLOAT float
#define MLO_CONV_BIAS 0
#define UNUSED __attribute((__unused__))

inline void AtomicAdd(volatile __global _FLOAT* source, const _FLOAT operand)
{
    union
    {
        unsigned int intVal;
        _FLOAT floatVal;
    } newVal;
    union
    {
        unsigned int intVal;
        _FLOAT floatVal;
    } prevVal;

    prevVal.floatVal = *source;
    while(true)
    {
        newVal.floatVal = prevVal.floatVal + operand;
        newVal.intVal =
            atomic_cmpxchg((volatile __global unsigned int*)source, prevVal.intVal, newVal.intVal);

        // equal to pass
        if(newVal.intVal == prevVal.intVal)
            break;

        prevVal.intVal = newVal.intVal;
    }
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    uint grp_id0       = get_group_id(0);
    uint out_grp_block = grp_id0 % MLO_N_OUT_GROUPS;
    uint in_grp_block  = (uint)(grp_id0 / MLO_N_OUT_GROUPS) % MLO_N_IN_GROUPS;
    uint grp_id0_faked = (uint)(grp_id0 / MLO_N_OUT_GROUPS) / MLO_N_IN_GROUPS;

    uint local_id0 = get_local_id(0);
#if MLO_CHEAT_SHADER_COMPILER == 1
    uint grp_id2 = get_group_id(2);
#endif

    uint pos      = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) % MLO_OUT_CHANNEL_STRIDE;
    uint batch_id = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) / MLO_OUT_CHANNEL_STRIDE;

    if(batch_id >= BATCHSIZE)
        return;

    uint out_id = out_grp_block * MLO_N_LCL_OUT_MAPS;

    short out_pos_x = pos % W_out;
    short out_pos_y = pos / W_out;

    uint in_pos = out_pos_x * MLO_FILTER_STRIDE0 + out_pos_y * MLO_FILTER_STRIDE1 * W;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE +
                      in_grp_block * MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE + in_pos;

    uint wei_off = out_id * MLO_WEI_CHANNEL_STRIDE + in_grp_block * MLO_N_LCL_IN_MAPS;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS];
    _FLOAT weights[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat2[MLO_N_LCL_IN_MAPS_ONCE];

//

// ATOMIC is needed if INPUTS in many waves
#if(MLO_N_LCL_IN_MAPS != MLO_N_INPUTS)

    if(in_grp_block == 0)
    {
        uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
        __global _FLOAT* q = out_ptr + gbl_out_off;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            *q = 0;
            q += MLO_OUT_CHANNEL_STRIDE;
        }
    }
#endif

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        accum[o] = 0;
    }

#if MLO_N_INPUTS == ((MLO_N_INPUTS / MLO_N_LCL_IN_MAPS) * MLO_N_LCL_IN_MAPS)
    // if(1)
    int loops = MLO_CLOOP0;

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif
#else
    int loops = MLO_CLOOP0;

    if(in_grp_block == (MLO_N_IN_GROUPS - 1))
    {
        loops = MLO_CLOOP2;
    }

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif

#endif
    {
        __global const float* p = in_ptr + gbl_in_off;
        __constant _FLOAT* w    = wei_ptr + wei_off;

        // read data
        for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
        {

            dat[j] = *p;
            p += MLO_IN_CHANNEL_STRIDE;
        }

        for(uint ci = 0; ci < (loops - 2); ci += 2)
        {
            // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }

        //
        // last 2 iterations
        { // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }
    }

    uint gbl_out_off   = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
    __global _FLOAT* q = out_ptr + gbl_out_off;

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
#if(MLO_N_LCL_IN_MAPS == MLO_N_INPUTS)
        *q = accum[o];
        q += MLO_OUT_CHANNEL_STRIDE;
#else
        AtomicAdd(q, accum[o]);
        q += MLO_OUT_CHANNEL_STRIDE;
#endif
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

// Trying to use float ATOMIC_ADD to increase total waves
// For example,  7x7 : 49
// Global 0:  H * W * N * (K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1)
// N * H * W =
//    [groupId / ((K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1))] * 64 + localId
// For example, H7*W7*N*C256*K64
// FIRST 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD
// Assembly Shader can utlize LDS to Reductiion
// However Shader Compiler of OpenCL will compile any localId[1] to FLAT_BUFFER_LOAD not constant
// load

// WAVE 0  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 0-127
// WAVE 1  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 0-127
// WAVE 2  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 0-127
// WAVE 3  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 0-127
// 2nd 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD

// WAVE 4  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 128-255
// WAVE 5  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 128-255
// WAVE 6  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 128-255
// WAVE 7  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 128-255

//STRIDE Mode: Global 0:  H_out * W_out * N * (K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1)

// example
#if  0//ndef MLopen_RUNNING
#define MLO_FILTER_STRIDE0 2
#define MLO_FILTER_STRIDE1 2
#define MLO_N_LCL_IN_MAPS_ONCE 8

#define H 28
#define W 28
#define C 192
#define K 64

#define MLO_IN_HEIGHT H
#define MLO_IN_WIDTH W
#define MLO_N_INPUTS C

//128 or MLO_N_INPUTS
#define MLO_N_LCL_IN_MAPS 192

#define MLO_N_OUTPUTS K

#define H_out 28
#define W_out 28
#define MLO_N_LCL_OUT_MAPS 16

#define MLO_N_IN_GROUPS ((MLO_N_INPUTS + MLO_N_LCL_IN_MAPS - 1) / MLO_N_LCL_IN_MAPS)
#define MLO_CLOOP0 (MLO_N_LCL_IN_MAPS / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CLOOP2 \
    ((MLO_N_INPUTS - MLO_N_LCL_IN_MAPS * (MLO_N_IN_GROUPS - 1)) / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CHEAT_SHADER_COMPILER 1

#endif

#define MLO_IN_CHANNEL_STRIDE (H * W)
#define MLO_IN_BATCH_STRIDE (H * W * C)

#define MLO_WEI_BSTRIDE (1 * 1 * C * K)
#define MLO_WEI_CHANNEL_STRIDE (1 * 1 * C)

#define MLO_OUT_BATCH_STRIDE (H_out * W_out * K)
#define MLO_OUT_CHANNEL_STRIDE (H_out * W_out)

#define FIXED_WORKGROUP_SIZE 64

#define MLO_N_OUT_GROUPS (MLO_N_OUTPUTS / MLO_N_LCL_OUT_MAPS)

#define MLO_GRP_SZ0 64
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1

#define _FLOAT float
#define MLO_CONV_BIAS 0
#define UNUSED __attribute((__unused__))

inline void AtomicAdd(volatile __global _FLOAT* source, const _FLOAT operand)
{
    union
    {
        unsigned int intVal;
        _FLOAT floatVal;
    } newVal;
    union
    {
        unsigned int intVal;
        _FLOAT floatVal;
    } prevVal;

    prevVal.floatVal = *source;
    while(true)
    {
        newVal.floatVal = prevVal.floatVal + operand;
        newVal.intVal =
            atomic_cmpxchg((volatile __global unsigned int*)source, prevVal.intVal, newVal.intVal);

        // equal to pass
        if(newVal.intVal == prevVal.intVal)
            break;

        prevVal.intVal = newVal.intVal;
    }
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    uint grp_id0       = get_group_id(0);
    uint out_grp_block = grp_id0 % MLO_N_OUT_GROUPS;
    uint in_grp_block  = (uint)(grp_id0 / MLO_N_OUT_GROUPS) % MLO_N_IN_GROUPS;
    uint grp_id0_faked = (uint)(grp_id0 / MLO_N_OUT_GROUPS) / MLO_N_IN_GROUPS;

    uint local_id0 = get_local_id(0);
#if MLO_CHEAT_SHADER_COMPILER == 1
    uint grp_id2 = get_group_id(2);
#endif

    uint pos      = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) % MLO_OUT_CHANNEL_STRIDE;
    uint batch_id = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) / MLO_OUT_CHANNEL_STRIDE;

    if(batch_id >= BATCHSIZE)
        return;

    uint out_id = out_grp_block * MLO_N_LCL_OUT_MAPS;

	short out_pos_x = pos % W_out;
    short out_pos_y = pos / W_out;

    uint in_pos = out_pos_x * MLO_FILTER_STRIDE0  + out_pos_y * MLO_FILTER_STRIDE1 * W ;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE +
                      in_grp_block * MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE + in_pos;

    uint wei_off = out_id * MLO_WEI_CHANNEL_STRIDE + in_grp_block * MLO_N_LCL_IN_MAPS;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS];
    _FLOAT weights[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat2[MLO_N_LCL_IN_MAPS_ONCE];

//

// ATOMIC is needed if INPUTS in many waves
#if(MLO_N_LCL_IN_MAPS != MLO_N_INPUTS)

    if(in_grp_block == 0)
    {
        uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
        __global _FLOAT* q = out_ptr + gbl_out_off;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            *q = 0;
            q += MLO_OUT_CHANNEL_STRIDE;
        }
    }
#endif

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        accum[o] = 0;
    }

#if MLO_N_INPUTS == ((MLO_N_INPUTS / MLO_N_LCL_IN_MAPS) * MLO_N_LCL_IN_MAPS)
    // if(1)
    int loops = MLO_CLOOP0;

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif
#else
    int loops = MLO_CLOOP0;

    if(in_grp_block == (MLO_N_IN_GROUPS - 1))
    {
        loops = MLO_CLOOP2;
    }

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif

#endif
    {
        __global const float* p = in_ptr + gbl_in_off;
        __constant _FLOAT* w    = wei_ptr + wei_off;

        // read data
        for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
        {

            dat[j] = *p;
            p += MLO_IN_CHANNEL_STRIDE;
        }

        for(uint ci = 0; ci < (loops - 2); ci += 2)
        {
            // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }

        //
        // last 2 iterations
        { // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }
    }

    uint gbl_out_off   = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
    __global _FLOAT* q = out_ptr + gbl_out_off;

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
#if(MLO_N_LCL_IN_MAPS == MLO_N_INPUTS)
        *q = accum[o];
        q += MLO_OUT_CHANNEL_STRIDE;
#else
        AtomicAdd(q, accum[o]);
        q += MLO_OUT_CHANNEL_STRIDE;
#endif
    }
}


adding rad16 version of 4096, is more performant than rad8