restore 1x1

restore 1x1

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

#if MLO_N_MAPS_PERGROUP > 1
#define MLO_WEIGHTS_PER_LOOP_MAX (8)
#else
#define MLO_WEIGHTS_PER_LOOP_MAX (16)
#endif
#if(MLO_N_MAPS_PERGROUP * MLO_N_LCL_IN_MAPS < MLO_N_INPUTS)
#define MLO_LCL_IN_ROW (MLO_N_MAPS_PERGROUP * MLO_N_LCL_IN_MAPS)
#else
#define MLO_LCL_IN_ROW (MLO_N_INPUTS)
#endif

#define MLO_WEIGHTS_PER_LOOP_TMP ((MLO_N_INPUTS + MLO_LCL_IN_ROW - 1) / MLO_LCL_IN_ROW)

#if(MLO_WEIGHTS_PER_LOOP_TMP < MLO_WEIGHTS_PER_LOOP_MAX)
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_TMP)
#else
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_MAX)
#endif

#define MLO_LCL_WEIGHTS_ROW (MLO_WEIGHTS_PER_LOOP * MLO_LCL_IN_ROW)

#define MLO_IN_LOOP ((MLO_N_INPUTS + MLO_LCL_WEIGHTS_ROW - 1) / MLO_LCL_WEIGHTS_ROW)

#define MLO_WEIGHTS_ROW (MLO_LCL_WEIGHTS_ROW * MLO_WEI_CHANNEL_STRIDE)

// size of the area for weights prefetch
#define MLO_WEIGHTS_LCL_SZ (MLO_WEIGHTS_ROW * MLO_N_LCL_OUT_MAPS)

// size of area for exchanging partial sums
#define MLO_EXCHNGE_SZ4 (MLO_MAP_SZ4 * MLO_EXCHANGE_STEP * MLO_N_MAPS_PERGROUP)

#if MLO_N_MAPS_PERGROUP > 1 && ((MLO_EXCHNGE_SZ4 * MLO_READ_UNIT) > MLO_WEIGHTS_LCL_SZ)
#define MLO_LCL_MEM_SZ (MLO_EXCHNGE_SZ4 * MLO_READ_UNIT)
#else
#define MLO_LCL_MEM_SZ MLO_WEIGHTS_LCL_SZ
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

/*
Layout:
assuming NCHW data layout.

Data:
data has been fetch by 4 floats sequentially.
MLO_MAP_SZ4 = (map_width*map_height + 3)/4.
in case of total size not a multiple of 4 the the last pixel has a special treatment.
There are 2 cases:
MLO_N_MAPS_PERGROUP == 1
and
MLO_N_MAPS_PERGROUP > 1, when MLO_MAP_SZ4 <= GPROUP_SIZE/2, in other words when more than 1 map can
be held by a group.
Case MLO_N_MAPS_PERGROUP == 1:
Data, by 4 floats, may come from MLO_N_LCL_IN_MAPS sequential input maps from MLO_N_LCL_BATCHS
neighboring batches.
Weigts:
on each MLO_WEIGHTS_PER_LOOP input loop set of weight are prefetched for another
MLO_WEIGHTS_PER_LOOP loops.
Each input map contributes to partial sums of MLO_N_LCL_OUT_MAPS output maps.
Case MLO_N_MAPS_PERGROUP > 1:
Similar to a previous case.
The difference is that several input sequential input maps are kept by group.
Each taking part in the calculation of partial sums of the same MLO_N_LCL_OUT_MAPS output maps.
After completion of the main MLO_IN_LOOP loop partial sums have been summed up in parallel.

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              const __global _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{
    // KERNEL
    // private buffers
    __private _FLOAT in_stage[MLO_N_LCL_BATCHS][MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];
    __private _FLOAT wei_stage;
    __private _FLOAT out_tiles[MLO_N_LCL_BATCHS][MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    __local _FLOAT lcl_wei_stage[MLO_LCL_MEM_SZ];

#if MLO_N_MAPS_PERGROUP > 1
    __local _FLOAT* lcl_out_stage = lcl_wei_stage;

#endif

    uint lcl_id0       = get_local_id(0);
    int in_map_id      = 0;                    // map
    uint pix_id        = get_global_id(0);     // inside map
    in_map_id          = pix_id / MLO_MAP_SZ4; // mad id inside group
    uint out_grp_block = get_group_id(1);      // block of outputs for the entire group
    uint out_block     = out_grp_block;
    uint batch_block   = get_group_id(2); // block of batchs
                                          // multipe maps per group

    pix_id = (pix_id - in_map_id * MLO_MAP_SZ4); // pixel inside map

    uint in_map_off_id = (in_map_id >= MLO_N_MAPS_PERGROUP) ? MLO_N_MAPS_PERGROUP - 1 : in_map_id;

    uint in_off = batch_block * MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE +
                  in_map_off_id * MLO_IN_CHANNEL_STRIDE + pix_id * MLO_READ_UNIT;

    uint wei_off = out_grp_block * MLO_N_LCL_OUT_MAPS *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;
    for(uint j = 0; j < MLO_N_LCL_BATCHS; ++j)
    {
        for(uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
        {
            for(uint k = 0; k < MLO_READ_UNIT; ++k)
            {
                out_tiles[j][i][k] = 0;
            }
        }
    }
    // over all input maps; with step == MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP; MLO_IN_LOOP
    for(uint wc = 0; wc < MLO_IN_LOOP; ++wc)
    {
        // read array of weights
        barrier(CLK_LOCAL_MEM_FENCE);

        for(uint w = lcl_id0; w < MLO_WEIGHTS_LCL_SZ; w += MLO_GRP_SZ0)
        {

#if(MLO_WEIGHTS_ROW) & (MLO_WEIGHTS_ROW - 1)

            uint oi  = iDiv(w, MLO_WEIGHTS_ROW);
            uint lwi = iMod(w, oi, MLO_WEIGHTS_ROW);
#else
            uint oi  = (w / MLO_WEIGHTS_ROW);
            uint lwi = (w & (MLO_WEIGHTS_ROW - 1));
#endif

            uint wi = (wc * MLO_LCL_WEIGHTS_ROW + lwi) *
#if MLO_DIR_FORWARD == 1
                      MLO_WEI_CHANNEL_STRIDE;
#else
                      MLO_WEI_BSTRIDE;
#endif

            // out of range check
            uint wei_off1 = wei_off + wi +
                            oi *
#if MLO_DIR_FORWARD == 1
                                MLO_WEI_BSTRIDE;
#else
                                MLO_WEI_CHANNEL_STRIDE;
#endif

            uint wei_off_r   = (wei_off1 < MLO_N_OUTPUTS * MLO_N_INPUTS) ? wei_off1 : 0;
            _FLOAT wei_val   = wei_ptr[wei_off_r];
            wei_val          = (wei_off1 < MLO_N_OUTPUTS * MLO_N_INPUTS) ? wei_val : 0;
            lcl_wei_stage[w] = wei_val;
        }

        barrier(CLK_LOCAL_MEM_FENCE);
        uint c = 0;
#if MLO_WEIGHTS_PER_LOOP > 7
#pragma unroll(MLO_WEIGHTS_PER_LOOP / 8)
#endif
        for(uint ci = 0; ci < MLO_WEIGHTS_PER_LOOP;
            ++ci, in_off += MLO_IN_CHANNEL_STRIDE * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP)
        {
            c             = wc * MLO_WEIGHTS_PER_LOOP + ci;
            uint wei_indx = ci;

            // read data
            // over all local batchs
            uint in_off1 = in_off;
            uint in_off2 = 0;
            for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib, in_off1 += MLO_IN_BATCH_STRIDE)
            {
                in_off2 = in_off1;
                // lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP
                for(uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS;
                    ++ilc, in_off2 += MLO_IN_CHANNEL_STRIDE * MLO_N_MAPS_PERGROUP)
                {
                    bool v =
#if MLO_BATCH_ALIGNED == 0
                        (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ) &&
#endif
                        c * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP + in_map_id +
                                ilc * MLO_N_MAPS_PERGROUP <
                            MLO_N_INPUTS;
#ifndef __AMDGCN__
                    in_off2 = (v) ? in_off2 : 0;
#endif
                    __global const _FLOAT* in_p = &in_ptr[in_off2];
#if MLO_C1x1_PIXLEFT > 0
                    // if the last one
                    if(pix_id == MLO_MAP_SZ4 - 1)
                    {

                        for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                        {
#ifdef __AMDGCN__
                            in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
                            _FLOAT val           = in_p[i];
                            in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE
                            if(in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                    else

#endif
                    {
                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
#ifdef __AMDGCN__
                            in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
                            _FLOAT val           = in_p[i];
                            in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE

                            if(in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                }
            }

            // convolve
            for(uint olc         = 0,
                     lcl_wei_off = wei_indx * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP *
                                   MLO_WEI_CHANNEL_STRIDE;
                olc < MLO_N_LCL_OUT_MAPS;
                ++olc, lcl_wei_off += MLO_WEIGHTS_ROW)
            {
                // lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP, weights are
                // mapped accordingly
                uint lcl_wei_off1 = lcl_wei_off;
                for(uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS;
                    ++ilc, lcl_wei_off1 += MLO_N_MAPS_PERGROUP * MLO_WEI_CHANNEL_STRIDE)
                {
                    // read weights
                    uint lcl_wei_off2 =
                        lcl_wei_off1 + mul24(in_map_id, (int)MLO_WEI_CHANNEL_STRIDE);
                    wei_stage = lcl_wei_stage[lcl_wei_off2];
                    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
                    {
                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            out_tiles[ib][olc][i] += in_stage[ib][ilc][i] * wei_stage;

#if 0 // MLO_DIR_FORWARD == 0
							if ( in_stage[ib][ilc][i] * wei_stage!= 0 && out_grp_block * MLO_N_LCL_OUT_MAPS + olc == 0 && i == 0 && get_global_id(0) == 0 && get_global_id(1) == 0 && get_global_id(2) == 0)
							{
								printf("K:c: %d %d %d %d   %f %f %f %f\n",
								wc,
								MLO_IN_LOOP,
								MLO_WEIGHTS_PER_LOOP,
								MLO_WEIGHTS_ROW,

								out_tiles[ib][olc][i],
								in_stage[ib][ilc][i] * wei_stage,
								in_stage[ib][ilc][i],
								 wei_stage
								);
							}
#endif
                        }
                    }
                }
            }
        }
    }

    // out of range check
    if(in_map_id >= MLO_N_MAPS_PERGROUP || in_map_id * MLO_N_LCL_IN_MAPS >= MLO_N_INPUTS)
    {
        return;
    }

    out_block    = out_grp_block * MLO_N_LCL_OUT_MAPS;
    uint out_off = batch_block * MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE +
                   out_block * MLO_OUT_CHANNEL_STRIDE + pix_id * MLO_READ_UNIT;

// small groups
#if MLO_N_MAPS_PERGROUP > 1

    // calculate reduction over all partial sums
    // MLO_N_LCL_OUT_MAPS is multiple of MLO_EXCHANGE_STEP
    // write data into local memory

    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
    {
        for(uint t = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)
        {

            barrier(CLK_LOCAL_MEM_FENCE);

            if(lcl_id0 < MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP)
            {
                for(uint om = 0; om < MLO_EXCHANGE_STEP; ++om)
                {
                    uint lcl_off = (om * MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP +
                                    in_map_id * MLO_MAP_SZ4 + pix_id) *
                                   MLO_READ_UNIT;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        lcl_out_stage[lcl_off + i] = out_tiles[ib][t + om][i];
                    }
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            // sum partial sum
            // MLO_N_MAPS_PERGROUP >= MLO_EXCHANGE_STEP
            // in_map_id is an index of the output map now.
            if(in_map_id < MLO_EXCHANGE_STEP)
            {
                _FLOAT sum[MLO_READ_UNIT];
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    sum[i] = 0;
                }

                for(uint s = 0; s < MLO_N_MAPS_PERGROUP; ++s)
                {
                    uint imp     = in_map_id + s;
                    imp          = (imp >= MLO_N_MAPS_PERGROUP) ? imp - MLO_N_MAPS_PERGROUP : imp;
                    uint lcl_off = (in_map_id * MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP +
                                    imp * MLO_MAP_SZ4 + pix_id) *
                                   MLO_READ_UNIT;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        sum[i] += lcl_out_stage[lcl_off + i];
                    }
                }

                // write it out
                uint olc = t + in_map_id;

                if(true
#if MLO_BATCH_ALIGNED == 0
                   &&
                   (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
                   &&
                   out_block + olc < MLO_N_OUTPUTS
#endif
                   )
                {

                    uint out_off2 =
                        out_off + ib * MLO_OUT_BATCH_STRIDE + olc * MLO_OUT_CHANNEL_STRIDE;
                    __global _FLOAT* out_p = &out_ptr[out_off2];

#if MLO_CONV_BIAS
                    _FLOAT bias_val = 0;
                    bias_val        = bias[out_block * MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

                    // if the last one
                    if(pix_id == MLO_MAP_SZ4 - 1)
                    {
                        for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                        {
                            out_p[i] = sum[i]
#if MLO_CONV_BIAS
                                       + bias_val
#endif
                                ;

#if DBG_OUT_OF_RNGE
                            if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:out-of-range\n");
                            }
#endif
                        }
                    }
                    else
#endif
                    {

                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            out_p[i] = sum[i]
#if MLO_CONV_BIAS
                                       + bias_val
#endif
                                ;
#if DBG_OUT_OF_RNGE
                            if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:out-of-range\n");
                            }
#endif
                        }
                    }

                } // if (true

            } // if (in_map_id < MLO_EXCHANGE_STEP)

        } // for (int t = 0, p = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)

    } // 	for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)

#else // #if MLO_N_MAPS_PERGROUP > 1

    uint out_off1 = out_off;
    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib, out_off1 += MLO_OUT_BATCH_STRIDE)
    {

        uint out_off2 = out_off1;
        for(uint olc = 0; olc < MLO_N_LCL_OUT_MAPS; ++olc, out_off2 += MLO_OUT_CHANNEL_STRIDE)
        {
            __global _FLOAT* out_p = &out_ptr[out_off2];

            if(true
#if MLO_BATCH_ALIGNED == 0
               &&
               (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
               &&
               (out_block + olc < MLO_N_OUTPUTS)
#endif
                   )
            {
#if MLO_CONV_BIAS
                _FLOAT bias_val = 0;
                bias_val = bias[out_block * MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

                // if the last one
                if(pix_id == MLO_MAP_SZ4 - 1)
                {
                    for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                    {
                        out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
                                   + bias_val
#endif
                            ;
#if DBG_OUT_OF_RNGE
                        if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:out-of-range\n");
                        }
#endif
                    }
                }
                else
#endif
                {
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {

                        out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
                                   + bias_val
#endif
                            ;
#if DBG_OUT_OF_RNGE
                        if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:out-of-range\n");
                        }
#endif
                    }
                }
            }
        }
    }

#endif // #if MLO_N_MAPS_PERGROUP > 1
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

/*
Layout:
assuming NCHW data layout.

Data:
data has been fetch by 4 floats sequentially.
MLO_MAP_SZ4 = (map_width*map_height + 3)/4.
in case of total size not a multiple of 4 the the last pixel has a special treatment.
There are 2 cases:
MLO_N_MAPS_PERGROUP == 1
and
MLO_N_MAPS_PERGROUP > 1, when MLO_MAP_SZ4 <= GPROUP_SIZE/2, in other words when more than 1 map can
be held by a group.
Case MLO_N_MAPS_PERGROUP == 1:
Data, by 4 floats, may come from MLO_N_LCL_IN_MAPS sequential input maps from MLO_N_LCL_BATCHS
neighboring batches.
Weigts:
on each MLO_WEIGHTS_PER_LOOP input loop set of weight are prefetched for another
MLO_WEIGHTS_PER_LOOP loops.
Each input map contributes to partial sums of MLO_N_LCL_OUT_MAPS output maps.
Case MLO_N_MAPS_PERGROUP > 1:
Similar to a previous case.
The difference is that several input sequential input maps are kept by group.
Each taking part in the calculation of partial sums of the same MLO_N_LCL_OUT_MAPS output maps.
After completion of the main MLO_IN_LOOP loop partial sums have been summed up in parallel.

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    uint gbl_id0       = get_global_id(0);
    uint batch_id      = gbl_id0 / MLO_MAP_SZ4; // batch
	uint pos           = gbl_id0 % MLO_MAP_SZ4;

    uint out_grp_block = get_group_id(1);      // block of outputs for the entire group
	uint out_id = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE + pos * MLO_READ_UNIT;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

	_FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
	_FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];
	_FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

	for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
	{
		for(uint i = 0; i < MLO_READ_UNIT; ++i)
		{
			accum[o][i] = 0;
		}
	}

    for(uint ci = 0; ci < MLO_CLOOP0; ++ci,
// move input offset
				 gbl_in_off += MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE,

// move weights offset
				 wei_off +=
				MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
				MLO_WEI_CHANNEL_STRIDE
#else
				MLO_WEI_BSTRIDE
#endif
	)
	{

// read weights
		for(uint o = 0, wei_off1 = wei_off; o < MLO_N_LCL_OUT_MAPS; ++o, wei_off1 +=
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
		)
		{
			for(uint c = 0, wei_off2 = wei_off1; c < MLO_N_LCL_IN_MAPS; ++c, wei_off2 +=
#if MLO_DIR_FORWARD == 1
				MLO_WEI_CHANNEL_STRIDE
#else
				MLO_WEI_BSTRIDE
#endif
			)
			{
				weights[o][c] = wei_ptr[wei_off2];
			}
		}
// read data
		for(uint j = 0, gbl_in_off1 = gbl_in_off; j < MLO_N_LCL_IN_MAPS; ++j, gbl_in_off1 += MLO_IN_CHANNEL_STRIDE)
		{
			for(uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				dat[j][i] = in_ptr[gbl_in_off1 + i];
			}
		}

// convolve
		for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for(uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					accum[o][i] += dat[c][i] * weights[o][c];
				}
			}
		}

	}

    uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + pos * MLO_READ_UNIT + out_id * MLO_OUT_CHANNEL_STRIDE;
	for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS; ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
		for(uint i = 0; i < MLO_READ_UNIT; ++i)
		{
		 out_ptr[gbl_out_off1 + i] = accum[o][i];
		}
	}

}


lib/kernel/svm_atomics.cl: add prototypes that accept operand with opposite sign

e.g. atomic_fetch_add(unsigned long volatile *A, long B)

.. required for CTS 3.0 C11 atomics test

/* OpenCL built-in library: OpenCL 2.0 Atomics (C11 subset)

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   These implementations merely lower the call to a device-specific
   call (prefixed with "__pocl_atomic_")

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#ifndef _SVM_ATOMICS_H
#include "svm_atomics.h"
#define _SVM_ATOMICS_H
#endif

#if !defined(Q)

#  define Q __global
#  define QUAL(f) f ## __global
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#  define Q __local
#  define QUAL(f) f ## __local
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#elif !defined(ATOMIC_TYPE)

bool _CL_OVERLOADABLE
atomic_flag_test_and_set_explicit (volatile Q atomic_flag *object,
                                   memory_order order, memory_scope scope)
{
  return QUAL (__pocl_atomic_flag_test_and_set) (object, order, scope);
}

bool _CL_OVERLOADABLE atomic_flag_test_and_set_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  return atomic_flag_test_and_set_explicit(object, order, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_flag_test_and_set (volatile Q atomic_flag *object)
{
  return atomic_flag_test_and_set_explicit (object, memory_order_seq_cst);
}

void _CL_OVERLOADABLE
atomic_flag_clear_explicit (volatile Q atomic_flag *object, memory_order order,
                            memory_scope scope)
{
  return QUAL (__pocl_atomic_flag_clear) (object, order, scope);
}

void _CL_OVERLOADABLE atomic_flag_clear_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  atomic_flag_clear_explicit(object, order, memory_scope_device);
}

void _CL_OVERLOADABLE
atomic_flag_clear (volatile Q atomic_flag *object)
{
  atomic_flag_clear_explicit (object, memory_order_seq_cst);
}

#  define ATOMIC_TYPE atomic_int
#  define NONATOMIC_TYPE int
#  define IS_INT
#  include "svm_atomics.cl"
#  undef IS_INT
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_uint
#  define NONATOMIC_TYPE uint
#  define IS_UINT
#  include "svm_atomics.cl"
#  undef IS_UINT
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_float
#  define NONATOMIC_TYPE float
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)

#  define ATOMIC_TYPE atomic_long
#  define NONATOMIC_TYPE long
#  define INV_TYPE ulong
#  include "svm_atomics.cl"
#  undef INV_TYPE
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_ulong
#  define NONATOMIC_TYPE ulong
#  define INV_TYPE long
#  include "svm_atomics.cl"
#  undef INV_TYPE
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#ifdef cl_khr_fp64

#  define ATOMIC_TYPE atomic_double
#  define NONATOMIC_TYPE double
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#else

void _CL_OVERLOADABLE
atomic_store_explicit (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired,
                       memory_order order, memory_scope scope)
{
  QUAL (__pocl_atomic_store) (object, desired, order, scope);
}

void _CL_OVERLOADABLE atomic_store_explicit (  volatile Q ATOMIC_TYPE  *object,
                              NONATOMIC_TYPE  desired,
                              memory_order order)
{
  atomic_store_explicit(object, desired, order, memory_scope_device);
}

void _CL_OVERLOADABLE
atomic_store (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired)
{
  atomic_store_explicit (object, desired, memory_order_seq_cst);
}

void _CL_OVERLOADABLE atomic_init (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE value)
{
  atomic_store_explicit(object, value, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_load_explicit (volatile Q ATOMIC_TYPE *object, memory_order order,
                      memory_scope scope)
{
  return QUAL (__pocl_atomic_load) (object, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_load_explicit ( volatile Q ATOMIC_TYPE  *object,
  memory_order order)
{
  return atomic_load_explicit(object, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_load (volatile Q ATOMIC_TYPE *object)
{
  return atomic_load_explicit (object, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_exchange_explicit (volatile Q ATOMIC_TYPE *object,
                          NONATOMIC_TYPE desired, memory_order order,
                          memory_scope scope)
{
  return QUAL (__pocl_atomic_exchange) (object, desired, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_exchange_explicit (volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  desired,
  memory_order order)
{
  return atomic_exchange_explicit(object, desired, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_exchange (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired)
{
  return atomic_exchange_explicit (object, desired, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_strong_explicit (volatile Q ATOMIC_TYPE *object,
                                         private NONATOMIC_TYPE *expected,
                                         NONATOMIC_TYPE desired,
                                         memory_order success,
                                         memory_order failure,
                                         memory_scope scope)
{
  return QUAL (__pocl_atomic_compare_exchange_strong) (object, expected, desired,
                                                     success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_strong_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_strong_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_strong (volatile Q ATOMIC_TYPE *object,
                                private NONATOMIC_TYPE *expected,
                                NONATOMIC_TYPE desired)
{
  return atomic_compare_exchange_strong_explicit (
      object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_weak_explicit (volatile Q ATOMIC_TYPE *object,
                                       private NONATOMIC_TYPE *expected,
                                       NONATOMIC_TYPE desired,
                                       memory_order success,
                                       memory_order failure,
                                       memory_scope scope)
{
  return QUAL (__pocl_atomic_compare_exchange_weak) (object, expected, desired,
                                                   success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_weak_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_weak_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_weak (volatile Q ATOMIC_TYPE *object,
                              private NONATOMIC_TYPE *expected,
                              NONATOMIC_TYPE desired)
{
  return atomic_compare_exchange_weak_explicit (
      object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

#ifndef NON_INTEGER

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_add) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_add_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_add_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_sub) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_sub_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_sub_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or_explicit (volatile Q ATOMIC_TYPE *object,
                          NONATOMIC_TYPE operand, memory_order order,
                          memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_or) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_or_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_or_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_xor) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_xor_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_xor_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_and) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_and_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_and_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_min) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_min_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_min_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_max) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_max_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_max_explicit (object, operand, memory_order_seq_cst);
}

#ifdef INV_TYPE

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add_explicit (volatile Q ATOMIC_TYPE *object,
                           INV_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_add) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_add_explicit(object, (NONATOMIC_TYPE)operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_add_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub_explicit (volatile Q ATOMIC_TYPE *object,
                           INV_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_sub) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_sub_explicit(object, (NONATOMIC_TYPE)operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_sub_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or_explicit (volatile Q ATOMIC_TYPE *object,
                          INV_TYPE operand, memory_order order,
                          memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_or) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_or_explicit(object, (NONATOMIC_TYPE)operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_or_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor_explicit (volatile Q ATOMIC_TYPE *object,
                           INV_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_xor) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_xor_explicit(object, (NONATOMIC_TYPE)operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_xor_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and_explicit (volatile Q ATOMIC_TYPE *object,
                           INV_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_and) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_and_explicit(object, (NONATOMIC_TYPE)operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_and_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min_explicit (volatile Q ATOMIC_TYPE *object,
                           INV_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_min) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_min_explicit(object, (NONATOMIC_TYPE)operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_min_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max_explicit (volatile Q ATOMIC_TYPE *object,
                           INV_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_max) (object, (NONATOMIC_TYPE)operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max_explicit ( volatile Q ATOMIC_TYPE  *object,
  INV_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_max_explicit(object, operand, (NONATOMIC_TYPE)order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max (volatile Q ATOMIC_TYPE *object, INV_TYPE operand)
{
  return atomic_fetch_max_explicit (object, (NONATOMIC_TYPE)operand, memory_order_seq_cst);
}

#endif

#endif

#endif

/* OpenCL built-in library: OpenCL 2.0 Atomics (C11 subset)

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   These implementations merely lower the call to a device-specific
   call (prefixed with "__pocl_atomic_")

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#ifndef _SVM_ATOMICS_H
#include "svm_atomics.h"
#define _SVM_ATOMICS_H
#endif

#if !defined(Q)

#  define Q __global
#  define QUAL(f) f ## __global
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#  define Q __local
#  define QUAL(f) f ## __local
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#elif !defined(ATOMIC_TYPE)

bool _CL_OVERLOADABLE
atomic_flag_test_and_set_explicit (volatile Q atomic_flag *object,
                                   memory_order order, memory_scope scope)
{
  return QUAL (__pocl_atomic_flag_test_and_set) (object, order, scope);
}

bool _CL_OVERLOADABLE atomic_flag_test_and_set_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  return atomic_flag_test_and_set_explicit(object, order, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_flag_test_and_set (volatile Q atomic_flag *object)
{
  return atomic_flag_test_and_set_explicit (object, memory_order_seq_cst);
}

void _CL_OVERLOADABLE
atomic_flag_clear_explicit (volatile Q atomic_flag *object, memory_order order,
                            memory_scope scope)
{
  return QUAL (__pocl_atomic_flag_clear) (object, order, scope);
}

void _CL_OVERLOADABLE atomic_flag_clear_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  atomic_flag_clear_explicit(object, order, memory_scope_device);
}

void _CL_OVERLOADABLE
atomic_flag_clear (volatile Q atomic_flag *object)
{
  atomic_flag_clear_explicit (object, memory_order_seq_cst);
}

#  define ATOMIC_TYPE atomic_int
#  define NONATOMIC_TYPE int
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_uint
#  define NONATOMIC_TYPE uint
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_float
#  define NONATOMIC_TYPE float
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)

#  define ATOMIC_TYPE atomic_long
#  define NONATOMIC_TYPE long
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_ulong
#  define NONATOMIC_TYPE ulong
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#ifdef cl_khr_fp64

#  define ATOMIC_TYPE atomic_double
#  define NONATOMIC_TYPE double
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#else

void _CL_OVERLOADABLE
atomic_store_explicit (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired,
                       memory_order order, memory_scope scope)
{
  QUAL (__pocl_atomic_store) (object, desired, order, scope);
}

void _CL_OVERLOADABLE atomic_store_explicit (  volatile Q ATOMIC_TYPE  *object,
                              NONATOMIC_TYPE  desired,
                              memory_order order)
{
  atomic_store_explicit(object, desired, order, memory_scope_device);
}

void _CL_OVERLOADABLE
atomic_store (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired)
{
  atomic_store_explicit (object, desired, memory_order_seq_cst);
}

void _CL_OVERLOADABLE atomic_init (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE value)
{
  atomic_store_explicit(object, value, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_load_explicit (volatile Q ATOMIC_TYPE *object, memory_order order,
                      memory_scope scope)
{
  return QUAL (__pocl_atomic_load) (object, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_load_explicit ( volatile Q ATOMIC_TYPE  *object,
  memory_order order)
{
  return atomic_load_explicit(object, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_load (volatile Q ATOMIC_TYPE *object)
{
  return atomic_load_explicit (object, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_exchange_explicit (volatile Q ATOMIC_TYPE *object,
                          NONATOMIC_TYPE desired, memory_order order,
                          memory_scope scope)
{
  return QUAL (__pocl_atomic_exchange) (object, desired, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_exchange_explicit (volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  desired,
  memory_order order)
{
  return atomic_exchange_explicit(object, desired, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_exchange (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired)
{
  return atomic_exchange_explicit (object, desired, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_strong_explicit (volatile Q ATOMIC_TYPE *object,
                                         private NONATOMIC_TYPE *expected,
                                         NONATOMIC_TYPE desired,
                                         memory_order success,
                                         memory_order failure,
                                         memory_scope scope)
{
  return QUAL (__pocl_atomic_compare_exchange_strong) (object, expected, desired,
                                                     success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_strong_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_strong_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_strong (volatile Q ATOMIC_TYPE *object,
                                private NONATOMIC_TYPE *expected,
                                NONATOMIC_TYPE desired)
{
  return atomic_compare_exchange_strong_explicit (
      object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_weak_explicit (volatile Q ATOMIC_TYPE *object,
                                       private NONATOMIC_TYPE *expected,
                                       NONATOMIC_TYPE desired,
                                       memory_order success,
                                       memory_order failure,
                                       memory_scope scope)
{
  return QUAL (__pocl_atomic_compare_exchange_weak) (object, expected, desired,
                                                   success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_weak_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_weak_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_weak (volatile Q ATOMIC_TYPE *object,
                              private NONATOMIC_TYPE *expected,
                              NONATOMIC_TYPE desired)
{
  return atomic_compare_exchange_weak_explicit (
      object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

#ifndef NON_INTEGER

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_add) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_add_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_add_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_sub) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_sub_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_sub_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or_explicit (volatile Q ATOMIC_TYPE *object,
                          NONATOMIC_TYPE operand, memory_order order,
                          memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_or) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_or_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_or_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_xor) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_xor_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_xor_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_and) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_and_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_and_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_min) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_min_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_min_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (__pocl_atomic_fetch_max) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_max_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_max_explicit (object, operand, memory_order_seq_cst);
}

#endif

#endif


Fixed header marker left by mistake

Fixed header marker left by mistake

/** Erosion of src image by mask. Result is stored in dst image.

  */

// ARRAY convolution_window [window_size_x*window_size_y]
// SCALAR window_size_x
// SCALAR window_size_y

__kernel void vglClFuzzyGeoErode(__read_only image2d_t img_input,
                                __write_only image2d_t img_output,
                                __constant float* convolution_window,
                                int window_size_x,
                                int window_size_y)
{
	int2 coords = (int2)(get_global_id(0), get_global_id(1));
	const sampler_t smp = CLK_NORMALIZED_COORDS_FALSE | //Natural coordinates
                              CLK_ADDRESS_CLAMP_TO_EDGE |   //Clamp to next edge
                              CLK_FILTER_NEAREST;           //Don't interpolate

	int factorx = floor((float)window_size_x / 2.0f);
	int factory = floor((float)window_size_y / 2.0f);
	int conv_controller = 0;
	float4 pmin = (1.0,1.0,1.0,1.0);
  for(int j = -factory; j <= factory; j++)
	{
		for(int i = -factorx; i <= factorx; i++)
		{
			float4 a = read_imagef(img_input, smp, (int2)(coords.x + i,coords.y + j));
			float b = 1 - convolution_window[conv_controller];
			float4 S = 1 - sqrt((1-a)*(1-b));
			pmin = min(pmin,S);
			conv_controller++;
		}
	}
	write_imagef(img_output,coords,pmin);
}

/** Erosion of src image by mask. Result is stored in dst image.

  */

// ARRAY convolution_window [window_size_x*window_size_y]
// SCALAR window_size_x
// SCALAR window_size_y

__kernel void vglClFuzzyGeoErode(__read_only image2d_t img_input,
                                __write_only image2d_t img_output,
                                __constant float* convolution_window,
                                int window_size_x,
                                int window_size_y)
{
<<<<<<< HEAD
	int2 coords = (int2)(get_global_id(0), get_global_id(1));
	const sampler_t smp = CLK_NORMALIZED_COORDS_FALSE | //Natural coordinates
                              CLK_ADDRESS_CLAMP_TO_EDGE |   //Clamp to next edge
                              CLK_FILTER_NEAREST;           //Don't interpolate

	int factorx = floor((float)window_size_x / 2.0f);
	int factory = floor((float)window_size_y / 2.0f);
	int conv_controller = 0;
	float4 pmin = (1.0,1.0,1.0,1.0);
  for(int j = -factory; j <= factory; j++)
	{
		for(int i = -factorx; i <= factorx; i++)
		{
			float4 a = read_imagef(img_input, smp, (int2)(coords.x + i,coords.y + j));
			float b = 1 - convolution_window[conv_controller];
			float4 S = 1 - sqrt((1-a)*(1-b));
			pmin = min(pmin,S);
			conv_controller++;
		}
	}
	write_imagef(img_output,coords,pmin);
}


Same as master branch: use vector coordinates

__kernel void
downsampledgauss5x5_cols(
        __write_only image2d_t output_image,
        __read_only image2d_t input_image)
{
    const float mask[5][5] = {
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        2.34375e-02f, 9.37500e-02f, 1.40625e-01f, 9.37500e-02f, 2.34375e-02f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
    };

    const sampler_t sampler =
        CLK_FILTER_LINEAR|CLK_NORMALIZED_COORDS_FALSE|CLK_ADDRESS_CLAMP_TO_EDGE;

    int2 outcoord = (int2)(get_global_id(0), get_global_id(1));

    float2 incoord = (float2)(
        (float)(2 * get_global_id(0) + (get_global_id(1) & 0x1)),
        (float)get_global_id(1));

    float4 c = 0.f;

    if (outcoord.x >= get_image_width(output_image)
            || outcoord.y >= get_image_height(output_image)) {
        return;
    }

    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, -2.0f))) * mask[0][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, -2.0f))) * mask[1][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, -2.0f))) * mask[2][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, -2.0f))) * mask[3][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+2.0f, -2.0f))) * mask[4][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, -1.0f))) * mask[0][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, -1.0f))) * mask[1][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, -1.0f))) * mask[2][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, -1.0f))) * mask[3][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+2.0f, -1.0f))) * mask[4][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +0.0f))) * mask[0][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +0.0f))) * mask[1][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, +0.0f))) * mask[2][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, +0.0f))) * mask[3][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+2.0f, +0.0f))) * mask[4][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +1.0f))) * mask[0][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +1.0f))) * mask[1][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, +1.0f))) * mask[2][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, +1.0f))) * mask[3][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+2.0f, +1.0f))) * mask[4][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +2.0f))) * mask[0][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +2.0f))) * mask[1][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, +2.0f))) * mask[2][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, +2.0f))) * mask[3][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+2.0f, +2.0f))) * mask[4][4];

    /* barrier(CLK_LOCAL_MEM_FENCE); */ /* Normaly not necessary */

    write_imageui(output_image, outcoord, convert_uint4(c));
}

__kernel void
downsampledgauss5x5_rows(
        __write_only image2d_t output_image,
        __read_only image2d_t input_image)
{
    const float mask[5][5] = {
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        2.34375e-02f, 9.37500e-02f, 1.40625e-01f, 9.37500e-02f, 2.34375e-02f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
    };

    const sampler_t sampler =
        CLK_FILTER_LINEAR|CLK_NORMALIZED_COORDS_FALSE|CLK_ADDRESS_CLAMP_TO_EDGE;

    int2 outcoord = (int2)(get_global_id(0), get_global_id(1));

    float2 incoord = (float2)(
        (float)get_global_id(0),
        (float)(2 * get_global_id(1)));

    float4 c = 0.f;

    if (outcoord.x >= get_image_width(output_image)
            || outcoord.y >= get_image_height(output_image)) {
        return;
    }

    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, -4.0f))) * mask[0][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, -3.0f))) * mask[0][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, -2.0f))) * mask[0][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, -1.0f))) * mask[0][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+2.0f, +0.0f))) * mask[0][5];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, -3.0f))) * mask[1][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, -2.0f))) * mask[1][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, -1.0f))) * mask[1][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, +0.0f))) * mask[1][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+1.0f, +1.0f))) * mask[1][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, -2.0f))) * mask[2][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, -1.0f))) * mask[2][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, +0.0f))) * mask[2][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, +1.0f))) * mask[2][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(+0.0f, +2.0f))) * mask[2][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, -1.0f))) * mask[3][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +0.0f))) * mask[3][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +1.0f))) * mask[3][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +2.0f))) * mask[3][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-1.0f, +3.0f))) * mask[3][4];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +0.0f))) * mask[4][0];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +1.0f))) * mask[4][1];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +2.0f))) * mask[4][2];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +3.0f))) * mask[4][3];
    c += convert_float4(read_imageui(input_image, sampler, incoord+(float2)(-2.0f, +4.0f))) * mask[4][4];

    /* barrier(CLK_LOCAL_MEM_FENCE); */ /* Normaly not necessary */

    write_imageui(output_image, outcoord, convert_uint4(c));
}

__kernel void
downsampledgauss5x5_cols(
        __write_only image2d_t output_image,
        __read_only image2d_t input_image)
{
    const float mask[5][5] = {
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        2.34375e-02f, 9.37500e-02f, 1.40625e-01f, 9.37500e-02f, 2.34375e-02f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
    };

    const sampler_t sampler =
        CLK_FILTER_LINEAR|CLK_NORMALIZED_COORDS_FALSE|CLK_ADDRESS_CLAMP_TO_EDGE;

    int x_in_output = get_global_id(0);
    int y_in_output = get_global_id(1);

    float x_in_input = (float)(2 * get_global_id(0) + (get_global_id(1) & 0x1));
    float y_in_input = (float)get_global_id(1);

    float4 c = 0.f;

    if (x_in_output >= get_image_width(output_image)
            || y_in_output >= get_image_height(output_image)) {
        return;
    }

    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input-2.0f))) * mask[0][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input-2.0f))) * mask[1][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input-2.0f))) * mask[2][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input-2.0f))) * mask[3][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.0f, y_in_input-2.0f))) * mask[4][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input-1.0f))) * mask[0][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input-1.0f))) * mask[1][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input-1.0f))) * mask[2][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input-1.0f))) * mask[3][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.0f, y_in_input-1.0f))) * mask[4][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+0.0f))) * mask[0][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+0.0f))) * mask[1][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input+0.0f))) * mask[2][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input+0.0f))) * mask[3][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.0f, y_in_input+0.0f))) * mask[4][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+1.0f))) * mask[0][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+1.0f))) * mask[1][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input+1.0f))) * mask[2][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input+1.0f))) * mask[3][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.0f, y_in_input+1.0f))) * mask[4][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+2.0f))) * mask[0][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+2.0f))) * mask[1][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input+2.0f))) * mask[2][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input+2.0f))) * mask[3][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.0f, y_in_input+2.0f))) * mask[4][4];

    /* barrier(CLK_LOCAL_MEM_FENCE); */ /* Normaly not necessary */

    write_imageui(
            output_image,
            (int2)(x_in_output, y_in_output),
            convert_uint4(c));
}

__kernel void
downsampledgauss5x5_rows(
        __write_only image2d_t output_image,
        __read_only image2d_t input_image)
{
    const float mask[5][5] = {
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        2.34375e-02f, 9.37500e-02f, 1.40625e-01f, 9.37500e-02f, 2.34375e-02f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
    };

    const sampler_t sampler =
        CLK_FILTER_LINEAR|CLK_NORMALIZED_COORDS_FALSE|CLK_ADDRESS_CLAMP_TO_EDGE;

    int x_in_output = get_global_id(0);
    int y_in_output = get_global_id(1);

    float x_in_input = (float)get_global_id(0);
    float y_in_input = (float)(2 * get_global_id(1));

    float4 c = 0.f;

    if (x_in_output >= get_image_width(output_image)
            || y_in_output >= get_image_height(output_image)) {
        return;
    }

    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input-4.0f))) * mask[0][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input-3.0f))) * mask[0][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input-2.0f))) * mask[0][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input-1.0f))) * mask[0][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.0f, y_in_input+0.0f))) * mask[0][5];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input-3.0f))) * mask[1][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input-2.0f))) * mask[1][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input-1.0f))) * mask[1][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input+0.0f))) * mask[1][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+1.0f, y_in_input+1.0f))) * mask[1][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input-2.0f))) * mask[2][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input-1.0f))) * mask[2][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input+0.0f))) * mask[2][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input+1.0f))) * mask[2][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.0f, y_in_input+2.0f))) * mask[2][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input-1.0f))) * mask[3][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+0.0f))) * mask[3][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+1.0f))) * mask[3][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+2.0f))) * mask[3][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.0f, y_in_input+3.0f))) * mask[3][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+0.0f))) * mask[4][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+1.0f))) * mask[4][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+2.0f))) * mask[4][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+3.0f))) * mask[4][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-2.0f, y_in_input+4.0f))) * mask[4][4];

    /* barrier(CLK_LOCAL_MEM_FENCE); */ /* Normaly not necessary */

    write_imageui(
            output_image,
            (int2)(x_in_output, y_in_output),
            convert_uint4(c));
}