clean up

clean up

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0
// number of filter taps in the processing wk_item
//#define MLO_WEI_WKITEM 5

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM - 1) / MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1 * MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ / MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_WIDTH + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_OUT_WEI_SCAN_BLK (MLO_OUT_WEI_EXT_SCAN_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_WIDTH + MLO_OUT_WEI_SCAN_BLK - 1) / MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ * MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

#if MLO_OUT_HORIZ_PIX_SZ < (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#else
#define MLO_OUT_HORIZ_PIX_EXT_SZ MLO_OUT_HORIZ_PIX_SZ
#endif
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_BATCHS * MLO_N_LCL_OUT_MAPS * MLO_OUT_LCL_SZ)
#if((MLO_OUT_HEIGHT / MLO_N_ALIGNED_OUT_SCAN_BLK) * MLO_N_ALIGNED_OUT_SCAN_BLK == MLO_OUT_HEIGHT)
#define MLO_BLK_ALIGNED 1
#else
#define MLO_BLK_ALIGNED 0
#endif

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK - 1) * MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS \
    (MLO_IN_WIDTH) //((MLO_OUT_WIDTH-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0 - 2 * MLO_FILTER_PAD0)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF \
    (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS / MLO_READ_UNIT) * MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + MLO_FILTER_SIZE0 - 1)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ / MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS * MLO_N_LCL_IN_MAPS * MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ / MLO_N_IN_HORIZ_READS)

#if(MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = v / d;
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24((uint)u, (uint)d);
    return (r);
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split filter taps into sub-tiles along x and y axis with number of tap groups muliples of stride
or 1
// for example
// the 5x10 filter has been split into 10 sub-tiles 1x5 each, 2 tap in y direction and 5 taps in x
direction.
// those horizontal taps are 0, 2, 4, 6, 8 and 1, 3, 5, 7, 9
// a single vertical tap is 0 or 1 or 2 or 3 or 4.
// one may say sub-tiles are indexed by a vertical tap.
// the partial sum has been calculated into those 10 sub-tiles in parallel.
// the full filter has been calulated by reducing all sub-tiles into a single filter per group.
// teh accumulation has been done over all pixels of several outputs being shared with a single
input.
// the accuulation has been done per batch.
//
// the total reduction over all batches has been doesn a separete kerenel.
//
// alg
//
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//		for j in output scans
//			for i in output scan interval
//				accumulate the weights into sub-tiles
//
//		reduce sub-tiles into a single filter for each output
//		write accululated weights
//
// group layout
// 0 - n waves * wave size (n_waves has been defined by host)
// 1 - input channel index
// 2 - output channel/batch index
//
//
// for each batch
//	 accumulate all weights per input/output pair

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenCvBwdWrW(const __global _FLOAT* __restrict top_df,
               const __global _FLOAT* __restrict bot,
               __global _FLOAT* __restrict weights_df,
#if MLO_CONV_BIAS
               __global _FLOAT* __restrict bias_df,
#endif
               UNUSED _FLOAT padding_val)
{

    // input/output tiles + reduce buffer

    __local _FLOAT lcl[(MLO_LCL_SZ)];
    __local _FLOAT* lcl_bot = lcl;
    __local _FLOAT* lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

    uint c_idx_base = get_group_id(0); // input map index base

    uint o_idx_base = get_group_id(1); // output map index base
    uint ib_base    = get_group_id(2); // batch index base

    uint ib = ib_base * (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS);

    uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

    uint o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

    uint gbl_in_off  = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
    uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

    uint lcl_id = get_local_id(0);

// weight tile id
#if MLO_WEI_BLK_SZ & (MLO_WEI_BLK_SZ - 1)
    uint w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
    uint w_idx     = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
#else
    uint w_blk_idx              = lcl_id / MLO_WEI_BLK_SZ;
    uint w_idx                  = lcl_id & (MLO_WEI_BLK_SZ - 1);
#endif
#if MLO_WEI_BLK_SZ0 & (MLO_WEI_BLK_SZ0 - 1)
    // weight y
    uint w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
    // weight starting x
    uint w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);
#else
    uint w_y                    = w_idx / MLO_WEI_BLK_SZ0;
    uint w_x0                   = w_idx & (MLO_WEI_BLK_SZ0 - 1);
#endif

    __private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

    for(uint i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
    {
        pvt_accum[i] = 0;
    }

    // zero out LDS
    for(uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
    {
        lcl[i] = 0;
    }

    // over all batches

    for(uint b = 0; b < MLO_N_BATCH_LOOPS; ++b,
             gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
             gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE)
    {

        barrier(CLK_LOCAL_MEM_FENCE);

        uint in_y  = 0;
        uint out_y = 0;

        // prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
        __private _FLOAT in_rd_data[MLO_READ_UNIT];

        uint gbl_in_scan_off  = gbl_in_off;
        uint gbl_out_scan_off = gbl_out_off;
        uint c_scan           = 0;

        for(uint p4 = lcl_id;
            p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1);
            p4 += MLO_GRP_SZ)
        {
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
            c_scan      = iDiv(p4, MLO_N_IN_HORIZ_READS);
            uint c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);
#else
            c_scan              = p4 / MLO_N_IN_HORIZ_READS;
            uint c_pix4         = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                in_rd_data[i] = 0;
            }
            if(in_y + c_scan < MLO_IN_HEIGHT)
            {
                uint bot_off = gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4 * MLO_READ_UNIT;
                const __global _FLOAT* bot_p = &bot[bot_off];
// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
                if(c_pix4 == MLO_N_IN_HORIZ_READS - 1)
                {
                    uint i = 0;
                    for(; i < MLO_IN_N_PIXS_OFF; ++i)
                    {
                        in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                        if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:in-of-range\n");
                        }
#endif
                    }
                }
                else
#endif
                {
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                        if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:in-of-range\n");
                        }
#endif
                    }
                }
            }
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                lcl_bot[(c_scan + MLO_FILTER_PAD1) * MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 +
                        c_pix4 * MLO_READ_UNIT + i] = in_rd_data[i];
            }
        }

        in_y += MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;

        // TO DO: HANDLE PADDING
        // over all out blocks
        // processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

        for(uint ob = 0; ob < MLO_N_OUT_BLK; ++ob,
                 in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1),
                 out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
        {

            barrier(CLK_LOCAL_MEM_FENCE);

            // fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
            // TODO:: HANDLE multiple INPUTS
            // an overshoot has to be handled by zero outing output
            gbl_in_scan_off = gbl_in_off + mul24(in_y, (uint)MLO_IN_STRIDE);
            uint c_scan2    = 0;
            for(uint p4 = lcl_id;
                p4 <
                MLO_N_IN_HORIZ_READS * (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1);
                p4 += MLO_GRP_SZ)
            {
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
                c_scan2     = iDiv(p4, MLO_N_IN_HORIZ_READS);
                uint c_pix4 = iMod(p4, c_scan2, MLO_N_IN_HORIZ_READS);
#else
                c_scan2         = p4 / MLO_N_IN_HORIZ_READS;
                uint c_pix4     = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    in_rd_data[i] = 0;
                }

                if(in_y + c_scan2 < MLO_IN_HEIGHT)
                {
                    uint bot_off =
                        gbl_in_scan_off + c_scan2 * MLO_IN_STRIDE + c_pix4 * MLO_READ_UNIT;
                    const __global _FLOAT* bot_p = &bot[bot_off];

#if MLO_IN_N_PIXS_OFF > 0
                    if(c_pix4 == MLO_N_IN_HORIZ_READS - 1)
                    {

                        uint i = 0;
                        for(; i < MLO_IN_N_PIXS_OFF; ++i)
                        {
                            in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                            if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                    else
#endif
                    {

                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                            if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }

                } // if (in_y + c_scan2 < MLO_IN_HEIGHT)

                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    uint lcl_off =
                        (c_scan2 + MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1) * MLO_IN_LCL_WIDTH +
                        MLO_FILTER_PAD0 + c_pix4 * MLO_READ_UNIT;
                    lcl_bot[lcl_off + i] = in_rd_data[i];
                }
            }

            gbl_out_scan_off = gbl_out_off + mul24(out_y, (uint)MLO_OUT_STRIDE);

            // over all outputs groups
            // MLO_N_OUT_BLK_GRP outputs reuse the same input
            // each output blk is MLO_N_LCL_OUT_MAPS outputs
            // MLO_N_LCL_OUT_MAPS nuber is restricted by LDS size

            uint gbl_out_scan_off1 = gbl_out_scan_off;
            for(uint og = 0; og < MLO_N_OUT_BLK_GRP;
                ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS * MLO_OUT_CHANNEL_STRIDE)
            {
                barrier(CLK_LOCAL_MEM_FENCE);

                // fetch output. MLO_N_ALIGNED_OUT_SCAN_BLK output scans, each of size
                // MLO_N_OUT_HORIZ_READS

                __private _FLOAT out_rd_data[MLO_READ_UNIT];

                for(uint oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS * MLO_N_ALIGNED_OUT_SCAN_BLK *
                                                  MLO_N_OUT_HORIZ_READS);
                    oo_p4 += MLO_GRP_SZ)
                {
#if(MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS) & \
    ((MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS) - 1)
                    uint o = iDiv(oo_p4, (MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS));
                    uint o_pX4 =
                        iMod(oo_p4, o, (MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS));
#else
                    uint o      = oo_p4 / (MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS);
                    uint o_pX4  = oo_p4 & ((MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS) - 1);
#endif
#if MLO_N_OUT_HORIZ_READS & (MLO_N_OUT_HORIZ_READS - 1)
                    uint o_scan = iDiv(o_pX4, MLO_N_OUT_HORIZ_READS);
                    uint o_pix4 = iMod(o_pX4, o_scan, MLO_N_OUT_HORIZ_READS);
#else
                    uint o_scan = o_pX4 / MLO_N_OUT_HORIZ_READS;
                    uint o_pix4 = o_pX4 & (MLO_N_OUT_HORIZ_READS - 1);
#endif
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        out_rd_data[i] = 0;
                    }
                    uint top_df_off = gbl_out_scan_off1 + o * MLO_OUT_CHANNEL_STRIDE +
                                      o_scan * MLO_OUT_STRIDE + o_pix4 * MLO_READ_UNIT;
                    const __global _FLOAT* top_df_p = &top_df[top_df_off];

                    if(out_y + o_scan < MLO_OUT_HEIGHT &&
                       o_idx + og * MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS)
                    {
// scan has been fetch by 4
// here the non-multiple of 4 scan has been handled
// also makes sure the input garbage hs been multipled by 0
#if MLO_OUT_N_PIXS_OFF > 0
                        if(o_pix4 == (MLO_N_OUT_HORIZ_READS - 1))
                        {
                            uint i = 0;
                            for(; i < MLO_OUT_N_PIXS_OFF; ++i)
                            {
                                out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
                                if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                                {
                                    printf("k:err:out-of-range\n");
                                }
#endif
                            }
                        }

                        else
#endif

                        {
                            for(uint i = 0; i < MLO_READ_UNIT; ++i)
                            {
                                out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
                                if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                                {
                                    printf("k:err:out-of-range\n");
                                }
#endif
                            }
                        }

                    } // if (out_y + o_scan < MLO_OUT_HEIGHT && o_idx + og *MLO_N_LCL_OUT_MAPS + o <
                      // MLO_N_OUTPUTS)

                    // write into LDS with MLO_OUT_HORIZ_PIX_EXT_SZ stride to zero out weights block
                    // overshoot

                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        uint lcl_off = o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ +
                                       o_pix4 * MLO_READ_UNIT;
                        lcl_top[lcl_off + i] = out_rd_data[i];
                    }

                } //	for (uint oo_p4 = lcl_id; oo_p4 <
                  //(MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS); oo_p4 +=
                  // MLO_GRP_SZ)

                barrier(CLK_LOCAL_MEM_FENCE);

                // process
                // algorithm

                // over all input scans in LDS
                for(uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
                {

                    // prefetch proper inputs pixels.
                    // they are MLO_WEI_BLK_SZ0 apart taps of the filter

                    _FLOAT i_vals[MLO_WEI_WKITEM];

                    for(uint w = 0; w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0));
                        ++w)
                    {
                        uint w_x   = w_x0 + w * MLO_WEI_BLK_SZ0;
                        uint i_off = (j * MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH +
                                     (w_blk_idx * MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 +
                                     w_x;
                        _FLOAT i_val = lcl_bot[i_off];

                        i_vals[w] = i_val;
                    }

                    // if we overshoot the scanline
                    // out data will be 0 by initial setting
                    for(uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
                    {

                        // read the current input pixel
                        for(uint w = (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0));
                            w < MLO_WEI_WKITEM;
                            ++w)
                        {
                            uint w_x = w_x0 + w * MLO_WEI_BLK_SZ0;
                            uint i_off =
                                (j * MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH +
                                (w_blk_idx * MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
                            _FLOAT i_val = lcl_bot[i_off];

                            i_vals[w] = i_val;
                        }
                        // for each output accumulate a proper filter tap
                        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
                        {

                            // read with MLO_OUT_HORIX_PIX_EXT_SZ stride
                            _FLOAT o_val =
                                lcl_top[(o)*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ +
                                        (w_blk_idx * MLO_OUT_WEI_SCAN_BLK + i)];

                            uint w = 0;
                            _FLOAT i_val;
                            for(/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)
                            {

                                i_val = i_vals[w];

                                pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] +=
                                    i_val * o_val;

                            } // for (/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)

                        } // for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)

                        for(uint w = 0;
                            w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0));
                            ++w)
                        {
                            i_vals[w] = i_vals[w + (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)];
                        }
                    } // for (uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
                }     // for (uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)

            } // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

            // move the input data tail inside LDS to reduce mem bandwidth
            for(uint c_scan3 = 0; c_scan3 < (uint)(MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1);
                ++c_scan3)
            {
                barrier(CLK_LOCAL_MEM_FENCE);

                for(uint p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS; p4 += MLO_GRP_SZ)
                {

                    uint c_pix4 = p4;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        lcl_bot[c_scan3 * (MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 +
                                c_pix4 * MLO_READ_UNIT + i] =
                            lcl_bot[(c_scan3 +
                                     (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1)) *
                                        (MLO_IN_LCL_WIDTH) +
                                    MLO_FILTER_PAD0 + c_pix4 * MLO_READ_UNIT + i];
                    }
                }
            }

        } // for (uint ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT -
          // MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
    }     // for (uint b = 0;

    // send it out

    uint wei_lcl_off = 0;
    _FLOAT final_sum = 0;

    // save in lcl and orgnize in a proper order
    // outputs
    //	filter size1
    //	  filter size0

    // TO DO:: DEPENDING ON THE GROUP SIZE
    for(uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
    {
        barrier(CLK_LOCAL_MEM_FENCE);

        // o_base may be larger than MLO_N_OUTPUTS, so o_number may be negative
        uint o_base  = o_idx + og * MLO_N_LCL_OUT_MAPS;
        int o_number = min((int)MLO_N_LCL_OUT_MAPS, (int)MLO_N_OUTPUTS - (int)o_base);

        for(uint o = 0; (int)o < o_number; ++o)
        {
            if(w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
            {
                uint w = 0;
                for(; w < MLO_WEI_WKITEM; ++w)
                {
                    // save "virtual" filter table
                    uint w_x = w_x0 + w * MLO_WEI_BLK_SZ0;
                    wei_lcl_off =
                        ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1 + w_y) *
                            (MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM) +
                        w_x;
                    lcl[wei_lcl_off] =
                        pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
                }
            }
        }

        barrier(CLK_LOCAL_MEM_FENCE);

        // read into real filter table
        for(uint l = lcl_id; (int)l < (o_number * MLO_WEI_CHANNEL_STRIDE); l += MLO_GRP_SZ)
        {
#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
            uint oo    = iDiv(l, MLO_WEI_CHANNEL_STRIDE);
            uint wei_i = iMod(l, oo, MLO_WEI_CHANNEL_STRIDE);
#else
            uint oo             = l / MLO_WEI_CHANNEL_STRIDE;
            uint wei_i          = l & MLO_WEI_CHANNEL_STRIDE - 1;
#endif
#if(MLO_FILTER_SIZE0) & ((MLO_FILTER_SIZE0)-1)
            uint wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
            uint wei_i_x = iMod(wei_i, wei_i_y, (MLO_FILTER_SIZE0));
#else
            uint wei_i_y        = wei_i / (MLO_FILTER_SIZE0);
            uint wei_i_x        = wei_i & ((MLO_FILTER_SIZE0)-1);
#endif
            // send it out
            // inputs are outputs
            uint wei_df_off = ((ib_base * MLO_N_OUTPUTS + o_idx) * (uint)MLO_WEI_BATCH_STRIDE)
                              // this input channel
                              + mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);

            final_sum = 0;
            for(uint i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
            {
                final_sum += lcl[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) *
                                     (MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM) +
                                 wei_i_x];
            }

            uint wei_out_off =
                wei_df_off + (og * MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i;
            weights_df[wei_out_off] = final_sum; // lcl_bot[lcl_id]; //

#if DBG_OUT_OF_RNGE
            // assured
            if(wei_out_off >= MLO_WEI_BATCH_STRIDE * MLO_N_OUTPUTS * MLO_N_BATCH_BLKS)
            {
                printf("k:err:interm-output-of-range\n");
            }
#endif
        }

    } // for(uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1))) __kernel void
MIOpenCvBwdWrW_rdc(const __global _FLOAT* __restrict weight_df_tmp,
                   __global _FLOAT* __restrict weights_df)
{
    uint gbl_id   = get_global_id(0);
    uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
    uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
    uint wei_idx     = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
    uint wei_blk_idx            = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
    uint wei_idx                = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

    _FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
    for(uint i = 0; i < MLO_UT_READ_UNIT; ++i)
    {
        pvt_accum_wei[i] = 0;
    }

    uint batch_loop = MLO_N_BATCH_BLKS;

    for(uint i = 0; i < batch_loop; ++i)
    {
        for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
        {
            pvt_accum_wei[j] += weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE +
                                               i * MLO_N_OUTPUTS * MLO_WEI_BATCH_STRIDE) +
                                              wei_idx + j];
        }
    }

    for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
    {
        weights_df[wei_idx0 + j] = pvt_accum_wei[j];
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0
// number of filter taps in the processing wk_item
//#define MLO_WEI_WKITEM 5

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM - 1) / MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1 * MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ / MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_WIDTH + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_OUT_WEI_SCAN_BLK (MLO_OUT_WEI_EXT_SCAN_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_WIDTH + MLO_OUT_WEI_SCAN_BLK - 1) / MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ * MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

#if MLO_OUT_HORIZ_PIX_SZ < (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#else
#define MLO_OUT_HORIZ_PIX_EXT_SZ MLO_OUT_HORIZ_PIX_SZ
#endif
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_BATCHS * MLO_N_LCL_OUT_MAPS * MLO_OUT_LCL_SZ)
#if((MLO_OUT_HEIGHT / MLO_N_ALIGNED_OUT_SCAN_BLK) * MLO_N_ALIGNED_OUT_SCAN_BLK == MLO_OUT_HEIGHT)
#define MLO_BLK_ALIGNED 1
#else
#define MLO_BLK_ALIGNED 0
#endif

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK - 1) * MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS \
    (MLO_IN_WIDTH) //((MLO_OUT_WIDTH-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0 - 2 * MLO_FILTER_PAD0)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF \
    (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS / MLO_READ_UNIT) * MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + MLO_FILTER_SIZE0 - 1)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ / MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS * MLO_N_LCL_IN_MAPS * MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ / MLO_N_IN_HORIZ_READS)

#if(MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = v / d;
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24((uint)u, (uint)d);
    return (r);
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split filter taps into sub-tiles along x and y axis with number of tap groups muliples of stride
or 1
// for example
// the 5x10 filter has been split into 10 sub-tiles 1x5 each, 2 tap in y direction and 5 taps in x
direction.
// those horizontal taps are 0, 2, 4, 6, 8 and 1, 3, 5, 7, 9
// a single vertical tap is 0 or 1 or 2 or 3 or 4.
// one may say sub-tiles are indexed by a vertical tap.
// the partial sum has been calculated into those 10 sub-tiles in parallel.
// the full filter has been calulated by reducing all sub-tiles into a single filter per group.
// teh accumulation has been done over all pixels of several outputs being shared with a single
input.
// the accuulation has been done per batch.
//
// the total reduction over all batches has been doesn a separete kerenel.
//
// alg
//
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//		for j in output scans
//			for i in output scan interval
//				accumulate the weights into sub-tiles
//
//		reduce sub-tiles into a single filter for each output
//		write accululated weights
//
// group layout
// 0 - n waves * wave size (n_waves has been defined by host)
// 1 - input channel index
// 2 - output channel/batch index
//
//
// for each batch
//	 accumulate all weights per input/output pair

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenCvBwdWrW(const __global _FLOAT* __restrict top_df,
               const __global _FLOAT* __restrict bot,
               __global _FLOAT* __restrict weights_df,
#if MLO_CONV_BIAS
               __global _FLOAT* __restrict bias_df,
#endif
               UNUSED _FLOAT padding_val)
{

    // input/output tiles + reduce buffer

    __local _FLOAT lcl[(MLO_LCL_SZ)];
    __local _FLOAT* lcl_bot = lcl;
    __local _FLOAT* lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

    uint c_idx_base = get_group_id(0); // input map index base

    uint o_idx_base = get_group_id(1); // output map index base
    uint ib_base    = get_group_id(2); // batch index base

    uint ib = ib_base * (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS);

    uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

    uint o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

    uint gbl_in_off  = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
    uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

    uint lcl_id = get_local_id(0);

// weight tile id
#if MLO_WEI_BLK_SZ & (MLO_WEI_BLK_SZ - 1)
    uint w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
    uint w_idx     = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
#else
    uint w_blk_idx              = lcl_id / MLO_WEI_BLK_SZ;
    uint w_idx                  = lcl_id & (MLO_WEI_BLK_SZ - 1);
#endif
#if MLO_WEI_BLK_SZ0 & (MLO_WEI_BLK_SZ0 - 1)
    // weight y
    uint w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
    // weight starting x
    uint w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);
#else
    uint w_y                    = w_idx / MLO_WEI_BLK_SZ0;
    uint w_x0                   = w_idx & (MLO_WEI_BLK_SZ0 - 1);
#endif

    __private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

    for(uint i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
    {
        pvt_accum[i] = 0;
    }

    // zero out LDS
    for(uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
    {
        lcl[i] = 0;
    }

    // over all batches

    for(uint b = 0; b < MLO_N_BATCH_LOOPS; ++b,
             gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
             gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE)
    {

        barrier(CLK_LOCAL_MEM_FENCE);

        uint in_y  = 0;
        uint out_y = 0;

        // prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
        __private _FLOAT in_rd_data[MLO_READ_UNIT];

        uint gbl_in_scan_off  = gbl_in_off;
        uint gbl_out_scan_off = gbl_out_off;
        uint c_scan           = 0;

        for(uint p4 = lcl_id;
            p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1);
            p4 += MLO_GRP_SZ)
        {
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
            c_scan      = iDiv(p4, MLO_N_IN_HORIZ_READS);
            uint c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);
#else
            c_scan              = p4 / MLO_N_IN_HORIZ_READS;
            uint c_pix4         = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                in_rd_data[i] = 0;
            }
            if(in_y + c_scan < MLO_IN_HEIGHT)
            {
                uint bot_off = gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4 * MLO_READ_UNIT;
                const __global _FLOAT* bot_p = &bot[bot_off];
// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
                if(c_pix4 == MLO_N_IN_HORIZ_READS - 1)
                {
                    uint i = 0;
                    for(; i < MLO_IN_N_PIXS_OFF; ++i)
                    {
                        in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                        if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:in-of-range\n");
                        }
#endif
                    }
                }
                else
#endif
                {
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                        if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:in-of-range\n");
                        }
#endif
                    }
                }
            }
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                lcl_bot[(c_scan + MLO_FILTER_PAD1) * MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 +
                        c_pix4 * MLO_READ_UNIT + i] = in_rd_data[i];
            }
        }

        in_y += MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;

        // TO DO: HANDLE PADDING
        // over all out blocks
        // processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

        for(uint ob = 0; ob < MLO_N_OUT_BLK; ++ob,
                 in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1),
                 out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
        {

            barrier(CLK_LOCAL_MEM_FENCE);

            // fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
            // TODO:: HANDLE multiple INPUTS
            // an overshoot has to be handled by zero outing output
            gbl_in_scan_off = gbl_in_off + mul24(in_y, (uint)MLO_IN_STRIDE);
            uint c_scan2    = 0;
            for(uint p4 = lcl_id;
                p4 <
                MLO_N_IN_HORIZ_READS * (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1);
                p4 += MLO_GRP_SZ)
            {
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
                c_scan2     = iDiv(p4, MLO_N_IN_HORIZ_READS);
                uint c_pix4 = iMod(p4, c_scan2, MLO_N_IN_HORIZ_READS);
#else
                c_scan2         = p4 / MLO_N_IN_HORIZ_READS;
                uint c_pix4     = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    in_rd_data[i] = 0;
                }

                if(in_y + c_scan2 < MLO_IN_HEIGHT)
                {
                    uint bot_off =
                        gbl_in_scan_off + c_scan2 * MLO_IN_STRIDE + c_pix4 * MLO_READ_UNIT;
                    const __global _FLOAT* bot_p = &bot[bot_off];

#if MLO_IN_N_PIXS_OFF > 0
                    if(c_pix4 == MLO_N_IN_HORIZ_READS - 1)
                    {

                        uint i = 0;
                        for(; i < MLO_IN_N_PIXS_OFF; ++i)
                        {
                            in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                            if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                    else
#endif
                    {

                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
                            if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }

                } // if (in_y + c_scan2 < MLO_IN_HEIGHT)

                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    uint lcl_off =
                        (c_scan2 + MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1) * MLO_IN_LCL_WIDTH +
                        MLO_FILTER_PAD0 + c_pix4 * MLO_READ_UNIT;
                    lcl_bot[lcl_off + i] = in_rd_data[i];
                }
            }

            gbl_out_scan_off = gbl_out_off + mul24(out_y, (uint)MLO_OUT_STRIDE);

            // over all outputs groups
            // MLO_N_OUT_BLK_GRP outputs reuse the same input
            // each output blk is MLO_N_LCL_OUT_MAPS outputs
            // MLO_N_LCL_OUT_MAPS nuber is restricted by LDS size

            uint gbl_out_scan_off1 = gbl_out_scan_off;
            for(uint og = 0; og < MLO_N_OUT_BLK_GRP;
                ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS * MLO_OUT_CHANNEL_STRIDE)
            {
                barrier(CLK_LOCAL_MEM_FENCE);

                // fetch output. MLO_N_ALIGNED_OUT_SCAN_BLK output scans, each of size
                // MLO_N_OUT_HORIZ_READS

                __private _FLOAT out_rd_data[MLO_READ_UNIT];

                for(uint oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS * MLO_N_ALIGNED_OUT_SCAN_BLK *
                                                  MLO_N_OUT_HORIZ_READS);
                    oo_p4 += MLO_GRP_SZ)
                {
#if(MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS) & \
    ((MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS) - 1)
                    uint o = iDiv(oo_p4, (MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS));
                    uint o_pX4 =
                        iMod(oo_p4, o, (MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS));
#else
                    uint o      = oo_p4 / (MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS);
                    uint o_pX4  = oo_p4 & ((MLO_N_ALIGNED_OUT_SCAN_BLK * MLO_N_OUT_HORIZ_READS) - 1);
#endif
#if MLO_N_OUT_HORIZ_READS & (MLO_N_OUT_HORIZ_READS - 1)
                    uint o_scan = iDiv(o_pX4, MLO_N_OUT_HORIZ_READS);
                    uint o_pix4 = iMod(o_pX4, o_scan, MLO_N_OUT_HORIZ_READS);
#else
                    uint o_scan = o_pX4 / MLO_N_OUT_HORIZ_READS;
                    uint o_pix4 = o_pX4 & (MLO_N_OUT_HORIZ_READS - 1);
#endif
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        out_rd_data[i] = 0;
                    }
                    uint top_df_off = gbl_out_scan_off1 + o * MLO_OUT_CHANNEL_STRIDE +
                                      o_scan * MLO_OUT_STRIDE + o_pix4 * MLO_READ_UNIT;
                    const __global _FLOAT* top_df_p = &top_df[top_df_off];

                    if(out_y + o_scan < MLO_OUT_HEIGHT &&
                       o_idx + og * MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS)
                    {
// scan has been fetch by 4
// here the non-multiple of 4 scan has been handled
// also makes sure the input garbage hs been multipled by 0
#if MLO_OUT_N_PIXS_OFF > 0
                        if(o_pix4 == (MLO_N_OUT_HORIZ_READS - 1))
                        {
                            uint i = 0;
                            for(; i < MLO_OUT_N_PIXS_OFF; ++i)
                            {
                                out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
                                if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                                {
                                    printf("k:err:out-of-range\n");
                                }
#endif
                            }
                        }

                        else
#endif

                        {
                            for(uint i = 0; i < MLO_READ_UNIT; ++i)
                            {
                                out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
                                if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                                {
                                    printf("k:err:out-of-range\n");
                                }
#endif
                            }
                        }

                    } // if (out_y + o_scan < MLO_OUT_HEIGHT && o_idx + og *MLO_N_LCL_OUT_MAPS + o <
                      // MLO_N_OUTPUTS)

                    // write into LDS with MLO_OUT_HORIZ_PIX_EXT_SZ stride to zero out weights block
                    // overshoot

                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        uint lcl_off = o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ +
                                       o_pix4 * MLO_READ_UNIT;
                        lcl_top[lcl_off + i] = out_rd_data[i];
                    }

                } //	for (uint oo_p4 = lcl_id; oo_p4 <
                  //(MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS); oo_p4 +=
                  // MLO_GRP_SZ)

                barrier(CLK_LOCAL_MEM_FENCE);

                // process
                // algorithm

                // over all input scans in LDS
                for(uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
                {

                    // prefetch proper inputs pixels.
                    // they are MLO_WEI_BLK_SZ0 apart taps of the filter

                    _FLOAT i_vals[MLO_WEI_WKITEM];

                    for(uint w = 0; w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0));
                        ++w)
                    {
                        uint w_x   = w_x0 + w * MLO_WEI_BLK_SZ0;
                        uint i_off = (j * MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH +
                                     (w_blk_idx * MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 +
                                     w_x;
                        _FLOAT i_val = lcl_bot[i_off];

                        i_vals[w] = i_val;
                    }

                    // if we overshoot the scanline
                    // out data will be 0 by initial setting
                    for(uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
                    {

                        // read the current input pixel
                        for(uint w = (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0));
                            w < MLO_WEI_WKITEM;
                            ++w)
                        {
                            uint w_x = w_x0 + w * MLO_WEI_BLK_SZ0;
                            uint i_off =
                                (j * MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH +
                                (w_blk_idx * MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
                            _FLOAT i_val = lcl_bot[i_off];

                            i_vals[w] = i_val;
                        }
                        // for each output accumulate a proper filter tap
                        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
                        {

                            // read with MLO_OUT_HORIX_PIX_EXT_SZ stride
                            _FLOAT o_val =
                                lcl_top[(o)*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ +
                                        (w_blk_idx * MLO_OUT_WEI_SCAN_BLK + i)];

                            uint w = 0;
                            _FLOAT i_val;
                            for(/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)
                            {

                                i_val = i_vals[w];

                                pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] +=
                                    i_val * o_val;

                            } // for (/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)

                        } // for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)

                        for(uint w = 0;
                            w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0));
                            ++w)
                        {
                            i_vals[w] = i_vals[w + (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)];
                        }
                    } // for (uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
                }     // for (uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)

            } // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

            // move the input data tail inside LDS to reduce mem bandwidth
            for(uint c_scan3 = 0; c_scan3 < (uint)(MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1);
                ++c_scan3)
            {
                barrier(CLK_LOCAL_MEM_FENCE);

                for(uint p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS; p4 += MLO_GRP_SZ)
                {

                    uint c_pix4 = p4;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        lcl_bot[c_scan3 * (MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 +
                                c_pix4 * MLO_READ_UNIT + i] =
                            lcl_bot[(c_scan3 +
                                     (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1)) *
                                        (MLO_IN_LCL_WIDTH) +
                                    MLO_FILTER_PAD0 + c_pix4 * MLO_READ_UNIT + i];
                    }
                }
            }

        } // for (uint ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT -
          // MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
    }     // for (uint b = 0;

    // send it out

    uint wei_lcl_off = 0;
    _FLOAT final_sum = 0;

    // save in lcl and orgnize in a proper order
    // outputs
    //	filter size1
    //	  filter size0

    // TO DO:: DEPENDING ON THE GROUP SIZE
    for(uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
    {
        barrier(CLK_LOCAL_MEM_FENCE);

#if 1 // debug
        // o_base may be larger than MLO_N_OUTPUTS, so o_number may be negative
        uint o_base  = o_idx + og * MLO_N_LCL_OUT_MAPS;
        int o_number = min((int)MLO_N_LCL_OUT_MAPS, (int)MLO_N_OUTPUTS - (int)o_base);
#endif

#if 0 // debug
        if(w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
        {
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {
#else
        for(uint o = 0; (int)o < o_number; ++o)
        {
            if(w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
            {
#endif
        uint w = 0;
        for(; w < MLO_WEI_WKITEM; ++w)
        {
            // save "virtual" filter table
            uint w_x    = w_x0 + w * MLO_WEI_BLK_SZ0;
            wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1 + w_y) *
                              (MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM) +
                          w_x;
            lcl[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
        }
    }
}

barrier(CLK_LOCAL_MEM_FENCE);

// read into real filter table
#if 0 // debug
        for(uint l = lcl_id; l < (MLO_N_LCL_OUT_MAPS * MLO_WEI_CHANNEL_STRIDE); l += MLO_GRP_SZ)
#else
        for(uint l = lcl_id; (int)l < (o_number * MLO_WEI_CHANNEL_STRIDE); l += MLO_GRP_SZ)
#endif
{
#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
    uint oo    = iDiv(l, MLO_WEI_CHANNEL_STRIDE);
    uint wei_i = iMod(l, oo, MLO_WEI_CHANNEL_STRIDE);
#else
            uint oo      = l / MLO_WEI_CHANNEL_STRIDE;
            uint wei_i   = l & MLO_WEI_CHANNEL_STRIDE - 1;
#endif
#if(MLO_FILTER_SIZE0) & ((MLO_FILTER_SIZE0)-1)
    uint wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
    uint wei_i_x = iMod(wei_i, wei_i_y, (MLO_FILTER_SIZE0));
#else
            uint wei_i_y = wei_i / (MLO_FILTER_SIZE0);
            uint wei_i_x = wei_i & ((MLO_FILTER_SIZE0)-1);
#endif
    // send it out
    // inputs are outputs
    uint wei_df_off = ((ib_base * MLO_N_OUTPUTS + o_idx) * (uint)MLO_WEI_BATCH_STRIDE)
                      // this input channel
                      + mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);

    final_sum = 0;
    for(uint i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
    {
        final_sum += lcl[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) *
                             (MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM) +
                         wei_i_x];
    }

    uint wei_out_off = wei_df_off + (og * MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i;
#if 0 // no longer needed
            if(wei_out_off < MLO_WEI_BATCH_STRIDE * MLO_N_OUTPUTS * MLO_N_BATCH_BLKS)
#endif
    {
        weights_df[wei_out_off] = final_sum; // lcl_bot[lcl_id]; //

#if DBG_OUT_OF_RNGE
        // assured
        if(wei_out_off >= MLO_WEI_BATCH_STRIDE * MLO_N_OUTPUTS * MLO_N_BATCH_BLKS)
        {
            printf("k:err:interm-output-of-range\n");
        }
#endif
    }
}

} // for(uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1))) __kernel void
MIOpenCvBwdWrW_rdc(const __global _FLOAT* __restrict weight_df_tmp,
                   __global _FLOAT* __restrict weights_df)
{
    uint gbl_id   = get_global_id(0);
    uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
    uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
    uint wei_idx     = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
    uint wei_blk_idx     = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
    uint wei_idx         = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

    _FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
    for(uint i = 0; i < MLO_UT_READ_UNIT; ++i)
    {
        pvt_accum_wei[i] = 0;
    }

    uint batch_loop = MLO_N_BATCH_BLKS;

    for(uint i = 0; i < batch_loop; ++i)
    {
        for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
        {
            pvt_accum_wei[j] += weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE +
                                               i * MLO_N_OUTPUTS * MLO_WEI_BATCH_STRIDE) +
                                              wei_idx + j];
        }
    }

    for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
    {
        weights_df[wei_idx0 + j] = pvt_accum_wei[j];
    }
}


lib/kernel/svm_atomics.cl: fix the order of definitions

the incorrect order was causing unnecessary generic pointers

/* OpenCL built-in library: OpenCL 2.0 Atomics (C11 subset)

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   These implementations merely lower the call to a device-specific
   call (prefixed with "pocl_atomic_")

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#ifndef _SVM_ATOMICS_H
#include "svm_atomics.h"
#define _SVM_ATOMICS_H
#endif

#if !defined(Q)

#  define Q __global
#  define QUAL(f) f ## __global
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#  define Q __local
#  define QUAL(f) f ## __local
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#elif !defined(ATOMIC_TYPE)

bool _CL_OVERLOADABLE
atomic_flag_test_and_set_explicit (volatile Q atomic_flag *object,
                                   memory_order order, memory_scope scope)
{
  return QUAL (pocl_atomic_flag_test_and_set) (object, order, scope);
}

bool _CL_OVERLOADABLE atomic_flag_test_and_set_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  return atomic_flag_test_and_set_explicit(object, order, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_flag_test_and_set (volatile Q atomic_flag *object)
{
  return atomic_flag_test_and_set_explicit (object, memory_order_seq_cst);
}

void _CL_OVERLOADABLE
atomic_flag_clear_explicit (volatile Q atomic_flag *object, memory_order order,
                            memory_scope scope)
{
  return QUAL (pocl_atomic_flag_clear) (object, order, scope);
}

void _CL_OVERLOADABLE atomic_flag_clear_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  atomic_flag_clear_explicit(object, order, memory_scope_device);
}

void _CL_OVERLOADABLE
atomic_flag_clear (volatile Q atomic_flag *object)
{
  atomic_flag_clear_explicit (object, memory_order_seq_cst);
}

#  define ATOMIC_TYPE atomic_int
#  define NONATOMIC_TYPE int
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_uint
#  define NONATOMIC_TYPE uint
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_float
#  define NONATOMIC_TYPE float
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)

#  define ATOMIC_TYPE atomic_long
#  define NONATOMIC_TYPE long
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_ulong
#  define NONATOMIC_TYPE ulong
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#ifdef cl_khr_fp64

#  define ATOMIC_TYPE atomic_double
#  define NONATOMIC_TYPE double
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#else

void _CL_OVERLOADABLE
atomic_store_explicit (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired,
                       memory_order order, memory_scope scope)
{
  QUAL (pocl_atomic_store) (object, desired, order, scope);
}

void _CL_OVERLOADABLE atomic_store_explicit (  volatile Q ATOMIC_TYPE  *object,
                              NONATOMIC_TYPE  desired,
                              memory_order order)
{
  atomic_store_explicit(object, desired, order, memory_scope_device);
}

void _CL_OVERLOADABLE
atomic_store (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired)
{
  atomic_store_explicit (object, desired, memory_order_seq_cst);
}

void _CL_OVERLOADABLE atomic_init (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE value)
{
  atomic_store_explicit(object, value, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_load_explicit (volatile Q ATOMIC_TYPE *object, memory_order order,
                      memory_scope scope)
{
  return QUAL (pocl_atomic_load) (object, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_load_explicit ( volatile Q ATOMIC_TYPE  *object,
  memory_order order)
{
  return atomic_load_explicit(object, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_load (volatile Q ATOMIC_TYPE *object)
{
  return atomic_load_explicit (object, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_exchange_explicit (volatile Q ATOMIC_TYPE *object,
                          NONATOMIC_TYPE desired, memory_order order,
                          memory_scope scope)
{
  return QUAL (pocl_atomic_exchange) (object, desired, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_exchange_explicit (volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  desired,
  memory_order order)
{
  return atomic_exchange_explicit(object, desired, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_exchange (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE desired)
{
  return atomic_exchange_explicit (object, desired, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_strong_explicit (volatile Q ATOMIC_TYPE *object,
                                         private NONATOMIC_TYPE *expected,
                                         NONATOMIC_TYPE desired,
                                         memory_order success,
                                         memory_order failure,
                                         memory_scope scope)
{
  return QUAL (pocl_atomic_compare_exchange_strong) (object, expected, desired,
                                                     success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_strong_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_strong_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_strong (volatile Q ATOMIC_TYPE *object,
                                private NONATOMIC_TYPE *expected,
                                NONATOMIC_TYPE desired)
{
  return atomic_compare_exchange_strong_explicit (
      object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_weak_explicit (volatile Q ATOMIC_TYPE *object,
                                       private NONATOMIC_TYPE *expected,
                                       NONATOMIC_TYPE desired,
                                       memory_order success,
                                       memory_order failure,
                                       memory_scope scope)
{
  return QUAL (pocl_atomic_compare_exchange_weak) (object, expected, desired,
                                                   success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_weak_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_weak_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE
atomic_compare_exchange_weak (volatile Q ATOMIC_TYPE *object,
                              private NONATOMIC_TYPE *expected,
                              NONATOMIC_TYPE desired)
{
  return atomic_compare_exchange_weak_explicit (
      object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

#ifndef NON_INTEGER

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_add) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_add_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_add (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_add_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_sub) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_sub_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_sub (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_sub_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or_explicit (volatile Q ATOMIC_TYPE *object,
                          NONATOMIC_TYPE operand, memory_order order,
                          memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_or) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_or_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_or (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_or_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_xor) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_xor_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_xor (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_xor_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_and) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_and_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_and (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_and_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_min) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_min_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_min (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_min_explicit (object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max_explicit (volatile Q ATOMIC_TYPE *object,
                           NONATOMIC_TYPE operand, memory_order order,
                           memory_scope scope)
{
  return QUAL (pocl_atomic_fetch_max) (object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_max_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE
atomic_fetch_max (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE operand)
{
  return atomic_fetch_max_explicit (object, operand, memory_order_seq_cst);
}

#endif

#endif

/* OpenCL built-in library: OpenCL 2.0 Atomics (C11 subset)

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   These implementations merely lower the call to a device-specific
   call (prefixed with "pocl_atomic_")

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#ifndef _SVM_ATOMICS_H
#include "svm_atomics.h"
#define _SVM_ATOMICS_H
#endif

#if !defined(Q)

#  define Q __global
#  define QUAL(f) f ## __global
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#  define Q __local
#  define QUAL(f) f ## __local
#  include "svm_atomics.cl"
#  undef Q
#  undef QUAL

#elif !defined(ATOMIC_TYPE)

bool _CL_OVERLOADABLE atomic_flag_test_and_set ( volatile Q atomic_flag  *object )
{
  return atomic_flag_test_and_set_explicit(object, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE atomic_flag_test_and_set_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  return atomic_flag_test_and_set_explicit(object, order, memory_scope_device);
}

bool _CL_OVERLOADABLE atomic_flag_test_and_set_explicit ( volatile Q atomic_flag  *object ,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_flag_test_and_set)(object, order, scope);
}

void _CL_OVERLOADABLE atomic_flag_clear ( volatile Q atomic_flag  *object )
{
  atomic_flag_clear_explicit(object, memory_order_seq_cst);
}

void _CL_OVERLOADABLE atomic_flag_clear_explicit ( volatile Q atomic_flag  *object ,
  memory_order order)
{
  atomic_flag_clear_explicit(object, order, memory_scope_device);
}

void _CL_OVERLOADABLE atomic_flag_clear_explicit ( volatile Q atomic_flag  *object ,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_flag_clear)(object, order, scope);
}

#  define ATOMIC_TYPE atomic_int
#  define NONATOMIC_TYPE int
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_uint
#  define NONATOMIC_TYPE uint
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_float
#  define NONATOMIC_TYPE float
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)

#  define ATOMIC_TYPE atomic_long
#  define NONATOMIC_TYPE long
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#  define ATOMIC_TYPE atomic_ulong
#  define NONATOMIC_TYPE ulong
#  include "svm_atomics.cl"
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#ifdef cl_khr_fp64

#  define ATOMIC_TYPE atomic_double
#  define NONATOMIC_TYPE double
#  define NON_INTEGER
#  include "svm_atomics.cl"
#  undef NON_INTEGER
#  undef ATOMIC_TYPE
#  undef NONATOMIC_TYPE

#endif

#else

void _CL_OVERLOADABLE atomic_store ( volatile Q ATOMIC_TYPE  *object,
                    NONATOMIC_TYPE  desired)
{
  atomic_store_explicit(object, desired, memory_order_seq_cst);
}

void _CL_OVERLOADABLE atomic_store_explicit (  volatile Q ATOMIC_TYPE  *object,
                              NONATOMIC_TYPE  desired,
                              memory_order order)
{
  atomic_store_explicit(object, desired, order, memory_scope_device);
}

void _CL_OVERLOADABLE atomic_store_explicit (  volatile Q ATOMIC_TYPE  *object,
                              NONATOMIC_TYPE  desired,
                              memory_order order,
                              memory_scope scope)
{
  QUAL(pocl_atomic_store)(object, desired, order, scope);
}

void _CL_OVERLOADABLE atomic_init (volatile Q ATOMIC_TYPE *object, NONATOMIC_TYPE value)
{
  atomic_store_explicit(object, value, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_load (  volatile Q ATOMIC_TYPE  *object)
{
  return atomic_load_explicit(object, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_load_explicit ( volatile Q ATOMIC_TYPE  *object,
  memory_order order)
{
  return atomic_load_explicit(object, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_load_explicit ( volatile Q ATOMIC_TYPE  *object,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_load)(object, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_exchange ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  desired)
{
  return atomic_exchange_explicit(object, desired, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_exchange_explicit (volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  desired,
  memory_order order)
{
  return atomic_exchange_explicit(object, desired, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_exchange_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  desired,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_exchange)(object, desired, order, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_strong ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired)
{
  return atomic_compare_exchange_strong_explicit(
        object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_strong_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_strong_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_strong_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure,
  memory_scope scope)
{
  return QUAL(pocl_atomic_compare_exchange_strong)(object, expected, desired, success, failure, scope);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_weak ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired)
{
  return atomic_compare_exchange_weak_explicit(
        object, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_weak_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure)
{
  return atomic_compare_exchange_weak_explicit(
        object, expected, desired, success, failure, memory_scope_device);
}

bool _CL_OVERLOADABLE atomic_compare_exchange_weak_explicit ( volatile Q ATOMIC_TYPE  *object,
  private NONATOMIC_TYPE  *expected,
  NONATOMIC_TYPE  desired,
  memory_order success,
  memory_order failure,
  memory_scope scope)
{
  return QUAL(pocl_atomic_compare_exchange_weak)(object, expected, desired, success, failure, scope);
}

#ifndef NON_INTEGER

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_add_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_add_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_add_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_add)(object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_sub_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_sub_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_sub)(object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_or_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_or_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_or_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_or)(object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_xor_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_xor_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_xor)(object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_and_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_and_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_and_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_and)(object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_min_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_min_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_min_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_min)(object, operand, order, scope);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand)
{
  return atomic_fetch_max_explicit(object, operand, memory_order_seq_cst);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order)
{
  return atomic_fetch_max_explicit(object, operand, order, memory_scope_device);
}

NONATOMIC_TYPE _CL_OVERLOADABLE atomic_fetch_max_explicit ( volatile Q ATOMIC_TYPE  *object,
  NONATOMIC_TYPE  operand,
  memory_order order,
  memory_scope scope)
{
  return QUAL(pocl_atomic_fetch_max)(object, operand, order, scope);
}

#endif

#endif