minor fix

minor fix

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

#if MLO_N_MAPS_PERGROUP > 1
#define MLO_WEIGHTS_PER_LOOP_MAX (8)
#else
#define MLO_WEIGHTS_PER_LOOP_MAX (16)
#endif
#if(MLO_N_MAPS_PERGROUP * MLO_N_LCL_IN_MAPS < MLO_N_INPUTS)
#define MLO_LCL_IN_ROW (MLO_N_MAPS_PERGROUP * MLO_N_LCL_IN_MAPS)
#else
#define MLO_LCL_IN_ROW (MLO_N_INPUTS)
#endif

#define MLO_WEIGHTS_PER_LOOP_TMP ((MLO_N_INPUTS + MLO_LCL_IN_ROW - 1) / MLO_LCL_IN_ROW)

#if(MLO_WEIGHTS_PER_LOOP_TMP < MLO_WEIGHTS_PER_LOOP_MAX)
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_TMP)
#else
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_MAX)
#endif

#define MLO_LCL_WEIGHTS_ROW (MLO_WEIGHTS_PER_LOOP * MLO_LCL_IN_ROW)

#define MLO_IN_LOOP ((MLO_N_INPUTS + MLO_LCL_WEIGHTS_ROW - 1) / MLO_LCL_WEIGHTS_ROW)

#define MLO_WEIGHTS_ROW (MLO_LCL_WEIGHTS_ROW * MLO_WEI_CHANNEL_STRIDE)

// size of the area for weights prefetch
#define MLO_WEIGHTS_LCL_SZ (MLO_WEIGHTS_ROW * MLO_N_LCL_OUT_MAPS)

// size of area for exchanging partial sums
#define MLO_EXCHNGE_SZ4 (MLO_MAP_SZ4 * MLO_EXCHANGE_STEP * MLO_N_MAPS_PERGROUP)

#if MLO_N_MAPS_PERGROUP > 1 && ((MLO_EXCHNGE_SZ4 * MLO_READ_UNIT) > MLO_WEIGHTS_LCL_SZ)
#define MLO_LCL_MEM_SZ (MLO_EXCHNGE_SZ4 * MLO_READ_UNIT)
#else
#define MLO_LCL_MEM_SZ MLO_WEIGHTS_LCL_SZ
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

/*
Layout:
assuming NCHW data layout.

Data:
data has been fetch by 4 floats sequentially.
MLO_MAP_SZ4 = (map_width*map_height + 3)/4.
in case of total size not a multiple of 4 the the last pixel has a special treatment.
There are 2 cases:
MLO_N_MAPS_PERGROUP == 1
and
MLO_N_MAPS_PERGROUP > 1, when MLO_MAP_SZ4 <= GPROUP_SIZE/2, in other words when more than 1 map can
be held by a group.
Case MLO_N_MAPS_PERGROUP == 1:
Data, by 4 floats, may come from MLO_N_LCL_IN_MAPS sequential input maps from MLO_N_LCL_BATCHS
neighboring batches.
Weigts:
on each MLO_WEIGHTS_PER_LOOP input loop set of weight are prefetched for another
MLO_WEIGHTS_PER_LOOP loops.
Each input map contributes to partial sums of MLO_N_LCL_OUT_MAPS output maps.
Case MLO_N_MAPS_PERGROUP > 1:
Similar to a previous case.
The difference is that several input sequential input maps are kept by group.
Each taking part in the calculation of partial sums of the same MLO_N_LCL_OUT_MAPS output maps.
After completion of the main MLO_IN_LOOP loop partial sums have been summed up in parallel.

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              const __global _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{
    // KERNEL
    // private buffers
    __private _FLOAT in_stage[MLO_N_LCL_BATCHS][MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];
    __private _FLOAT wei_stage;
    __private _FLOAT out_tiles[MLO_N_LCL_BATCHS][MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    __local _FLOAT lcl_wei_stage[MLO_LCL_MEM_SZ];

#if MLO_N_MAPS_PERGROUP > 1
    __local _FLOAT* lcl_out_stage = lcl_wei_stage;

#endif

    uint lcl_id0       = get_local_id(0);
    int in_map_id      = 0;                    // map
    uint pix_id        = get_global_id(0);     // inside map
    in_map_id          = pix_id / MLO_MAP_SZ4; // mad id inside group
    uint out_grp_block = get_group_id(1);      // block of outputs for the entire group
    uint out_block     = out_grp_block;
    uint batch_block   = get_group_id(2); // block of batchs
                                          // multipe maps per group

    pix_id = (pix_id - in_map_id * MLO_MAP_SZ4); // pixel inside map

    uint in_map_off_id = (in_map_id >= MLO_N_MAPS_PERGROUP) ? MLO_N_MAPS_PERGROUP - 1 : in_map_id;

    uint in_off = batch_block * MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE +
                  in_map_off_id * MLO_IN_CHANNEL_STRIDE + pix_id * MLO_READ_UNIT;

    uint wei_off = out_grp_block * MLO_N_LCL_OUT_MAPS *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;
    for(uint j = 0; j < MLO_N_LCL_BATCHS; ++j)
    {
        for(uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
        {
            for(uint k = 0; k < MLO_READ_UNIT; ++k)
            {
                out_tiles[j][i][k] = 0;
            }
        }
    }
    // over all input maps; with step == MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP; MLO_IN_LOOP
    for(uint wc = 0; wc < MLO_IN_LOOP; ++wc)
    {
        // read array of weights
        barrier(CLK_LOCAL_MEM_FENCE);

        for(uint w = lcl_id0; w < MLO_WEIGHTS_LCL_SZ; w += MLO_GRP_SZ0)
        {

#if(MLO_WEIGHTS_ROW) & (MLO_WEIGHTS_ROW - 1)

            uint oi  = iDiv(w, MLO_WEIGHTS_ROW);
            uint lwi = iMod(w, oi, MLO_WEIGHTS_ROW);
#else
            uint oi  = (w / MLO_WEIGHTS_ROW);
            uint lwi = (w & (MLO_WEIGHTS_ROW - 1));
#endif

            uint wi = (wc * MLO_LCL_WEIGHTS_ROW + lwi) *
#if MLO_DIR_FORWARD == 1
                      MLO_WEI_CHANNEL_STRIDE;
#else
                      MLO_WEI_BSTRIDE;
#endif

            // out of range check
            uint wei_off1 = wei_off + wi +
                            oi *
#if MLO_DIR_FORWARD == 1
                                MLO_WEI_BSTRIDE;
#else
                                MLO_WEI_CHANNEL_STRIDE;
#endif

            uint wei_off_r   = (wei_off1 < MLO_N_OUTPUTS * MLO_N_INPUTS) ? wei_off1 : 0;
            _FLOAT wei_val   = wei_ptr[wei_off_r];
            wei_val          = (wei_off1 < MLO_N_OUTPUTS * MLO_N_INPUTS) ? wei_val : 0;
            lcl_wei_stage[w] = wei_val;
        }

        barrier(CLK_LOCAL_MEM_FENCE);
        uint c = 0;
#if MLO_WEIGHTS_PER_LOOP > 7
#pragma unroll(MLO_WEIGHTS_PER_LOOP / 8)
#endif
        for(uint ci = 0; ci < MLO_WEIGHTS_PER_LOOP;
            ++ci, in_off += MLO_IN_CHANNEL_STRIDE * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP)
        {
            c             = wc * MLO_WEIGHTS_PER_LOOP + ci;
            uint wei_indx = ci;

            // read data
            // over all local batchs
            uint in_off1 = in_off;
            uint in_off2 = 0;
            for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib, in_off1 += MLO_IN_BATCH_STRIDE)
            {
                in_off2 = in_off1;
                // lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP
                for(uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS;
                    ++ilc, in_off2 += MLO_IN_CHANNEL_STRIDE * MLO_N_MAPS_PERGROUP)
                {
                    bool v =
#if MLO_BATCH_ALIGNED == 0
                        (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ) &&
#endif
                        c * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP + in_map_id +
                                ilc * MLO_N_MAPS_PERGROUP <
                            MLO_N_INPUTS;
#ifndef __AMDGCN__
                    in_off2 = (v) ? in_off2 : 0;
#endif
                    __global const _FLOAT* in_p = &in_ptr[in_off2];
#if MLO_C1x1_PIXLEFT > 0
                    // if the last one
                    if(pix_id == MLO_MAP_SZ4 - 1)
                    {

                        for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                        {
#ifdef __AMDGCN__
                            in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
                            _FLOAT val           = in_p[i];
                            in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE
                            if(in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                    else

#endif
                    {
                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
#ifdef __AMDGCN__
                            in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
                            _FLOAT val           = in_p[i];
                            in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE

                            if(in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                }
            }

            // convolve
            for(uint olc         = 0,
                     lcl_wei_off = wei_indx * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP *
                                   MLO_WEI_CHANNEL_STRIDE;
                olc < MLO_N_LCL_OUT_MAPS;
                ++olc, lcl_wei_off += MLO_WEIGHTS_ROW)
            {
                // lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP, weights are
                // mapped accordingly
                uint lcl_wei_off1 = lcl_wei_off;
                for(uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS;
                    ++ilc, lcl_wei_off1 += MLO_N_MAPS_PERGROUP * MLO_WEI_CHANNEL_STRIDE)
                {
                    // read weights
                    uint lcl_wei_off2 =
                        lcl_wei_off1 + mul24(in_map_id, (int)MLO_WEI_CHANNEL_STRIDE);
                    wei_stage = lcl_wei_stage[lcl_wei_off2];
                    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
                    {
                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            out_tiles[ib][olc][i] += in_stage[ib][ilc][i] * wei_stage;

#if 0 // MLO_DIR_FORWARD == 0
							if ( in_stage[ib][ilc][i] * wei_stage!= 0 && out_grp_block * MLO_N_LCL_OUT_MAPS + olc == 0 && i == 0 && get_global_id(0) == 0 && get_global_id(1) == 0 && get_global_id(2) == 0)
							{
								printf("K:c: %d %d %d %d   %f %f %f %f\n",
								wc,
								MLO_IN_LOOP,
								MLO_WEIGHTS_PER_LOOP,
								MLO_WEIGHTS_ROW,

								out_tiles[ib][olc][i],
								in_stage[ib][ilc][i] * wei_stage,
								in_stage[ib][ilc][i],
								 wei_stage
								);
							}
#endif
                        }
                    }
                }
            }
        }
    }

    // out of range check
    if(in_map_id >= MLO_N_MAPS_PERGROUP || in_map_id * MLO_N_LCL_IN_MAPS >= MLO_N_INPUTS)
    {
        return;
    }

    out_block    = out_grp_block * MLO_N_LCL_OUT_MAPS;
    uint out_off = batch_block * MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE +
                   out_block * MLO_OUT_CHANNEL_STRIDE + pix_id * MLO_READ_UNIT;

// small groups
#if MLO_N_MAPS_PERGROUP > 1

    // calculate reduction over all partial sums
    // MLO_N_LCL_OUT_MAPS is multiple of MLO_EXCHANGE_STEP
    // write data into local memory

    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
    {
        for(uint t = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)
        {

            barrier(CLK_LOCAL_MEM_FENCE);

            if(lcl_id0 < MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP)
            {
                for(uint om = 0; om < MLO_EXCHANGE_STEP; ++om)
                {
                    uint lcl_off = (om * MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP +
                                    in_map_id * MLO_MAP_SZ4 + pix_id) *
                                   MLO_READ_UNIT;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        lcl_out_stage[lcl_off + i] = out_tiles[ib][t + om][i];
                    }
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            // sum partial sum
            // MLO_N_MAPS_PERGROUP >= MLO_EXCHANGE_STEP
            // in_map_id is an index of the output map now.
            if(in_map_id < MLO_EXCHANGE_STEP)
            {
                _FLOAT sum[MLO_READ_UNIT];
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    sum[i] = 0;
                }

                for(uint s = 0; s < MLO_N_MAPS_PERGROUP; ++s)
                {
                    uint imp     = in_map_id + s;
                    imp          = (imp >= MLO_N_MAPS_PERGROUP) ? imp - MLO_N_MAPS_PERGROUP : imp;
                    uint lcl_off = (in_map_id * MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP +
                                    imp * MLO_MAP_SZ4 + pix_id) *
                                   MLO_READ_UNIT;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        sum[i] += lcl_out_stage[lcl_off + i];
                    }
                }

                // write it out
                uint olc = t + in_map_id;

                if(true
#if MLO_BATCH_ALIGNED == 0
                   &&
                   (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
                   &&
                   out_block + olc < MLO_N_OUTPUTS
#endif
                   )
                {

                    uint out_off2 =
                        out_off + ib * MLO_OUT_BATCH_STRIDE + olc * MLO_OUT_CHANNEL_STRIDE;
                    __global _FLOAT* out_p = &out_ptr[out_off2];

#if MLO_CONV_BIAS
                    _FLOAT bias_val = 0;
                    bias_val        = bias[out_block * MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

                    // if the last one
                    if(pix_id == MLO_MAP_SZ4 - 1)
                    {
                        for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                        {
                            out_p[i] = sum[i]
#if MLO_CONV_BIAS
                                       + bias_val
#endif
                                ;

#if DBG_OUT_OF_RNGE
                            if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:out-of-range\n");
                            }
#endif
                        }
                    }
                    else
#endif
                    {

                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            out_p[i] = sum[i]
#if MLO_CONV_BIAS
                                       + bias_val
#endif
                                ;
#if DBG_OUT_OF_RNGE
                            if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:out-of-range\n");
                            }
#endif
                        }
                    }

                } // if (true

            } // if (in_map_id < MLO_EXCHANGE_STEP)

        } // for (int t = 0, p = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)

    } // 	for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)

#else // #if MLO_N_MAPS_PERGROUP > 1

    uint out_off1 = out_off;
    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib, out_off1 += MLO_OUT_BATCH_STRIDE)
    {

        uint out_off2 = out_off1;
        for(uint olc = 0; olc < MLO_N_LCL_OUT_MAPS; ++olc, out_off2 += MLO_OUT_CHANNEL_STRIDE)
        {
            __global _FLOAT* out_p = &out_ptr[out_off2];

            if(true
#if MLO_BATCH_ALIGNED == 0
               &&
               (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
               &&
               (out_block + olc < MLO_N_OUTPUTS)
#endif
                   )
            {
#if MLO_CONV_BIAS
                _FLOAT bias_val = 0;
                bias_val = bias[out_block * MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

                // if the last one
                if(pix_id == MLO_MAP_SZ4 - 1)
                {
                    for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                    {
                        out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
                                   + bias_val
#endif
                            ;
#if DBG_OUT_OF_RNGE
                        if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:out-of-range\n");
                        }
#endif
                    }
                }
                else
#endif
                {
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {

                        out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
                                   + bias_val
#endif
                            ;
#if DBG_OUT_OF_RNGE
                        if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:out-of-range\n");
                        }
#endif
                    }
                }
            }
        }
    }

#endif // #if MLO_N_MAPS_PERGROUP > 1
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

#if MLO_N_MAPS_PERGROUP > 1
#define MLO_WEIGHTS_PER_LOOP_MAX (8)
#else
#define MLO_WEIGHTS_PER_LOOP_MAX (16)
#endif
#if(MLO_N_MAPS_PERGROUP * MLO_N_LCL_IN_MAPS < MLO_N_INPUTS)
#define MLO_LCL_IN_ROW (MLO_N_MAPS_PERGROUP * MLO_N_LCL_IN_MAPS)
#else
#define MLO_LCL_IN_ROW (MLO_N_INPUTS)
#endif

#define MLO_WEIGHTS_PER_LOOP_TMP ((MLO_N_INPUTS + MLO_LCL_IN_ROW - 1) / MLO_LCL_IN_ROW)

#if(MLO_WEIGHTS_PER_LOOP_TMP < MLO_WEIGHTS_PER_LOOP_MAX)
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_TMP)
#else
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_MAX)
#endif

#define MLO_LCL_WEIGHTS_ROW (MLO_WEIGHTS_PER_LOOP * MLO_LCL_IN_ROW)

#define MLO_IN_LOOP ((MLO_N_INPUTS + MLO_LCL_WEIGHTS_ROW - 1) / MLO_LCL_WEIGHTS_ROW)

#define MLO_WEIGHTS_ROW (MLO_LCL_WEIGHTS_ROW * MLO_WEI_CHANNEL_STRIDE)

// size of the area for weights prefetch
#define MLO_WEIGHTS_LCL_SZ (MLO_WEIGHTS_ROW * MLO_N_LCL_OUT_MAPS)

// size of area for exchanging partial sums
#define MLO_EXCHNGE_SZ4 (MLO_MAP_SZ4 * MLO_EXCHANGE_STEP * MLO_N_MAPS_PERGROUP)

#if MLO_N_MAPS_PERGROUP > 1 && ((MLO_EXCHNGE_SZ4 * MLO_READ_UNIT) > MLO_WEIGHTS_LCL_SZ)
#define MLO_LCL_MEM_SZ (MLO_EXCHNGE_SZ4 * MLO_READ_UNIT)
#else
#define MLO_LCL_MEM_SZ MLO_WEIGHTS_LCL_SZ
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

/*
Layout:
assuming NCHW data layout.

Data:
data has been fetch by 4 floats sequentially.
MLO_MAP_SZ4 = (map_width*map_height + 3)/4.
in case of total size not a multiple of 4 the the last pixel has a special treatment.
There are 2 cases:
MLO_N_MAPS_PERGROUP == 1
and
MLO_N_MAPS_PERGROUP > 1, when MLO_MAP_SZ4 <= GPROUP_SIZE/2, in other words when more than 1 map can
be held by a group.
Case MLO_N_MAPS_PERGROUP == 1:
Data, by 4 floats, may come from MLO_N_LCL_IN_MAPS sequential input maps from MLO_N_LCL_BATCHS
neighboring batches.
Weigts:
on each MLO_WEIGHTS_PER_LOOP input loop set of weight are prefetched for another
MLO_WEIGHTS_PER_LOOP loops.
Each input map contributes to partial sums of MLO_N_LCL_OUT_MAPS output maps.
Case MLO_N_MAPS_PERGROUP > 1:
Similar to a previous case.
The difference is that several input sequential input maps are kept by group.
Each taking part in the calculation of partial sums of the same MLO_N_LCL_OUT_MAPS output maps.
After completion of the main MLO_IN_LOOP loop partial sums have been summed up in parallel.

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              const __global _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{
    // KERNEL
    // private buffers
    __private _FLOAT in_stage[MLO_N_LCL_BATCHS][MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];
    __private _FLOAT wei_stage;
    __private _FLOAT out_tiles[MLO_N_LCL_BATCHS][MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    __local _FLOAT lcl_wei_stage[MLO_LCL_MEM_SZ];

#if MLO_N_MAPS_PERGROUP > 1
    __local _FLOAT* lcl_out_stage = lcl_wei_stage;

#endif

    uint lcl_id0       = get_local_id(0);
    int in_map_id      = 0;                    // map
    uint pix_id        = get_global_id(0);     // inside map
    in_map_id          = pix_id / MLO_MAP_SZ4; // mad id inside group
    uint out_grp_block = get_group_id(1);      // block of outputs for the entire group
    uint out_block     = out_grp_block;
    uint batch_block   = get_group_id(2); // block of batchs
                                          // multipe maps per group

    pix_id = (pix_id - in_map_id * MLO_MAP_SZ4); // pixel inside map

    uint in_map_off_id = (in_map_id >= MLO_N_MAPS_PERGROUP) ? MLO_N_MAPS_PERGROUP - 1 : in_map_id;

    uint in_off = batch_block * MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE +
                  in_map_off_id * MLO_IN_CHANNEL_STRIDE + pix_id * MLO_READ_UNIT;

    uint wei_off = out_grp_block * MLO_N_LCL_OUT_MAPS *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;
    for(uint j = 0; j < MLO_N_LCL_BATCHS; ++j)
    {
        for(uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
        {
            for(uint k = 0; k < MLO_READ_UNIT; ++k)
            {
                out_tiles[j][i][k] = 0;
            }
        }
    }
    // over all input maps; with step == MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP; MLO_IN_LOOP
    for(uint wc = 0; wc < MLO_IN_LOOP; ++wc)
    {
        // read array of weights
        barrier(CLK_LOCAL_MEM_FENCE);

        for(uint w = lcl_id0; w < MLO_WEIGHTS_LCL_SZ; w += MLO_GRP_SZ0)
        {

#if(MLO_WEIGHTS_ROW) & (MLO_WEIGHTS_ROW - 1)

            uint oi  = iDiv(w, MLO_WEIGHTS_ROW);
            uint lwi = iMod(w, oi, MLO_WEIGHTS_ROW);
#else
            uint oi  = (w / MLO_WEIGHTS_ROW);
            uint lwi = (w & (MLO_WEIGHTS_ROW - 1));
#endif

            uint wi = (wc * MLO_LCL_WEIGHTS_ROW + lwi) *
#if MLO_DIR_FORWARD == 1
                      MLO_WEI_CHANNEL_STRIDE;
#else
                      MLO_WEI_BSTRIDE;
#endif

            // out of range check
            uint wei_off1 = wei_off + wi +
                            oi *
#if MLO_DIR_FORWARD == 1
                                MLO_WEI_BSTRIDE;
#else
                                MLO_WEI_CHANNEL_STRIDE;
#endif

            uint wei_off_r   = (wei_off1 < MLO_N_OUTPUTS * MLO_N_INPUTS) ? wei_off1 : 0;
            _FLOAT wei_val   = wei_ptr[wei_off_r];
            wei_val          = (wei_off1 < MLO_N_OUTPUTS * MLO_N_INPUTS) ? wei_val : 0;
            lcl_wei_stage[w] = wei_val;
        }

        barrier(CLK_LOCAL_MEM_FENCE);
        uint c = 0;
#if MLO_WEIGHTS_PER_LOOP > 7
#pragma unroll(MLO_WEIGHTS_PER_LOOP / 8)
#endif
        for(uint ci = 0; ci < MLO_WEIGHTS_PER_LOOP;
            ++ci, in_off += MLO_IN_CHANNEL_STRIDE * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP)
        {
            c             = wc * MLO_WEIGHTS_PER_LOOP + ci;
            uint wei_indx = ci;

            // read data
            // over all local batchs
            uint in_off1 = in_off;
            uint in_off2 = 0;
            for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib, in_off1 += MLO_IN_BATCH_STRIDE)
            {
                in_off2 = in_off1;
                // lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP
                for(uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS;
                    ++ilc, in_off2 += MLO_IN_CHANNEL_STRIDE * MLO_N_MAPS_PERGROUP)
                {
                    bool v =
#if MLO_BATCH_ALIGNED == 0
                        (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ) &&
#endif
                        c * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP + in_map_id +
                                ilc * MLO_N_MAPS_PERGROUP <
                            MLO_N_INPUTS;
#ifndef __AMDGCN__
                    in_off2 = (v) ? in_off2 : 0;
#endif
                    __global const _FLOAT* in_p = &in_ptr[in_off2];
#if MLO_C1x1_PIXLEFT > 0
                    // if the last one
                    if(pix_id == MLO_MAP_SZ4 - 1)
                    {

                        for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                        {
#ifdef __AMDGCN__
                            in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
                            _FLOAT val           = in_p[i];
                            in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE
                            if(in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range %d %d %d\n");
                            }
#endif
                        }
                    }
                    else

#endif
                    {
                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
#ifdef __AMDGCN__
                            in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
                            _FLOAT val           = in_p[i];
                            in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE

                            if(in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:in-of-range\n");
                            }
#endif
                        }
                    }
                }
            }

            // convolve
            for(uint olc         = 0,
                     lcl_wei_off = wei_indx * MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP *
                                   MLO_WEI_CHANNEL_STRIDE;
                olc < MLO_N_LCL_OUT_MAPS;
                ++olc, lcl_wei_off += MLO_WEIGHTS_ROW)
            {
                // lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP, weights are
                // mapped accordingly
                uint lcl_wei_off1 = lcl_wei_off;
                for(uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS;
                    ++ilc, lcl_wei_off1 += MLO_N_MAPS_PERGROUP * MLO_WEI_CHANNEL_STRIDE)
                {
                    // read weights
                    uint lcl_wei_off2 =
                        lcl_wei_off1 + mul24(in_map_id, (int)MLO_WEI_CHANNEL_STRIDE);
                    wei_stage = lcl_wei_stage[lcl_wei_off2];
                    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
                    {
                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            out_tiles[ib][olc][i] += in_stage[ib][ilc][i] * wei_stage;

#if 0 // MLO_DIR_FORWARD == 0
							if ( in_stage[ib][ilc][i] * wei_stage!= 0 && out_grp_block * MLO_N_LCL_OUT_MAPS + olc == 0 && i == 0 && get_global_id(0) == 0 && get_global_id(1) == 0 && get_global_id(2) == 0)
							{
								printf("K:c: %d %d %d %d   %f %f %f %f\n",
								wc,
								MLO_IN_LOOP,
								MLO_WEIGHTS_PER_LOOP,
								MLO_WEIGHTS_ROW,

								out_tiles[ib][olc][i],
								in_stage[ib][ilc][i] * wei_stage,
								in_stage[ib][ilc][i],
								 wei_stage
								);
							}
#endif
                        }
                    }
                }
            }
        }
    }

    // out of range check
    if(in_map_id >= MLO_N_MAPS_PERGROUP || in_map_id * MLO_N_LCL_IN_MAPS >= MLO_N_INPUTS)
    {
        return;
    }

    out_block    = out_grp_block * MLO_N_LCL_OUT_MAPS;
    uint out_off = batch_block * MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE +
                   out_block * MLO_OUT_CHANNEL_STRIDE + pix_id * MLO_READ_UNIT;

// small groups
#if MLO_N_MAPS_PERGROUP > 1

    // calculate reduction over all partial sums
    // MLO_N_LCL_OUT_MAPS is multiple of MLO_EXCHANGE_STEP
    // write data into local memory

    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
    {
        for(uint t = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)
        {

            barrier(CLK_LOCAL_MEM_FENCE);

            if(lcl_id0 < MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP)
            {
                for(uint om = 0; om < MLO_EXCHANGE_STEP; ++om)
                {
                    uint lcl_off = (om * MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP +
                                    in_map_id * MLO_MAP_SZ4 + pix_id) *
                                   MLO_READ_UNIT;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        lcl_out_stage[lcl_off + i] = out_tiles[ib][t + om][i];
                    }
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            // sum partial sum
            // MLO_N_MAPS_PERGROUP >= MLO_EXCHANGE_STEP
            // in_map_id is an index of the output map now.
            if(in_map_id < MLO_EXCHANGE_STEP)
            {
                _FLOAT sum[MLO_READ_UNIT];
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    sum[i] = 0;
                }

                for(uint s = 0; s < MLO_N_MAPS_PERGROUP; ++s)
                {
                    uint imp     = in_map_id + s;
                    imp          = (imp >= MLO_N_MAPS_PERGROUP) ? imp - MLO_N_MAPS_PERGROUP : imp;
                    uint lcl_off = (in_map_id * MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP +
                                    imp * MLO_MAP_SZ4 + pix_id) *
                                   MLO_READ_UNIT;
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {
                        sum[i] += lcl_out_stage[lcl_off + i];
                    }
                }

                // write it out
                uint olc = t + in_map_id;

                if(true
#if MLO_BATCH_ALIGNED == 0
                   &&
                   (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
                   &&
                   out_block + olc < MLO_N_OUTPUTS
#endif
                   )
                {

                    uint out_off2 =
                        out_off + ib * MLO_OUT_BATCH_STRIDE + olc * MLO_OUT_CHANNEL_STRIDE;
                    __global _FLOAT* out_p = &out_ptr[out_off2];

#if MLO_CONV_BIAS
                    _FLOAT bias_val = 0;
                    bias_val        = bias[out_block * MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

                    // if the last one
                    if(pix_id == MLO_MAP_SZ4 - 1)
                    {
                        for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                        {
                            out_p[i] = sum[i]
#if MLO_CONV_BIAS
                                       + bias_val
#endif
                                ;

#if DBG_OUT_OF_RNGE
                            if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:out-of-range\n");
                            }
#endif
                        }
                    }
                    else
#endif
                    {

                        for(uint i = 0; i < MLO_READ_UNIT; ++i)
                        {
                            out_p[i] = sum[i]
#if MLO_CONV_BIAS
                                       + bias_val
#endif
                                ;
#if DBG_OUT_OF_RNGE
                            if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                            {
                                printf("k:err:out-of-range\n");
                            }
#endif
                        }
                    }

                } // if (true

            } // if (in_map_id < MLO_EXCHANGE_STEP)

        } // for (int t = 0, p = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)

    } // 	for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)

#else // #if MLO_N_MAPS_PERGROUP > 1

    uint out_off1 = out_off;
    for(uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib, out_off1 += MLO_OUT_BATCH_STRIDE)
    {

        uint out_off2 = out_off1;
        for(uint olc = 0; olc < MLO_N_LCL_OUT_MAPS; ++olc, out_off2 += MLO_OUT_CHANNEL_STRIDE)
        {
            __global _FLOAT* out_p = &out_ptr[out_off2];

            if(true
#if MLO_BATCH_ALIGNED == 0
               &&
               (batch_block * MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
               &&
               (out_block + olc < MLO_N_OUTPUTS)
#endif
                   )
            {
#if MLO_CONV_BIAS
                _FLOAT bias_val = 0;
                bias_val = bias[out_block * MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

                // if the last one
                if(pix_id == MLO_MAP_SZ4 - 1)
                {
                    for(uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
                    {
                        out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
                                   + bias_val
#endif
                            ;
#if DBG_OUT_OF_RNGE
                        if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:out-of-range\n");
                        }
#endif
                    }
                }
                else
#endif
                {
                    for(uint i = 0; i < MLO_READ_UNIT; ++i)
                    {

                        out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
                                   + bias_val
#endif
                            ;
#if DBG_OUT_OF_RNGE
                        if(out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
                        {
                            printf("k:err:out-of-range\n");
                        }
#endif
                    }
                }
            }
        }
    }

#endif // #if MLO_N_MAPS_PERGROUP > 1
}


Optimize pyrlk.

Optimize pyrlk.

1. Remove uncessary index calculation.
2. Use mad/mad24 as possible.

Signed-off-by: Yan Wang <8ca54580088a7ffe1dd243d1c16c6d1f04463d55@linux.intel.com>

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Dachuan Zhao, dachuan@multicorewareinc.com
//    Yao Wang, bitwangyaoyao@gmail.com
//    Xiaopeng Fu, fuxiaopeng2222@163.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#define GRIDSIZE    3
#define LSx 8
#define LSy 8
// defeine local memory sizes
#define LM_W (LSx*GRIDSIZE+2)
#define LM_H (LSy*GRIDSIZE+2)
#define BUFFER  (LSx*LSy)
#define BUFFER2 BUFFER>>1
#ifndef WAVE_SIZE
#define WAVE_SIZE 1
#endif

#ifdef CPU

inline void reduce3(float val1, float val2, float val3,  __local float* smem1,  __local float* smem2,  __local float* smem3, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    smem3[tid] = val3;
    barrier(CLK_LOCAL_MEM_FENCE);

    for(int i = BUFFER2; i > 0; i >>= 1)
    {
        if(tid < i)
        {
            smem1[tid] += smem1[tid + i];
            smem2[tid] += smem2[tid + i];
            smem3[tid] += smem3[tid + i];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

inline void reduce2(float val1, float val2, volatile __local float* smem1, volatile __local float* smem2, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    barrier(CLK_LOCAL_MEM_FENCE);

    for(int i = BUFFER2; i > 0; i >>= 1)
    {
        if(tid < i)
        {
            smem1[tid] += smem1[tid + i];
            smem2[tid] += smem2[tid + i];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

inline void reduce1(float val1, volatile __local float* smem1, int tid)
{
    smem1[tid] = val1;
    barrier(CLK_LOCAL_MEM_FENCE);

    for(int i = BUFFER2; i > 0; i >>= 1)
    {
        if(tid < i)
        {
            smem1[tid] += smem1[tid + i];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}
#else
inline void reduce3(float val1, float val2, float val3,
             __local volatile float* smem1, __local volatile float* smem2, __local volatile float* smem3, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    smem3[tid] = val3;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (tid < 32)
    {
        smem1[tid] += smem1[tid + 32];
        smem2[tid] += smem2[tid + 32];
        smem3[tid] += smem3[tid + 32];
#if WAVE_SIZE < 32
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid < 16)
    {
#endif
        smem1[tid] += smem1[tid + 16];
        smem2[tid] += smem2[tid + 16];
        smem3[tid] += smem3[tid + 16];
#if WAVE_SIZE <16
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid<1)
    {
#endif
        local float8* m1 = (local float8*)smem1;
        local float8* m2 = (local float8*)smem2;
        local float8* m3 = (local float8*)smem3;
        float8 t1 = m1[0]+m1[1];
        float8 t2 = m2[0]+m2[1];
        float8 t3 = m3[0]+m3[1];
        float4 t14 = t1.lo + t1.hi;
        float4 t24 = t2.lo + t2.hi;
        float4 t34 = t3.lo + t3.hi;
        smem1[0] = t14.x+t14.y+t14.z+t14.w;
        smem2[0] = t24.x+t24.y+t24.z+t24.w;
        smem3[0] = t34.x+t34.y+t34.z+t34.w;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}

inline void reduce2(float val1, float val2, __local volatile float* smem1, __local volatile float* smem2, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (tid < 32)
    {
        smem1[tid] += smem1[tid + 32];
        smem2[tid] += smem2[tid + 32];
#if WAVE_SIZE < 32
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid < 16)
    {
#endif
        smem1[tid] += smem1[tid + 16];
        smem2[tid] += smem2[tid + 16];
#if WAVE_SIZE <16
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid<1)
    {
#endif
        local float8* m1 = (local float8*)smem1;
        local float8* m2 = (local float8*)smem2;
        float8 t1 = m1[0]+m1[1];
        float8 t2 = m2[0]+m2[1];
        float4 t14 = t1.lo + t1.hi;
        float4 t24 = t2.lo + t2.hi;
        smem1[0] = t14.x+t14.y+t14.z+t14.w;
        smem2[0] = t24.x+t24.y+t24.z+t24.w;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}

inline void reduce1(float val1, __local volatile float* smem1, int tid)
{
    smem1[tid] = val1;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (tid < 32)
    {
        smem1[tid] += smem1[tid + 32];
#if WAVE_SIZE < 32
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid < 16)
    {
#endif
        smem1[tid] += smem1[tid + 16];
#if WAVE_SIZE <16
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid<1)
    {
#endif
        local float8* m1 = (local float8*)smem1;
        float8 t1 = m1[0]+m1[1];
        float4 t14 = t1.lo + t1.hi;
        smem1[0] = t14.x+t14.y+t14.z+t14.w;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#define SCALE (1.0f / (1 << 20))
#define	THRESHOLD	0.01f

// Image read mode
__constant sampler_t sampler    = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_LINEAR;

// macro to get pixel value from local memory

#define VAL(_y,_x,_yy,_xx)    (IPatchLocal[mad24(((_y) + (_yy)), LM_W, ((_x) + (_xx)))])
inline void SetPatch(local float* IPatchLocal, int TileY, int TileX,
              float* Pch, float* Dx, float* Dy,
              float* A11, float* A12, float* A22, float w)
{
    int xid=get_local_id(0);
    int yid=get_local_id(1);
    int xBase = mad24(TileX, LSx, (xid + 1));
    int yBase = mad24(TileY, LSy, (yid + 1));

    *Pch = VAL(yBase,xBase,0,0);

    *Dx = mad((VAL(yBase,xBase,-1,1) + VAL(yBase,xBase,+1,1) - VAL(yBase,xBase,-1,-1) - VAL(yBase,xBase,+1,-1)), 3.0f, (VAL(yBase,xBase,0,1) - VAL(yBase,xBase,0,-1)) * 10.0f) * w;
    *Dy = mad((VAL(yBase,xBase,1,-1) + VAL(yBase,xBase,1,+1) - VAL(yBase,xBase,-1,-1) - VAL(yBase,xBase,-1,+1)), 3.0f, (VAL(yBase,xBase,1,0) - VAL(yBase,xBase,-1,0)) * 10.0f) * w;

    *A11 = mad(*Dx, *Dx, *A11);
    *A12 = mad(*Dx, *Dy, *A12);
    *A22 = mad(*Dy, *Dy, *A22);
}
#undef VAL

inline void GetPatch(image2d_t J, float x, float y,
              float* Pch, float* Dx, float* Dy,
              float* b1, float* b2)
{
    float J_val = read_imagef(J, sampler, (float2)(x, y)).x;
    float diff = (J_val - *Pch) * 32.0f;
    *b1 = mad(diff, *Dx, *b1);
    *b2 = mad(diff, *Dy, *b2);
}

inline void GetError(image2d_t J, const float x, const float y, const float* Pch, float* errval)
{
    float diff = read_imagef(J, sampler, (float2)(x,y)).x-*Pch;
    *errval += fabs(diff);
}

//macro to read pixel value into local memory.
#define READI(_y,_x) IPatchLocal[mad24(mad24((_y), LSy, yid), LM_W, mad24((_x), LSx, xid))] = read_imagef(I, sampler, (float2)(mad((_x), LSx, Point.x + xid - 0.5f), mad((_y), LSy, Point.y + yid - 0.5f))).x;
void ReadPatchIToLocalMem(image2d_t I, float2 Point, local float* IPatchLocal)
{
    int xid=get_local_id(0);
    int yid=get_local_id(1);
    //read (3*LSx)*(3*LSy) window. each macro call read LSx*LSy pixels block
    READI(0,0);READI(0,1);READI(0,2);
    READI(1,0);READI(1,1);READI(1,2);
    READI(2,0);READI(2,1);READI(2,2);
    if(xid<2)
    {// read last 2 columns border. each macro call reads 2*LSy pixels block
        READI(0,3);
        READI(1,3);
        READI(2,3);
    }

    if(yid<2)
    {// read last 2 row. each macro call reads LSx*2 pixels block
        READI(3,0);READI(3,1);READI(3,2);
    }

    if(yid<2 && xid<2)
    {// read right bottom 2x2 corner. one macro call reads 2*2 pixels block
        READI(3,3);
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}
#undef READI

__attribute__((reqd_work_group_size(LSx, LSy, 1)))
__kernel void lkSparse(image2d_t I, image2d_t J,
                       __global const float2* prevPts, __global float2* nextPts, __global uchar* status, __global float* err,
                       const int level, const int rows, const int cols, int PATCH_X, int PATCH_Y, int c_winSize_x, int c_winSize_y, int c_iters, char calcErr)
{
    __local float smem1[BUFFER];
    __local float smem2[BUFFER];
    __local float smem3[BUFFER];

    int xid=get_local_id(0);
    int yid=get_local_id(1);
    int gid=get_group_id(0);
    int xsize=get_local_size(0);
    int ysize=get_local_size(1);
    int k;
    int xBase = mad24(xsize, 2, xid);
    int yBase = mad24(ysize, 2, yid);
    float wx = (xBase < c_winSize_x) ? 1 : 0;
    float wy = (yBase < c_winSize_y) ? 1 : 0;

    float2 c_halfWin = (float2)((c_winSize_x - 1)>>1, (c_winSize_y - 1)>>1);

    const int tid = mad24(yid, xsize, xid);

    float2 prevPt = prevPts[gid] / (float2)(1 << level);

    if (prevPt.x < 0 || prevPt.x >= cols || prevPt.y < 0 || prevPt.y >= rows)
    {
        if (tid == 0 && level == 0)
        {
            status[gid] = 0;
        }

        return;
    }
    prevPt -= c_halfWin;

    // extract the patch from the first image, compute covariation matrix of derivatives

    float A11 = 0;
    float A12 = 0;
    float A22 = 0;

    float I_patch[GRIDSIZE][GRIDSIZE];
    float dIdx_patch[GRIDSIZE][GRIDSIZE];
    float dIdy_patch[GRIDSIZE][GRIDSIZE];

    // local memory to read image with border to calc sobels
    local float IPatchLocal[LM_W*LM_H];
    ReadPatchIToLocalMem(I,prevPt,IPatchLocal);

    {
        SetPatch(IPatchLocal, 0, 0,
                 &I_patch[0][0], &dIdx_patch[0][0], &dIdy_patch[0][0],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 0, 1,
                 &I_patch[0][1], &dIdx_patch[0][1], &dIdy_patch[0][1],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 0, 2,
                    &I_patch[0][2], &dIdx_patch[0][2], &dIdy_patch[0][2],
                    &A11, &A12, &A22,wx);
    }
    {
        SetPatch(IPatchLocal, 1, 0,
                 &I_patch[1][0], &dIdx_patch[1][0], &dIdy_patch[1][0],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 1,1,
                 &I_patch[1][1], &dIdx_patch[1][1], &dIdy_patch[1][1],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 1,2,
                    &I_patch[1][2], &dIdx_patch[1][2], &dIdy_patch[1][2],
                    &A11, &A12, &A22,wx);
    }
    {
        SetPatch(IPatchLocal, 2,0,
                 &I_patch[2][0], &dIdx_patch[2][0], &dIdy_patch[2][0],
                 &A11, &A12, &A22,wy);

        SetPatch(IPatchLocal, 2,1,
                 &I_patch[2][1], &dIdx_patch[2][1], &dIdy_patch[2][1],
                 &A11, &A12, &A22,wy);

        SetPatch(IPatchLocal, 2,2,
                    &I_patch[2][2], &dIdx_patch[2][2], &dIdy_patch[2][2],
                    &A11, &A12, &A22,wx*wy);
    }

    reduce3(A11, A12, A22, smem1, smem2, smem3, tid);

    A11 = smem1[0];
    A12 = smem2[0];
    A22 = smem3[0];
    barrier(CLK_LOCAL_MEM_FENCE);

    float D = mad(A11, A22, - A12 * A12);

    if (D < 1.192092896e-07f)
    {
        if (tid == 0 && level == 0)
            status[gid] = 0;

        return;
    }

    A11 /= D;
    A12 /= D;
    A22 /= D;

    prevPt = mad(nextPts[gid], 2.0f, - c_halfWin);

    float2 offset0 = (float2)(xid + 0.5f, yid + 0.5f);
    float2 offset1 = (float2)(xsize, ysize);
    float2 loc0 = prevPt + offset0;
    float2 loc1 = loc0 + offset1;
    float2 loc2 = loc1 + offset1;

    for (k = 0; k < c_iters; ++k)
    {
        if (prevPt.x < -c_halfWin.x || prevPt.x >= cols || prevPt.y < -c_halfWin.y || prevPt.y >= rows)
        {
            if (tid == 0 && level == 0)
                status[gid] = 0;
            break;
        }
        float b1 = 0;
        float b2 = 0;

        {
            GetPatch(J, loc0.x, loc0.y,
                     &I_patch[0][0], &dIdx_patch[0][0], &dIdy_patch[0][0],
                     &b1, &b2);

            GetPatch(J, loc1.x, loc0.y,
                     &I_patch[0][1], &dIdx_patch[0][1], &dIdy_patch[0][1],
                     &b1, &b2);

            GetPatch(J, loc2.x, loc0.y,
                        &I_patch[0][2], &dIdx_patch[0][2], &dIdy_patch[0][2],
                        &b1, &b2);
        }
        {
            GetPatch(J, loc0.x, loc1.y,
                     &I_patch[1][0], &dIdx_patch[1][0], &dIdy_patch[1][0],
                     &b1, &b2);

            GetPatch(J, loc1.x, loc1.y,
                     &I_patch[1][1], &dIdx_patch[1][1], &dIdy_patch[1][1],
                     &b1, &b2);

            GetPatch(J, loc2.x, loc1.y,
                        &I_patch[1][2], &dIdx_patch[1][2], &dIdy_patch[1][2],
                        &b1, &b2);
        }
        {
            GetPatch(J, loc0.x, loc2.y,
                     &I_patch[2][0], &dIdx_patch[2][0], &dIdy_patch[2][0],
                     &b1, &b2);

            GetPatch(J, loc1.x, loc2.y,
                     &I_patch[2][1], &dIdx_patch[2][1], &dIdy_patch[2][1],
                     &b1, &b2);

            GetPatch(J, loc2.x, loc2.y,
                        &I_patch[2][2], &dIdx_patch[2][2], &dIdy_patch[2][2],
                        &b1, &b2);
        }

        reduce2(b1, b2, smem1, smem2, tid);

        b1 = smem1[0];
        b2 = smem2[0];
        barrier(CLK_LOCAL_MEM_FENCE);

        float2 delta;
        delta.x = mad(A12, b2, - A22 * b1);
        delta.y = mad(A12, b1, - A11 * b2);

        prevPt += delta;
        loc0 += delta;
        loc1 += delta;
        loc2 += delta;

        if (fabs(delta.x) < THRESHOLD && fabs(delta.y) < THRESHOLD)
            break;
    }

    D = 0.0f;
    if (calcErr)
    {
        {
            GetError(J, loc0.x, loc0.y, &I_patch[0][0], &D);
            GetError(J, loc1.x, loc0.y, &I_patch[0][1], &D);
        }
        {
            GetError(J, loc0.x, loc1.y, &I_patch[1][0], &D);
            GetError(J, loc1.x, loc1.y, &I_patch[1][1], &D);
        }
        if(xBase < c_winSize_x)
        {
            GetError(J, loc2.x, loc0.y, &I_patch[0][2], &D);
            GetError(J, loc2.x, loc1.y, &I_patch[1][2], &D);
        }
        if(yBase < c_winSize_y)
        {
            GetError(J, loc0.x, loc2.y, &I_patch[2][0], &D);
            GetError(J, loc1.x, loc2.y, &I_patch[2][1], &D);
            if(xBase < c_winSize_x)
                GetError(J, loc2.x, loc2.y, &I_patch[2][2], &D);
        }

        reduce1(D, smem1, tid);
    }

    if (tid == 0)
    {
        prevPt += c_halfWin;

        nextPts[gid] = prevPt;

        if (calcErr)
            err[gid] = smem1[0] / (float)(c_winSize_x * c_winSize_y);
    }
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Dachuan Zhao, dachuan@multicorewareinc.com
//    Yao Wang, bitwangyaoyao@gmail.com
//    Xiaopeng Fu, fuxiaopeng2222@163.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#define GRIDSIZE    3
#define LSx 8
#define LSy 8
// defeine local memory sizes
#define LM_W (LSx*GRIDSIZE+2)
#define LM_H (LSy*GRIDSIZE+2)
#define BUFFER  (LSx*LSy)
#define BUFFER2 BUFFER>>1
#ifndef WAVE_SIZE
#define WAVE_SIZE 1
#endif

#ifdef CPU

inline void reduce3(float val1, float val2, float val3,  __local float* smem1,  __local float* smem2,  __local float* smem3, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    smem3[tid] = val3;
    barrier(CLK_LOCAL_MEM_FENCE);

    for(int i = BUFFER2; i > 0; i >>= 1)
    {
        if(tid < i)
        {
            smem1[tid] += smem1[tid + i];
            smem2[tid] += smem2[tid + i];
            smem3[tid] += smem3[tid + i];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

inline void reduce2(float val1, float val2, volatile __local float* smem1, volatile __local float* smem2, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    barrier(CLK_LOCAL_MEM_FENCE);

    for(int i = BUFFER2; i > 0; i >>= 1)
    {
        if(tid < i)
        {
            smem1[tid] += smem1[tid + i];
            smem2[tid] += smem2[tid + i];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

inline void reduce1(float val1, volatile __local float* smem1, int tid)
{
    smem1[tid] = val1;
    barrier(CLK_LOCAL_MEM_FENCE);

    for(int i = BUFFER2; i > 0; i >>= 1)
    {
        if(tid < i)
        {
            smem1[tid] += smem1[tid + i];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}
#else
inline void reduce3(float val1, float val2, float val3,
             __local volatile float* smem1, __local volatile float* smem2, __local volatile float* smem3, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    smem3[tid] = val3;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (tid < 32)
    {
        smem1[tid] += smem1[tid + 32];
        smem2[tid] += smem2[tid + 32];
        smem3[tid] += smem3[tid + 32];
#if WAVE_SIZE < 32
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid < 16)
    {
#endif
        smem1[tid] += smem1[tid + 16];
        smem2[tid] += smem2[tid + 16];
        smem3[tid] += smem3[tid + 16];
#if WAVE_SIZE <16
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid<1)
    {
#endif
        local float8* m1 = (local float8*)smem1;
        local float8* m2 = (local float8*)smem2;
        local float8* m3 = (local float8*)smem3;
        float8 t1 = m1[0]+m1[1];
        float8 t2 = m2[0]+m2[1];
        float8 t3 = m3[0]+m3[1];
        float4 t14 = t1.lo + t1.hi;
        float4 t24 = t2.lo + t2.hi;
        float4 t34 = t3.lo + t3.hi;
        smem1[0] = t14.x+t14.y+t14.z+t14.w;
        smem2[0] = t24.x+t24.y+t24.z+t24.w;
        smem3[0] = t34.x+t34.y+t34.z+t34.w;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}

inline void reduce2(float val1, float val2, __local volatile float* smem1, __local volatile float* smem2, int tid)
{
    smem1[tid] = val1;
    smem2[tid] = val2;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (tid < 32)
    {
        smem1[tid] += smem1[tid + 32];
        smem2[tid] += smem2[tid + 32];
#if WAVE_SIZE < 32
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid < 16)
    {
#endif
        smem1[tid] += smem1[tid + 16];
        smem2[tid] += smem2[tid + 16];
#if WAVE_SIZE <16
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid<1)
    {
#endif
        local float8* m1 = (local float8*)smem1;
        local float8* m2 = (local float8*)smem2;
        float8 t1 = m1[0]+m1[1];
        float8 t2 = m2[0]+m2[1];
        float4 t14 = t1.lo + t1.hi;
        float4 t24 = t2.lo + t2.hi;
        smem1[0] = t14.x+t14.y+t14.z+t14.w;
        smem2[0] = t24.x+t24.y+t24.z+t24.w;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}

inline void reduce1(float val1, __local volatile float* smem1, int tid)
{
    smem1[tid] = val1;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (tid < 32)
    {
        smem1[tid] += smem1[tid + 32];
#if WAVE_SIZE < 32
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid < 16)
    {
#endif
        smem1[tid] += smem1[tid + 16];
#if WAVE_SIZE <16
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (tid<1)
    {
#endif
        local float8* m1 = (local float8*)smem1;
        float8 t1 = m1[0]+m1[1];
        float4 t14 = t1.lo + t1.hi;
        smem1[0] = t14.x+t14.y+t14.z+t14.w;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#define SCALE (1.0f / (1 << 20))
#define	THRESHOLD	0.01f

// Image read mode
__constant sampler_t sampler    = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_LINEAR;

// macro to get pixel value from local memory

#define VAL(_y,_x,_yy,_xx)    (IPatchLocal[(yid+((_y)*LSy)+1+(_yy))*LM_W+(xid+((_x)*LSx)+1+(_xx))])
inline void SetPatch(local float* IPatchLocal, int TileY, int TileX,
              float* Pch, float* Dx, float* Dy,
              float* A11, float* A12, float* A22, float w)
{
    unsigned int xid=get_local_id(0);
    unsigned int yid=get_local_id(1);
    *Pch = VAL(TileY,TileX,0,0);

    float dIdx = (3.0f*VAL(TileY,TileX,-1,1)+10.0f*VAL(TileY,TileX,0,1)+3.0f*VAL(TileY,TileX,+1,1))-(3.0f*VAL(TileY,TileX,-1,-1)+10.0f*VAL(TileY,TileX,0,-1)+3.0f*VAL(TileY,TileX,+1,-1));
    float dIdy = (3.0f*VAL(TileY,TileX,1,-1)+10.0f*VAL(TileY,TileX,1,0)+3.0f*VAL(TileY,TileX,1,+1))-(3.0f*VAL(TileY,TileX,-1,-1)+10.0f*VAL(TileY,TileX,-1,0)+3.0f*VAL(TileY,TileX,-1,+1));

    dIdx *= w;
    dIdy *= w;

    *Dx = dIdx;
    *Dy = dIdy;

    *A11 += dIdx * dIdx;
    *A12 += dIdx * dIdy;
    *A22 += dIdy * dIdy;
}
#undef VAL

inline void GetPatch(image2d_t J, float x, float y,
              float* Pch, float* Dx, float* Dy,
              float* b1, float* b2)
{
    float J_val = read_imagef(J, sampler, (float2)(x, y)).x;
    float diff = (J_val - *Pch) * 32.0f;
    *b1 += diff**Dx;
    *b2 += diff**Dy;
}

inline void GetError(image2d_t J, const float x, const float y, const float* Pch, float* errval)
{
    float diff = read_imagef(J, sampler, (float2)(x,y)).x-*Pch;
    *errval += fabs(diff);
}

//macro to read pixel value into local memory.
#define READI(_y,_x)    IPatchLocal[(yid+((_y)*LSy))*LM_W+(xid+((_x)*LSx))] = read_imagef(I, sampler, (float2)(Point.x + xid+(_x)*LSx + 0.5f-1, Point.y + yid+(_y)*LSy+ 0.5f-1)).x;
void ReadPatchIToLocalMem(image2d_t I, float2 Point, local float* IPatchLocal)
{
    unsigned int xid=get_local_id(0);
    unsigned int yid=get_local_id(1);
    //read (3*LSx)*(3*LSy) window. each macro call read LSx*LSy pixels block
    READI(0,0);READI(0,1);READI(0,2);
    READI(1,0);READI(1,1);READI(1,2);
    READI(2,0);READI(2,1);READI(2,2);
    if(xid<2)
    {// read last 2 columns border. each macro call reads 2*LSy pixels block
        READI(0,3);
        READI(1,3);
        READI(2,3);
    }

    if(yid<2)
    {// read last 2 row. each macro call reads LSx*2 pixels block
        READI(3,0);READI(3,1);READI(3,2);
    }

    if(yid<2 && xid<2)
    {// read right bottom 2x2 corner. one macro call reads 2*2 pixels block
        READI(3,3);
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}
#undef READI

__attribute__((reqd_work_group_size(LSx, LSy, 1)))
__kernel void lkSparse(image2d_t I, image2d_t J,
                       __global const float2* prevPts, __global float2* nextPts, __global uchar* status, __global float* err,
                       const int level, const int rows, const int cols, int PATCH_X, int PATCH_Y, int c_winSize_x, int c_winSize_y, int c_iters, char calcErr)
{
    __local float smem1[BUFFER];
    __local float smem2[BUFFER];
    __local float smem3[BUFFER];

    unsigned int xid=get_local_id(0);
    unsigned int yid=get_local_id(1);
    unsigned int gid=get_group_id(0);
    unsigned int xsize=get_local_size(0);
    unsigned int ysize=get_local_size(1);
    int xBase, yBase, k;
    float wx = ((xid+2*xsize)<c_winSize_x)?1:0;
    float wy = ((yid+2*ysize)<c_winSize_y)?1:0;

    float2 c_halfWin = (float2)((c_winSize_x - 1)>>1, (c_winSize_y - 1)>>1);

    const int tid = mad24(yid, xsize, xid);

    float2 prevPt = prevPts[gid] / (float2)(1 << level);

    if (prevPt.x < 0 || prevPt.x >= cols || prevPt.y < 0 || prevPt.y >= rows)
    {
        if (tid == 0 && level == 0)
        {
            status[gid] = 0;
        }

        return;
    }
    prevPt -= c_halfWin;

    // extract the patch from the first image, compute covariation matrix of derivatives

    float A11 = 0;
    float A12 = 0;
    float A22 = 0;

    float I_patch[GRIDSIZE][GRIDSIZE];
    float dIdx_patch[GRIDSIZE][GRIDSIZE];
    float dIdy_patch[GRIDSIZE][GRIDSIZE];

    // local memory to read image with border to calc sobels
    local float IPatchLocal[LM_W*LM_H];
    ReadPatchIToLocalMem(I,prevPt,IPatchLocal);

    {
        SetPatch(IPatchLocal, 0, 0,
                 &I_patch[0][0], &dIdx_patch[0][0], &dIdy_patch[0][0],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 0, 1,
                 &I_patch[0][1], &dIdx_patch[0][1], &dIdy_patch[0][1],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 0, 2,
                    &I_patch[0][2], &dIdx_patch[0][2], &dIdy_patch[0][2],
                    &A11, &A12, &A22,wx);
    }
    {
        SetPatch(IPatchLocal, 1, 0,
                 &I_patch[1][0], &dIdx_patch[1][0], &dIdy_patch[1][0],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 1,1,
                 &I_patch[1][1], &dIdx_patch[1][1], &dIdy_patch[1][1],
                 &A11, &A12, &A22,1);

        SetPatch(IPatchLocal, 1,2,
                    &I_patch[1][2], &dIdx_patch[1][2], &dIdy_patch[1][2],
                    &A11, &A12, &A22,wx);
    }
    {
        SetPatch(IPatchLocal, 2,0,
                 &I_patch[2][0], &dIdx_patch[2][0], &dIdy_patch[2][0],
                 &A11, &A12, &A22,wy);

        SetPatch(IPatchLocal, 2,1,
                 &I_patch[2][1], &dIdx_patch[2][1], &dIdy_patch[2][1],
                 &A11, &A12, &A22,wy);

        SetPatch(IPatchLocal, 2,2,
                    &I_patch[2][2], &dIdx_patch[2][2], &dIdy_patch[2][2],
                    &A11, &A12, &A22,wx*wy);
    }

    reduce3(A11, A12, A22, smem1, smem2, smem3, tid);

    A11 = smem1[0];
    A12 = smem2[0];
    A22 = smem3[0];
    barrier(CLK_LOCAL_MEM_FENCE);

    float D = A11 * A22 - A12 * A12;

    if (D < 1.192092896e-07f)
    {
        if (tid == 0 && level == 0)
            status[gid] = 0;

        return;
    }

    A11 /= D;
    A12 /= D;
    A22 /= D;

    prevPt = nextPts[gid] * 2.0f - c_halfWin;

    for (k = 0; k < c_iters; ++k)
    {
        if (prevPt.x < -c_halfWin.x || prevPt.x >= cols || prevPt.y < -c_halfWin.y || prevPt.y >= rows)
        {
            if (tid == 0 && level == 0)
                status[gid] = 0;
            break;
        }
        float b1 = 0;
        float b2 = 0;

        yBase=yid;
        {
            xBase=xid;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[0][0], &dIdx_patch[0][0], &dIdy_patch[0][0],
                     &b1, &b2);

            xBase+=xsize;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[0][1], &dIdx_patch[0][1], &dIdy_patch[0][1],
                     &b1, &b2);

            xBase+=xsize;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                        &I_patch[0][2], &dIdx_patch[0][2], &dIdy_patch[0][2],
                        &b1, &b2);
        }
        yBase+=ysize;
        {
            xBase=xid;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[1][0], &dIdx_patch[1][0], &dIdy_patch[1][0],
                     &b1, &b2);

            xBase+=xsize;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[1][1], &dIdx_patch[1][1], &dIdy_patch[1][1],
                     &b1, &b2);

            xBase+=xsize;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                        &I_patch[1][2], &dIdx_patch[1][2], &dIdy_patch[1][2],
                        &b1, &b2);
        }
        yBase+=ysize;
        {
            xBase=xid;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[2][0], &dIdx_patch[2][0], &dIdy_patch[2][0],
                     &b1, &b2);

            xBase+=xsize;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[2][1], &dIdx_patch[2][1], &dIdy_patch[2][1],
                     &b1, &b2);

            xBase+=xsize;
            GetPatch(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                        &I_patch[2][2], &dIdx_patch[2][2], &dIdy_patch[2][2],
                        &b1, &b2);
        }

        reduce2(b1, b2, smem1, smem2, tid);

        b1 = smem1[0];
        b2 = smem2[0];
        barrier(CLK_LOCAL_MEM_FENCE);

        float2 delta;
        delta.x = A12 * b2 - A22 * b1;
        delta.y = A12 * b1 - A11 * b2;

        prevPt += delta;

        if (fabs(delta.x) < THRESHOLD && fabs(delta.y) < THRESHOLD)
            break;
    }

    D = 0.0f;
    if (calcErr)
    {
        yBase=yid;
        {
            xBase=xid;
            GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[0][0], &D);

            xBase+=xsize;
            GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[0][1], &D);

            xBase+=xsize;
            if(xBase<c_winSize_x)
                GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                         &I_patch[0][2], &D);
        }
        yBase+=ysize;
        {
            xBase=xid;
            GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[1][0], &D);

            xBase+=xsize;
            GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[1][1], &D);

            xBase+=xsize;
            if(xBase<c_winSize_x)
                GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                         &I_patch[1][2], &D);
        }
        yBase+=ysize;
        if(yBase<c_winSize_y)
        {
            xBase=xid;
            GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[2][0], &D);

            xBase+=xsize;
            GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                     &I_patch[2][1], &D);

            xBase+=xsize;
            if(xBase<c_winSize_x)
                GetError(J, prevPt.x + xBase + 0.5f, prevPt.y + yBase + 0.5f,
                         &I_patch[2][2], &D);
        }

        reduce1(D, smem1, tid);
    }

    if (tid == 0)
    {
        prevPt += c_halfWin;

        nextPts[gid] = prevPt;

        if (calcErr)
            err[gid] = smem1[0] / (float)(c_winSize_x * c_winSize_y);
    }
}


Fix bad shared memory access pattern

Fix bad shared memory access pattern

Since we do not explicitly synchronize threads after fill_block calls, we have to make sure we write into curr in the same pattern as when we read it later.

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, uint subblock, uint hash_lane,
       uint bw, uint bh, uint dx, uint dy, uint offset)
{
    uint index[4];
    for (uint i = 0; i < 4; i++) {
        uint bpos = (hash_lane + i * offset) % 4;
        uint x = (subblock * dy + i * dx) * bw + bpos % bw;
        uint y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, uint thread)
{
    uint subblock = (thread >> 2) & 0x7;
    uint hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    uint thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    uint thread = (uint)get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * (uint)lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (uint index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            uint addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            uint addr_index_x = addr_index % 16;
            uint addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                    uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, uint subblock, uint hash_lane,
       uint bw, uint bh, uint dx, uint dy, uint offset)
{
    uint index[4];
    for (uint i = 0; i < 4; i++) {
        uint bpos = (hash_lane + i * offset) % 4;
        uint x = (subblock * dy + i * dx) * bw + bpos % bw;
        uint y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, uint thread)
{
    uint subblock = (thread >> 2) & 0x7;
    uint hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    uint thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    uint thread = (uint)get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * (uint)lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (uint index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            uint addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            uint addr_index_x = addr_index % 16;
            uint addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                    uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}