Further MIOpenConvDirUniC perf improvements

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_PIX_TILE1*MLO_OUT_PIX_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)

#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  (MLO_N_INPUTS/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#define MLO_IN_LCL_WIDTH (MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1)  // here we use kernel size. it's important when padding == 0
#define MLO_IN_LCL_HEIGHT (MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1)
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#define MLO_PVT_IN_WIDTH (MLO_FILTER_SIZE0 + MLO_OUT_PIX_TILE0 - 1)
#define MLO_PVT_IN_HEIGHT (MLO_OUT_PIX_TILE1)

#define MLO_LCL_WEIGHTS 1

#if defined(__AMDGCN__)
extern uint __llvm_amdgcn_readfirstlane(uint) __asm("llvm.amdgcn.readfirstlane");
#define uniform(x) __llvm_amdgcn_readfirstlane(x)
#else
#define uniform(x) (x)
#endif

static inline void calculateXYPos(uint linPos, uint width, uint * __restrict x, uint * __restrict y)
{

	(*y) = (uint)((float)linPos * (1.0f / (float)width) + 0.00001f);

	(*x) = linPos - (*y) * width;
}

static inline uint calculateOffset(uint stride, uint x, uint y)
{
	uint ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(uint linPos,__local _FLOAT *lcl_data, uint lcl_base, UNUSED uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 UNUSED bool debug)
{
	uint x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	uint gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	uint gbl_off = gbl_off0 + gbl_base;

#if MLO_LARGE_MAP == 1
	uint lcl_off = lcl_base + linPos;
	(void)lcl_stride;
	(void)lcl_x;
	(void)lcl_y;
#else
	uint l_x = x + lcl_x;
	uint l_y = y + lcl_y;
	uint lcl_off = lcl_base + mad24(l_y, lcl_stride, l_x);
#endif

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#else
	 (void)gbl_width;
	 (void)gbl_height;
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(uint lcl_id, uint size, uint lcl_p_stride, __local _FLOAT *lcl_data, uint lcl_base, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(uint i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(uint lcl_id, uint lcl_p_stride,
					__local _FLOAT * lcl_data,
					uint lcl_off, uint lcl_size, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_bot_y, uint lcl_bot_x,
					const __global _FLOAT * gbl_data,
					uint gbl_off, uint gbl_size, uint gbl_height, uint glb_width, uint gbl_stride, int gbl_bot_y, int gbl_bot_x,
					uint buf_block_ind, uint max_n_bufs, uint lcl_n_bufs,
					bool debug)
{

	for(uint c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(uint o_map_base,
				uint in_stg_off,
				 _FLOAT * pvt_in_stage, __local _FLOAT * __restrict lcl_indata,
				 _FLOAT * pvt_wei_stage, __local _FLOAT * __restrict lcl_wei,
				 _FLOAT * pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		uint in_stg_off1 = in_stg_off;
		for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			uint wei_stg_base_off = mad24(o_map_base, (uint)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(uint)MLO_FILTER_SZ));
			uint in_stg_off2 = in_stg_off1;
			for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#ifdef __AMDGCN__
			#pragma unroll
#endif
			for(uint k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
			{
				uint k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(uint i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					uint wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(uint i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] = lcl_wei[wei_stg_off + i];
					}

		// actual conv

					for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j)
					{
						for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
						{
							for(uint l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								uint l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

								pvt_accum[(o_c * MLO_OUT_PIX_TILE1 + j) * MLO_OUT_PIX_TILE0 + i]
									 += pvt_in_stage[j * MLO_PVT_IN_WIDTH + i + l] * pvt_wei_stage[l_act];

							}

						}
					}

				} // for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

//				mem_fence(CLK_LOCAL_MEM_FENCE);

			} // for(uint k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenConvUniC(
       const __global _FLOAT * __restrict in,
       const __global _FLOAT * __restrict weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
	__global _FLOAT * __restrict out,
	UNUSED _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	uint grp_id0 = get_group_id(0);
#if MLO_OUT_WIDTH == MLO_IN_TILE0
	uint y_tile_blk = grp_id0;
	uint x_tile_blk = 0;
#else
#if MLO_N_OUT_TILE_BLOCKS0 & (MLO_N_OUT_TILE_BLOCKS0 - 1)
	uint y_tile_blk = (uint)((float)grp_id0 * (1.0f / (float) MLO_N_OUT_TILE_BLOCKS0) + 0.00001f);
	int x_tile_blk = grp_id0 - mul24(y_tile_blk, (uint)MLO_N_OUT_TILE_BLOCKS0);
#else
	uint y_tile_blk = grp_id0 / MLO_N_OUT_TILE_BLOCKS0;
	int x_tile_blk = grp_id0 & (MLO_N_OUT_TILE_BLOCKS0 - 1);
#endif
#endif
	uint o_pack = get_group_id(1); // block of outputs
	uint b_pack = get_group_id(2); // batch block

	uint lcl_id = get_local_id(0);
#if MLO_ALUTILES_STACK_SZ & (MLO_ALUTILES_STACK_SZ - 1)
	uint stack = (uint)((float)lcl_id * (1.0f / (float)MLO_ALUTILES_STACK_SZ) + 0.00001f);  // stack
	uint alu_stack_id = lcl_id - mul24(stack, (uint)MLO_ALUTILES_STACK_SZ);  // alu index in stack
#else
	uint stack = lcl_id / MLO_ALUTILES_STACK_SZ;  // stack
	uint alu_stack_id = lcl_id & (MLO_ALUTILES_STACK_SZ - 1);  // alu index in stack
#if MLO_ALUTILES_STACK_SZ >= 64
	stack = uniform(stack);
#endif
#endif
// ALU plane inside stack
#if MLO_ALU_TILE_SZ & (MLO_ALU_TILE_SZ - 1)
	uint alu_out_plane_id = (uint)((float)alu_stack_id * (1.0f / (float)MLO_ALU_TILE_SZ) + 0.00001f);  // alu output plane index
	uint alu_out_id = alu_stack_id - mul24(alu_out_plane_id, (uint)MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
#else
	uint alu_out_plane_id = alu_stack_id / MLO_ALU_TILE_SZ;  // alu output plane index
	uint alu_out_id = alu_stack_id & (MLO_ALU_TILE_SZ - 1); // alu index inside an ALU output plane
#endif
// pos inside ALU tile
#if MLO_ALU_VTILE0 & (MLO_ALU_VTILE0 - 1)
	uint alu_tl1 = (uint)((float)alu_out_id * (1.0f / (float)MLO_ALU_VTILE0) + 0.00001f);
	uint alu_tl0 = alu_out_id - mul24(alu_tl1, (uint)MLO_ALU_VTILE0);
#else
	uint alu_tl1 = alu_out_id / MLO_ALU_VTILE0;
	uint alu_tl0 = alu_out_id & (MLO_ALU_VTILE0 - 1);
#endif

	uint o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	uint o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	uint o_map = o_map_plane + o_map_base; // output map index per ALU plane
	uint b_index = b_pack * MLO_N_STACKS;

#if MLO_LARGE_MAP != 1
#if MLO_GRP_SZ <= MLO_N_READ_PROCS
	uint wave_id = 0;
	uint wave_lcl_id = lcl_id;
#elif MLO_N_READ_PROCS & (MLO_N_READ_PROCS - 1)
	uint wave_id = (uint)((float)lcl_id * (1.0f / (float)MLO_N_READ_PROCS) + 0.00001f);
	uint wave_lcl_id = lcl_id - mul24(wave_id, (uint)MLO_N_READ_PROCS);
#else
	uint wave_id = (uint)((uint)lcl_id / MLO_N_READ_PROCS);
	uint wave_lcl_id = lcl_id & (MLO_N_READ_PROCS - 1);
#if MLO_N_READ_PROCS >= 64
        wave_id = uniform(wave_id);
#endif
#endif
#endif

	int x_grp = x_tile_blk * MLO_IN_TILE0;
	uint y_grp = y_tile_blk * MLO_IN_TILE1;

// TO DO: scale
#if MLO_LARGE_MAP == 1
	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;
#endif

	uint x_in_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_in_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

// base offset to read data from local input data
	uint in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    uint in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	uint wei_off = mul24(o_map_plane, (uint)(MLO_N_INPUTS * MLO_FILTER_SZ));
#else
	uint wei_off = mul24(o_map_plane,(uint)MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(uint i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(uint i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(uint ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		uint in_lcl_off1 = 0;
		uint in_off1 = in_off;
		for(uint i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			uint in_off2 = in_off1;
			uint in_lcl_off2 = in_lcl_off1;
			for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				uint elem_id = lcl_id;
				uint lcl_p_stride = MLO_GRP_SZ0;
				uint lcl_base = 0;
				uint lcl_y = 0;
				uint lcl_x = 0;
				uint gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(uint i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
		//(MLO_N_STACKS * MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_IN_TILES_PERSTACK & (MLO_N_IN_TILES_PERSTACK - 1)
			uint i_b = (uint)((float)i * (1.0f / (float)MLO_N_IN_TILES_PERSTACK) + 0.00001f);
			uint i_c = i - mul24(i_b, (uint)MLO_N_IN_TILES_PERSTACK);
#else
			uint i_b = (uint)i / MLO_N_IN_TILES_PERSTACK;
			uint i_c = i & (MLO_N_IN_TILES_PERSTACK - 1);
#endif

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			uint in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			uint in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			uint elem_id = wave_lcl_id;
			uint lcl_p_stride = MLO_N_READ_PROCS;
			uint lcl_base = 0;
			uint lcl_y = MLO_FILTER_PAD1;
			uint lcl_x = MLO_FILTER_PAD0;
			uint gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(uint i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
#if (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = (uint)((float)i * (1.0f / (float)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ)) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
			if((wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i) < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ))
				lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i];
			else
				lcl_wei[i] = weights[0];
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)

#if (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = (uint)((float)i * (1.0f / (float)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ)) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
#if MLO_FILTER_SZ & (MLO_FILTER_SZ - 1)
			uint lcl_c = (uint)((float)gbl_i * (1.0f / (float)MLO_FILTER_SZ) + 0.00001f);
			uint lcl_i = gbl_i - mul24(lcl_c, (uint)MLO_FILTER_SZ);
#else
			uint lcl_c = gbl_i / MLO_FILTER_SZ;
			uint lcl_i = gbl_i & (MLO_FILTER_SZ - 1);
#endif

			uint lcl_we_off = mad24(mad24(lcl_c, (uint)MLO_N_IN_TILES_PERSTACK, lcl_o), (uint)MLO_FILTER_SZ, lcl_i);
			uint gbl_we_off = mad24(mad24(lcl_o, (uint)MLO_N_OUTPUTS, lcl_c), (uint)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
		if (lcl_id < MLO_ACTIVE_ALUS)
#endif
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}

#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
	if (lcl_id >= MLO_ACTIVE_ALUS)
	{
		return;
	}
#endif
// write results out
	uint x_out_grp = x_grp;
	uint y_out_grp = y_grp;
	uint x_out_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_out_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

    uint out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		uint out_off1 = out_off;
		for(uint o = 0; o < MLO_N_OUT_TILES
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile
			_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS
			bias_val = bias[o_map + o];
#endif
			uint out_off2 = out_off1;
			for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				__global _FLOAT *out_p = &out[out_off2];
				for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
				{
					if ( true
#if 1 //MLO_OUT_ALIGNED == 0
					&& y_out_lcl + j < MLO_OUT_TILE1 && y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT &&  x_out_lcl + i < MLO_OUT_TILE0 && x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH
#endif
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif
					)
					{
						out_p[i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i] + bias_val;
#if 0
						if ( out_off2 + i == 1 /*y_out_grp + y_out_lcl + j == 2 && x_out_grp + x_out_lcl + i == 0*/)
						{
							printf("K:out: %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d    %f %f %f\n",
								MLO_OUT_TILE1,
								MLO_OUT_TILE0,
								grp_id0,
								lcl_id,
								alu_out_plane_id,
								alu_out_id,
								b_index + stack,
								o,
								o_map,
								out_off,
								out_off1,
								out_off2,
								y_out_grp,
								y_out_lcl,
								x_out_grp,
								x_out_lcl,
								j,
								i,
								pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i],
								 bias_val,
								out[out_off2 + i]
								);
						}
#endif
					}

				}
			}

		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_PIX_TILE1*MLO_OUT_PIX_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)

#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  (MLO_N_INPUTS/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#define MLO_IN_LCL_WIDTH (MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1)  // here we use kernel size. it's important when padding == 0
#define MLO_IN_LCL_HEIGHT (MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1)
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#define MLO_PVT_IN_WIDTH (MLO_FILTER_SIZE0 + MLO_OUT_PIX_TILE0 - 1)
#define MLO_PVT_IN_HEIGHT (MLO_OUT_PIX_TILE1)

#define MLO_LCL_WEIGHTS 1

#if defined(__AMDGCN__)
extern uint __llvm_amdgcn_readfirstlane(uint) __asm("llvm.amdgcn.readfirstlane");
#define uniform(x) __llvm_amdgcn_readfirstlane(x)
#else
#define uniform(x) (x)
#endif

static inline void calculateXYPos(uint linPos, uint width, uint * __restrict x, uint * __restrict y)
{

	(*y) = (uint)((float)linPos * (1.0f / (float)width) + 0.00001f);

	(*x) = linPos - (*y) * width;
}

static inline uint calculateOffset(uint stride, uint x, uint y)
{
	uint ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(uint linPos,__local _FLOAT *lcl_data, uint lcl_base, UNUSED uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 UNUSED bool debug)
{
	uint x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	uint gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	uint gbl_off = gbl_off0 + gbl_base;

#if MLO_LARGE_MAP == 1
	uint lcl_off = lcl_base + linPos;
	(void)lcl_stride;
	(void)lcl_x;
	(void)lcl_y;
#else
	uint l_x = x + lcl_x;
	uint l_y = y + lcl_y;
	uint lcl_off = lcl_base + mad24(l_y, lcl_stride, l_x);
#endif

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#else
	 (void)gbl_width;
	 (void)gbl_height;
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(uint lcl_id, uint size, uint lcl_p_stride, __local _FLOAT *lcl_data, uint lcl_base, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(uint i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(uint lcl_id, uint lcl_p_stride,
					__local _FLOAT * lcl_data,
					uint lcl_off, uint lcl_size, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_bot_y, uint lcl_bot_x,
					const __global _FLOAT * gbl_data,
					uint gbl_off, uint gbl_size, uint gbl_height, uint glb_width, uint gbl_stride, int gbl_bot_y, int gbl_bot_x,
					uint buf_block_ind, uint max_n_bufs, uint lcl_n_bufs,
					bool debug)
{

	for(uint c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(uint o_map_base,
				uint in_stg_off,
				 _FLOAT * pvt_in_stage, __local _FLOAT * __restrict lcl_indata,
				 _FLOAT * pvt_wei_stage, __local _FLOAT * __restrict lcl_wei,
				 _FLOAT * pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		uint in_stg_off1 = in_stg_off;
		for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			uint wei_stg_base_off = mad24(o_map_base, (uint)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(uint)MLO_FILTER_SZ));
			uint in_stg_off2 = in_stg_off1;
			for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#ifdef __AMDGCN__
			#pragma unroll
#endif
			for(uint k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
			{
				uint k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(uint i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					uint wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(uint i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] = lcl_wei[wei_stg_off + i];
					}

		// actual conv

					for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j)
					{
						for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
						{
							for(uint l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								uint l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

								pvt_accum[(o_c * MLO_OUT_PIX_TILE1 + j) * MLO_OUT_PIX_TILE0 + i]
									 += pvt_in_stage[j * MLO_PVT_IN_WIDTH + i + l] * pvt_wei_stage[l_act];

							}

						}
					}

				} // for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

//				mem_fence(CLK_LOCAL_MEM_FENCE);

			} // for(uint k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenConvUniC(
       const __global _FLOAT * __restrict in,
       const __global _FLOAT * __restrict weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
	__global _FLOAT * __restrict out,
	UNUSED _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	uint grp_id0 = get_group_id(0);
#if MLO_OUT_WIDTH == MLO_IN_TILE0
	uint y_tile_blk = grp_id0;
	uint x_tile_blk = 0;
#else
	uint y_tile_blk = (uint)((float)grp_id0 / (float) MLO_N_OUT_TILE_BLOCKS0 + 0.00001f);
	int x_tile_blk = grp_id0 - mul24(y_tile_blk, (uint)MLO_N_OUT_TILE_BLOCKS0);
#endif
	uint o_pack = get_group_id(1); // block of outputs
	uint b_pack = get_group_id(2); // batch block

	uint lcl_id = get_local_id(0);
	uint stack = (uint)((float)lcl_id/(float)MLO_ALUTILES_STACK_SZ + 0.00001f);  // stack
	uint alu_stack_id = lcl_id - mul24(stack, (uint)MLO_ALUTILES_STACK_SZ);  // alu index in stack
// ALU plane inside stack
	uint alu_out_plane_id = (uint)((float)alu_stack_id / (float)MLO_ALU_TILE_SZ + 0.00001f);  // alu output plane index
	uint alu_out_id = alu_stack_id - mul24(alu_out_plane_id, (uint)MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
// pos inside ALU tile
	uint alu_tl1 = (uint)((float)alu_out_id/(float)MLO_ALU_VTILE0 + 0.00001f);
	uint alu_tl0 = alu_out_id - mul24(alu_tl1, (uint)MLO_ALU_VTILE0);

	uint o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	uint o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	uint o_map = o_map_plane + o_map_base; // output map index per ALU plane
	uint b_index = b_pack * MLO_N_STACKS;

#if MLO_N_READ_PROCS & (MLO_N_READ_PROCS - 1)
	uint wave_id = (uint)((float)lcl_id / (float)MLO_N_READ_PROCS + 0.00001f);
	uint wave_lcl_id = lcl_id - mul24(wave_id, (uint)MLO_N_READ_PROCS);
#else
	uint wave_id = (uint)((uint)lcl_id / MLO_N_READ_PROCS);
#if MLO_LARGE_MAP != 1
	uint wave_lcl_id = lcl_id & (MLO_N_READ_PROCS - 1);
#endif
#if MLO_N_READ_PROCS >= 64
        wave_id = uniform(wave_id);
#endif
#endif

	int x_grp = x_tile_blk * MLO_IN_TILE0;
	uint y_grp = y_tile_blk * MLO_IN_TILE1;

// TO DO: scale
#if MLO_LARGE_MAP == 1
	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;
#endif

	uint x_in_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_in_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

// base offset to read data from local input data
	uint in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    uint in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	uint wei_off = mul24(o_map_plane, (uint)(MLO_N_INPUTS * MLO_FILTER_SZ));
#else
	uint wei_off = mul24(o_map_plane,(uint)MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(uint i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(uint i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(uint ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		uint in_lcl_off1 = 0;
		uint in_off1 = in_off;
		for(uint i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			uint in_off2 = in_off1;
			uint in_lcl_off2 = in_lcl_off1;
			for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				uint elem_id = lcl_id;
				uint lcl_p_stride = MLO_GRP_SZ0;
				uint lcl_base = 0;
				uint lcl_y = 0;
				uint lcl_x = 0;
				uint gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(uint i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
		//(MLO_N_STACKS * MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_IN_TILES_PERSTACK & (MLO_N_IN_TILES_PERSTACK - 1)
			uint i_b = (uint)((float)i / (float)MLO_N_IN_TILES_PERSTACK + 0.00001f);
			uint i_c = i - mul24(i_b, (uint)MLO_N_IN_TILES_PERSTACK);
#else
			uint i_b = (uint)i / MLO_N_IN_TILES_PERSTACK;
			uint i_c = i & (MLO_N_IN_TILES_PERSTACK - 1);
#endif

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			uint in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			uint in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			uint elem_id = wave_lcl_id;
			uint lcl_p_stride = MLO_N_READ_PROCS;
			uint lcl_base = 0;
			uint lcl_y = MLO_FILTER_PAD1;
			uint lcl_x = MLO_FILTER_PAD0;
			uint gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(uint i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
			uint lcl_o = (uint)((float)i / (float)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			if((wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i) < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ))
				lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i];
			else
				lcl_wei[i] = weights[0];
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)

			uint lcl_o = (uint)((float)i/ (float)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			uint lcl_c = (uint)((float)gbl_i/ (float)MLO_FILTER_SZ + 0.00001f);
			uint lcl_i = gbl_i - mul24(lcl_c, (uint)MLO_FILTER_SZ);

			uint lcl_we_off = mad24(mad24(lcl_c, (uint)MLO_N_IN_TILES_PERSTACK, lcl_o), (uint)MLO_FILTER_SZ, lcl_i);
			uint gbl_we_off = mad24(mad24(lcl_o, (uint)MLO_N_OUTPUTS, lcl_c), (uint)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
		if (lcl_id < MLO_ACTIVE_ALUS)
#endif
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}

#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
	if (lcl_id >= MLO_ACTIVE_ALUS)
	{
		return;
	}
#endif
// write results out
	uint x_out_grp = x_grp;
	uint y_out_grp = y_grp;
	uint x_out_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_out_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

    uint out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		uint out_off1 = out_off;
		for(uint o = 0; o < MLO_N_OUT_TILES
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile
			_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS
			bias_val = bias[o_map + o];
#endif
			uint out_off2 = out_off1;
			for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
				{
					if ( true
#if 1 //MLO_OUT_ALIGNED == 0
					&& y_out_lcl + j < MLO_OUT_TILE1 && y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT &&  x_out_lcl + i < MLO_OUT_TILE0 && x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH
#endif
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif
					)
					{
						out[out_off2 + i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i] + bias_val;
#if 0
						if ( out_off2 + i == 1 /*y_out_grp + y_out_lcl + j == 2 && x_out_grp + x_out_lcl + i == 0*/)
						{
							printf("K:out: %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d    %f %f %f\n",
								MLO_OUT_TILE1,
								MLO_OUT_TILE0,
								grp_id0,
								lcl_id,
								alu_out_plane_id,
								alu_out_id,
								b_index + stack,
								o,
								o_map,
								out_off,
								out_off1,
								out_off2,
								y_out_grp,
								y_out_lcl,
								x_out_grp,
								x_out_lcl,
								j,
								i,
								pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i],
								 bias_val,
								out[out_off2 + i]
								);
						}
#endif
					}

				}
			}

		}
	}

}


Jump by local size instead of global. No cache invalidates on cpu

MSTRINGIFY(

\n#define FETCH_2(sum, id, A)      sum += A[id];   id += get_local_size(0);   sum += A[id];   id += get_local_size(0);
\n#define FETCH_8(sum, id, A)      FETCH_2(sum, id, A);   FETCH_2(sum, id, A);   FETCH_2(sum, id, A);   FETCH_2(sum, id, A);
\n
\n
\n#define FETCH_PER_WI  16
\n

__kernel void bandwidth_v1(__global float *A, __global float *B)
{
    int id = (get_group_id(0) * get_local_size(0) * FETCH_PER_WI) + get_local_id(0);
    float sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = sum;
}

__kernel void bandwidth_v2(__global float2 *A, __global float *B)
{
    int id = (get_group_id(0) * get_local_size(0) * FETCH_PER_WI) + get_local_id(0);
    float2 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = (sum.S0) + (sum.S1);
}

__kernel void bandwidth_v4(__global float4 *A, __global float *B)
{
    int id = (get_group_id(0) * get_local_size(0) * FETCH_PER_WI) + get_local_id(0);
    float4 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = (sum.S0) + (sum.S1) + (sum.S2) + (sum.S3);
}

__kernel void bandwidth_v8(__global float8 *A, __global float *B)
{
    int id = (get_group_id(0) * get_local_size(0) * FETCH_PER_WI) + get_local_id(0);
    float8 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = (sum.S0) + (sum.S1) + (sum.S2) + (sum.S3) + (sum.S4) + (sum.S5) + (sum.S6) + (sum.S7);
}

__kernel void bandwidth_v16(__global float16 *A, __global float *B)
{
    int id = (get_group_id(0) * get_local_size(0) * FETCH_PER_WI) + get_local_id(0);
    float16 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    float t = (sum.S0) + (sum.S1) + (sum.S2) + (sum.S3) + (sum.S4) + (sum.S5) + (sum.S6) + (sum.S7);
    t += (sum.S8) + (sum.S9) + (sum.SA) + (sum.SB) + (sum.SC) + (sum.SD) + (sum.SE) + (sum.SF);
    B[get_global_id(0)] = t;
}

)

MSTRINGIFY(

\n#define FETCH_2(sum, id, A)      sum += A[id];   id += get_global_size(0);   sum += A[id];   id += get_global_size(0);
\n#define FETCH_8(sum, id, A)      FETCH_2(sum, id, A);   FETCH_2(sum, id, A);   FETCH_2(sum, id, A);   FETCH_2(sum, id, A);
\n

__kernel void bandwidth_v1(__global float *A, __global float *B)
{
    int id = get_global_id(0);
    float sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = sum;
}

__kernel void bandwidth_v2(__global float2 *A, __global float *B)
{
    int id = get_global_id(0);
    float2 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = (sum.S0) + (sum.S1);
}

__kernel void bandwidth_v4(__global float4 *A, __global float *B)
{
    int id = get_global_id(0);
    float4 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = (sum.S0) + (sum.S1) + (sum.S2) + (sum.S3);
}

__kernel void bandwidth_v8(__global float8 *A, __global float *B)
{
    int id = get_global_id(0);
    float8 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    B[get_global_id(0)] = (sum.S0) + (sum.S1) + (sum.S2) + (sum.S3) + (sum.S4) + (sum.S5) + (sum.S6) + (sum.S7);
}

__kernel void bandwidth_v16(__global float16 *A, __global float *B)
{
    int id = get_global_id(0);
    float16 sum = 0;

    FETCH_8(sum, id, A);
    FETCH_8(sum, id, A);

    float t = (sum.S0) + (sum.S1) + (sum.S2) + (sum.S3) + (sum.S4) + (sum.S5) + (sum.S6) + (sum.S7);
    t += (sum.S8) + (sum.S9) + (sum.SA) + (sum.SB) + (sum.SC) + (sum.SD) + (sum.SE) + (sum.SF);
    B[get_global_id(0)] = t;
}

)


Optimize shared memory access

Optimize shared memory access

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, size_t subblock, size_t hash_lane,
       size_t bw, size_t bh, size_t dx, size_t dy, size_t offset)
{
    size_t index[4];
    for (size_t i = 0; i < 4; i++) {
        size_t bpos = (hash_lane + i * offset) % 4;
        size_t x = (subblock * dy + i * dx) * bw + bpos % bw;
        size_t y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;

        if (i == 1) {
            //printf("[%i, %i] x: %2i, y: %2i\n", (uint)subblock, (uint)hash_lane, (uint)x, (uint)y);
        }
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7;
    size_t hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        //printf("[%2i, %i] pos_l: %3i\n", (uint)thread, (uint)i, (uint)pos_l);
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            size_t addr_index_x = addr_index % 16;
            size_t addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
            }

            ++mem_curr;
        }
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

void g(__local uint *v_lo, __local uint *v_hi, uint4 i)
{
    ulong a, b, c, d;
    a = upsample(v_hi[i.s0], v_lo[i.s0]);
    b = upsample(v_hi[i.s1], v_lo[i.s1]);
    c = upsample(v_hi[i.s2], v_lo[i.s2]);
    d = upsample(v_hi[i.s3], v_lo[i.s3]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    v_lo[i.s0] = (uint)a;
    v_lo[i.s1] = (uint)b;
    v_lo[i.s2] = (uint)c;
    v_lo[i.s3] = (uint)d;

    v_hi[i.s0] = (uint)(a >> 32);
    v_hi[i.s1] = (uint)(b >> 32);
    v_hi[i.s2] = (uint)(c >> 32);
    v_hi[i.s3] = (uint)(d >> 32);
}

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7U;
    uint hash_lane = (thread >> 0) & 0x3U;
    uint4 u0123 = (uint4)(0U, 1U, 2U, 3U);
    uint4 index0 = 4 * u0123 + hash_lane;
    uint4 index1 = 4 * u0123 + (u0123 + hash_lane) % 4;

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index1);

    barrier(CLK_LOCAL_MEM_FENCE);

    index0 = 16 * (index0 / 2) + index0 % 2;
    index1 = 16 * (index1 / 2) + index1 % 2;

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] = prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] = prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] ^= prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] ^= prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = mem_prev->data[pos];
        prev->lo[pos] = (uint)in;
        prev->hi[pos] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos = i * THREADS_PER_LANE + thread;
                    ulong in = mem_curr->data[pos];
                    curr->lo[pos] = (uint)in;
                    curr->hi[pos] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos = i * THREADS_PER_LANE + thread;
                mem_curr->data[pos] = upsample(curr->hi[pos], curr->lo[pos]);
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
            }

            ++mem_curr;
        }
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
        mem_curr = mem_lane;
    }
}


fix potential race condition in canny.cl.

See the below code snippet:

while(l_counter != 0)
{
    int mod = l_counter % LOCAL_TOTAL;
    int pix_per_thr = l_counter / LOCAL_TOTAL + ((lid < mod) ? 1 : 0);

    for (int i = 0; i < pix_per_thr; ++i)
    {
        int index = atomic_dec(&l_counter) - 1;
        ....
    }
    ....
    barrier(CLK_LOCAL_MEM_FENCE);
}

If we don't put a barrier before the for loop, then there is a possiblity
that some work item enter this loop but the others are not, the the l_counter
will be reduced in the for loop and may be changed to zero, and the other
work items may can't enter the while loop. If this happens, it breaks the
barrier's rule which requires all the work items reach the same barrier.
And it may hang the GPU depends on the implementation of opencl platform.

This issue is raised at:
https://github.com/Itseez/opencv/issues/5175

Signed-off-by: Zhigang Gong <e04a7b9b70b1e4c6318cf117dcd1a9056e14b97a@linux.intel.com>

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Peng Xiao, pengxiao@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#define TG22 0.4142135623730950488016887242097f
#define TG67 2.4142135623730950488016887242097f

#ifdef WITH_SOBEL

#if cn == 1
#define loadpix(addr) convert_floatN(*(__global const TYPE *)(addr))
#else
#define loadpix(addr) convert_floatN(vload3(0, (__global const TYPE *)(addr)))
#endif
#define storepix(value, addr) *(__global int *)(addr) = (int)(value)

/*
    stage1_with_sobel:
        Sobel operator
        Calc magnitudes
        Non maxima suppression
        Double thresholding
*/

__constant int prev[4][2] = {
    { 0, -1 },
    { -1, 1 },
    { -1, 0 },
    { -1, -1 }
};

__constant int next[4][2] = {
    { 0, 1 },
    { 1, -1 },
    { 1, 0 },
    { 1, 1 }
};

inline float3 sobel(int idx, __local const floatN *smem)
{
    // result: x, y, mag
    float3 res;

    floatN dx = fma(2, smem[idx + GRP_SIZEX + 6] - smem[idx + GRP_SIZEX + 4],
        smem[idx + 2] - smem[idx] + smem[idx + 2 * GRP_SIZEX + 10] - smem[idx + 2 * GRP_SIZEX + 8]);

    floatN dy = fma(2, smem[idx + 1] - smem[idx + 2 * GRP_SIZEX + 9],
        smem[idx + 2] - smem[idx + 2 * GRP_SIZEX + 10] + smem[idx] - smem[idx + 2 * GRP_SIZEX + 8]);

#ifdef L2GRAD
    floatN magN = fma(dx, dx, dy * dy);
#else
    floatN magN = fabs(dx) + fabs(dy);
#endif
#if cn == 1
    res.z = magN;
    res.x = dx;
    res.y = dy;
#else
    res.z = max(magN.x, max(magN.y, magN.z));
    if (res.z == magN.y)
    {
        dx.x = dx.y;
        dy.x = dy.y;
    }
    else if (res.z == magN.z)
    {
        dx.x = dx.z;
        dy.x = dy.z;
    }
    res.x = dx.x;
    res.y = dy.x;
#endif

    return res;
}

__kernel void stage1_with_sobel(__global const uchar *src, int src_step, int src_offset, int rows, int cols,
                                __global uchar *map, int map_step, int map_offset,
                                float low_thr, float high_thr)
{
    __local floatN smem[(GRP_SIZEX + 4) * (GRP_SIZEY + 4)];

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int start_x = GRP_SIZEX * get_group_id(0);
    int start_y = GRP_SIZEY * get_group_id(1);

    int i = lidx + lidy * GRP_SIZEX;
    for (int j = i;  j < (GRP_SIZEX + 4) * (GRP_SIZEY + 4); j += GRP_SIZEX * GRP_SIZEY)
    {
        int x = clamp(start_x - 2 + (j % (GRP_SIZEX + 4)), 0, cols - 1);
        int y = clamp(start_y - 2 + (j / (GRP_SIZEX + 4)), 0, rows - 1);
        smem[j] = loadpix(src + mad24(y, src_step, mad24(x, cn * (int)sizeof(TYPE), src_offset)));
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    //// Sobel, Magnitude
    //

    __local float mag[(GRP_SIZEX + 2) * (GRP_SIZEY + 2)];

    lidx++;
    lidy++;

    if (i < GRP_SIZEX + 2)
    {
        int grp_sizey = min(GRP_SIZEY + 1, rows - start_y);
        mag[i] = (sobel(i, smem)).z;
        mag[i + grp_sizey * (GRP_SIZEX + 2)] = (sobel(i + grp_sizey * (GRP_SIZEX + 4), smem)).z;
    }
    if (i < GRP_SIZEY + 2)
    {
        int grp_sizex = min(GRP_SIZEX + 1, cols - start_x);
        mag[i * (GRP_SIZEX + 2)] = (sobel(i * (GRP_SIZEX + 4), smem)).z;
        mag[i * (GRP_SIZEX + 2) + grp_sizex] = (sobel(i * (GRP_SIZEX + 4) + grp_sizex, smem)).z;
    }

    int idx = lidx + lidy * (GRP_SIZEX + 4);
    i = lidx + lidy * (GRP_SIZEX + 2);

    float3 res = sobel(idx, smem);
    mag[i] = res.z;
    barrier(CLK_LOCAL_MEM_FENCE);

    int x = (int) res.x;
    int y = (int) res.y;

    //// Threshold + Non maxima suppression
    //

    /*
        Sector numbers

        3   2   1
         *  *  *
          * * *
        0*******0
          * * *
         *  *  *
        1   2   3

        We need to determine arctg(dy / dx) to one of the four directions: 0, 45, 90 or 135 degrees.
        Therefore if abs(dy / dx) belongs to the interval
        [0, tg(22.5)]           -> 0 direction
        [tg(22.5), tg(67.5)]    -> 1 or 3
        [tg(67,5), +oo)         -> 2

        Since tg(67.5) = 1 / tg(22.5), if we take
        a = abs(dy / dx) * tg(22.5) and b = abs(dy / dx) * tg(67.5)
        we can get another intervals

        in case a:
        [0, tg(22.5)^2]     -> 0
        [tg(22.5)^2, 1]     -> 1, 3
        [1, +oo)            -> 2

        in case b:
        [0, 1]              -> 0
        [1, tg(67.5)^2]     -> 1,3
        [tg(67.5)^2, +oo)   -> 2

        that can help to find direction without conditions.

        0 - might belong to an edge
        1 - pixel doesn't belong to an edge
        2 - belong to an edge
    */

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if (gidx >= cols || gidy >= rows)
        return;

    float mag0 = mag[i];

    int value = 1;
    if (mag0 > low_thr)
    {
        int a = (y / (float)x) * TG22;
        int b = (y / (float)x) * TG67;

        a = min((int)abs(a), 1) + 1;
        b = min((int)abs(b), 1);

        //  a = { 1, 2 }
        //  b = { 0, 1 }
        //  a * b = { 0, 1, 2 } - directions that we need ( + 3 if x ^ y < 0)

        int dir3 = (a * b) & (((x ^ y) & 0x80000000) >> 31); // if a = 1, b = 1, dy ^ dx < 0
        int dir = a * b + 2 * dir3;
        float prev_mag = mag[(lidy + prev[dir][0]) * (GRP_SIZEX + 2) + lidx + prev[dir][1]];
        float next_mag = mag[(lidy + next[dir][0]) * (GRP_SIZEX + 2) + lidx + next[dir][1]] + (dir & 1);

        if (mag0 > prev_mag && mag0 >= next_mag)
        {
            value = (mag0 > high_thr) ? 2 : 0;
        }
    }

    storepix(value, map + mad24(gidy, map_step, mad24(gidx, (int)sizeof(int), map_offset)));
}

#elif defined WITHOUT_SOBEL

/*
    stage1_without_sobel:
        Calc magnitudes
        Non maxima suppression
        Double thresholding
*/

#define loadpix(addr) (__global short *)(addr)
#define storepix(val, addr) *(__global int *)(addr) = (int)(val)

#ifdef L2GRAD
#define dist(x, y) ((int)(x) * (x) + (int)(y) * (y))
#else
#define dist(x, y) (abs(x) + abs(y))
#endif

__constant int prev[4][2] = {
    { 0, -1 },
    { -1, -1 },
    { -1, 0 },
    { -1, 1 }
};

__constant int next[4][2] = {
    { 0, 1 },
    { 1, 1 },
    { 1, 0 },
    { 1, -1 }
};

__kernel void stage1_without_sobel(__global const uchar *dxptr, int dx_step, int dx_offset,
                                   __global const uchar *dyptr, int dy_step, int dy_offset,
                                   __global uchar *map, int map_step, int map_offset, int rows, int cols,
                                   int low_thr, int high_thr)
{
    int start_x = get_group_id(0) * GRP_SIZEX;
    int start_y = get_group_id(1) * GRP_SIZEY;

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    __local int mag[(GRP_SIZEX + 2) * (GRP_SIZEY + 2)];
    __local short2 sigma[(GRP_SIZEX + 2) * (GRP_SIZEY + 2)];

#pragma unroll
    for (int i = lidx + lidy * GRP_SIZEX; i < (GRP_SIZEX + 2) * (GRP_SIZEY + 2); i += GRP_SIZEX * GRP_SIZEY)
    {
        int x = clamp(start_x - 1 + i % (GRP_SIZEX + 2), 0, cols - 1);
        int y = clamp(start_y - 1 + i / (GRP_SIZEX + 2), 0, rows - 1);

        int dx_index = mad24(y, dx_step, mad24(x, cn * (int)sizeof(short), dx_offset));
        int dy_index = mad24(y, dy_step, mad24(x, cn * (int)sizeof(short), dy_offset));

        __global short *dx = loadpix(dxptr + dx_index);
        __global short *dy = loadpix(dyptr + dy_index);

        int mag0 = dist(dx[0], dy[0]);
#if cn > 1
        short cdx = dx[0], cdy = dy[0];
#pragma unroll
        for (int j = 1; j < cn; ++j)
        {
            int mag1 = dist(dx[j], dy[j]);
            if (mag1 > mag0)
            {
                mag0 = mag1;
                cdx = dx[j];
                cdy = dy[j];
            }
        }
        dx[0] = cdx;
        dy[0] = cdy;
#endif
        mag[i] = mag0;
        sigma[i] = (short2)(dx[0], dy[0]);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if (gidx >= cols || gidy >= rows)
        return;

    lidx++;
    lidy++;

    int mag0 = mag[lidx + lidy * (GRP_SIZEX + 2)];
    short x = (sigma[lidx + lidy * (GRP_SIZEX + 2)]).x;
    short y = (sigma[lidx + lidy * (GRP_SIZEX + 2)]).y;

    int value = 1;
    if (mag0 > low_thr)
    {
        int a = (y / (float)x) * TG22;
        int b = (y / (float)x) * TG67;

        a = min((int)abs(a), 1) + 1;
        b = min((int)abs(b), 1);

        int dir3 = (a * b) & (((x ^ y) & 0x80000000) >> 31);
        int dir = a * b + 2 * dir3;
        int prev_mag = mag[(lidy + prev[dir][0]) * (GRP_SIZEX + 2) + lidx + prev[dir][1]];
        int next_mag = mag[(lidy + next[dir][0]) * (GRP_SIZEX + 2) + lidx + next[dir][1]] + (dir & 1);

        if (mag0 > prev_mag && mag0 >= next_mag)
        {
            value = (mag0 > high_thr) ? 2 : 0;
        }
    }

    storepix(value, map + mad24(gidy, map_step, mad24(gidx, (int)sizeof(int), map_offset)));
}

#undef TG22
#undef CANNY_SHIFT

#elif defined STAGE2
/*
    stage2:
        hysteresis (add edges labeled 0 if they are connected with an edge labeled 2)
*/

#define loadpix(addr) *(__global int *)(addr)
#define storepix(val, addr) *(__global int *)(addr) = (int)(val)
#define LOCAL_TOTAL (LOCAL_X*LOCAL_Y)
#define l_stack_size (4*LOCAL_TOTAL)
#define p_stack_size 8

__constant short move_dir[2][8] = {
    { -1, -1, -1, 0, 0, 1, 1, 1 },
    { -1, 0, 1, -1, 1, -1, 0, 1 }
};

__kernel void stage2_hysteresis(__global uchar *map_ptr, int map_step, int map_offset, int rows, int cols)
{
    map_ptr += map_offset;

    int x = get_global_id(0);
    int y = get_global_id(1) * PIX_PER_WI;

    int lid = get_local_id(0) + get_local_id(1) * LOCAL_X;

    __local ushort2 l_stack[l_stack_size];
    __local int l_counter;

    if (lid == 0)
        l_counter = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (x < cols)
    {
        __global uchar* map = map_ptr + mad24(y, map_step, x * (int)sizeof(int));

        #pragma unroll
        for (int cy = 0; cy < PIX_PER_WI; ++cy)
        {
            if (y < rows)
            {
                int type = loadpix(map);
                if (type == 2)
                {
                    l_stack[atomic_inc(&l_counter)] = (ushort2)(x, y);
                }

                y++;
                map += map_step;
            }
        }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    ushort2 p_stack[p_stack_size];
    int p_counter = 0;

    while(l_counter != 0)
    {
        int mod = l_counter % LOCAL_TOTAL;
        int pix_per_thr = l_counter / LOCAL_TOTAL + ((lid < mod) ? 1 : 0);

        barrier(CLK_LOCAL_MEM_FENCE);
        for (int i = 0; i < pix_per_thr; ++i)
        {
            int index = atomic_dec(&l_counter) - 1;
            if (index < 0)
               continue;
            ushort2 pos = l_stack[ index ];

            #pragma unroll
            for (int j = 0; j < 8; ++j)
            {
                ushort posx = pos.x + move_dir[0][j];
                ushort posy = pos.y + move_dir[1][j];
                if (posx < 0 || posy < 0 || posx >= cols || posy >= rows)
                    continue;
                __global uchar *addr = map_ptr + mad24(posy, map_step, posx * (int)sizeof(int));
                int type = loadpix(addr);
                if (type == 0)
                {
                    p_stack[p_counter++] = (ushort2)(posx, posy);
                    storepix(2, addr);
                }
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);
        if (l_counter < 0)
            l_counter = 0;
        barrier(CLK_LOCAL_MEM_FENCE);

        while (p_counter > 0)
        {
            l_stack[ atomic_inc(&l_counter) ] = p_stack[--p_counter];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

#elif defined GET_EDGES

// Get the edge result. egde type of value 2 will be marked as an edge point and set to 255. Otherwise 0.
// map      edge type mappings
// dst      edge output

__kernel void getEdges(__global const uchar *mapptr, int map_step, int map_offset, int rows, int cols,
                       __global uchar *dst, int dst_step, int dst_offset)
{
    int x = get_global_id(0);
    int y = get_global_id(1) * PIX_PER_WI;

    if (x < cols)
    {
        int map_index = mad24(map_step, y, mad24(x, (int)sizeof(int), map_offset));
        int dst_index = mad24(dst_step, y, x + dst_offset);

        #pragma unroll
        for (int cy = 0; cy < PIX_PER_WI; ++cy)
        {
            if (y < rows)
            {
                __global const int * map = (__global const int *)(mapptr + map_index);
                dst[dst_index] = (uchar)(-(map[0] >> 1));

                y++;
                map_index += map_step;
                dst_index += dst_step;
            }
        }
    }
}

#endif

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Peng Xiao, pengxiao@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#define TG22 0.4142135623730950488016887242097f
#define TG67 2.4142135623730950488016887242097f

#ifdef WITH_SOBEL

#if cn == 1
#define loadpix(addr) convert_floatN(*(__global const TYPE *)(addr))
#else
#define loadpix(addr) convert_floatN(vload3(0, (__global const TYPE *)(addr)))
#endif
#define storepix(value, addr) *(__global int *)(addr) = (int)(value)

/*
    stage1_with_sobel:
        Sobel operator
        Calc magnitudes
        Non maxima suppression
        Double thresholding
*/

__constant int prev[4][2] = {
    { 0, -1 },
    { -1, 1 },
    { -1, 0 },
    { -1, -1 }
};

__constant int next[4][2] = {
    { 0, 1 },
    { 1, -1 },
    { 1, 0 },
    { 1, 1 }
};

inline float3 sobel(int idx, __local const floatN *smem)
{
    // result: x, y, mag
    float3 res;

    floatN dx = fma(2, smem[idx + GRP_SIZEX + 6] - smem[idx + GRP_SIZEX + 4],
        smem[idx + 2] - smem[idx] + smem[idx + 2 * GRP_SIZEX + 10] - smem[idx + 2 * GRP_SIZEX + 8]);

    floatN dy = fma(2, smem[idx + 1] - smem[idx + 2 * GRP_SIZEX + 9],
        smem[idx + 2] - smem[idx + 2 * GRP_SIZEX + 10] + smem[idx] - smem[idx + 2 * GRP_SIZEX + 8]);

#ifdef L2GRAD
    floatN magN = fma(dx, dx, dy * dy);
#else
    floatN magN = fabs(dx) + fabs(dy);
#endif
#if cn == 1
    res.z = magN;
    res.x = dx;
    res.y = dy;
#else
    res.z = max(magN.x, max(magN.y, magN.z));
    if (res.z == magN.y)
    {
        dx.x = dx.y;
        dy.x = dy.y;
    }
    else if (res.z == magN.z)
    {
        dx.x = dx.z;
        dy.x = dy.z;
    }
    res.x = dx.x;
    res.y = dy.x;
#endif

    return res;
}

__kernel void stage1_with_sobel(__global const uchar *src, int src_step, int src_offset, int rows, int cols,
                                __global uchar *map, int map_step, int map_offset,
                                float low_thr, float high_thr)
{
    __local floatN smem[(GRP_SIZEX + 4) * (GRP_SIZEY + 4)];

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int start_x = GRP_SIZEX * get_group_id(0);
    int start_y = GRP_SIZEY * get_group_id(1);

    int i = lidx + lidy * GRP_SIZEX;
    for (int j = i;  j < (GRP_SIZEX + 4) * (GRP_SIZEY + 4); j += GRP_SIZEX * GRP_SIZEY)
    {
        int x = clamp(start_x - 2 + (j % (GRP_SIZEX + 4)), 0, cols - 1);
        int y = clamp(start_y - 2 + (j / (GRP_SIZEX + 4)), 0, rows - 1);
        smem[j] = loadpix(src + mad24(y, src_step, mad24(x, cn * (int)sizeof(TYPE), src_offset)));
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    //// Sobel, Magnitude
    //

    __local float mag[(GRP_SIZEX + 2) * (GRP_SIZEY + 2)];

    lidx++;
    lidy++;

    if (i < GRP_SIZEX + 2)
    {
        int grp_sizey = min(GRP_SIZEY + 1, rows - start_y);
        mag[i] = (sobel(i, smem)).z;
        mag[i + grp_sizey * (GRP_SIZEX + 2)] = (sobel(i + grp_sizey * (GRP_SIZEX + 4), smem)).z;
    }
    if (i < GRP_SIZEY + 2)
    {
        int grp_sizex = min(GRP_SIZEX + 1, cols - start_x);
        mag[i * (GRP_SIZEX + 2)] = (sobel(i * (GRP_SIZEX + 4), smem)).z;
        mag[i * (GRP_SIZEX + 2) + grp_sizex] = (sobel(i * (GRP_SIZEX + 4) + grp_sizex, smem)).z;
    }

    int idx = lidx + lidy * (GRP_SIZEX + 4);
    i = lidx + lidy * (GRP_SIZEX + 2);

    float3 res = sobel(idx, smem);
    mag[i] = res.z;
    barrier(CLK_LOCAL_MEM_FENCE);

    int x = (int) res.x;
    int y = (int) res.y;

    //// Threshold + Non maxima suppression
    //

    /*
        Sector numbers

        3   2   1
         *  *  *
          * * *
        0*******0
          * * *
         *  *  *
        1   2   3

        We need to determine arctg(dy / dx) to one of the four directions: 0, 45, 90 or 135 degrees.
        Therefore if abs(dy / dx) belongs to the interval
        [0, tg(22.5)]           -> 0 direction
        [tg(22.5), tg(67.5)]    -> 1 or 3
        [tg(67,5), +oo)         -> 2

        Since tg(67.5) = 1 / tg(22.5), if we take
        a = abs(dy / dx) * tg(22.5) and b = abs(dy / dx) * tg(67.5)
        we can get another intervals

        in case a:
        [0, tg(22.5)^2]     -> 0
        [tg(22.5)^2, 1]     -> 1, 3
        [1, +oo)            -> 2

        in case b:
        [0, 1]              -> 0
        [1, tg(67.5)^2]     -> 1,3
        [tg(67.5)^2, +oo)   -> 2

        that can help to find direction without conditions.

        0 - might belong to an edge
        1 - pixel doesn't belong to an edge
        2 - belong to an edge
    */

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if (gidx >= cols || gidy >= rows)
        return;

    float mag0 = mag[i];

    int value = 1;
    if (mag0 > low_thr)
    {
        int a = (y / (float)x) * TG22;
        int b = (y / (float)x) * TG67;

        a = min((int)abs(a), 1) + 1;
        b = min((int)abs(b), 1);

        //  a = { 1, 2 }
        //  b = { 0, 1 }
        //  a * b = { 0, 1, 2 } - directions that we need ( + 3 if x ^ y < 0)

        int dir3 = (a * b) & (((x ^ y) & 0x80000000) >> 31); // if a = 1, b = 1, dy ^ dx < 0
        int dir = a * b + 2 * dir3;
        float prev_mag = mag[(lidy + prev[dir][0]) * (GRP_SIZEX + 2) + lidx + prev[dir][1]];
        float next_mag = mag[(lidy + next[dir][0]) * (GRP_SIZEX + 2) + lidx + next[dir][1]] + (dir & 1);

        if (mag0 > prev_mag && mag0 >= next_mag)
        {
            value = (mag0 > high_thr) ? 2 : 0;
        }
    }

    storepix(value, map + mad24(gidy, map_step, mad24(gidx, (int)sizeof(int), map_offset)));
}

#elif defined WITHOUT_SOBEL

/*
    stage1_without_sobel:
        Calc magnitudes
        Non maxima suppression
        Double thresholding
*/

#define loadpix(addr) (__global short *)(addr)
#define storepix(val, addr) *(__global int *)(addr) = (int)(val)

#ifdef L2GRAD
#define dist(x, y) ((int)(x) * (x) + (int)(y) * (y))
#else
#define dist(x, y) (abs(x) + abs(y))
#endif

__constant int prev[4][2] = {
    { 0, -1 },
    { -1, -1 },
    { -1, 0 },
    { -1, 1 }
};

__constant int next[4][2] = {
    { 0, 1 },
    { 1, 1 },
    { 1, 0 },
    { 1, -1 }
};

__kernel void stage1_without_sobel(__global const uchar *dxptr, int dx_step, int dx_offset,
                                   __global const uchar *dyptr, int dy_step, int dy_offset,
                                   __global uchar *map, int map_step, int map_offset, int rows, int cols,
                                   int low_thr, int high_thr)
{
    int start_x = get_group_id(0) * GRP_SIZEX;
    int start_y = get_group_id(1) * GRP_SIZEY;

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    __local int mag[(GRP_SIZEX + 2) * (GRP_SIZEY + 2)];
    __local short2 sigma[(GRP_SIZEX + 2) * (GRP_SIZEY + 2)];

#pragma unroll
    for (int i = lidx + lidy * GRP_SIZEX; i < (GRP_SIZEX + 2) * (GRP_SIZEY + 2); i += GRP_SIZEX * GRP_SIZEY)
    {
        int x = clamp(start_x - 1 + i % (GRP_SIZEX + 2), 0, cols - 1);
        int y = clamp(start_y - 1 + i / (GRP_SIZEX + 2), 0, rows - 1);

        int dx_index = mad24(y, dx_step, mad24(x, cn * (int)sizeof(short), dx_offset));
        int dy_index = mad24(y, dy_step, mad24(x, cn * (int)sizeof(short), dy_offset));

        __global short *dx = loadpix(dxptr + dx_index);
        __global short *dy = loadpix(dyptr + dy_index);

        int mag0 = dist(dx[0], dy[0]);
#if cn > 1
        short cdx = dx[0], cdy = dy[0];
#pragma unroll
        for (int j = 1; j < cn; ++j)
        {
            int mag1 = dist(dx[j], dy[j]);
            if (mag1 > mag0)
            {
                mag0 = mag1;
                cdx = dx[j];
                cdy = dy[j];
            }
        }
        dx[0] = cdx;
        dy[0] = cdy;
#endif
        mag[i] = mag0;
        sigma[i] = (short2)(dx[0], dy[0]);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if (gidx >= cols || gidy >= rows)
        return;

    lidx++;
    lidy++;

    int mag0 = mag[lidx + lidy * (GRP_SIZEX + 2)];
    short x = (sigma[lidx + lidy * (GRP_SIZEX + 2)]).x;
    short y = (sigma[lidx + lidy * (GRP_SIZEX + 2)]).y;

    int value = 1;
    if (mag0 > low_thr)
    {
        int a = (y / (float)x) * TG22;
        int b = (y / (float)x) * TG67;

        a = min((int)abs(a), 1) + 1;
        b = min((int)abs(b), 1);

        int dir3 = (a * b) & (((x ^ y) & 0x80000000) >> 31);
        int dir = a * b + 2 * dir3;
        int prev_mag = mag[(lidy + prev[dir][0]) * (GRP_SIZEX + 2) + lidx + prev[dir][1]];
        int next_mag = mag[(lidy + next[dir][0]) * (GRP_SIZEX + 2) + lidx + next[dir][1]] + (dir & 1);

        if (mag0 > prev_mag && mag0 >= next_mag)
        {
            value = (mag0 > high_thr) ? 2 : 0;
        }
    }

    storepix(value, map + mad24(gidy, map_step, mad24(gidx, (int)sizeof(int), map_offset)));
}

#undef TG22
#undef CANNY_SHIFT

#elif defined STAGE2
/*
    stage2:
        hysteresis (add edges labeled 0 if they are connected with an edge labeled 2)
*/

#define loadpix(addr) *(__global int *)(addr)
#define storepix(val, addr) *(__global int *)(addr) = (int)(val)
#define LOCAL_TOTAL (LOCAL_X*LOCAL_Y)
#define l_stack_size (4*LOCAL_TOTAL)
#define p_stack_size 8

__constant short move_dir[2][8] = {
    { -1, -1, -1, 0, 0, 1, 1, 1 },
    { -1, 0, 1, -1, 1, -1, 0, 1 }
};

__kernel void stage2_hysteresis(__global uchar *map_ptr, int map_step, int map_offset, int rows, int cols)
{
    map_ptr += map_offset;

    int x = get_global_id(0);
    int y = get_global_id(1) * PIX_PER_WI;

    int lid = get_local_id(0) + get_local_id(1) * LOCAL_X;

    __local ushort2 l_stack[l_stack_size];
    __local int l_counter;

    if (lid == 0)
        l_counter = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (x < cols)
    {
        __global uchar* map = map_ptr + mad24(y, map_step, x * (int)sizeof(int));

        #pragma unroll
        for (int cy = 0; cy < PIX_PER_WI; ++cy)
        {
            if (y < rows)
            {
                int type = loadpix(map);
                if (type == 2)
                {
                    l_stack[atomic_inc(&l_counter)] = (ushort2)(x, y);
                }

                y++;
                map += map_step;
            }
        }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    ushort2 p_stack[p_stack_size];
    int p_counter = 0;

    while(l_counter != 0)
    {
        int mod = l_counter % LOCAL_TOTAL;
        int pix_per_thr = l_counter / LOCAL_TOTAL + ((lid < mod) ? 1 : 0);

        for (int i = 0; i < pix_per_thr; ++i)
        {
            int index = atomic_dec(&l_counter) - 1;
            if (index < 0)
               continue;
            ushort2 pos = l_stack[ index ];

            #pragma unroll
            for (int j = 0; j < 8; ++j)
            {
                ushort posx = pos.x + move_dir[0][j];
                ushort posy = pos.y + move_dir[1][j];
                if (posx < 0 || posy < 0 || posx >= cols || posy >= rows)
                    continue;
                __global uchar *addr = map_ptr + mad24(posy, map_step, posx * (int)sizeof(int));
                int type = loadpix(addr);
                if (type == 0)
                {
                    p_stack[p_counter++] = (ushort2)(posx, posy);
                    storepix(2, addr);
                }
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);
        if (l_counter < 0)
            l_counter = 0;
        barrier(CLK_LOCAL_MEM_FENCE);

        while (p_counter > 0)
        {
            l_stack[ atomic_inc(&l_counter) ] = p_stack[--p_counter];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

#elif defined GET_EDGES

// Get the edge result. egde type of value 2 will be marked as an edge point and set to 255. Otherwise 0.
// map      edge type mappings
// dst      edge output

__kernel void getEdges(__global const uchar *mapptr, int map_step, int map_offset, int rows, int cols,
                       __global uchar *dst, int dst_step, int dst_offset)
{
    int x = get_global_id(0);
    int y = get_global_id(1) * PIX_PER_WI;

    if (x < cols)
    {
        int map_index = mad24(map_step, y, mad24(x, (int)sizeof(int), map_offset));
        int dst_index = mad24(dst_step, y, x + dst_offset);

        #pragma unroll
        for (int cy = 0; cy < PIX_PER_WI; ++cy)
        {
            if (y < rows)
            {
                __global const int * map = (__global const int *)(mapptr + map_index);
                dst[dst_index] = (uchar)(-(map[0] >> 1));

                y++;
                map_index += map_step;
                dst_index += dst_step;
            }
        }
    }
}

#endif