remove printf

remove printf

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if  MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS ((int)(MLO_IN_N_VERT_LOOPS - 2))
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
static inline void readInput(int lcl_id, int gbl_in_scan_off, int n_v_reads, const __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		int c = 0;
		int c_scan = iDiv(p4, (MLO_N_IN_HORIZ_READS));

		int c_pix4 = iMod(p4, c_scan, (MLO_N_IN_HORIZ_READS));

//		if (c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
			}

// MLO_N_LCL_IN_MAPS inputs
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
static inline void Processing(int sc, int sc_lcl_off, int top_lim, int bot_lim, __private _FLOAT * pvt_accum, __local _FLOAT * lcl_bot, __private _FLOAT * top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (int m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				_FLOAT bot_val = lcl_bot[sc_lcl_off + n + m];
				for(int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int pvt_top_off =  k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
					int pvt_accum_off = k*MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;

					_FLOAT top_val = top_dat[pvt_top_off];

					pvt_accum[pvt_accum_off]
						// each wk-it process an input
						+= bot_val*top_val;
#if 0
					if (bot_val * top_val != 0 && get_global_id(1) == 0 && get_global_id(2) == 0 && (get_local_id(0) == 0 || get_local_id(0) == 1) && k == 0 && l == 0 && n == 0)
					{
						printf("G: %d %d %d %d  %f %f %f %f\n",
							get_local_id(0),
							sc,
							sc_lcl_off + n + m,
							pvt_top_off,
							pvt_accum[pvt_accum_off],
							bot_val * top_val,
							bot_val,
							top_val
						);
					}
#endif
				}
			}

		}

	}

}

static inline void moveOutputUp(__private _FLOAT * top_dat)
{
	// move up output to reduce overfetch
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
		{
			for (int i = 0; i < MLO_IN_TILE0; ++i)
			{
				int pvt_off_n = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0 + i;
				int pvt_off_o = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) *MLO_IN_TILE0 + i;
				top_dat[pvt_off_n] = top_dat[pvt_off_o];
			}
		}
	}
}

static inline void spanRightSiding5x5(int k, int top_df_off, int j, _FLOAT mask, __private _FLOAT * top_dat, __global const _FLOAT * top_df)
{
	int i = 0;
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	for (; i < MLO_OUT_N_PIXS_OFF; ++i)
	{
		top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
	}
	for (; i < MLO_IN_TILE0; ++i)
	{
		top_dat[pvt_off + i] = 0;
	}

}
#if 1
#if (MLO_IN_TILE0 - MLO_OUT_N_PIXS_OFF <= MLO_FILTER_PAD0)
#define MLO_OUT_MASK_SZ (MLO_IN_TILE0 - MLO_OUT_N_PIXS_OFF)
#else
#define MLO_OUT_MASK_SZ (MLO_FILTER_PAD0)
#endif
#endif

//#define MLO_OUT_MASK_SZ (1)
static inline void spanReadingOutput3x3(int k, int j, int top_df_off, _FLOAT mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
	__private _FLOAT * out_mask,
#endif
	__private _FLOAT * top_dat, const __global _FLOAT * top_df)
{
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	for (int i = 0; i < MLO_IN_TILE0; ++i)
	{
		top_dat[pvt_off + i] = top_df[top_df_off + i] * mask
			;
	}
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16

	for (int i = MLO_OUT_N_PIXS_OFF; i < MLO_OUT_N_PIXS_OFF + MLO_OUT_MASK_SZ; ++i)
	{
		top_dat[pvt_off + i] *= out_mask[i - MLO_OUT_N_PIXS_OFF];

	}
#endif
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
	// 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
	// TODO: more than 1 input
	int o = iDiv(lcl_id, MLO_N_SPANS_PER_SCAN);
	//	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);
	int spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);

	int lcl_bot_off = spn * MLO_IN_TILE0;
	int out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
	gbl_out_off += out_wk_item_off;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

// 3x3 out mask
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16

	_FLOAT out_mask[MLO_OUT_MASK_SZ];
	if (spn == MLO_N_SPANS_PER_SCAN - 1)
	{
		for (int i = 0; i < MLO_OUT_MASK_SZ; ++i)
		{
			out_mask[i] = 0;
		}
	}
	else
	{
		for (int i = 0; i < MLO_OUT_MASK_SZ; ++i)
		{
			out_mask[i] = 1;
		}
	}

#endif

	// over all batches
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// top border input block
		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input map
		readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);
		// move input pointer
		gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

		for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
		{
			top_dat[i] = 0;
		}

		// prefetch output

		int gbl_out_scan_off1 = gbl_out_scan_off;
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
		{
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				// loop around all output maps
				int top_df_off = gbl_out_scan_off1 + j*MLO_OUT_STRIDE;
				_FLOAT mask = 1;
#if MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

// 5x5 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) > 16
				if (spn == MLO_N_SPANS_PER_SCAN - 1)
				{

					spanRightSiding5x5(k, top_df_off, j, mask, top_dat, top_df);

				}
				else
#endif
				{
					spanReadingOutput3x3(k, j, top_df_off, mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
						out_mask,
#endif
						top_dat, top_df);
				}

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

		int sc = 0;
		int sc_lcl_off = lcl_bot_off;

		// prolog
		// handling padding

		// pad0
		for (; sc < MLO_FILTER_PAD1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
		}

		for (; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
			- MLO_FILTER_PAD1
			// 3x3 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
			- 1
#endif
#endif
			; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
				// move in the last output scans
// 5x5 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) > 16
				if (spn == MLO_N_SPANS_PER_SCAN - 1)
				{
					spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
				}
				else
#endif
				{
					spanReadingOutput3x3(k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
						out_mask,
#endif
						top_dat, top_df);
				}

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);
		}

// non-border input blocks
		for (int i_loop = 0;i_loop < MLO_N_GENERIC_LOOPS; ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);

			readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (; sc < (i_loop + 2) * MLO_IN_EXTENT1
				// 3x3 out of range
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					// move in the last output scans
					// 5x5 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) > 16
					if (spn == MLO_N_SPANS_PER_SCAN - 1)
					{
						spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
					}
					else
#endif
					{

						spanReadingOutput3x3(k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
							out_mask,
#endif
							top_dat, top_df);

					}

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// bottom border block

		for (int i_loop = 0; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1); ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			// read 1 scan line less
			// padding processing takes care of the bottom border.
			// do need sync with the real read: non intersecting areas.
#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))
#if 0 //MLO_IN_N_VERT_LOOPS > 1
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (int i = lcl_id; i < MLO_IN_VERT_READS - MLO_LAST_VERT_READS; i += MLO_GRP_SZ)
				{
					lcl_bot[c*MLO_IN_LCL_SZ + MLO_LAST_VERT_READS *MLO_IN_LCL_WIDTH + i] = 0;
				}
			}
#endif

			readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

			// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1
				// 3x3 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
				- 1
#endif
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					// move in the last output scans

					if (spn == MLO_N_SPANS_PER_SCAN - 1)
					{
						spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
					}
					else
					{
						int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (MLO_FILTER_SIZE1 - 1) *MLO_IN_TILE0;
						for (int i = 0; i < MLO_IN_TILE0; ++i)
						{
							top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
						}
					}

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// handling 3x3 out of range
#if MLO_IN_N_VERT_LOOPS == 1 && MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
		{
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
				// move in the last output scans

				if (spn == MLO_N_SPANS_PER_SCAN - 1 )
				{
					spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
				}
				else
				{

					int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (MLO_FILTER_SIZE1 - 1) *MLO_IN_TILE0;
					for (int i = 0; i < MLO_IN_TILE0; ++i)
					{
						top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
					}
				}

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		}

		++sc; gbl_out_scan_off += MLO_OUT_STRIDE; sc_lcl_off += MLO_IN_LCL_WIDTH;
#endif

		// epilog
		// handling padding
		// pad0/1

		for (; sc < MLO_OUT_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, (MLO_FILTER_PAD1 + 1 - (MLO_OUT_HEIGHT - sc)), pvt_accum, lcl_bot, top_dat);
			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

	// final summation over all output maps and each filter row
	// this coudl be done with log but it negligeble anyway
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				int pvt_off = k*MLO_FILTER_SIZE0 * MLO_FILTER_SIZE1 + l*MLO_FILTER_SIZE0 + n;
				lcl[lcl_id * MLO_FILTER_SIZE0 + n] =
					pvt_accum[pvt_off];

			}

			barrier(CLK_LOCAL_MEM_FENCE);

			if (spn == 0)
			{
				for (int s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
				{

					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						int pvt_off = k*MLO_FILTER_SIZE0 * MLO_FILTER_SIZE1 + l*MLO_FILTER_SIZE0 + n;
						pvt_accum[pvt_off]
							+= lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
#if 0
						if (/*fabs(pvt_accum[pvt_off] - 0.020364f) < 0.0001f*/ pvt_off == 12 && get_global_id(1) == 0 && get_global_id(2) == 0 && get_local_id(0) == 0 && k == 0/* && l == 2 && n == 2*/)
						{
							printf("G:s: %d %d %d  %f %f\n",
								get_local_id(0),
								(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n,
								pvt_off,
								pvt_accum[pvt_off],
								lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n]
							);
						}
#endif

					}

				}
			}

//			barrier(CLK_LOCAL_MEM_FENCE);
		}
	}

// output
// inputs are outputs
// TODO : for more than 1 input
	int c = 0;

	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx + o) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24((c_idx + c), (int)MLO_WEI_CHANNEL_STRIDE);
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		if (spn == 0 && o_idx + o + k*MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
		{

			for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
			{
				weights_df[wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + i] = pvt_accum[k*MLO_FILTER_SIZE0 * MLO_FILTER_SIZE1 + i];
#if 0
				if (wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + i == 12)
				{
					printf("G:o: %d %d  %f\n",
						get_local_id(0),
						k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + i,
						pvt_accum[k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + i]
					);
				}
#endif
			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if  MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS ((int)(MLO_IN_N_VERT_LOOPS - 2))
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
static inline void readInput(int lcl_id, int gbl_in_scan_off, int n_v_reads, const __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		int c = 0;
		int c_scan = iDiv(p4, (MLO_N_IN_HORIZ_READS));

		int c_pix4 = iMod(p4, c_scan, (MLO_N_IN_HORIZ_READS));

//		if (c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
			}

// MLO_N_LCL_IN_MAPS inputs
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
static inline void Processing(int sc, int sc_lcl_off, int top_lim, int bot_lim, __private _FLOAT * pvt_accum, __local _FLOAT * lcl_bot, __private _FLOAT * top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (int m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				_FLOAT bot_val = lcl_bot[sc_lcl_off + n + m];
				for(int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int pvt_top_off =  k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
					int pvt_accum_off = k*MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;

					_FLOAT top_val = top_dat[pvt_top_off];

					pvt_accum[pvt_accum_off]
						// each wk-it process an input
						+= bot_val*top_val;
#if 0
					if (bot_val * top_val != 0 && get_global_id(1) == 0 && get_global_id(2) == 0 && (get_local_id(0) == 0 || get_local_id(0) == 1) && k == 0 && l == 0 && n == 0)
					{
						printf("G: %d %d %d %d  %f %f %f %f\n",
							get_local_id(0),
							sc,
							sc_lcl_off + n + m,
							pvt_top_off,
							pvt_accum[pvt_accum_off],
							bot_val * top_val,
							bot_val,
							top_val
						);
					}
#endif
				}
			}

		}

	}

}

static inline void moveOutputUp(__private _FLOAT * top_dat)
{
	// move up output to reduce overfetch
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
		{
			for (int i = 0; i < MLO_IN_TILE0; ++i)
			{
				int pvt_off_n = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0 + i;
				int pvt_off_o = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) *MLO_IN_TILE0 + i;
				top_dat[pvt_off_n] = top_dat[pvt_off_o];
			}
		}
	}
}

static inline void spanRightSiding5x5(int k, int top_df_off, int j, _FLOAT mask, __private _FLOAT * top_dat, __global const _FLOAT * top_df)
{
	int i = 0;
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	for (; i < MLO_OUT_N_PIXS_OFF; ++i)
	{
		top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
	}
	for (; i < MLO_IN_TILE0; ++i)
	{
		top_dat[pvt_off + i] = 0;
	}

}
#if 1
#if (MLO_IN_TILE0 - MLO_OUT_N_PIXS_OFF <= MLO_FILTER_PAD0)
#define MLO_OUT_MASK_SZ (MLO_IN_TILE0 - MLO_OUT_N_PIXS_OFF)
#else
#define MLO_OUT_MASK_SZ (MLO_FILTER_PAD0)
#endif
#endif

//#define MLO_OUT_MASK_SZ (1)
static inline void spanReadingOutput3x3(int k, int j, int top_df_off, _FLOAT mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
	__private _FLOAT * out_mask,
#endif
	__private _FLOAT * top_dat, const __global _FLOAT * top_df)
{
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	for (int i = 0; i < MLO_IN_TILE0; ++i)
	{
		top_dat[pvt_off + i] = top_df[top_df_off + i] * mask
			;
	}
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16

	for (int i = MLO_OUT_N_PIXS_OFF; i < MLO_OUT_N_PIXS_OFF + MLO_OUT_MASK_SZ; ++i)
	{
		top_dat[pvt_off + i] *= out_mask[i - MLO_OUT_N_PIXS_OFF];

	}
#endif
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
	// 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
	// TODO: more than 1 input
	int o = iDiv(lcl_id, MLO_N_SPANS_PER_SCAN);
	//	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);
	int spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);

	int lcl_bot_off = spn * MLO_IN_TILE0;
	int out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
	gbl_out_off += out_wk_item_off;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

// 3x3 out mask
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16

	_FLOAT out_mask[MLO_OUT_MASK_SZ];
	if (spn == MLO_N_SPANS_PER_SCAN - 1)
	{
		for (int i = 0; i < MLO_OUT_MASK_SZ; ++i)
		{
			out_mask[i] = 0;
		}
	}
	else
	{
		for (int i = 0; i < MLO_OUT_MASK_SZ; ++i)
		{
			out_mask[i] = 1;
		}
	}

#endif

	// over all batches
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// top border input block
		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input map
		readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);
		// move input pointer
		gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

		for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
		{
			top_dat[i] = 0;
		}

		// prefetch output

		int gbl_out_scan_off1 = gbl_out_scan_off;
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
		{
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				// loop around all output maps
				int top_df_off = gbl_out_scan_off1 + j*MLO_OUT_STRIDE;
				_FLOAT mask = 1;
#if MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

// 5x5 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) > 16
				if (spn == MLO_N_SPANS_PER_SCAN - 1)
				{

					spanRightSiding5x5(k, top_df_off, j, mask, top_dat, top_df);

				}
				else
#endif
				{
					spanReadingOutput3x3(k, j, top_df_off, mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
						out_mask,
#endif
						top_dat, top_df);
				}

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

		int sc = 0;
		int sc_lcl_off = lcl_bot_off;

		// prolog
		// handling padding

		// pad0
		for (; sc < MLO_FILTER_PAD1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
		}

		for (; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
			- MLO_FILTER_PAD1
			// 3x3 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
			- 1
#endif
#endif
			; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
				// move in the last output scans
// 5x5 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) > 16
				if (spn == MLO_N_SPANS_PER_SCAN - 1)
				{
					spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
				}
				else
#endif
				{
					spanReadingOutput3x3(k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
						out_mask,
#endif
						top_dat, top_df);
				}

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);
		}

// non-border input blocks
		for (int i_loop = 0;i_loop < MLO_N_GENERIC_LOOPS; ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);

			readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (; sc < (i_loop + 2) * MLO_IN_EXTENT1
				// 3x3 out of range
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					// move in the last output scans
					// 5x5 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) > 16
					if (spn == MLO_N_SPANS_PER_SCAN - 1)
					{
						spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
					}
					else
#endif
					{

						spanReadingOutput3x3(k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
#if MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
							out_mask,
#endif
							top_dat, top_df);

					}

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// bottom border block

		for (int i_loop = 0; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1); ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			// read 1 scan line less
			// padding processing takes care of the bottom border.
			// do need sync with the real read: non intersecting areas.
#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))
#if 0 //MLO_IN_N_VERT_LOOPS > 1
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (int i = lcl_id; i < MLO_IN_VERT_READS - MLO_LAST_VERT_READS; i += MLO_GRP_SZ)
				{
					lcl_bot[c*MLO_IN_LCL_SZ + MLO_LAST_VERT_READS *MLO_IN_LCL_WIDTH + i] = 0;
				}
			}
#endif

			readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

			// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1
				// 3x3 out of range
#if MLO_OUT_N_PIXS_OFF > 0 && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
				- 1
#endif
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					// move in the last output scans

					if (spn == MLO_N_SPANS_PER_SCAN - 1)
					{
						spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
					}
					else
					{
						int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (MLO_FILTER_SIZE1 - 1) *MLO_IN_TILE0;
						for (int i = 0; i < MLO_IN_TILE0; ++i)
						{
							top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
						}
					}

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// handling 3x3 out of range
#if MLO_IN_N_VERT_LOOPS == 1 && MLO_OUT_N_PIXS_OFF > 0  && (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) <= 16
		{
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_FILTER_SIZE1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
				// move in the last output scans

				if (spn == MLO_N_SPANS_PER_SCAN - 1 )
				{
					spanRightSiding5x5(k, top_df_off, (MLO_FILTER_SIZE1 - 1), mask, top_dat, top_df);
				}
				else
				{
#if 1
					if (get_global_id(1) == 0 && get_global_id(2) == 0 && (get_local_id(0) == 0 || get_local_id(0) == 1))
					{
						printf("G:b21: %d %d\n",
							get_local_id(0),
							sc
						);
					}
#endif
					int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (MLO_FILTER_SIZE1 - 1) *MLO_IN_TILE0;
					for (int i = 0; i < MLO_IN_TILE0; ++i)
					{
						top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
					}
				}

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		}

		++sc; gbl_out_scan_off += MLO_OUT_STRIDE; sc_lcl_off += MLO_IN_LCL_WIDTH;
#endif

		// epilog
		// handling padding
		// pad0/1

		for (; sc < MLO_OUT_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, (MLO_FILTER_PAD1 + 1 - (MLO_OUT_HEIGHT - sc)), pvt_accum, lcl_bot, top_dat);
			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

	// final summation over all output maps and each filter row
	// this coudl be done with log but it negligeble anyway
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				int pvt_off = k*MLO_FILTER_SIZE0 * MLO_FILTER_SIZE1 + l*MLO_FILTER_SIZE0 + n;
				lcl[lcl_id * MLO_FILTER_SIZE0 + n] =
					pvt_accum[pvt_off];

			}

			barrier(CLK_LOCAL_MEM_FENCE);

			if (spn == 0)
			{
				for (int s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
				{

					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						int pvt_off = k*MLO_FILTER_SIZE0 * MLO_FILTER_SIZE1 + l*MLO_FILTER_SIZE0 + n;
						pvt_accum[pvt_off]
							+= lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
#if 0
						if (/*fabs(pvt_accum[pvt_off] - 0.020364f) < 0.0001f*/ pvt_off == 12 && get_global_id(1) == 0 && get_global_id(2) == 0 && get_local_id(0) == 0 && k == 0/* && l == 2 && n == 2*/)
						{
							printf("G:s: %d %d %d  %f %f\n",
								get_local_id(0),
								(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n,
								pvt_off,
								pvt_accum[pvt_off],
								lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n]
							);
						}
#endif

					}

				}
			}

//			barrier(CLK_LOCAL_MEM_FENCE);
		}
	}

// output
// inputs are outputs
// TODO : for more than 1 input
	int c = 0;

	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx + o) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24((c_idx + c), (int)MLO_WEI_CHANNEL_STRIDE);
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		if (spn == 0 && o_idx + o + k*MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
		{

			for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
			{
				weights_df[wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + i] = pvt_accum[k*MLO_FILTER_SIZE0 * MLO_FILTER_SIZE1 + i];
#if 0
				if (wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + i == 12)
				{
					printf("G:o: %d %d  %f\n",
						get_local_id(0),
						k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + i,
						pvt_accum[k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + i]
					);
				}
#endif
			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}


Finished debugging opencl c spearman.

Finished debugging opencl c spearman.

Finished debugging opencl kernel that does basic spearman analysis.
Still need to adjust for bigger sizes, but everything else is done.

void switch_f(__local float* a, __local float* b)
{
   float c = *a;
   *a = *b;
   *b = c;
}

void switch_i(__local int* a, __local int* b)
{
   int c = *a;
   *a = *b;
   *b = c;
}

void build_lists(int aInd, int bInd, int size, __local float* alist,
                 __local float* blist, __global float* clist, __local int* rank)
{
   int ix = get_local_id(0)<<1;
   if (ix<size)
   {
      alist[ix] = clist[ix+aInd];
      blist[ix] = clist[ix+bInd];
      rank[ix] = ix+1;
   }
   else
   {
      alist[ix] = INFINITY;
      blist[ix] = INFINITY;
      rank[ix] = 0;
   }
   if ((ix+1)<size)
   {
      alist[ix+1] = clist[ix+aInd+1];
      blist[ix+1] = clist[ix+bInd+1];
      rank[ix+1] = ix+2;
   }
   else
   {
      alist[ix+1] = INFINITY;
      blist[ix+1] = INFINITY;
      rank[ix+1] = 0;
   }
   barrier(CLK_LOCAL_MEM_FENCE);
}

void bitonic_sort_ff(__local float* sort, __local float* extra)
{
   int i = get_local_id(0);
   int bsize = get_local_size(0)<<1;
   int ob,ib,dir,a,b,t;
   for (ob=2;ob<=bsize;ob<<=1)
   {
      dir = -((i/(ob>>1))&0x1);
      for (ib=ob;ib>=2;ib>>=1)
      {
         t = ib>>1;
         a = (i/t)*ib+(i%t);
         b = a + t;
         if (((sort[a]>sort[b])&&!dir)||((sort[a]<sort[b])&&dir))
         {
            switch_f(&sort[a],&sort[b]);
            switch_f(&extra[a],&extra[b]);
         }
         barrier(CLK_LOCAL_MEM_FENCE);
      }
   }
}

void bitonic_sort_fi(__local float* sort, __local int* extra)
{
   int i = get_local_id(0);
   int bsize = get_local_size(0)<<1;
   int ob,ib,dir,a,b,t;
   for (ob=2;ob<=bsize;ob<<=1)
   {
      dir = -((i/(ob>>1))&0x1);
      for (ib=ob;ib>=2;ib>>=1)
      {
         t = ib>>1;
         a = (i/t)*ib+(i%t);
         b = a + t;
         if (((sort[a]>sort[b])&&!dir)||((sort[a]<sort[b])&&dir))
         {
            switch_f(&sort[a],&sort[b]);
            switch_i(&extra[a],&extra[b]);
         }
         barrier(CLK_LOCAL_MEM_FENCE);
      }
   }
}

long calc_ranks(int size, __local long* sums, __local int* ranks)
{
   int ix = get_local_id(0)<<1;
   long c;
   if (ix<size)
   {
      c = ranks[ix]-(ix+1);
      sums[ix] = c*c;
   }
   else
   {
      sums[ix] = 0.0;
   }
   if ((ix+1)<size)
   {
      c = ranks[ix+1]-(ix+2);
      sums[ix+1] = c*c;
   }
   else
   {
      sums[ix+1] = 0.0;
   }
   barrier(CLK_LOCAL_MEM_FENCE);
}

void accumulate(__local long* summation)
{
   int i = get_local_id(0);
   int hbsize = get_local_size(0);
   int b;
   for (b=hbsize;b>=1;b>>=1)
   {
      if (i<b)
      {
         summation[i] += summation[i+b];
      }
      barrier(CLK_LOCAL_MEM_FENCE);
   }
}

__kernel void spearman
(
      int aInd,
      int bInd,
      int size,
      __global float* correlations,
      __global float* result,
      __local float* alist,
      __local float* blist,
      __local int* rank,
      __local long* summation
)
{
   build_lists(aInd,bInd,size,alist,blist,correlations,rank);
   bitonic_sort_ff(alist,blist);
   bitonic_sort_fi(blist,rank);
   calc_ranks(size,summation,rank);
   accumulate(summation);
   if (get_local_id(0)==0)
   {
      *result = 1.0-(6.0*summation[0]/((float)(size*(size*size-1))));
   }
}

void switch_f(__local float* a, __local float* b)
{
   float c = *a;
   *a = *b;
   *b = c;
}

void switch_i(__local int* a, __local int* b)
{
   int c = *a;
   *a = *b;
   *b = c;
}

void bitonic_sort_ff(__local float* sort, __local float* extra)
{
   int i = get_local_id(0);
   int bsize = get_local_size(0)<<1;
   int ob,ib,dir,a,b,t;
   for (ob=2;ob<=bsize;ob<<=1)
   {
      dir = -((i/(ob>>1))&0x1);
      for (ib=ob;ib>=2;ib>>=1)
      {
         t = ib>>1;
         a = (i/t)*ib+(i%t);
         b = a + t;
         if ((sort[a]>sort[b])^dir)
         {
            switch_f(&sort[a],&sort[b]);
            switch_f(&extra[a],&extra[b]);
         }
         barrier(CLK_LOCAL_MEM_FENCE);
      }
   }
}

void bitonic_sort_fi(__local float* sort, __local int* extra)
{
   int i = get_local_id(0);
   int bsize = get_local_size(0)<<1;
   int ob,ib,dir,a,b,t;
   for (ob=2;ob<=bsize;ob<<=1)
   {
      dir = -((i/(ob>>1))&0x1);
      for (ib=ob;ib>=2;ib>>=1)
      {
         t = ib>>1;
         a = (i/t)*ib+(i%t);
         b = a + t;
         if ((sort[a]>sort[b])^dir)
         {
            switch_f(&sort[a],&sort[b]);
            switch_i(&extra[a],&extra[b]);
         }
         barrier(CLK_LOCAL_MEM_FENCE);
      }
   }
}

float grab_correlation(__global float* correlations, int i, int size)
{
   if (i<size)
   {
      return correlations[i];
   }
   else
   {
      return INFINITY;
   }
}

long calc_rank_diff(long a, long b, int size)
{
   if (b<size)
   {
      return (a-b)*(a-b);
   }
   else
   {
      return 0;
   }
}

void accumulate(__local long* summation)
{
   int i = get_local_id(0);
   int hbsize = get_local_size(0);
   int b;
   for (b=hbsize;b>=1;b>>=1)
   {
      if (i<b)
      {
         summation[i] += summation[i+b];
      }
      barrier(CLK_LOCAL_MEM_FENCE);
   }
}

__kernel void spearman
(
      int aInd,
      int bInd,
      int size,
      __global float* correlations,
      __global float* result,
      __local float* alist,
      __local float* blist,
      __local int* rank,
      __local long* summation
)
{
   int ix = get_local_id(0)<<1;
   int nsize = get_local_size(0)<<1;
   alist[ix] = grab_correlation(correlations,ix+aInd,size);
   alist[ix+1] = grab_correlation(correlations,ix+aInd+1,size);
   blist[ix] = grab_correlation(correlations,ix+bInd,size);
   blist[ix+1] = grab_correlation(correlations,ix+bInd+1,size);
   rank[ix] = ix+1;
   rank[ix+1] = ix+2;
   barrier(CLK_LOCAL_MEM_FENCE);
   bitonic_sort_ff(alist,blist);
   bitonic_sort_fi(blist,rank);
   summation[ix] = calc_rank_diff(rank[ix],ix+1,size);
   summation[ix+1] = calc_rank_diff(rank[ix+1],ix+2,size);
   barrier(CLK_LOCAL_MEM_FENCE);
   accumulate(summation);
   if (get_local_id(0)==0)
   {
      *result = 1.0-(6.0*summation[0]/(float)(nsize*(nsize-1)));
   }
}


exact.cl: Remove all branching

exact.cl: Remove all branching

All branching is now replaced by the appropriate OpenCL relational intrinsics
(select, isequal, etc.).  This speeds up ./exact test from 1.873 s to 1.471 s.

// Kernel code for exact poker winning probabilities

#include "exact.h"

// OpenCL whines if we don't have prototypes
inline score_t drop_bit(score_t x);
inline score_t drop_two_bits(score_t x);
inline cards_t count_suits(cards_t cards);
inline uint32_t cards_with_suit(cards_t cards, cards_t suits);
inline score_t all_straights(score_t unique);
inline score_t max_bit2(score_t x);
inline score_t max_bit3(score_t x);
score_t score_hand(cards_t cards);
inline uint64_t compare_cards(cards_t alice_cards, cards_t bob_cards, __global const cards_t* free, struct five_subset_t set);
inline cards_t mostly_random_set(uint64_t r);

// Drop the lowest bit, assuming a nonzero input
inline score_t drop_bit(score_t x) {
    return x-min_bit(x);
}

// Drop the two lowest bits, assuming at least two bits set
inline score_t drop_two_bits(score_t x) {
    return drop_bit(drop_bit(x));
}

// Count the number of cards in each suit in parallel
inline cards_t count_suits(cards_t cards) {
    const cards_t suits = 1+((cards_t)1<<13)+((cards_t)1<<26)+((cards_t)1<<39);
    cards_t s = cards; // initially, each suit has 13 single bit chunks
    s = (s&suits*0x1555)+(s>>1&suits*0x0555); // reduce each suit to 1 single bit and 6 2-bit chunks
    s = (s&suits*0x1333)+(s>>2&suits*0x0333); // reduce each suit to 1 single bit and 3 4-bit chunks
    s = (s&suits*0x0f0f)+(s>>4&suits*0x010f); // reduce each suit to 2 8-bit chunks
    s = (s+(s>>8))&suits*0xf; // reduce each suit to 1 16-bit count (only 4 bits of which can be nonzero)
    return s;
}

// Given a set of cards and a set of suits, find the set of cards with that suit
inline uint32_t cards_with_suit(cards_t cards, cards_t suits) {
    cards_t c = cards&suits*0x1fff;
    c |= c>>13;
    c |= c>>26;
    return c&0x1fff;
}

// Find all straights in a (suited) set of cards, assuming cards == cards&0x1111111111111
inline score_t all_straights(score_t unique) {
    const score_t wheel = (1<<12)|1|2|4|8;
    const score_t u = unique&unique<<1;
    return (u&u>>2&unique>>3)|((unique&wheel)==wheel);
}

// Non-branching ternary operators.  All the 0* stuff is to make overload resolution work.  It should disappear at compile time.
#define if_nz(c,a,b) select((a),(b),0*(a)+isequal((c),0*(c)))
#define if_eq(x,y,a,b) select((b),(a),0*(a)+isequal((x),(y)))
#define if_ne(x,y,a,b) select((b),(a),0*(a)+isnotequal((x),(y)))
#define if_gt(x,y,a,b) select((b),(a),0*(a)+isgreater((x),(y)))
#define if_nz1(c,a) if_nz(c,a,0*(a))
#define if_ne1(x,y,a) if_ne(x,y,a,0*(a))

// Find the maximum bit set of x, assuming x has at most 2 bit sets
inline score_t max_bit2(score_t x) {
    score_t m = min_bit(x);
    return if_eq(x,m,x,x-m);
}

// Find the maximum bit set of x, assuming x has at most 3 bit sets
inline score_t max_bit3(score_t x) {
    return max_bit2(max_bit2(x));
}

// Determine the best possible five card hand out of a bit set of seven cards
score_t score_hand(cards_t cards) {
    #define SCORE(type,c0,c1) ((type)|((c0)<<14)|(c1))
    const score_t each_card = 0x1fff;
    const cards_t each_suit = 1+((cards_t)1<<13)+((cards_t)1<<26)+((cards_t)1<<39);

    // Check for straight flushes
    const cards_t suits = count_suits(cards);
    const cards_t flushes = each_suit&(suits>>2)&(suits>>1|suits); // Detect suits with at least 5 cards
    const score_t straight_flushes = all_straights(cards_with_suit(cards,flushes));
    score_t score = if_nz1(straight_flushes,SCORE(STRAIGHT_FLUSH,0,max_bit3(straight_flushes)));

    // Check for four of a kind
    const score_t cand = cards&cards>>26;
    const score_t cor  = (cards|cards>>26)&each_card*(1+(1<<13));
    const score_t quads = cand&cand>>13;
    const score_t unique = each_card&(cor|cor>>13);
    score = max(score,if_nz1(quads,SCORE(QUADS,quads,max_bit3(unique-quads))));

    // Check for a full house
    const score_t trips = (cand&cor>>13)|(cor&cand>>13);
    const score_t pairs = each_card&~trips&(cand|cand>>13|(cor&cor>>13));
    const score_t min_trips = min_bit(trips);
    score = max(score,if_nz1(trips,
        if_nz(pairs,SCORE(FULL_HOUSE,trips,max_bit2(pairs)), // If there are pairs, there can't be two kinds of trips
        if_ne1(trips,min_trips,SCORE(FULL_HOUSE,trips-min_trips,min_trips))))); // Two kind of trips: use only two of the lower one

    // Check for flushes
    const int suit_count = cards_with_suit(suits,flushes);
    score_t suited = cards_with_suit(cards,flushes);
    suited = if_gt(suit_count,5u,suited-min_bit(suited),suited);
    suited = if_gt(suit_count,6u,suited-min_bit(suited),suited);
    score = max(score,if_nz1(flushes,SCORE(FLUSH,0,suited)));

    // Check for straights
    const score_t straights = all_straights(unique);
    score = max(score,if_nz1(straights,SCORE(STRAIGHT,0,max_bit3(straights))));

    // Check for three of a kind
    score = max(score,if_nz1(trips,SCORE(TRIPS,trips,drop_two_bits(unique-trips))));

    // Check for pair or two pair
    const score_t high_pairs = drop_bit(pairs);
    score = max(score,if_nz1(pairs,
        if_eq(pairs,min_bit(pairs),SCORE(PAIR,pairs,drop_two_bits(unique-pairs)),
        if_eq(high_pairs,min_bit(high_pairs),SCORE(TWO_PAIR,pairs,drop_two_bits(unique-pairs)),
        SCORE(TWO_PAIR,high_pairs,drop_bit(unique-high_pairs))))));

    // Nothing interesting happened, so high cards win
    score = max(score,SCORE(HIGH_CARD,0,drop_two_bits(unique)));
    return score;
    #undef SCORE
}

// Evaluate a full set of hands and shared cards
inline uint64_t compare_cards(cards_t alice_cards, cards_t bob_cards, __global const cards_t* free, struct five_subset_t set) {
    const cards_t shared_cards = free_set(free,set);
    const score_t alice_score = score_hand(shared_cards|alice_cards),
                  bob_score   = score_hand(shared_cards|bob_cards);
    return if_gt(alice_score,bob_score,(uint64_t)1<<32,
           if_gt(bob_score,alice_score,(uint64_t)1,(uint64_t)0));
}

// Score a bunch of hands
__kernel void score_hands_kernel(__global const cards_t* cards, __global score_t* results) {
    const int id = get_global_id(0);
    results[id] = score_hand(cards[id]);
}

// Given Alice's and Bob's hands, determine outcomes for one block of shared cards
__kernel void compare_cards_kernel(__global const struct five_subset_t* five_subsets, __global const cards_t* free, __global uint64_t* results, const cards_t alice_cards, const cards_t bob_cards) {
    const int id = get_global_id(0);
    const int offset = id*BLOCK_SIZE;
    //const int bound = min(BLOCK_SIZE,NUM_FIVE_SUBSETS-offset);
    uint64_t sum = 0;
    for (int i = 0; i < BLOCK_SIZE; i++)
        sum += compare_cards(alice_cards,bob_cards,free,five_subsets[offset+i]);
    results[id] = sum;
}

inline cards_t mostly_random_set(uint64_t r) {
    cards_t cards = 0;
    #define ADD(a) \
        int i##a = (r>>(6*a)&0x3f)%52; \
        const cards_t b##a = (cards_t)1<<i##a; \
        cards |= if_ne(cards&b##a,(cards_t)0,min_bit(~cards),b##a);
    ADD(0) ADD(1) ADD(2) ADD(3) ADD(4) ADD(5) ADD(6)
    #undef ADD
    return cards;
}

// Hash a bunch of mostly random hand scores
__kernel void hash_scores_kernel(__global uint64_t* results, const uint64_t offset) {
    const int id = get_global_id(0), i = offset+id;
    uint64_t h = 0;
    for (int j = 0; j < 1024; j++)
        h = hash2(h,score_hand(mostly_random_set(hash2(i,j))));
    results[id] = h;
}

// Kernel code for exact poker winning probabilities

#include "exact.h"

// OpenCL whines if we don't have prototypes
inline score_t drop_bit(score_t x);
inline score_t drop_two_bits(score_t x);
inline cards_t count_suits(cards_t cards);
inline uint32_t cards_with_suit(cards_t cards, cards_t suits);
inline score_t all_straights(score_t unique);
inline score_t max_bit(score_t x, int n);
score_t score_hand(cards_t cards);
inline uint64_t compare_cards(cards_t alice_cards, cards_t bob_cards, __global const cards_t* free, struct five_subset_t set);
inline cards_t mostly_random_set(uint64_t r);

// Drop the lowest bit, assuming a nonzero input
inline score_t drop_bit(score_t x) {
    return x-min_bit(x);
}

// Drop the two lowest bits, assuming at least two bits set
inline score_t drop_two_bits(score_t x) {
    return drop_bit(drop_bit(x));
}

// Count the number of cards in each suit in parallel
inline cards_t count_suits(cards_t cards) {
    const cards_t suits = 1+((cards_t)1<<13)+((cards_t)1<<26)+((cards_t)1<<39);
    cards_t s = cards; // initially, each suit has 13 single bit chunks
    s = (s&suits*0x1555)+(s>>1&suits*0x0555); // reduce each suit to 1 single bit and 6 2-bit chunks
    s = (s&suits*0x1333)+(s>>2&suits*0x0333); // reduce each suit to 1 single bit and 3 4-bit chunks
    s = (s&suits*0x0f0f)+(s>>4&suits*0x010f); // reduce each suit to 2 8-bit chunks
    s = (s+(s>>8))&suits*0xf; // reduce each suit to 1 16-bit count (only 4 bits of which can be nonzero)
    return s;
}

// Given a set of cards and a set of suits, find the set of cards with that suit
inline uint32_t cards_with_suit(cards_t cards, cards_t suits) {
    cards_t c = cards&suits*0x1fff;
    c |= c>>13;
    c |= c>>26;
    return c&0x1fff;
}

// Find all straights in a (suited) set of cards, assuming cards == cards&0x1111111111111
inline score_t all_straights(score_t unique) {
    const score_t wheel = (1<<12)|1|2|4|8;
    const score_t u = unique&unique<<1;
    return (u&u>>2&unique>>3)|((unique&wheel)==wheel);
}

// Find the maximum bit set of x, assuming x has at most n bits set, where n <= 3
inline score_t max_bit(score_t x, int n) {
    if (n>1 && x!=min_bit(x)) x -= min_bit(x);
    if (n>2 && x!=min_bit(x)) x -= min_bit(x);
    return x;
}

// Determine the best possible five card hand out of a bit set of seven cards
score_t score_hand(cards_t cards) {
    #define SCORE(type,c0,c1) ((type)|((c0)<<14)|(c1))
    const score_t each_card = 0x1fff;
    const cards_t each_suit = 1+((cards_t)1<<13)+((cards_t)1<<26)+((cards_t)1<<39);

    // Check for straight flushes
    const cards_t suits = count_suits(cards);
    const cards_t flushes = each_suit&(suits>>2)&(suits>>1|suits); // Detect suits with at least 5 cards
    if (flushes) {
        const score_t straight_flushes = all_straights(cards_with_suit(cards,flushes));
        if (straight_flushes)
            return SCORE(STRAIGHT_FLUSH,0,max_bit(straight_flushes,3));
    }

    // Check for four of a kind
    const score_t cand = cards&cards>>26;
    const score_t cor  = (cards|cards>>26)&each_card*(1+(1<<13));
    const score_t quads = cand&cand>>13;
    const score_t unique = each_card&(cor|cor>>13);
    if (quads)
        return SCORE(QUADS,quads,max_bit(unique-quads,3));

    // Check for a full house
    const score_t trips = (cand&cor>>13)|(cor&cand>>13);
    const score_t pairs = each_card&~trips&(cand|cand>>13|(cor&cor>>13));
    if (trips) {
        if (pairs) // If there are pairs, there can't be two kinds of trips
            return SCORE(FULL_HOUSE,trips,max_bit(pairs,2));
        else if (trips!=min_bit(trips)) // Two kind of trips: use only two of the lower one
            return SCORE(FULL_HOUSE,trips-min_bit(trips),min_bit(trips));
    }

    // Check for flushes
    if (flushes) {
        const int count = cards_with_suit(suits,flushes);
        score_t best = cards_with_suit(cards,flushes);
        if (count>5) best -= min_bit(best);
        if (count>6) best -= min_bit(best);
        return SCORE(FLUSH,0,best);
    }

    // Check for straights
    const score_t straights = all_straights(unique);
    if (straights)
        return SCORE(STRAIGHT,0,max_bit(straights,3));

    // Check for three of a kind
    if (trips)
        return SCORE(TRIPS,trips,drop_two_bits(unique-trips));

    // Check for pair or two pair
    if (pairs) {
        if (pairs==min_bit(pairs))
            return SCORE(PAIR,pairs,drop_two_bits(unique-pairs));
        const cards_t high_pairs = drop_bit(pairs);
        if (high_pairs==min_bit(high_pairs))
            return SCORE(TWO_PAIR,pairs,drop_two_bits(unique-pairs));
        return SCORE(TWO_PAIR,high_pairs,drop_bit(unique-high_pairs));
    }

    // Nothing interesting happened, so high cards win
    return SCORE(HIGH_CARD,0,drop_two_bits(unique));
    #undef SCORE
}

// Evaluate a full set of hands and shared cards
inline uint64_t compare_cards(cards_t alice_cards, cards_t bob_cards, __global const cards_t* free, struct five_subset_t set) {
    const cards_t shared_cards = free_set(free,set);
    const score_t alice_score = score_hand(shared_cards|alice_cards),
                  bob_score   = score_hand(shared_cards|bob_cards);
    return alice_score>bob_score?(uint64_t)1<<32:alice_score<bob_score?1u:0;
}

// Score a bunch of hands
__kernel void score_hands_kernel(__global const cards_t* cards, __global score_t* results) {
    const int id = get_global_id(0);
    results[id] = score_hand(cards[id]);
}

// Given Alice's and Bob's hands, determine outcomes for one block of shared cards
__kernel void compare_cards_kernel(__global const struct five_subset_t* five_subsets, __global const cards_t* free, __global uint64_t* results, const cards_t alice_cards, const cards_t bob_cards) {
    const int id = get_global_id(0);
    const int offset = id*BLOCK_SIZE;
    //const int bound = min(BLOCK_SIZE,NUM_FIVE_SUBSETS-offset);
    uint64_t sum = 0;
    for (int i = 0; i < BLOCK_SIZE; i++)
        sum += compare_cards(alice_cards,bob_cards,free,five_subsets[offset+i]);
    results[id] = sum;
}

inline cards_t mostly_random_set(uint64_t r) {
    cards_t cards = 0;
    #define ADD(a) \
        int i##a = (r>>(6*a)&0x3f)%52; \
        cards_t b##a = (cards_t)1<<i##a; \
        cards |= cards&b##a?min_bit(~cards):b##a;
    ADD(0) ADD(1) ADD(2) ADD(3) ADD(4) ADD(5) ADD(6)
    #undef ADD
    return cards;
}

// Hash a bunch of mostly random hand scores
__kernel void hash_scores_kernel(__global uint64_t* results, const uint64_t offset) {
    const int id = get_global_id(0), i = offset+id;
    uint64_t h = 0;
    for (int j = 0; j < 1024; j++)
        h = hash2(h,score_hand(mostly_random_set(hash2(i,j))));
    results[id] = h;
}


Update prngcl_example_pi.cl

Update prngcl_example_pi.cl

/******************************************************************************
 * @file     prngcl_example_pi.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1.1
 *
 * @brief    [PRNGCL library]
 *           EXAMPLE: OpenCL implementation of MRG32k3a pseudo-random number generator
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2014, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_EXAMPLE_PI_CL
#define PRNGCL_EXAMPLE_PI_CL

#include "prngcl_common.cl"

__kernel void
calculate_pi(__global hgpu_float4* randoms,
             __global float* acceptance,
             const uint N)
{
    uint index = GID;
    float count = 0;
    hgpu_float4 rnd,rnd2;

    for (uint i=0; i<N; i++) {
        rnd = randoms[index];
        rnd2 = rnd * rnd;
        if ((rnd2.x+rnd2.y) <= 1.0) count += 1.0;
        if ((rnd2.z+rnd2.w) <= 1.0) count += 1.0;

        index += GID_SIZE;
    }
    acceptance[GID] += count;
}

#endif

/******************************************************************************
 * @file     prngcl_example_pi.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1
 *
 * @brief    [PRNGCL library]
 *           EXAMPLE: OpenCL implementation of MRG32k3a pseudo-random number generator
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2014, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_EXAMPLE_PI_CL
#define PRNGCL_EXAMPLE_PI_CL

#include "prngcl_common.cl"

__kernel void
calculate_pi(__global hgpu_float4* randoms,
             __global float* acceptance,
             const uint N)
{
    uint index = GID;
    float count = 0;
    hgpu_float4 rnd,rnd2;

    for (uint i=0; i<N; i++) {
        rnd = randoms[index];
        rnd2 = rnd * rnd;
        if ((rnd2.x+rnd2.y) <= 1.0) count += 1.0;
        if ((rnd2.z+rnd2.w) <= 1.0) count += 1.0;

        index += GID_SIZE;
    }
    acceptance[GID] += count;
}

#endif


Remove old code, add comment

Remove old code, add comment

/******************************************************************************
 * function: radix sort                                                       *
 * input:                                                                     *
 * output:                                                                    *
 ******************************************************************************/

 // Adapted from: https://github.com/Schw1ng/GPURadixSort
 // Also, see:
 // - http://http.developer.nvidia.com/GPUGems3/gpugems3_ch32.html
 // - http://www.heterogeneouscompute.org/wordpress/wp-content/uploads/2011/06/RadixSort.pdf
 // TODO: Can we replace some barriers with mem_fences?

typedef struct radix_config {
  uint radices;  // number of radices
  uint blocks;   // number of blocks
  uint gpb;      // groups per block
  uint tpg;      // threads per group
  uint epg;      // elements per group
  uint rpb;      // radices per block
  uint mask;     // bit mask
  uint l_val;    // L
  uint tpb;      // threads per block =?= get_local_size(0)
  uint size;     // total elements
} configuration;

kernel void zeroes(global uint* counters) {
  counters[get_global_id(0)] = 0;
}

/*
Start index for threads:
(block * groups_per_block + (thread / threads_per_group))
  * elements_per_group
  + (thread % threads_per_group)

then, increment by threads_per_group
*/

inline void prefix_sum(local uint* data, int len, int threads) {
  uint thread = get_local_id(0);
  int inc = 2;
  // reduce
  while (inc <= len) {
    int j = inc >> 1;
    for (int i = (j - 1) + (thread * inc); (i + inc) < len; i += (threads * inc))
      data[i + j] = data[i] + data[i + j];
    inc = inc << 1;
    barrier(CLK_LOCAL_MEM_FENCE);
  }
  // downsweep
  data[len - 1] = 0;
  barrier(CLK_LOCAL_MEM_FENCE);
  while (inc >= 2) {
    int j = inc >> 1;
    for (int i = (j - 1) + (thread * inc); (i + j) <= len; i += (threads * inc)) {
      uint tmp = data[i + j];
      data[i + j] = data[i] + data[i + j];
      data[i] = tmp;
    }
    inc = inc >> 1;
    barrier(CLK_LOCAL_MEM_FENCE);
  }
}

// Optimization: count in local memory and copy to global when the block is finished
kernel void count(global uint* cell_in, global volatile uint* counters,
                  configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  // Each group has its own counter for each radix.
  // Position in counter for the first group in this block.
  const uint group_offset = block * conf.gpb;
  // Thread id related to group (not block).
  const uint group_thread = thread % conf.tpg;
  // Start of the elements inside the block
  const uint elem_offset = (group_offset + group) * conf.epg;
  // Number of counters for each radix.
  const uint entries = conf.blocks * conf.gpb;
  // Startindex for the current thread.
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    const uint index = (bits * entries) + group_offset + group;
    // The following code ensures that the counters of each thread group are
    // sequentially incremented. Only one thread per group writes in each loop.
    for (uint j = 0; j < conf.tpg; ++j) {
      if (group_thread == j)
        ++counters[index];
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}

kernel void scan(global uint* cell_in, global volatile uint* counters,
                 global uint* prefixes, local uint* l_counters,
                 configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint entries = conf.blocks * conf.gpb; // one entry per group
  const uint radix_base = conf.rpb * block; // radix we handle here
  for (uint i = 0; i < conf.rpb; ++i) {
    const uint radix = radix_base + i;
    const uint start = (radix * entries) + thread;
    const uint end = (radix + 1) * entries;
    const uint prior = counters[((radix + 1) * entries) - 1];
    // Copy entries counters of the radix to local memory
    int k = 0;
    for (uint j = start; j < end; j+= conf.tpb) {
      l_counters[conf.tpb * k + thread] = counters[j];
      ++k;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    prefix_sum(l_counters, entries, conf.tpb);
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread == 1)
      prefixes[radix] = l_counters[entries - 1] + prior;
    barrier(CLK_GLOBAL_MEM_FENCE);
    k = 0;
    for (uint j = start; j < end; j += conf.tpb) {
      counters[j] = l_counters[thread + conf.tpb * k];
      ++k;
    }
    barrier(CLK_GLOBAL_MEM_FENCE);
  }
}

kernel void reorder(global uint* cell_in, global uint* cell_out,
                    global volatile uint* counters, global uint* prefixes,
                    local uint* l_counters, local uint* l_prefixes,
                    configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  const uint radix = conf.rpb * block;
  const uint entries = conf.gpb * conf.blocks;
  const uint rc_offset = radix * entries;
  // Finish phase 2 by combining the sum of each radix
  for (uint i = thread; i < conf.radices; i += conf.tpb) {
    l_prefixes[i] = prefixes[i];
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  prefix_sum(l_prefixes, conf.radices, conf.tpb);
  barrier(CLK_LOCAL_MEM_FENCE);
  // Load (groups per block * radices) counters, i.e., the block column
  for (uint i = 0; i < conf.gpb; ++i) {
    if (thread < conf.radices) {
      l_counters[thread * conf.gpb + i]
        = counters[thread * entries + block * conf.gpb + i] + l_prefixes[thread];
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  const uint group_offset = block * conf.gpb;
  const uint group_thread = thread % conf.tpg;
  const uint elem_offset = (group_offset + group) * conf.epg;
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    //const uint index = (bits * entries) + group_offset + group;
    const uint index = (bits * conf.gpb) + group;
    for (uint j = 0; j < conf.tpg; ++j) {
      if (group_thread == j) {
        cell_out[l_counters[index]] = cell_in[i];
        ++l_counters[index];
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }
  }
}

/*
kernel void reorder(global uint* cell_in, global uint* cell_out,
                    global volatile uint* counters, global uint* prefixes,
                    local uint* l_counters, local uint* l_prefixes,
                    configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  const uint radix = conf.rpb * block;
  const uint entries = conf.gpb * conf.blocks;
  const uint rc_offset = radix * entries;
  // Final step of sum operation
  for (uint i = 0 ; i < conf.rpb ; ++i) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      l_counters[i * entries + j] = counters[i * entries + rc_offset + j];
    }
  }
  // Read radix prefixes to localMemory
  for (uint i = thread; i < conf.radices; i += conf.tpb)
    l_prefixes[i] = prefixes[i];
  // prefix sum on the prefixes in local memory
  barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
  prefix_sum(l_prefixes, conf.radices, conf.tpb);
  barrier(CLK_LOCAL_MEM_FENCE);
  // add prefix sum to the subcounters
  for (uint i = 0; i < conf.rpb; ++i) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      atomic_add(&l_counters[i * entries + j], l_prefixes[radix + i]);
      //l_counters[i * entries + j] += l_prefixes[radix + i];
      //barrier(CLK_LOCAL_MEM_FENCE);
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
  // Write back radices at the calculated offsets
  for (uint i = 0; i < conf.rpb; ++i) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      counters[i * entries + rc_offset + j] = l_counters[i * entries + j];
    }
  }
  barrier(CLK_GLOBAL_MEM_FENCE | CLK_LOCAL_MEM_FENCE);
  const uint group_offset = block * conf.gpb;
  const uint group_thread = thread % conf.tpg;
  const uint elem_offset = (group_offset + group) * conf.epg;
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    const uint index = (bits * entries) + group_offset + group;
    //const uint old = atomic_inc(&counters[index]);
    //cell_out[old] = cell_in[i];
    //barrier(CLK_GLOBAL_MEM_FENCE);
    for (uint j = 0; j < conf.tpg; ++j) {
      if (group_thread == j) {
        ++counters[index];
        cell_out[counters[index] - 1] = cell_in[i];
        //atomic_inc(&counters[index]);
      }
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}
*/

kernel void reorder_kv(global uint* cell_in, global uint* cell_out,
                       global uint* value_in, global uint* value_out,
                       global uint* counters, global uint* prefixes,
                       local uint* l_counters, local uint* l_prefixes,
                       configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  const uint radix = conf.rpb * block;
  const uint entries = conf.gpb * conf.blocks;
  const uint rc_offset = radix * entries;
  // Finish the radix sum
  for (uint i = 0 ; i < conf.rpb ; i++) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      l_counters[i * entries + j] = counters[rc_offset + i * entries + j];
    }
  }
  // Read radix prefixes to localMemory
  for (uint i = thread; i < conf.radices; i+= conf.tpb) {
    l_prefixes[i] = prefixes[i];
  }
  // build prefix sum over radix counters?
  // (seems to be the prefixes, not counters)
  barrier(CLK_GLOBAL_MEM_FENCE);
  prefix_sum(l_prefixes, conf.radices, conf.tpb);
  barrier(CLK_GLOBAL_MEM_FENCE);
  // Add the sum of the radix to all subounter of the radixes
  for (uint i = 0; i < conf.rpb; i++) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      l_counters[i * entries + j] += l_prefixes[radix + i];
    }
    // barrier(CLK_GLOBAL_MEM_FENCE);
  }
  // Write back the radixes to their respectve offers
  for (uint i = 0; i < conf.rpb; i++) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      // The entries counters of the radix are read from global memory to
      // shared memory.
      counters[rc_offset + i * entries + j] = l_counters[i * entries + j];
    }
  }
  barrier(CLK_GLOBAL_MEM_FENCE);
  const uint group_offset = block * conf.gpb;
  const uint group_thread = thread % conf.tpg;
  const uint elem_offset = (group_offset + group) * conf.epg;
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    const uint index = bits * entries + group_offset + group;
    for (uint j = 0; j < conf.tpg; ++j) {
      if (thread % conf.tpg == j) {
        cell_out[counters[index]] = cell_in[i];
        value_out[counters[index]] = value_in[i];
        //atomic_inc(&counters[index]);
        ++counters[index];
      }
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}

/******************************************************************************
 * function: radix sort                                                       *
 * input:                                                                     *
 * output:                                                                    *
 ******************************************************************************/

 // Adapted from: https://github.com/Schw1ng/GPURadixSort
 // Also, see:
 // - http://http.developer.nvidia.com/GPUGems3/gpugems3_ch32.html
 // - http://www.heterogeneouscompute.org/wordpress/wp-content/uploads/2011/06/RadixSort.pdf
 // TODO: Can we replace some barriers with mem_fences?

typedef struct radix_config {
  uint radices;  // number of radices
  uint blocks;   // number of blocks
  uint gpb;      // groups per block
  uint tpg;      // threads per group
  uint epg;      // elements per group
  uint rpb;      // radices per block
  uint mask;     // bit mask
  uint l_val;    // L
  uint tpb;      // threads per block =?= get_local_size(0)
  uint size;     // total elements
} configuration;

kernel void zeroes(global uint* counters) {
  counters[get_global_id(0)] = 0;
}

/*
Start index for threads:
(block * groups_per_block + (thread / threads_per_group))
  * elements_per_group
  + (thread % threads_per_group)

then, increment by threads_per_group
*/

inline void prefix_sum(local uint* data, int len, int threads) {
  uint thread = get_local_id(0);
  int inc = 2;
  // reduce
  while (inc <= len) {
    int j = inc >> 1;
    for (int i = (j - 1) + (thread * inc); (i + inc) < len; i += (threads * inc))
      data[i + j] = data[i] + data[i + j];
    inc = inc << 1;
    barrier(CLK_LOCAL_MEM_FENCE);
  }
  // downsweep
  data[len - 1] = 0;
  barrier(CLK_LOCAL_MEM_FENCE);
  while (inc >= 2) {
    int j = inc >> 1;
    for (int i = (j - 1) + (thread * inc); (i + j) <= len; i += (threads * inc)) {
      uint tmp = data[i + j];
      data[i + j] = data[i] + data[i + j];
      data[i] = tmp;
    }
    inc = inc >> 1;
    barrier(CLK_LOCAL_MEM_FENCE);
  }
}

// Optimization: count in local memory and copy to global when the block is finished
kernel void count(global uint* cell_in, global volatile uint* counters,
                  configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  // Each group has its own counter for each radix.
  // Position in counter for the first group in this block.
  const uint group_offset = block * conf.gpb;
  // Thread id related to group (not block).
  const uint group_thread = thread % conf.tpg;
  // Start of the elements inside the block
  const uint elem_offset = (group_offset + group) * conf.epg;
  // Number of counters for each radix.
  const uint entries = conf.blocks * conf.gpb;
  // Startindex for the current thread.
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    const uint index = (bits * entries) + group_offset + group;
    // The following code ensures that the counters of each thread group are
    // sequentially incremented. Only one thread per group writes in each loop.
    for (uint j = 0; j < conf.tpg; ++j) {
      if (group_thread == j)
        ++counters[index];
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}

kernel void scan(global uint* cell_in, global volatile uint* counters,
                 global uint* prefixes, local uint* l_counters,
                 configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint entries = conf.blocks * conf.gpb; // one entry per group
  const uint radix_base = conf.rpb * block; // radix we handle here
  for (uint i = 0; i < conf.rpb; ++i) {
    const uint radix = radix_base + i;
    const uint start = (radix * entries) + thread;
    const uint end = (radix + 1) * entries;
    const uint prior = counters[((radix + 1) * entries) - 1];
    // Copy entries counters of the radix to local memory
    int k = 0;
    for (uint j = start; j < end; j+= conf.tpb) {
      l_counters[conf.tpb * k + thread] = counters[j];
      ++k;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    prefix_sum(l_counters, entries, conf.tpb);
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread == 1)
      prefixes[radix] = l_counters[entries - 1] + prior;
    barrier(CLK_GLOBAL_MEM_FENCE);
    k = 0;
    for (uint j = start; j < end; j += conf.tpb) {
      counters[j] = l_counters[thread + conf.tpb * k];
      ++k;
    }
    barrier(CLK_GLOBAL_MEM_FENCE);
  }
}

kernel void reorder(global uint* cell_in, global uint* cell_out,
                    global volatile uint* counters, global uint* prefixes,
                    local uint* l_counters, local uint* l_prefixes,
                    configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  const uint radix = conf.rpb * block;
  const uint entries = conf.gpb * conf.blocks;
  const uint rc_offset = radix * entries;
  // Finish phase 2 by combining the sum of each radix
  for (uint i = thread; i < conf.radices; i += conf.tpb) {
    l_prefixes[i] = prefixes[i];
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  prefix_sum(l_prefixes, conf.radices, conf.tpb);
  barrier(CLK_LOCAL_MEM_FENCE);
  // Load (groups per block * radices) counters, i.e., one colum
  // Load counters, adding the global radix prefixes
  for (uint i = 0; i < conf.gpb; ++i) {
    if (thread < conf.radices) {
      l_counters[thread * conf.gpb + i]
        = counters[thread * entries + block * conf.gpb + i] + l_prefixes[thread];
    }
    //l_counters[i * entries + j] = counters[i * entries + rc_offset + j]
    //                              + l_prefixes[radix + i];
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  const uint group_offset = block * conf.gpb;
  const uint group_thread = thread % conf.tpg;
  const uint elem_offset = (group_offset + group) * conf.epg;
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    //const uint index = (bits * entries) + group_offset + group;
    const uint index = (bits * conf.gpb) + group;
    for (uint j = 0; j < conf.tpg; ++j) {
      if (group_thread == j) {
        cell_out[l_counters[index]] = cell_in[i];
        ++l_counters[index];
        //uint old = atomic_inc(&counters[index]);
        //cell_out[old] = cell_in[i];
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }
  }
}

/*
kernel void reorder(global uint* cell_in, global uint* cell_out,
                    global volatile uint* counters, global uint* prefixes,
                    local uint* l_counters, local uint* l_prefixes,
                    configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  const uint radix = conf.rpb * block;
  const uint entries = conf.gpb * conf.blocks;
  const uint rc_offset = radix * entries;
  // Final step of sum operation
  for (uint i = 0 ; i < conf.rpb ; ++i) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      l_counters[i * entries + j] = counters[i * entries + rc_offset + j];
    }
  }
  // Read radix prefixes to localMemory
  for (uint i = thread; i < conf.radices; i += conf.tpb)
    l_prefixes[i] = prefixes[i];
  // prefix sum on the prefixes in local memory
  barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
  prefix_sum(l_prefixes, conf.radices, conf.tpb);
  barrier(CLK_LOCAL_MEM_FENCE);
  // add prefix sum to the subcounters
  for (uint i = 0; i < conf.rpb; ++i) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      atomic_add(&l_counters[i * entries + j], l_prefixes[radix + i]);
      //l_counters[i * entries + j] += l_prefixes[radix + i];
      //barrier(CLK_LOCAL_MEM_FENCE);
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
  // Write back radices at the calculated offsets
  for (uint i = 0; i < conf.rpb; ++i) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      counters[i * entries + rc_offset + j] = l_counters[i * entries + j];
    }
  }
  barrier(CLK_GLOBAL_MEM_FENCE | CLK_LOCAL_MEM_FENCE);
  const uint group_offset = block * conf.gpb;
  const uint group_thread = thread % conf.tpg;
  const uint elem_offset = (group_offset + group) * conf.epg;
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    const uint index = (bits * entries) + group_offset + group;
    //const uint old = atomic_inc(&counters[index]);
    //cell_out[old] = cell_in[i];
    //barrier(CLK_GLOBAL_MEM_FENCE);
    for (uint j = 0; j < conf.tpg; ++j) {
      if (group_thread == j) {
        ++counters[index];
        cell_out[counters[index] - 1] = cell_in[i];
        //atomic_inc(&counters[index]);
      }
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}
*/

kernel void reorder_kv(global uint* cell_in, global uint* cell_out,
                       global uint* value_in, global uint* value_out,
                       global uint* counters, global uint* prefixes,
                       local uint* l_counters, local uint* l_prefixes,
                       configuration conf, uint offset) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint group = thread / conf.tpg;
  const uint radix = conf.rpb * block;
  const uint entries = conf.gpb * conf.blocks;
  const uint rc_offset = radix * entries;
  // Finish the radix sum
  for (uint i = 0 ; i < conf.rpb ; i++) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      l_counters[i * entries + j] = counters[rc_offset + i * entries + j];
    }
  }
  // Read radix prefixes to localMemory
  for (uint i = thread; i < conf.radices; i+= conf.tpb) {
    l_prefixes[i] = prefixes[i];
  }
  // build prefix sum over radix counters?
  // (seems to be the prefixes, not counters)
  barrier(CLK_GLOBAL_MEM_FENCE);
  prefix_sum(l_prefixes, conf.radices, conf.tpb);
  barrier(CLK_GLOBAL_MEM_FENCE);
  // Add the sum of the radix to all subounter of the radixes
  for (uint i = 0; i < conf.rpb; i++) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      l_counters[i * entries + j] += l_prefixes[radix + i];
    }
    // barrier(CLK_GLOBAL_MEM_FENCE);
  }
  // Write back the radixes to their respectve offers
  for (uint i = 0; i < conf.rpb; i++) {
    for (uint j = thread; j < entries; j += conf.tpb) {
      // The entries counters of the radix are read from global memory to
      // shared memory.
      counters[rc_offset + i * entries + j] = l_counters[i * entries + j];
    }
  }
  barrier(CLK_GLOBAL_MEM_FENCE);
  const uint group_offset = block * conf.gpb;
  const uint group_thread = thread % conf.tpg;
  const uint elem_offset = (group_offset + group) * conf.epg;
  const uint start = elem_offset + group_thread;
  const uint end = min(elem_offset + conf.epg, conf.size);
  for (uint i = start; i < end; i += conf.tpg) {
    const uint bits = (cell_in[i] >> offset) & conf.mask;
    const uint index = bits * entries + group_offset + group;
    for (uint j = 0; j < conf.tpg; ++j) {
      if (thread % conf.tpg == j) {
        cell_out[counters[index]] = cell_in[i];
        value_out[counters[index]] = value_in[i];
        //atomic_inc(&counters[index]);
        ++counters[index];
      }
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}