cleaned 1x1 WrW

cleaned 1x1 WrW

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readData(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * __restrict g_data, __private _FLOAT * __restrict p_data, bool last_pixel)
{
	for (int j = 0; j < n; ++j)
	{
		int gbl_data_off0 = (j*map_stride + map_base < map_limit) ? gbl_data_off + j*map_stride*gbl_data_stride : 0;

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = 0;
			}

		}
		else
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}
		}

	} // for (int j = 0; j < n; ++j)

}

__attribute__((always_inline))
void readDataFlex(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * __restrict g_data, __local _FLOAT * __restrict l_data)
{

	for( int l = get_local_id(0); l < n*map_stride* MLO_MAP_WK_SZ; l += MLO_GRP_SZ)
	{

	    int r = 0;
		int k0 = l;
#if MLO_N_LCL_OUT_MAPS > 1
#if (map_stride* MLO_MAP_WK_SZ) &  (map_stride*MLO_MAP_WK_SZ - 1)
		r = iDiv(l, map_stride* MLO_MAP_WK_SZ);  // maps row
		k0 = iMod(l, r, map_stride* MLO_MAP_WK_SZ);
#else
		r = ((uint)l / (map_stride* MLO_MAP_WK_SZ));  // maps row
		k0 = ((uint)l & (map_stride* MLO_MAP_WK_SZ-1));

#endif
#endif

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		int k = iDiv(k0, MLO_MAP_WK_SZ);  // map column
		int p4 = iMod(k0, k, MLO_MAP_WK_SZ); // pixel block
#else
		int k = ((uint)k0 / MLO_MAP_WK_SZ);  // map column
		int p4 = ((uint)k0 & (MLO_MAP_WK_SZ-1)); // pixel block

#endif

		bool last_pixel = (p4 == MLO_MAP_WK_SZ -1);

		int gbl_data_off0 = (r*map_stride + k + map_base < map_limit) ? gbl_data_off + (r*map_stride + k)*gbl_data_stride + p4*MLO_READ_UNIT : 0;
		__private _FLOAT p_data[MLO_READ_UNIT];

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = 0;
			}

		}
		else
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = g_data[gbl_data_off0 + i];
			}
		}

		for(int i = 0; i < MLO_READ_UNIT; ++i)
		{
			l_data[(k*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i] = p_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_IN_STACKS per group, MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// loop over MLO_OUT_STACKS of output

// convolve with MLO_N_LCL_IN_MAPS per wk-item

// reduce with transform

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	_FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;
	__local _FLOAT * proc_mem = lcl_mem;

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_IN_STACKS * MLO_N_LCL_IN_MAPS); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	// map id inside group
	int m_idx = iDiv(lcl_id, MLO_MAP_WK_SZ);
	// read pixel inside the map
	int p4 = iMod(lcl_id, m_idx, MLO_MAP_WK_SZ);

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

	gbl_in_off += m_idx * MLO_IN_CHANNEL_STRIDE + p4*MLO_READ_UNIT;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS * MLO_OUT_STACKS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}
	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input maps
		readData(MLO_N_LCL_IN_MAPS, gbl_in_scan_off, MLO_IN_CHANNEL_STRIDE, MLO_IN_STACKS, (m_idx + c_idx), MLO_N_INPUTS, bot, bot_dat, last_pixel);

		// read output maps into LDS
		readDataFlex(MLO_N_LCL_OUT_MAPS, gbl_out_scan_off, MLO_OUT_CHANNEL_STRIDE, MLO_OUT_STACKS, (k_idx), MLO_N_OUTPUTS, top_df, proc_mem);

		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			/*
			core processing loop
			bot - input
			top - output diff

			do convolution with all available input maps

			*/

			for (int n = 0; n < MLO_OUT_STACKS; ++n)
			{
				__private _FLOAT pvt_top[MLO_READ_UNIT];
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_top[i] = proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i]; //top_df[gbl_out_scan_off + (k*MLO_OUT_STACKS + n0)*MLO_IN_CHANNEL_STRIDE + i]; //proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
				}

				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c]
							+= bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i];
#if 0

						if (k_idx + k*MLO_OUT_STACKS + n == 0 && c_idx + c * MLO_IN_STACKS + m_idx == 16 && m_idx < MLO_IN_STACKS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS)
						{
							printf("K:s: %d %d %d %d %d %d %d %d %f %f %f %f\n",
								MLO_OUT_STACKS,
								n,
								m_idx,
								gbl_out_scan_off,
								get_group_id(1),
								get_local_id(0),
								k,
								c,
								pvt_accum[(n* MLO_N_LCL_OUT_MAPS + k)*MLO_N_LCL_IN_MAPS + c],
								bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i],
								bot_dat[c*MLO_READ_UNIT + i],
								pvt_top[i]
							);
						}
#endif

					}

				}
			}
		}

	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_MAP_WK_SZ > 8
// transpose
	int red_base_off = m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		// write data
		red_mem[red_base_off + p4* MLO_ACCUM_SZ + l] = pvt_accum[l];

	}
	barrier(CLK_LOCAL_MEM_FENCE);

	// do final summation

	__private _FLOAT final_sum[1] = { 0 };

#if MLO_ACCUM_SZ & (MLO_ACCUM_SZ-1)
	int new_m_idx = iDiv(lcl_id, MLO_ACCUM_SZ);
	int new_p4 = iMod(lcl_id, new_m_idx, MLO_ACCUM_SZ);
#else
	int new_m_idx = ((uint)lcl_id / MLO_ACCUM_SZ);
	int new_p4 = ((uint)lcl_id & (MLO_ACCUM_SZ-1));

#endif

	int new_red_base_off = new_m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;

	for (int s = 0; s < MLO_MAP_WK_SZ; ++s)
	{
		final_sum[0] += red_mem[new_red_base_off + s*MLO_ACCUM_SZ + new_p4];
	}

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + new_m_idx) * MLO_WEI_CHANNEL_STRIDE);

#if MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS-1)

	int n = iDiv(new_p4, MLO_N_LCL_IN_MAPS);
	int c = iMod(new_p4, n, MLO_N_LCL_IN_MAPS);
#else

	int n = ((uint)new_p4 / MLO_N_LCL_IN_MAPS);
	int c = ((uint)new_p4 & (MLO_N_LCL_IN_MAPS-1));
#endif

	int k = 0;

	if (new_m_idx < MLO_IN_STACKS &&k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + new_m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
	{
		weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = final_sum[0];
	}
#else

#if 0 //MLO_IN_WIDTH > 24
// logar reduction
	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		red_mem[i] = 0;
	}

	int red_base_off = (m_idx >= MLO_OUT_STACKS) ? MLO_LCL_MEM_SZ : m_idx * MLO_POW2_MAP_WK_SZ;
	// final summation over each filter row
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// write data
		red_mem[red_base_off + p4] = pvt_accum[l];

		// barrier inside
		ReduceKernel(&red_mem[red_base_off], &pvt_accum[l], p4, p4, MLO_POW2_MAP_WK_SZ, 1, false);

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#else
	// direct reduction
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		lcl_mem[lcl_id] = pvt_accum[l];
		barrier(CLK_LOCAL_MEM_FENCE);
		if (p4 == 0)
		{
			for (int i = 1; i < MLO_MAP_WK_SZ; ++i)
			{
				pvt_accum[l] += lcl_mem[lcl_id + i];
			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

#endif

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + m_idx) * MLO_WEI_CHANNEL_STRIDE);

	if (p4 == 0)
	{
		for (int n = 0; n < MLO_OUT_STACKS  ; ++n)
		{
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if (m_idx < MLO_IN_STACKS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
					{
						weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}
	}

#endif
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
static inline void readData(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * g_data, __private _FLOAT * p_data, bool last_pixel)
{
	for (int j = 0; j < n; ++j)
	{
		int gbl_data_off0 = (j*map_stride + map_base < map_limit) ? gbl_data_off + j*map_stride*gbl_data_stride : 0;

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = 0;
			}

		}
		else
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}
		}

	} // for (int j = 0; j < n; ++j)

}

static inline void readDataFlex(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * g_data, __local _FLOAT * l_data)
{

	for( int l = get_local_id(0); l < n*map_stride* MLO_MAP_WK_SZ; l += MLO_GRP_SZ)
	{

	    int r = 0;
		int k0 = l;
#if MLO_N_LCL_OUT_MAPS > 1
		r = iDiv(l, map_stride* MLO_MAP_WK_SZ);  // maps row
		k0 = iMod(l, r, map_stride* MLO_MAP_WK_SZ);
#endif
		int k = iDiv(k0, MLO_MAP_WK_SZ);  // map column
		int p4 = iMod(k0, k, MLO_MAP_WK_SZ); // pixel block

		bool last_pixel = (p4 == MLO_MAP_WK_SZ -1);

		int gbl_data_off0 = (r*map_stride + k + map_base < map_limit) ? gbl_data_off + (r*map_stride + k)*gbl_data_stride + p4*MLO_READ_UNIT : 0;
		__private _FLOAT p_data[MLO_READ_UNIT];

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = 0;
			}

		}
		else
#endif
		{
//			*(MLO_READ_TYPE*)&p_data[j*MLO_READ_UNIT] = *(__global MLO_READ_TYPE*)&g_data[gbl_data_off + j*gbl_data_stride*MLO_OUT_STACKS];
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = g_data[gbl_data_off0 + i];
			}
		}

		for(int i = 0; i < MLO_READ_UNIT; ++i)
		{
			l_data[(k*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i] = p_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_IN_STACKS per group, MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// loop over MLO_OUT_STACKS of output

// convolve with MLO_N_LCL_IN_MAPS per wk-item

// reduce with transform

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;
	__local _FLOAT * proc_mem = lcl_mem;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int k_idx = get_group_id(0) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_IN_STACKS * MLO_N_LCL_IN_MAPS); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	// map id inside group
	int m_idx = iDiv(lcl_id, MLO_MAP_WK_SZ);
	// read pixel inside the map
	int p4 = iMod(lcl_id, m_idx, MLO_MAP_WK_SZ);

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

	gbl_in_off += m_idx * MLO_IN_CHANNEL_STRIDE + p4*MLO_READ_UNIT;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS * MLO_OUT_STACKS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}
	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input maps
		readData(MLO_N_LCL_IN_MAPS, gbl_in_scan_off, MLO_IN_CHANNEL_STRIDE, MLO_IN_STACKS, (m_idx + c_idx), MLO_N_INPUTS, bot, bot_dat, last_pixel);

		// read output maps into LDS
		readDataFlex(MLO_N_LCL_OUT_MAPS, gbl_out_scan_off, MLO_OUT_CHANNEL_STRIDE, MLO_OUT_STACKS, (k_idx), MLO_N_OUTPUTS, top_df, proc_mem);

		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			/*
			core processing loop
			bot - input
			top - output diff

			do convolution with all available input maps

			*/

			for (int n = 0; n < MLO_OUT_STACKS; ++n)
			{
				__private _FLOAT pvt_top[MLO_READ_UNIT];
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_top[i] = proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i]; //top_df[gbl_out_scan_off + (k*MLO_OUT_STACKS + n0)*MLO_IN_CHANNEL_STRIDE + i]; //proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
				}

				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c]
							+= bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i];
#if 0

						if (k_idx + k*MLO_OUT_STACKS + n == 0 && c_idx + c * MLO_IN_STACKS + m_idx == 16 && m_idx < MLO_IN_STACKS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS)
						{
							printf("K:s: %d %d %d %d %d %d %d %d %f %f %f %f\n",
								MLO_OUT_STACKS,
								n,
								m_idx,
								gbl_out_scan_off,
								get_group_id(1),
								get_local_id(0),
								k,
								c,
								pvt_accum[(n* MLO_N_LCL_OUT_MAPS + k)*MLO_N_LCL_IN_MAPS + c],
								bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i],
								bot_dat[c*MLO_READ_UNIT + i],
								pvt_top[i]
							);
						}
#endif

					}

				}
			}
		}

	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_MAP_WK_SZ > 8
// transpose
	int red_base_off = m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		// write data
		red_mem[red_base_off + p4* MLO_ACCUM_SZ + l] = pvt_accum[l];

	}
	barrier(CLK_LOCAL_MEM_FENCE);

	// do final summation

	__private _FLOAT final_sum[1] = { 0 };
	int new_m_idx = iDiv(lcl_id, MLO_ACCUM_SZ);
	int new_p4 = iMod(lcl_id, new_m_idx, MLO_ACCUM_SZ);
	int new_red_base_off = new_m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;

	for (int s = 0; s < MLO_MAP_WK_SZ; ++s)
	{
		final_sum[0] += red_mem[new_red_base_off + s*MLO_ACCUM_SZ + new_p4];
	}

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + new_m_idx) * MLO_WEI_CHANNEL_STRIDE);

	int n = iDiv(new_p4, MLO_N_LCL_IN_MAPS);
	int c = iMod(new_p4, n, MLO_N_LCL_IN_MAPS);
	int k = 0;

	if (new_m_idx < MLO_IN_STACKS &&k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + new_m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
	{
		weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = final_sum[0];
	}
#else

#if 0 //MLO_IN_WIDTH > 24
// logar reduction
	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		red_mem[i] = 0;
	}

	int red_base_off = (m_idx >= MLO_OUT_STACKS) ? MLO_LCL_MEM_SZ : m_idx * MLO_POW2_MAP_WK_SZ;
	// final summation over each filter row
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// write data
		red_mem[red_base_off + p4] = pvt_accum[l];

		// barrier inside
		ReduceKernel(&red_mem[red_base_off], &pvt_accum[l], p4, p4, MLO_POW2_MAP_WK_SZ, 1, false);

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#else
	// direct reduction
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		lcl_mem[lcl_id] = pvt_accum[l];
		barrier(CLK_LOCAL_MEM_FENCE);
		if (p4 == 0)
		{
			for (int i = 1; i < MLO_MAP_WK_SZ; ++i)
			{
				pvt_accum[l] += lcl_mem[lcl_id + i];
			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

#endif

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + m_idx) * MLO_WEI_CHANNEL_STRIDE);

	for (int n = 0; n < MLO_OUT_STACKS && p4 == 0 && m_idx < MLO_IN_STACKS; ++n)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS; ++c)
			{
				weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c];

			}

		}
	}

#endif
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}


BUGFIX: memcopy kernel was creating indices incorrectly

/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

typedef struct {
    dim_type dim[4];
} dims_t;

__kernel
void memcopy_kernel(__global T *out, dims_t ostrides,
                    __global const T *in, dims_t idims,
                    dims_t istrides, dim_type offset,
                    dim_type groups_0, dim_type groups_1)
{
    const dim_type lid0 = get_local_id(0);
    const dim_type lid1 = get_local_id(1);

    const dim_type id2 = get_group_id(0) / groups_0;
    const dim_type id3 = get_group_id(1) / groups_1;
    const dim_type group_id_0 = get_group_id(0) - groups_0 * id2;
    const dim_type group_id_1 = get_group_id(1) - groups_1 * id3;
    const dim_type id0 = group_id_0 * get_local_size(0) + lid0;
    const dim_type id1 = group_id_1 * get_local_size(1) + lid1;

    in += offset;

    // FIXME: Do more work per work group
    out += id3 * ostrides.dim[3] + id2 * ostrides.dim[2] + id1 * ostrides.dim[1];
    in  += id3 * istrides.dim[3] + id2 * istrides.dim[2] + id1 * istrides.dim[1];

    dim_type istride0 = istrides.dim[0];
    if (id0 < idims.dim[0] &&
        id1 < idims.dim[1] &&
        id2 < idims.dim[2] &&
        id3 < idims.dim[3]) {
        out[id0] = in[id0 * istride0];
    }
}

/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

typedef struct {
    dim_type dim[4];
} dims_t;

__kernel
void memcopy_kernel(__global T *out, dims_t ostrides,
                    __global const T *in, dims_t idims,
                    dims_t istrides, dim_type offset,
                    dim_type groups_0, dim_type groups_1)
{
    const dim_type lid0 = get_local_id(0);
    const dim_type lid1 = get_local_id(1);

    const dim_type id2 = get_group_id(0) / groups_0;
    const dim_type id3 = get_group_id(1) / groups_1;
    const dim_type group_id_0 = get_group_id(0) - groups_0 * id2;
    const dim_type group_id_1 = get_group_id(1) - groups_1 * id3;
    const dim_type id0 = group_id_0 * get_num_groups(0) + lid0;
    const dim_type id1 = group_id_1 * get_num_groups(1) + lid1;

    in += offset;

    // FIXME: Do more work per work group
    out += id3 * ostrides.dim[3] + id2 * ostrides.dim[2] + id1 * ostrides.dim[1];
    in  += id3 * istrides.dim[3] + id2 * istrides.dim[2] + id1 * istrides.dim[1];

    dim_type istride0 = istrides.dim[0];
    if (id0 < idims.dim[0] &&
        id1 < idims.dim[1] &&
        id2 < idims.dim[2] &&
        id3 < idims.dim[3]) {
        out[id0] = in[id0 * istride0];
    }
}


update

update

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}
static inline fetchWeights(int c, int f_s, int lcl_id, int wei_read, int gbl_wei_off, __local _FLOAT * wei_mem, const __global _FLOAT * weights)
{
	// read weights by stride
	for (int w = lcl_id; w < wei_read* MLO_N_LCL_OUT_MAPS; w += MLO_GRP_SZ)
	{
		int k = iDiv(w, wei_read);
		int t0 = iMod(w, k, wei_read);
		int j = iDiv(t0, MLO_FILTER_SIZE0);
		int i = iMod(t0, j, MLO_FILTER_SIZE0);
		int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

		if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
			if (ob == 0 && k == 1)
			{
				printf("G:w: %d %d %d %d   %f %f\n",
					//										lcl_id,
					//										w,
					//										f_s,
					//										j,
					//										i,
					//										k_idx,
					k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
					gbl_wei_off,
					wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
					weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
					wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
				);
			}

#endif

		}
		else
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
		}
	}
}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;
	int out_y = ob*MLO_OUT_EXTENT1;
	int in_y = out_y*MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;
	gbl_in_off += in_y * MLO_IN_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				int f_s = 0;
				for (; f_s < MLO_FILTER_STRIDE1 - 1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_WEI_SZ);
						int t0 = iMod(w, k, MLO_WEI_SZ);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
								if (ob==0 && k == 1)
								{
									printf("G:w: %d %d %d %d   %f %f\n",
//										lcl_id,
//										w,
//										f_s,
//										j,
//										i,
//										k_idx,
										k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
										gbl_wei_off,
										wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
										weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
										wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
									);
								}

#endif

						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}
					}

					int n_reads = MLO_IN_LCL_HEIGHT; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
//								for (; i < MLO_READ_UNIT; ++i)
//								{
//									in_rd_data[i] = 0;
//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l <  MLO_FILTER_STRIDE0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
// 3d
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0-1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

// last f_s
				{

					barrier(CLK_LOCAL_MEM_FENCE);
#define MLO_WEI_READ ((MLO_N_FILTER_SPLITS1 - 1)*MLO_WEI_LCL_WIDTH)
#if 1
					fetchWeights(c, f_s, lcl_id, MLO_WEI_READ, gbl_wei_off, wei_mem, weights);
#else
					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_READ* MLO_N_LCL_OUT_MAPS; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_WEI_READ);
						int t0 = iMod(w, k, MLO_WEI_READ);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
							if (ob == 0 && k == 1)
							{
								printf("G:w: %d %d %d %d   %f %f\n",
									//										lcl_id,
									//										w,
									//										f_s,
									//										j,
									//										i,
									//										k_idx,
									k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
									gbl_wei_off,
									wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
									weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
									wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
								);
							}

#endif

						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}
					}

#endif
					int n_reads = MLO_IN_LCL_HEIGHT - 1; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

									  // fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								//								for (; i < MLO_READ_UNIT; ++i)
								//								{
								//									in_rd_data[i] = 0;
								//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1 - 1; ++m)
					{
						// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l < MLO_FILTER_STRIDE0 - 1; ++l)
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
						  // 3d
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS && ex_row < MLO_OUT_EXTENT1 && (out_y + ex_row) < MLO_OUT_HEIGHT; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS && (k_idx + k) < MLO_N_OUTPUTS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}
static inline fetchWeights(int c, int f_s, int lcl_id, int wei_read, int gbl_wei_off, __local _FLOAT * wei_mem, const __global _FLOAT * weights)
{
	// read weights by stride
	for (int w = lcl_id; w < wei_read* MLO_N_LCL_OUT_MAPS; w += MLO_GRP_SZ)
	{
		int k = iDiv(w, wei_read);
		int t0 = iMod(w, k, wei_read);
		int j = iDiv(t0, MLO_FILTER_SIZE0);
		int i = iMod(t0, j, MLO_FILTER_SIZE0);
		int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

		if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
			if (ob == 0 && k == 1)
			{
				printf("G:w: %d %d %d %d   %f %f\n",
					//										lcl_id,
					//										w,
					//										f_s,
					//										j,
					//										i,
					//										k_idx,
					k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
					gbl_wei_off,
					wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
					weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
					wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
				);
			}

#endif

		}
		else
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
		}
	}
}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;
	int out_y = ob*MLO_OUT_EXTENT1;
	int in_y = out_y*MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;
	gbl_in_off += in_y * MLO_IN_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				int f_s = 0;
				for (; f_s < MLO_FILTER_STRIDE1 - 1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_WEI_SZ);
						int t0 = iMod(w, k, MLO_WEI_SZ);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
								if (ob==0 && k == 1)
								{
									printf("G:w: %d %d %d %d   %f %f\n",
//										lcl_id,
//										w,
//										f_s,
//										j,
//										i,
//										k_idx,
										k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
										gbl_wei_off,
										wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
										weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
										wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
									);
								}

#endif

						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}
					}

					int n_reads = MLO_IN_LCL_HEIGHT; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
//								for (; i < MLO_READ_UNIT; ++i)
//								{
//									in_rd_data[i] = 0;
//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l <  MLO_FILTER_STRIDE0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
// 3d
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0-1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

// last f_s
				{

					barrier(CLK_LOCAL_MEM_FENCE);
#define MLO_WEI_READ ((MLO_N_FILTER_SPLITS1 - 1)*MLO_WEI_LCL_WIDTH)
					// LDS size

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_READ* MLO_N_LCL_OUT_MAPS; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_WEI_READ);
						int t0 = iMod(w, k, MLO_WEI_READ);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
							if (ob == 0 && k == 1)
							{
								printf("G:w: %d %d %d %d   %f %f\n",
									//										lcl_id,
									//										w,
									//										f_s,
									//										j,
									//										i,
									//										k_idx,
									k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
									gbl_wei_off,
									wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
									weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
									wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
								);
							}

#endif

						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}
					}

					int n_reads = MLO_IN_LCL_HEIGHT - 1; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

									  // fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								//								for (; i < MLO_READ_UNIT; ++i)
								//								{
								//									in_rd_data[i] = 0;
								//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1 - 1; ++m)
					{
						// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l < MLO_FILTER_STRIDE0 - 1; ++l)
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
						  // 3d
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS && ex_row < MLO_OUT_EXTENT1 && (out_y + ex_row) < MLO_OUT_HEIGHT; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS && (k_idx + k) < MLO_N_OUTPUTS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}


testing stringify

testing stringify

#define STRINGIFY(src) #src

STRINGIFY(
//#ifndef A
//#define A(i, j)  A[j * rows + i]
//#endif
//#ifndef B
//#define B(i, j)  B[j * rows + i]
//#endif
//#ifndef C
//#define C(i, j)  C[j * rows + i]
//#endif
/**
 * Matrix addition on the GPU
 *
 * @param[out] C Output matrix.
 * @param[in] A LHS of matrix addition.
 * @param[in] B RHS of matrix addition.
 * @param rows Number of rows for matrix A.
 * @param cols Number of rows for matrix B.
 *
 */
__kernel void add(__global double *C, __global double *A, __global double *B,
  unsigned int rows, unsigned int cols) {
  int i = get_global_id(0);
  int j = get_global_id(1);
  if (i < rows && j < cols) {
    C[j * rows + i] = A[j * rows + i] + B[j * rows + i];
  }
}
);

R"(
#ifndef A
#define A(i, j)  A[j * rows + i]
#endif
#ifndef B
#define B(i, j)  B[j * rows + i]
#endif
#ifndef C
#define C(i, j)  C[j * rows + i]
#endif
/**
 * Matrix addition on the GPU
 *
 * @param[out] C Output matrix.
 * @param[in] A LHS of matrix addition.
 * @param[in] B RHS of matrix addition.
 * @param rows Number of rows for matrix A.
 * @param cols Number of rows for matrix B.
 *
 */
__kernel void add(__global double *C, __global double *A, __global double *B,
  unsigned int rows, unsigned int cols) {
  int i = get_global_id(0);
  int j = get_global_id(1);
  if (i < rows && j < cols) {
    C(i, j) = A(i, j) + B(i, j);
  }
};)"


OpenCL implementation of RANMAR PRNG

OpenCL implementation of RANMAR PRNG

/******************************************************************************
 * @file     prngcl_ranmar.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of RANMAR pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   George Marsaglia, Arif Zaman and Wai Wan Tsang,
 *   "Toward a universal random number generator",
 *   Statistics & Probability Letters, 9 (1990), 35--39.
 *
 *   F. James,
 *   "A review of pseudorandom number generators",
 *   Computer Physics Communications, 60 (1990), 329--344.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, 2014 Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_RANMAR_CL
#define PRNGCL_RANMAR_CL

#include "prngcl_common.cl"

#define RM_min    (1.0/16777216.0)
#define RM_max    (16777215.0/16777216.0)
#define RM_k      (5.9604644775390625E-8) // 1/2^24

#define RM_left   (RM_min+RM_k*RM_max)
#define RM_right  (RM_max+RM_k*RM_min)

#define RM_CD (7654321.0f / 16777216.0f)
#define RM_CM (16777213.0f /16777216.0f)

//________________________________________________________________________________________________________ RANMAR PRNG
__kernel void
ranmar_init(const __global uint4* seeds, __global float4* seedtable)
{
    float4 seed;
    //
    uint4 RM_seed1 = seeds[GID];
    uint4 RM_seed2 = seeds[GID + GID_SIZE];
    //
    uint4 i = ((RM_seed1 / 177) % 177) + 2;
    uint4 j = (RM_seed1 % 177) + 2;
    uint4 k = ((RM_seed2 / 169) % 178) + 1;
    uint4 l = (RM_seed2 % 169);
    for (int n = 0; n < 97; n++)
    {
        uint4 s = (uint4) 0;
        uint  t = 8388608;
        for (int m = 0; m < 24;  m++)
        {
            uint4 u = (i * j) % 179;
            u = (u * k) % 179;
            i = j;
            j = k;
            k = u;
            l = (53 * l + 1) % 169;
            uint4 lus = (l * u) % 64;

            if ( lus.x >= 32) s.x += t;
            if ( lus.y >= 32) s.y += t;
            if ( lus.z >= 32) s.z += t;
            if ( lus.w >= 32) s.w += t;
            t = t >> 1;
        }
        seed.x = ((float) s.x);
        seed.y = ((float) s.y);
        seed.z = ((float) s.z);
        seed.w = ((float) s.w);
        seedtable[GID + GID_SIZE * n] = seed / ((float4) 16777216.0f);
	}
    Uint_and_Float indx_I97, indx_J97;
    indx_I97.uint_value = 96;
    indx_J97.uint_value = 32;
    float4 indx;
    indx.x = indx_I97.float_value;
    indx.y = indx_J97.float_value;
    indx.z = (362436.0f / 16777216.0f);
    indx.w = 0.0f;

    seedtable[GID + GID_SIZE * 97] = indx;   // (RM_I97,RM_J97,RM_C,0)
}

__attribute__((always_inline)) float4
rm_step(__global float4 * seedtable,uint* RM_I97,uint* RM_J97,float* uniz)
{
        float4 uni;
        uni = seedtable[GID + GID_SIZE * (*RM_I97)] - seedtable[GID + GID_SIZE * (*RM_J97)];
	    uni = select(uni, (uni + 1.0f), uni < (float4) 0.0f);
        seedtable[GID + GID_SIZE * (*RM_I97)] = uni;

	    if ((*RM_I97) == 0) (*RM_I97) = 97;
	    if ((*RM_J97) == 0) (*RM_J97) = 97;
	    (*RM_I97)--;
	    (*RM_J97)--;

        (*uniz) -= RM_CD;
	    if ((*uniz) < 0.0) {(*uniz) += RM_CM;}
        uni -= (*uniz);
	    uni = select(uni, (uni + 1.0f), uni < (float4) 0.0f);
        return uni;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double4
rm_step_double(__global float4 * seedtable,uint* RM_I97,uint* RM_J97,float* uniz)
{
    hgpu_double4 result;
    float4 rnd1, rnd2;
    rnd1 = rm_step(seedtable,RM_I97,RM_J97,uniz);
#ifndef PRNG_SKIP_CHECK
    bool flag = true;
    while (flag) {
#endif
        rnd2 = rm_step(seedtable,RM_I97,RM_J97,uniz);
        result = hgpu_float4_to_double4(rnd1,rnd2,RM_min,RM_max,RM_k);

#ifndef PRNG_SKIP_CHECK
        if ((result.x>=RM_left) && (result.x<RM_right) &&
            (result.y>=RM_left) && (result.y<RM_right) &&
            (result.z>=RM_left) && (result.z<RM_right) &&
            (result.w>=RM_left) && (result.w<RM_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
#endif
    return result;
}
#endif

__kernel void
ranmar(__global float4 * seedtable,__global hgpu_float4 * prns, const uint samples)
{
    uint giddst = GID;
    float4 uni = (float4) 0.0;
    Uint_and_Float indx_I97, indx_J97;
    float4 indx = seedtable[GID + GID_SIZE * 97];
    indx_I97.float_value = indx.x;
    indx_J97.float_value = indx.y;
    uint RM_I97 = indx_I97.uint_value;
    uint RM_J97 = indx_J97.uint_value;
    float uniz = indx.z;
    for (uint i=0; i<samples; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        hgpu_double4 result = rm_step_double(seedtable,&RM_I97,&RM_J97,&uniz);
        prns[giddst] = result;
#else
        uni = rm_step(seedtable,&RM_I97,&RM_J97,&uniz);
        prns[giddst] = uni;
#endif
        giddst += GID_SIZE;
    }

    indx_I97.uint_value = RM_I97;
    indx_J97.uint_value = RM_J97;
    indx.z = uniz;
    indx.x = indx_I97.float_value;
    indx.y = indx_J97.float_value;
    seedtable[GID + GID_SIZE * 97] = indx;
}

#endif

/******************************************************************************
 * @file     prngcl_ranmar.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of RANMAR pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   George Marsaglia, Arif Zaman and Wai Wan Tsang,
 *   "Toward a universal random number generator",
 *   Statistics & Probability Letters, 9 (1990), 35--39.
 *
 *   F. James,
 *   "A review of pseudorandom number generators",
 *   Computer Physics Communications, 60 (1990), 329--344.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_RANMAR_CL
#define PRNGCL_RANMAR_CL

#include "prngcl_common.cl"

#define RM_min    (1.0/16777216.0)
#define RM_max    (16777215.0/16777216.0)
#define RM_k      (5.9604644775390625E-8) // 1/2^24

#define RM_left   (RM_min+RM_k*RM_max)
#define RM_right  (RM_max+RM_k*RM_min)

#define RM_CD (7654321.0f / 16777216.0f)
#define RM_CM (16777213.0f /16777216.0f)

//________________________________________________________________________________________________________ RANMAR PRNG
__kernel void
ranmar_init(const __global uint4* seeds, __global float4* seedtable)
{
    float4 seed;
    //
    uint4 RM_seed1 = seeds[GID];
    uint4 RM_seed2 = seeds[GID + GID_SIZE];
    //
    uint4 i = ((RM_seed1 / 177) % 177) + 2;
    uint4 j = (RM_seed1 % 177) + 2;
    uint4 k = ((RM_seed2 / 169) % 178) + 1;
    uint4 l = (RM_seed2 % 169);
    for (int n = 0; n < 97; n++)
    {
        uint4 s = (uint4) 0;
        uint  t = 8388608;
        for (int m = 0; m < 24;  m++)
        {
            uint4 u = (i * j) % 179;
            u = (u * k) % 179;
            i = j;
            j = k;
            k = u;
            l = (53 * l + 1) % 169;
            uint4 lus = (l * u) % 64;

            if ( lus.x >= 32) s.x += t;
            if ( lus.y >= 32) s.y += t;
            if ( lus.z >= 32) s.z += t;
            if ( lus.w >= 32) s.w += t;
            t = t >> 1;
        }
        seed.x = ((float) s.x);
        seed.y = ((float) s.y);
        seed.z = ((float) s.z);
        seed.w = ((float) s.w);
        seedtable[GID + GID_SIZE * n] = seed / ((float4) 16777216.0f);
	}
    Uint_and_Float indx_I97, indx_J97;
    indx_I97.uint_value = 96;
    indx_J97.uint_value = 32;
    float4 indx;
    indx.x = indx_I97.float_value;
    indx.y = indx_J97.float_value;
    indx.z = (362436.0f / 16777216.0f);
    indx.w = 0.0f;

    seedtable[GID + GID_SIZE * 97] = indx;   // (RM_I97,RM_J97,RM_C,0)
}

__attribute__((always_inline)) float4
rm_step(__global float4 * seedtable,uint* RM_I97,uint* RM_J97,float* uniz)
{
        float4 uni;
        uni = seedtable[GID + GID_SIZE * (*RM_I97)] - seedtable[GID + GID_SIZE * (*RM_J97)];
	    uni = select(uni, (uni + 1.0f), uni < (float4) 0.0f);
        seedtable[GID + GID_SIZE * (*RM_I97)] = uni;

	    if ((*RM_I97) == 0) (*RM_I97) = 97;
	    if ((*RM_J97) == 0) (*RM_J97) = 97;
	    (*RM_I97)--;
	    (*RM_J97)--;

        (*uniz) -= RM_CD;
	    if ((*uniz) < 0.0) {(*uniz) += RM_CM;}
        uni -= (*uniz);
	    uni = select(uni, (uni + 1.0f), uni < (float4) 0.0f);
        return uni;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double4
rm_step_double(__global float4 * seedtable,uint* RM_I97,uint* RM_J97,float* uniz)
{
    hgpu_double4 result;
    float4 rnd1, rnd2;
    bool flag = true;
    rnd1 = rm_step(seedtable,RM_I97,RM_J97,uniz);
    while (flag) {
        rnd2 = rm_step(seedtable,RM_I97,RM_J97,uniz);
        result = hgpu_float4_to_double4(rnd1,rnd2,RM_min,RM_max,RM_k);
        if ((result.x>=RM_left) && (result.x<RM_right) &&
            (result.y>=RM_left) && (result.y<RM_right) &&
            (result.z>=RM_left) && (result.z<RM_right) &&
            (result.w>=RM_left) && (result.w<RM_right))
            flag = false;
        else
            rnd1 = rnd2;
    }

    return result;
}
#endif

__kernel void
ranmar(__global float4 * seedtable,__global hgpu_float4 * prns, const uint samples)
{
    uint giddst = GID;
    float4 uni = (float4) 0.0;
    Uint_and_Float indx_I97, indx_J97;
    float4 indx = seedtable[GID + GID_SIZE * 97];
    indx_I97.float_value = indx.x;
    indx_J97.float_value = indx.y;
    uint RM_I97 = indx_I97.uint_value;
    uint RM_J97 = indx_J97.uint_value;
    float uniz = indx.z;
    for (uint i=0; i<samples; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        hgpu_double4 result = rm_step_double(seedtable,&RM_I97,&RM_J97,&uniz);
        prns[giddst] = result;
#else
        uni = rm_step(seedtable,&RM_I97,&RM_J97,&uniz);
        prns[giddst] = uni;
#endif
        giddst += GID_SIZE;
    }

    indx_I97.uint_value = RM_I97;
    indx_J97.uint_value = RM_J97;
    indx.z = uniz;
    indx.x = indx_I97.float_value;
    indx.y = indx_J97.float_value;
    seedtable[GID + GID_SIZE * 97] = indx;
}

#endif