Stas guidence changes

Stas guidence changes

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					__global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
					__global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

					__global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				__global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				__global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				__global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				__global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	int m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	int p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	int m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	int p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

	for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		int bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (int i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				int m = iDiv(p, MLO_MAP_WK_SZ);
				int pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				int m = ((uint)p / MLO_MAP_WK_SZ);
				int pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				int top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif
#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{
					for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df[top_off1 + i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (int i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df[top_off1 + i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			int split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			int split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			int split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			int split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (int j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= (MLO_REDUC_LOOP_STEP << rd) && p4 < (MLO_REDUC_LOOP_STEP << (rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP << rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	int gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (int pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		int b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		int p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		int b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		int p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		int gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		int gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;

#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df[top_off + i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df[top_off + i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

		// processing
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			int split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			int split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			int split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			int split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (int j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= (MLO_REDUC_LOOP_STEP << rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << rd))
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}


test_performance.cl: kernel clamped.O3.ll was not being run

// Test source code is from http://musicdsp.org/archive.php?classid=0#1 and
// modified to pass clang C compiler and changed TF to be local var in main
// to support ClampPointers pass.

// FormantsAM.cpp

// Thierry Rochebois' "Formantic Synthesis by Double Amplitude Modulation"

// Based on a tutorial by Thierry Rochebois.
// Comments by Paul Sernine.

// The spectral content of the signal is obtained by adding amplitude modulated formantic
// waveforms. The amplitude modulations spectraly shift the formantic waveforms.
// Continuous spectral shift, without losing the harmonic structure, is obtained
// by using crossfaded double carriers (multiple of the base frequency).
// To avoid  unwanted interference artifacts, phase relationships must be of the
// "cosine type".

// The output is a 44100Hz 16bit stereo PCM file.

// NOTE: for now looks like -O3 cripples safe exception load/store analysis
// TODO: get times from unclamped versions and compare to clamped ones and expect perf hit to be max 30%

// RUN: clang -target nvptx -S -c $TEST_SRC -O0 -emit-llvm -S -o $OUT_FILE.ll &&
// RUN: echo "Running and verifying 'Formantic Synthesis by Double Amplitude Modulation' case" &&
// RUN: opt -S -O3 $OUT_FILE.ll -o $OUT_FILE.O3.ll &&
// RUN: opt -debug -S -load $CLAMP_PLUGIN -clamp-pointers -allow-unsafe-exceptions $OUT_FILE.ll -o $OUT_FILE.clamped.ll &&
// RUN: opt -debug -S -load $CLAMP_PLUGIN -clamp-pointers -allow-unsafe-exceptions $OUT_FILE.O3.ll -o $OUT_FILE.O3.clamped.ll &&
// RUN: opt -S -O3 $OUT_FILE.clamped.ll -o $OUT_FILE.clamped.O3.ll &&
// RUN: echo "Running original.O0:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running original.O3:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.O3.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running clamped.O0:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.clamped.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running O3.clamped:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.O3.clamped.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running clamped.O3:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.clamped.O3.ll test_kernel 8 "(int,{0}):(int,1)")

typedef int int32_t;
typedef short int16_t;

float expf(float f);
float sinf(float f);
float cosf(float f);
float floorf(float f);
float powf(float a, float b);
float fmodf(float x, float y);

//Approximates cos(pi*x) for x in [-1,1].
float fast_cos(const float x)
{
  float x2 = x*x;
  return 1+x2*(-4+2*x2);
}

// Length of the table
#define L_TABLE (256+1) //The last entry of the table equals the first (to avoid a modulo)
// Maximal formant width
#define I_MAX 64

//Formantic function of width I (used to fill the table of formants)
float fonc_formant(float p,const float I)
{
  float a=0.5f;
  int32_t hmax=(int)(10*I)>L_TABLE/2?L_TABLE/2:(int32_t)(10*I);
  float phi=0.0f;
  for(int32_t h=1;h<hmax;h++)
    {
      phi+=3.14159265359f*p;
      float hann=0.5f+0.5f*fast_cos(h*(1.0f/hmax));
      float gaussienne=0.85f*expf(-h*h/(I*I));
      float jupe=0.15f;
      float harmonique=cosf(phi);
      a+=hann*(gaussienne+jupe)*harmonique;
    }
  return a;
}

//Initialisation of the table TF with the fonction fonc_formant.
void init_formant(float TF[])
{ float coef=2.0f/(L_TABLE-1);
  for(int32_t I=0;I<I_MAX;I++)
    for(int32_t P=0;P<L_TABLE;P++)
      TF[P+I*L_TABLE]=fonc_formant(-1+P*coef,(float)(I));
}

//This function emulates the function fonc_formant
// thanks to the table TF. A bilinear interpolation is
// performed
float formant(float p,float i, float TF[])
{
  i=i<0?0:i>I_MAX-2?I_MAX-2:i;    // width limitation
  float P=(L_TABLE-1)*(p+1)*0.5f; // phase normalisation
  int32_t P0=(int32_t)P;     float fP=P-P0;  // Integer and fractional
  int32_t I0=(int32_t)i;     float fI=i-I0;  // parts of the phase (p) and width (i).
  int32_t i00=P0+L_TABLE*I0;     int32_t i10=i00+L_TABLE;
  //bilinear interpolation.
  return (1-fI)*(TF[i00] + fP*(TF[i00+1]-TF[i00]))
    +    fI*(TF[i10] + fP*(TF[i10+1]-TF[i10]));
}

// Double carrier.
// h : position (float harmonic number)
// p : phase
float porteuse(const float h,const float p)
{
  float h0=floorf(h);  //integer and
  float hf=h-h0;      //decimal part of harmonic number.
  // modulos pour ramener p*h0 et p*(h0+1) dans [-1,1]
  float phi0=fmodf(p* h0   +1+1000,2.0f)-1.0f;
  float phi1=fmodf(p*(h0+1)+1+1000,2.0f)-1.0f;
  // two carriers.
  float Porteuse0=fast_cos(phi0);  float Porteuse1=fast_cos(phi1);
  // crossfade between the two carriers.
  return Porteuse0+hf*(Porteuse1-Porteuse0);
}

__kernel void test_kernel(__global int* out_val)
{
  //Formant table for various french vowels (you can add your own)
  float F1[]={  730,  200,  400,  250,  190,  350,  550,  550,  450};
  float A1[]={ 1.0f, 0.5f, 1.0f, 1.0f, 0.7f, 1.0f, 1.0f, 0.3f, 1.0f};
  float F2[]={ 1090, 2100,  900, 1700,  800, 1900, 1600,  850, 1100};
  float A2[]={ 2.0f, 0.5f, 0.7f, 0.7f,0.35f, 0.3f, 0.5f, 1.0f, 0.7f};
  float F3[]={ 2440, 3100, 2300, 2100, 2000, 2500, 2250, 1900, 1500};
  float A3[]={ 0.3f,0.15f, 0.2f, 0.4f, 0.1f, 0.3f, 0.7f, 0.2f, 0.2f};
  float F4[]={ 3400, 4700, 3000, 3300, 3400, 3700, 3200, 3000, 3000};
  float A4[]={ 0.2f, 0.1f, 0.2f, 0.3f, 0.1f, 0.1f, 0.3f, 0.2f, 0.3f};

  float f0,dp0,p0=0.0f;
  int32_t F=7; //number of the current formant preset
  float f1,f2,f3,f4,a1,a2,a3,a4;
  f1=f2=f3=f4=100.0f;a1=a2=a3=a4=0.0f;

  //Table of formants
  float TF[L_TABLE*I_MAX];

  init_formant(TF);

  for(int ns=0;ns<200*44100;ns++)
    {
      if(0==(ns%11025)){F++;F%=8;} //formant change
      f0=12*powf(2.0f,4-4*ns/(10*44100.0f)); //sweep
      f0*=(1.0f+0.01f*sinf(ns*0.0015f));        //vibrato
      dp0=f0*(1/22050.0f);
      float un_f0=1.0f/f0;
      p0+=dp0;  //phase increment
      p0-=2*(p0>1);
      { //smoothing of the commands.
        float r=0.001f;
        f1+=r*(F1[F]-f1);f2+=r*(F2[F]-f2);f3+=r*(F3[F]-f3);f4+=r*(F4[F]-f4);
        a1+=r*(A1[F]-a1);a2+=r*(A2[F]-a2);a3+=r*(A3[F]-a3);a4+=r*(A4[F]-a4);
      }

      //The f0/fn coefficients stand for a -3dB/oct spectral enveloppe
      float out=
        a1*(f0/f1)*formant(p0,100*un_f0,TF)*porteuse(f1*un_f0,p0)
        +0.7f*a2*(f0/f2)*formant(p0,120*un_f0,TF)*porteuse(f2*un_f0,p0)
        +     a3*(f0/f3)*formant(p0,150*un_f0,TF)*porteuse(f3*un_f0,p0)
        +     a4*(f0/f4)*formant(p0,300*un_f0,TF)*porteuse(f4*un_f0,p0);

      // write to volatile variable to test... (use puts to verify correctness)
      out_val[0] = (int16_t)(15000.0f*out);
    }
}

// Test source code is from http://musicdsp.org/archive.php?classid=0#1 and
// modified to pass clang C compiler and changed TF to be local var in main
// to support ClampPointers pass.

// FormantsAM.cpp

// Thierry Rochebois' "Formantic Synthesis by Double Amplitude Modulation"

// Based on a tutorial by Thierry Rochebois.
// Comments by Paul Sernine.

// The spectral content of the signal is obtained by adding amplitude modulated formantic
// waveforms. The amplitude modulations spectraly shift the formantic waveforms.
// Continuous spectral shift, without losing the harmonic structure, is obtained
// by using crossfaded double carriers (multiple of the base frequency).
// To avoid  unwanted interference artifacts, phase relationships must be of the
// "cosine type".

// The output is a 44100Hz 16bit stereo PCM file.

// NOTE: for now looks like -O3 cripples safe exception load/store analysis
// TODO: get times from unclamped versions and compare to clamped ones and expect perf hit to be max 30%

// RUN: clang -target nvptx -S -c $TEST_SRC -O0 -emit-llvm -S -o $OUT_FILE.ll &&
// RUN: echo "Running and verifying 'Formantic Synthesis by Double Amplitude Modulation' case" &&
// RUN: opt -S -O3 $OUT_FILE.ll -o $OUT_FILE.O3.ll &&
// RUN: opt -debug -S -load $CLAMP_PLUGIN -clamp-pointers -allow-unsafe-exceptions $OUT_FILE.ll -o $OUT_FILE.clamped.ll &&
// RUN: opt -debug -S -load $CLAMP_PLUGIN -clamp-pointers -allow-unsafe-exceptions $OUT_FILE.O3.ll -o $OUT_FILE.O3.clamped.ll &&
// RUN: opt -S -O3 $OUT_FILE.clamped.ll -o $OUT_FILE.clamped.O3.ll &&
// RUN: echo "Running original.O0:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running original.O3:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.O3.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running clamped.O0:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.clamped.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running O3.clamped:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.O3.clamped.ll test_kernel 8 "(int,{0}):(int,1)") &&
// RUN: echo "Running clamped.O3:" &&
// RUN: (BENCHMARK=1 $RUN_KERNEL $OUT_FILE.clamped.O3.ll)

typedef int int32_t;
typedef short int16_t;

float expf(float f);
float sinf(float f);
float cosf(float f);
float floorf(float f);
float powf(float a, float b);
float fmodf(float x, float y);

//Approximates cos(pi*x) for x in [-1,1].
float fast_cos(const float x)
{
  float x2 = x*x;
  return 1+x2*(-4+2*x2);
}

// Length of the table
#define L_TABLE (256+1) //The last entry of the table equals the first (to avoid a modulo)
// Maximal formant width
#define I_MAX 64

//Formantic function of width I (used to fill the table of formants)
float fonc_formant(float p,const float I)
{
  float a=0.5f;
  int32_t hmax=(int)(10*I)>L_TABLE/2?L_TABLE/2:(int32_t)(10*I);
  float phi=0.0f;
  for(int32_t h=1;h<hmax;h++)
    {
      phi+=3.14159265359f*p;
      float hann=0.5f+0.5f*fast_cos(h*(1.0f/hmax));
      float gaussienne=0.85f*expf(-h*h/(I*I));
      float jupe=0.15f;
      float harmonique=cosf(phi);
      a+=hann*(gaussienne+jupe)*harmonique;
    }
  return a;
}

//Initialisation of the table TF with the fonction fonc_formant.
void init_formant(float TF[])
{ float coef=2.0f/(L_TABLE-1);
  for(int32_t I=0;I<I_MAX;I++)
    for(int32_t P=0;P<L_TABLE;P++)
      TF[P+I*L_TABLE]=fonc_formant(-1+P*coef,(float)(I));
}

//This function emulates the function fonc_formant
// thanks to the table TF. A bilinear interpolation is
// performed
float formant(float p,float i, float TF[])
{
  i=i<0?0:i>I_MAX-2?I_MAX-2:i;    // width limitation
  float P=(L_TABLE-1)*(p+1)*0.5f; // phase normalisation
  int32_t P0=(int32_t)P;     float fP=P-P0;  // Integer and fractional
  int32_t I0=(int32_t)i;     float fI=i-I0;  // parts of the phase (p) and width (i).
  int32_t i00=P0+L_TABLE*I0;     int32_t i10=i00+L_TABLE;
  //bilinear interpolation.
  return (1-fI)*(TF[i00] + fP*(TF[i00+1]-TF[i00]))
    +    fI*(TF[i10] + fP*(TF[i10+1]-TF[i10]));
}

// Double carrier.
// h : position (float harmonic number)
// p : phase
float porteuse(const float h,const float p)
{
  float h0=floorf(h);  //integer and
  float hf=h-h0;      //decimal part of harmonic number.
  // modulos pour ramener p*h0 et p*(h0+1) dans [-1,1]
  float phi0=fmodf(p* h0   +1+1000,2.0f)-1.0f;
  float phi1=fmodf(p*(h0+1)+1+1000,2.0f)-1.0f;
  // two carriers.
  float Porteuse0=fast_cos(phi0);  float Porteuse1=fast_cos(phi1);
  // crossfade between the two carriers.
  return Porteuse0+hf*(Porteuse1-Porteuse0);
}

__kernel void test_kernel(__global int* out_val)
{
  //Formant table for various french vowels (you can add your own)
  float F1[]={  730,  200,  400,  250,  190,  350,  550,  550,  450};
  float A1[]={ 1.0f, 0.5f, 1.0f, 1.0f, 0.7f, 1.0f, 1.0f, 0.3f, 1.0f};
  float F2[]={ 1090, 2100,  900, 1700,  800, 1900, 1600,  850, 1100};
  float A2[]={ 2.0f, 0.5f, 0.7f, 0.7f,0.35f, 0.3f, 0.5f, 1.0f, 0.7f};
  float F3[]={ 2440, 3100, 2300, 2100, 2000, 2500, 2250, 1900, 1500};
  float A3[]={ 0.3f,0.15f, 0.2f, 0.4f, 0.1f, 0.3f, 0.7f, 0.2f, 0.2f};
  float F4[]={ 3400, 4700, 3000, 3300, 3400, 3700, 3200, 3000, 3000};
  float A4[]={ 0.2f, 0.1f, 0.2f, 0.3f, 0.1f, 0.1f, 0.3f, 0.2f, 0.3f};

  float f0,dp0,p0=0.0f;
  int32_t F=7; //number of the current formant preset
  float f1,f2,f3,f4,a1,a2,a3,a4;
  f1=f2=f3=f4=100.0f;a1=a2=a3=a4=0.0f;

  //Table of formants
  float TF[L_TABLE*I_MAX];

  init_formant(TF);

  for(int ns=0;ns<200*44100;ns++)
    {
      if(0==(ns%11025)){F++;F%=8;} //formant change
      f0=12*powf(2.0f,4-4*ns/(10*44100.0f)); //sweep
      f0*=(1.0f+0.01f*sinf(ns*0.0015f));        //vibrato
      dp0=f0*(1/22050.0f);
      float un_f0=1.0f/f0;
      p0+=dp0;  //phase increment
      p0-=2*(p0>1);
      { //smoothing of the commands.
        float r=0.001f;
        f1+=r*(F1[F]-f1);f2+=r*(F2[F]-f2);f3+=r*(F3[F]-f3);f4+=r*(F4[F]-f4);
        a1+=r*(A1[F]-a1);a2+=r*(A2[F]-a2);a3+=r*(A3[F]-a3);a4+=r*(A4[F]-a4);
      }

      //The f0/fn coefficients stand for a -3dB/oct spectral enveloppe
      float out=
        a1*(f0/f1)*formant(p0,100*un_f0,TF)*porteuse(f1*un_f0,p0)
        +0.7f*a2*(f0/f2)*formant(p0,120*un_f0,TF)*porteuse(f2*un_f0,p0)
        +     a3*(f0/f3)*formant(p0,150*un_f0,TF)*porteuse(f3*un_f0,p0)
        +     a4*(f0/f4)*formant(p0,300*un_f0,TF)*porteuse(f4*un_f0,p0);

      // write to volatile variable to test... (use puts to verify correctness)
      out_val[0] = (int16_t)(15000.0f*out);
    }
}


Fix a warning on test_ldexp.cl

Fix a warning on test_ldexp.cl

Fix a warning on test_ldexp.cl when cl_khr_fp64 is not available.

#include "common.cl"

DEFINE_BODY_V (
    test_ldexp, ({
      /* ldexp */
      Ivec ival2;
      ival2.v = (itype)4;
      res.v = ldexp (val.v, ival2.v);
      Tvec goodres;
      equal = true;
      for (int n = 0; n < vecsize; ++n)
        {
          if (ISNAN (val.s[n]))
            {
              goodres.s[n] = NAN;
            }
          else if (val.s[n] == (stype)0)
            {
              goodres.s[n] = val.s[n];
            }
          else if (isinf (val.s[n]))
            {
              goodres.s[n] = val.s[n];
            }
          else
            {
              goodres.s[n] = val.s[n] * (stype) (16.0f);
            }
          equal = equal && ISEQ (res.s[n], goodres.s[n]);
        }
      if (!equal)
        {
          for (int n = 0; n < vecsize; ++n)
            {
              printf ("FAIL: ldexp type=%s val=%.17g val2=%d res=%.17g "
                      "good=%.17g\n",
                      typename, val.s[n], (int)ival2.s[n], res.s[n],
                      goodres.s[n]);
            }
          return;
        }
    }))

kernel void
test_ldexp ()
{
  CALL_FUNC_V (test_ldexp)
}

#include "common.cl"

DEFINE_BODY_V (
    test_ldexp, ({
      /* ldexp */
      Ivec ival2;
      ival2.v = (itype)4;
      res.v = ldexp (val.v, ival2.v);
      Tvec goodres;
      equal = true;
      for (int n = 0; n < vecsize; ++n)
        {
          if (ISNAN (val.s[n]))
            {
              goodres.s[n] = NAN;
            }
          else if (val.s[n] == (stype)0)
            {
              goodres.s[n] = val.s[n];
            }
          else if (isinf (val.s[n]))
            {
              goodres.s[n] = val.s[n];
            }
          else
            {
              goodres.s[n] = val.s[n] * (stype) (16.0);
            }
          equal = equal && ISEQ (res.s[n], goodres.s[n]);
        }
      if (!equal)
        {
          for (int n = 0; n < vecsize; ++n)
            {
              printf ("FAIL: ldexp type=%s val=%.17g val2=%d res=%.17g "
                      "good=%.17g\n",
                      typename, val.s[n], (int)ival2.s[n], res.s[n],
                      goodres.s[n]);
            }
          return;
        }
    }))

kernel void
test_ldexp ()
{
  CALL_FUNC_V (test_ldexp)
}