random pragma

random pragma

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll 4
			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

				const __global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

#pragma unroll 4
			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

//#pragma unroll 2
		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
//#pragma unroll 4
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

				const __global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}


Minor changes.

Minor changes.

/*
  finite_sum_opencl.cl

  OpenCL code for computing the finite sum in the computation of the Riemann
  theta function.
*/

/*
 */
#define PI 3.141592653589793

float exppart(int, __global float*, __global float*, __global float*, int,
	      __global float*, int);

float normpart(int, __global float*, __global float*, int,
	       __global float* int);

/*
  Given integer vector S[i], compte the exppart of the sum.
*/
float exppart(int i,
	      __global float* X,
	      __global float* intshift,
	      __global float* x,
	      int N,
	      __global float* S,
	      int L)
{
  // perform dot product of matrix mult
  float sum = 0;
  float v_j, Xw_j;
  for (int j=0; j<N; j++) {
    // jth component of a-intshift
    v_j = S[i*N+j]-intshift[j];

    // jth component of (1/2) X*(a-intshift) + x
    Xw_j = 0;
    for (int k=0; k<N; k++) {
      Xw_j += 0.5 * X[j*N+k] * (S[i*N+k]-intshift[k]) + x[j];
    }

    // add the jth component to the dot product container
    sum += v_j*Xw_j
  }

  // scale and return
  sum *= 2*PI;
  return sum
}

/*
  Given integer vector S[i], compte the exppart of the sum.
*/
float normpart(int i,
	       __global float* T,
	       __global float* fracshift,
	       int N,
	       __global float* S,
	       int L);
{
  float sum = 0;
  float v_j;

  // loop over elements of two vectors
  for (int j=0; j<N; j++){
    // calculate jth component of v=T(a+fracshift)
    v_j = 0;
    for (int k=0; k<N; k++) {
      v_j += T[j*N+k] * (S[i*N+k]-fracshift[k]);
    }

    sum += -PI*v_j*v_j;
  }
}

__kernel void
finite_sum_without_derivs(__global float* X,
			  __global float* Yinv,
			  __global float* T,
			  __global float* x,
			  __global float* y,
			  int N
			  __global float* S,
			  int L
			  __global float fsum_real,
			  __global float fsum_imag)
{
  float ept, npt, cpart, spart;
  int i;
  i = get_global_id(0);

  // compute exponential and norm parts
  ept = exppart(i,X,intshift,x,N,S,L);
  npt = exp(normpart(i,T,fracshift,N,S,L));

  // split up into real and imaginary (cosine and sine) parts
  cpart = npt * cos(ept);
  spart = npt * sin(ept);

  // add to real and imaginary parts of summation
  fsum_real += cpart;
  fsum_imag += spart;
}

/*
  finite_sum_opencl.cl

  OpenCL code for computing the finite sum in the computation of the Riemann
  theta function.
*/

/*
 */
#define PI 3.141592653589793

float exppart(int, __global float*, __global float*, __global float*, int,
	      __global float*, int);

float normpart(int, __global float*, __global float*, int,
	       __global float* int);

/*
  Given integer vector S[i], compte the exppart of the sum.
*/
float exppart(int i,
	      __global float* X,
	      __global float* intshift,
	      __global float* x,
	      int N,
	      __global float* S,
	      int L)
{
  // perform dot product of matrix mult
  float sum = 0;
  float v_j, Xw_j;
  for (int j=0; j<N; j++) {
    // jth component of a-intshift
    vj = S[i*N+j]-intshift[j];

    // jth component of (1/2) X*(a-intshift) + x
    Xw_j = 0;
    for (int k=0; k<N; k++) {
      Xw_j += 0.5 * X[j + N*k] * (S[i*N+k]-intshift[k]) + x[j];
    }

    // add the jth component to the dot product container
    sum += v_j*Xw_j
  }

  // scale and return
  sum *= 2*PI;
  return sum
}

/*
  Given integer vector S[i], compte the exppart of the sum.
*/
float normpart(float* a,
	       __global float* T,
	       __global float* fracshift,
	       int N,
	       __global float* S,
	       int L);
{

}

__kernel void finite_sum(__global float* X,
			 __global float* Yinv,
			 __global float* T,
			 __global float* x,
			 __global float* y,
			 int N
			 __global float* S,
			 int L)
{
  int i = get_global_id(0);   // index of the integer point
}


Fixed crashing issues on AMD cards

Fixed crashing issues on AMD cards

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics: enable

#define __THREAD_FENCE_USED__

typedef struct {
    int nposi, nposj;
    int nmaxpos;
    float fmaxscore;
    int noutputlen;
}   MAX_INFO;

#define PATH_END 0
#define COALESCED_OFFSET 32

//Simple GPU barrier, with atomicAdd used
//Input: Local thread idx in a block, goal value
void __barrier_opencl_lock_based(int localID, int goalValue, volatile __global int *g_mutexOpencl)
{
    int tid = localID;
    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);

#ifdef __THREAD_FENCE_USED__
    write_mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
    //other options
    //read_mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
    //mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
#endif

    if (tid == 0)
    {
        atom_add(g_mutexOpencl, 1);
        mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
        while (atom_add(g_mutexOpencl,0) < goalValue)
        { }
    }

    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
}

__kernel void  MatchStringGPUSync(__global char  *pathFlag,
								  __global char  *extFlag,
								  __global float *nGapDist,
								  __global float *hGapDist,
								  __global float *vGapDist,
								  __global int   *diffPos,
								  __global int   *threadNum,
								  int            rowNum,
								  int            columnNum,
								  __global char  *seq1,
								  __global char  *seq2,
								  int            blosumWidth,
								  float          openPenalty,
								  float          extensionPenalty,
								  __global MAX_INFO *maxInfo,
								  __global float *blosum62D,
								  volatile __global int *mutexMem,
								  __local int* temp)
{
	int npos, ntablepos, tid;
	int npreposngap, npreposhgap, npreposvgap;
	int nLocalID = get_local_id(0);
	int blockNum = get_num_groups(0);
	int blockSize = get_local_size(0);
	int totalThreadNum = blockSize * blockNum;
	int threadid = get_global_id(0);

	int launchNo;
	int launchNum = rowNum + columnNum - 1;
	int indexi1 = -1;
	int indexj1 = 0;
	int indexi, indexj;
	int startPos = 2 * COALESCED_OFFSET;
	int noffset = 0;

	float fdist;
	float fdistngap, fdisthgap, fdistvgap;
	float ext_dist;
	float fmaxdist;

	//__local float temp[5000];
	temp[0] = 0.0f;

	for (launchNo = 2; launchNo < launchNum; launchNo++)
	{
		if (launchNo < rowNum)
		{
			indexi1++;
		}
		else if (launchNo == rowNum)
		{
			indexi1++;
			noffset = 1;
		}
		else
		{
			indexj1++;
		}

		for (tid = threadid; tid < threadNum[launchNo]; tid += totalThreadNum)
		{
			indexi = indexi1 - tid;
			indexj = indexj1 + tid;

			npos = startPos + tid;

			npreposhgap = npos - diffPos[launchNo];
			npreposvgap = npreposhgap - 1;
			npreposngap = npreposvgap - diffPos[launchNo - 1];

			ntablepos = seq1[indexi] * blosumWidth + seq2[indexj];
			//ntablepos = seq1C[indexi] * blosumWidth + seq2C[indexj];
			fdist = blosum62D[ntablepos];

			fmaxdist = nGapDist[npreposngap];
			fdistngap = fmaxdist + fdist;

			ext_dist  = hGapDist[npreposhgap] - extensionPenalty;
			fdisthgap = nGapDist[npreposhgap] - openPenalty;

			if (fdisthgap <= ext_dist)
			{
				fdisthgap = ext_dist;
				extFlag[npreposhgap] = 1;
			}

			ext_dist  = vGapDist[npreposvgap] - extensionPenalty;
			fdistvgap = nGapDist[npreposvgap] - openPenalty;

			if (fdistvgap <= ext_dist)
			{
				fdistvgap = ext_dist;
				pathFlag[npreposvgap] += 8;
			}

			fdistngap = (fdistngap < 0.0f) ? 0.0f : fdistngap;
			fdisthgap = (fdisthgap < 0.0f) ? 0.0f : fdisthgap;
			fdistvgap = (fdistvgap < 0.0f) ? 0.0f : fdistvgap;

			hGapDist[npos] = fdisthgap;
			vGapDist[npos] = fdistvgap;

			//priority 00, 01, 10
			if (fdistngap >= fdisthgap && fdistngap >= fdistvgap)
			{
				fmaxdist = fdistngap;
				pathFlag[npos] = 2;
			}
			else if (fdisthgap >= fdistngap && fdisthgap >= fdistvgap)
			{
				fmaxdist = fdisthgap;
				pathFlag[npos] = 1;
			}
			else //fdistvgap >= fdistngap && fdistvgap >= fdisthgap
			{
				fmaxdist = fdistvgap;
				pathFlag[npos] = 3;
			}

			nGapDist[npos] = fmaxdist;

			//Here, the maximum match distance is 0, which means
			//previous alignment is useless
			if (fmaxdist <= 0.00000001f)
			{
				pathFlag[npos] = PATH_END;
			}

			if (maxInfo[threadid].fmaxscore < fmaxdist)
			{
				maxInfo[threadid].nposi = indexi + 1;
				maxInfo[threadid].nposj = indexj + 1;
				maxInfo[threadid].nmaxpos = npos;
				maxInfo[threadid].fmaxscore = fmaxdist;
			}
		}

		//GPU synchronization
		__barrier_opencl_lock_based(nLocalID, (launchNo - 1) * blockNum, mutexMem);
		startPos += diffPos[launchNo + 1] + noffset;
	}
}

__kernel void trace_back2(__global char *str_npathflagp,
						  __global char *str_nExtFlagp,
						  __global int  *ndiffpos,
						  __global char *instr1D,
						  __global char *instr2D,
						  __global char *outstr1,
						  __global char *outstr2,
						  __global MAX_INFO * strMaxInfop,
						  int mfThreadNum)
{
	int i, j;
	int npos, maxPos, nlen;
	int npathflag;
	int nlaunchno;
	float maxScore;

	maxPos = 0;
	maxScore = strMaxInfop[0].fmaxscore;
	for (i = 1; i < mfThreadNum; i++)
	{
		if (maxScore < strMaxInfop[i].fmaxscore)
		{
			maxPos = i;
			maxScore = strMaxInfop[i].fmaxscore;
		}
	}

	npos = strMaxInfop[maxPos].nmaxpos;
	npathflag = str_npathflagp[npos] & 0x3;
	nlen = 0;

	i = strMaxInfop[maxPos].nposi;
	j = strMaxInfop[maxPos].nposj;
	nlaunchno = i + j;

	while (1)
	{
		if (npathflag == 3)
		{
			outstr1[nlen] = 23;
			outstr2[nlen] = instr2D[j - 1];
			nlen++;
			j--;

			//position in the transformed matrix
			npos = npos - ndiffpos[nlaunchno] - 1;
			nlaunchno--;
		}
		else if (npathflag == 1)
		{
			outstr1[nlen] = instr1D[i - 1];
			outstr2[nlen] = 23;
			nlen++;
			i--;

			//position in the transformed matrix
			npos = npos - ndiffpos[nlaunchno];
			nlaunchno--;
		}
		else if (npathflag == 2)
		{
			outstr1[nlen] = instr1D[i - 1];
			outstr2[nlen] = instr2D[j - 1];
			nlen++;
			i--;
			j--;

			//position in the transformed matrix
			npos = npos - ndiffpos[nlaunchno] - ndiffpos[nlaunchno - 1] - 1;
			nlaunchno = nlaunchno - 2;
		}
		else
		{
			//printf("npathflag = %d, npos = %d\n", npathflag, npos);
			//printf("find path error!\n");
			return;
		}

		//only if it is not an extension gap, will the path direction change
		//otherwise, back on the same direction
		int nExtFlag = str_npathflagp[npos] / 4;
		if (npathflag == 3 && (nExtFlag == 2 || nExtFlag == 3))
		{
			npathflag = 3;
		}
		//else if (npathflag == 1 && (nExtFlag == 1 || nExtFlag == 3))
		else if (npathflag == 1 && str_nExtFlagp[npos] == 1)
		{
			npathflag = 1;
		}
		else
		{
			npathflag = str_npathflagp[npos] & 0x3;
		}

		if (i == 0 || j == 0)
		{
			break;
		}

		if (npathflag == PATH_END)
		{
			break;
		}
	}

	i--;
	j--;

	while(i >= 0)
	{
		outstr1[nlen] = instr1D[i];
		outstr2[nlen] = 23;
		nlen++;
		i--;
	}

	while(j >= 0)
	{
		outstr1[nlen] = 23;
		outstr2[nlen] = instr2D[j];
		nlen++;
		j--;
	}

	strMaxInfop[0] = strMaxInfop[maxPos];
	strMaxInfop[0].noutputlen = nlen;

	return;
}

__kernel void setZero(__global char *a,
                      int arraySize)
{
    unsigned int index = get_global_id(0);
    if (index < arraySize)
    {
        a[index] = 0;
    }
}

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics: enable

#define __THREAD_FENCE_USED__

typedef struct {
    int nposi, nposj;
    int nmaxpos;
    float fmaxscore;
    int noutputlen;
}   MAX_INFO;

#define PATH_END 0
#define COALESCED_OFFSET 32

//Simple GPU barrier, with atomicAdd used
//Input: Local thread idx in a block, goal value
void __barrier_opencl_lock_based(int localID, int goalValue, volatile __global int *g_mutexOpencl)
{
    int tid = localID;
    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);

#ifdef __THREAD_FENCE_USED__
    write_mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
    //other options
    //read_mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
    //mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
#endif

    if (tid == 0)
    {
        atom_add(g_mutexOpencl, 1);
        mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
        while (*g_mutexOpencl < goalValue)
        { }
    }

    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
}

__kernel void  MatchStringGPUSync(__global char  *pathFlag,
								  __global char  *extFlag,
								  __global float *nGapDist,
								  __global float *hGapDist,
								  __global float *vGapDist,
								  __global int   *diffPos,
								  __global int   *threadNum,
								  int            rowNum,
								  int            columnNum,
								  __global char  *seq1,
								  __global char  *seq2,
								  int            blosumWidth,
								  float          openPenalty,
								  float          extensionPenalty,
								  __global MAX_INFO *maxInfo,
								  __global float *blosum62D,
								  volatile __global int *mutexMem,
								  __local int* temp)
{
	int npos, ntablepos, tid;
	int npreposngap, npreposhgap, npreposvgap;
	int nLocalID = get_local_id(0);
	int blockNum = get_num_groups(0);
	int blockSize = get_local_size(0);
	int totalThreadNum = blockSize * blockNum;
	int threadid = get_global_id(0);

	int launchNo;
	int launchNum = rowNum + columnNum - 1;
	int indexi1 = -1;
	int indexj1 = 0;
	int indexi, indexj;
	int startPos = 2 * COALESCED_OFFSET;
	int noffset = 0;

	float fdist;
	float fdistngap, fdisthgap, fdistvgap;
	float ext_dist;
	float fmaxdist;

	//__local float temp[5000];
	temp[0] = 0.0f;

	for (launchNo = 2; launchNo < launchNum; launchNo++)
	{
		if (launchNo < rowNum)
		{
			indexi1++;
		}
		else if (launchNo == rowNum)
		{
			indexi1++;
			noffset = 1;
		}
		else
		{
			indexj1++;
		}

		for (tid = threadid; tid < threadNum[launchNo]; tid += totalThreadNum)
		{
			indexi = indexi1 - tid;
			indexj = indexj1 + tid;

			npos = startPos + tid;

			npreposhgap = npos - diffPos[launchNo];
			npreposvgap = npreposhgap - 1;
			npreposngap = npreposvgap - diffPos[launchNo - 1];

			ntablepos = seq1[indexi] * blosumWidth + seq2[indexj];
			//ntablepos = seq1C[indexi] * blosumWidth + seq2C[indexj];
			fdist = blosum62D[ntablepos];

			fmaxdist = nGapDist[npreposngap];
			fdistngap = fmaxdist + fdist;

			ext_dist  = hGapDist[npreposhgap] - extensionPenalty;
			fdisthgap = nGapDist[npreposhgap] - openPenalty;

			if (fdisthgap <= ext_dist)
			{
				fdisthgap = ext_dist;
				extFlag[npreposhgap] = 1;
			}

			ext_dist  = vGapDist[npreposvgap] - extensionPenalty;
			fdistvgap = nGapDist[npreposvgap] - openPenalty;

			if (fdistvgap <= ext_dist)
			{
				fdistvgap = ext_dist;
				pathFlag[npreposvgap] += 8;
			}

			fdistngap = (fdistngap < 0.0f) ? 0.0f : fdistngap;
			fdisthgap = (fdisthgap < 0.0f) ? 0.0f : fdisthgap;
			fdistvgap = (fdistvgap < 0.0f) ? 0.0f : fdistvgap;

			hGapDist[npos] = fdisthgap;
			vGapDist[npos] = fdistvgap;

			//priority 00, 01, 10
			if (fdistngap >= fdisthgap && fdistngap >= fdistvgap)
			{
				fmaxdist = fdistngap;
				pathFlag[npos] = 2;
			}
			else if (fdisthgap >= fdistngap && fdisthgap >= fdistvgap)
			{
				fmaxdist = fdisthgap;
				pathFlag[npos] = 1;
			}
			else //fdistvgap >= fdistngap && fdistvgap >= fdisthgap
			{
				fmaxdist = fdistvgap;
				pathFlag[npos] = 3;
			}

			nGapDist[npos] = fmaxdist;

			//Here, the maximum match distance is 0, which means
			//previous alignment is useless
			if (fmaxdist <= 0.00000001f)
			{
				pathFlag[npos] = PATH_END;
			}

			if (maxInfo[threadid].fmaxscore < fmaxdist)
			{
				maxInfo[threadid].nposi = indexi + 1;
				maxInfo[threadid].nposj = indexj + 1;
				maxInfo[threadid].nmaxpos = npos;
				maxInfo[threadid].fmaxscore = fmaxdist;
			}
		}

		//GPU synchronization
		__barrier_opencl_lock_based(nLocalID, (launchNo - 1) * blockNum, mutexMem);
		startPos += diffPos[launchNo + 1] + noffset;
	}
}

__kernel void trace_back2(__global char *str_npathflagp,
						  __global char *str_nExtFlagp,
						  __global int  *ndiffpos,
						  __global char *instr1D,
						  __global char *instr2D,
						  __global char *outstr1,
						  __global char *outstr2,
						  __global MAX_INFO * strMaxInfop,
						  int mfThreadNum)
{
	int i, j;
	int npos, maxPos, nlen;
	int npathflag;
	int nlaunchno;
	float maxScore;

	maxPos = 0;
	maxScore = strMaxInfop[0].fmaxscore;
	for (i = 1; i < mfThreadNum; i++)
	{
		if (maxScore < strMaxInfop[i].fmaxscore)
		{
			maxPos = i;
			maxScore = strMaxInfop[i].fmaxscore;
		}
	}

	npos = strMaxInfop[maxPos].nmaxpos;
	npathflag = str_npathflagp[npos] & 0x3;
	nlen = 0;

	i = strMaxInfop[maxPos].nposi;
	j = strMaxInfop[maxPos].nposj;
	nlaunchno = i + j;

	while (1)
	{
		if (npathflag == 3)
		{
			outstr1[nlen] = 23;
			outstr2[nlen] = instr2D[j - 1];
			nlen++;
			j--;

			//position in the transformed matrix
			npos = npos - ndiffpos[nlaunchno] - 1;
			nlaunchno--;
		}
		else if (npathflag == 1)
		{
			outstr1[nlen] = instr1D[i - 1];
			outstr2[nlen] = 23;
			nlen++;
			i--;

			//position in the transformed matrix
			npos = npos - ndiffpos[nlaunchno];
			nlaunchno--;
		}
		else if (npathflag == 2)
		{
			outstr1[nlen] = instr1D[i - 1];
			outstr2[nlen] = instr2D[j - 1];
			nlen++;
			i--;
			j--;

			//position in the transformed matrix
			npos = npos - ndiffpos[nlaunchno] - ndiffpos[nlaunchno - 1] - 1;
			nlaunchno = nlaunchno - 2;
		}
		else
		{
			//printf("npathflag = %d, npos = %d\n", npathflag, npos);
			//printf("find path error!\n");
			return;
		}

		//only if it is not an extension gap, will the path direction change
		//otherwise, back on the same direction
		int nExtFlag = str_npathflagp[npos] / 4;
		if (npathflag == 3 && (nExtFlag == 2 || nExtFlag == 3))
		{
			npathflag = 3;
		}
		//else if (npathflag == 1 && (nExtFlag == 1 || nExtFlag == 3))
		else if (npathflag == 1 && str_nExtFlagp[npos] == 1)
		{
			npathflag = 1;
		}
		else
		{
			npathflag = str_npathflagp[npos] & 0x3;
		}

		if (i == 0 || j == 0)
		{
			break;
		}

		if (npathflag == PATH_END)
		{
			break;
		}
	}

	i--;
	j--;

	while(i >= 0)
	{
		outstr1[nlen] = instr1D[i];
		outstr2[nlen] = 23;
		nlen++;
		i--;
	}

	while(j >= 0)
	{
		outstr1[nlen] = 23;
		outstr2[nlen] = instr2D[j];
		nlen++;
		j--;
	}

	strMaxInfop[0] = strMaxInfop[maxPos];
	strMaxInfop[0].noutputlen = nlen;

	return;
}

__kernel void setZero(__global char *a,
                      int arraySize)
{
    unsigned int index = get_global_id(0);
    if (index < arraySize)
    {
        a[index] = 0;
    }
}


Ripple: bugfix + interpolate

Ripple: bugfix + interpolate

/*
 * This file is part of openfx-arena <https://github.com/olear/openfx-arena>,
 * Copyright (C) 2016 INRIA
 *
 * openfx-arena is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * openfx-arena is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with openfx-arena.  If not, see <http://www.gnu.org/licenses/gpl-2.0.html>
*/

const sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_FILTER_LINEAR | CLK_ADDRESS_CLAMP_TO_EDGE;

float2 effect(float x, float y, float a, float w) {
        float m = sqrt(x*x + y*y);
        float s = sin(w*m);
        float x1 = a*y*m*s;
        float y1 = -a*x*m*s;
        return (float2)(x1, y1);
}

float2 displacement(int xi, int yi, int width, int height, double waveAmp, double waveLength) {
        float aspect = ((float) height) / ((float) width);
        float x = ((float) xi) / ((float) width), y = ((float) yi) / ((float) height);
        return effect(x - 0.5, aspect*(y - 0.5 + 0.35), waveAmp, waveLength)
             - effect(x - 0.5, aspect*(y - 0.5 - 0.35), waveAmp, waveLength)
             + effect(x - 0.5, aspect*(y), 2, 5000);
}

kernel void filter(read_only image2d_t inputImage, write_only image2d_t outputImage, double waveAmp, double waveLength) {
   int2 dimensions = get_image_dim(inputImage);
   int width = dimensions.x, height = dimensions.y;
   int channelDataType = get_image_channel_data_type(inputImage);
   int channelOrder = get_image_channel_order(inputImage);
   int x = get_global_id(0), y = get_global_id(1);
   int2 coordinates = (int2)(x, y);
   float2 disp = displacement(x, y, width, height, waveAmp, waveLength);
   float fx = (float)coordinates.x;
   float fy = (float)coordinates.y;
   float2 fromCoordinates = (float2)(fx,fy) + disp;

   float newX = (float)(x/(float)width);
   float newY = (float)(y/(float)height);

   // Interpolate
   // Calculate the (x, y) coordinates of the transformation and keep
   // the fractional  in the delta variables
   float x0 = floor(fromCoordinates.x);
   float xf = x0 + 1.f;
   float y0 = floor(fromCoordinates.y);
   float yf = y0 + 1.f;
   float deltaX = fromCoordinates.x - x0;
   float deltaY = fromCoordinates.y - y0;

   float centerX = (float)(width)/2.f;
   float centerY = (float)(height)/2.f;

   // Calculte the array position for the pixels (x, y), (x + 1, y), (x, y + 1) and (x + 1, y + 1)
   float2 pos0 = (float2)(x0,y0);
   float2 pos1 = (float2)(x0+1,y0);
   float2 pos2 = (float2)(x0,y0+1);
   float2 pos3 = (float2)(x0+1,y0+1);

   float4 componentX0 = ((read_imagef(inputImage, sampler, pos1) - read_imagef(inputImage, sampler, pos0))*deltaX+read_imagef(inputImage, sampler, pos0));
   float4 componentX1 = ((read_imagef(inputImage, sampler, pos3) - read_imagef(inputImage, sampler, pos2))*deltaX+read_imagef(inputImage, sampler, pos2));
   float4 finalPixelComponent = (componentX1 - componentX0) * deltaY + componentX0;
   float4 minval = (float4)(0.f,0.f,0.f,0.f);
   float4 maxval = (float4)(1.f,1.f,1.f,1.f);
   finalPixelComponent = clamp(finalPixelComponent,minval,maxval);

   write_imagef(outputImage, coordinates, finalPixelComponent);
}

/*
 * This file is part of openfx-arena <https://github.com/olear/openfx-arena>,
 * Copyright (C) 2016 INRIA
 *
 * openfx-arena is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * openfx-arena is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with openfx-arena.  If not, see <http://www.gnu.org/licenses/gpl-2.0.html>
*/

const sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_FILTER_NEAREST | CLK_ADDRESS_CLAMP_TO_EDGE;
int2 effect(float x, float y, float a, float w) {
        float m = sqrt(x*x + y*y);
        float s = sin(w*m);
        float x1 = a*y*m*s;
        float y1 = -a*x*m*s;
        return (int2)(x1, y1);
}
int2 displacement(int xi, int yi, int width, int height, double waveAmp, double waveLength) {
        float aspect = ((float) height) / ((float) width);
        float x = ((float) xi) / ((float) width), y = ((float) yi) / ((float) height);
        return effect(x - 0.5, aspect*(y - 0.5 + 0.35), waveAmp, waveLength)
             - effect(x - 0.5, aspect*(y - 0.5 - 0.35), waveAmp, waveLength)
             + effect(x - 0.5, aspect*(y), 2, 5000);
}
kernel void filter(read_only image2d_t inputImage, write_only image2d_t outputImage, double waveAmp, double waveLength) {
   int2 dimensions = get_image_dim(inputImage);
   int width = dimensions.x, height = dimensions.y;
   int channelDataType = get_image_channel_data_type(inputImage);
   int channelOrder = get_image_channel_order(inputImage);
   int x = get_global_id(0), y = get_global_id(1);
   int2 coordinates = (int2)(x, y);
   int2 disp = displacement(x, y, width, height, waveAmp, waveLength);
   int2 fromCoordinates = coordinates + disp;
   float4 pixel = read_imagef(inputImage, sampler, fromCoordinates);
   float4 transformedPixel = pixel;
   write_imagef(outputImage, coordinates, transformedPixel);
}


reflection + perturbation seems to work

reflection + perturbation seems to work

#include "cl/mwc64x/cl/mwc64x.cl"

// Doesn't return 0 so that rays can bounce without bumping.
float sphere_toi(
  const float3 eye,
  const float3 look,

  const float3 center,
  const float radius)
{
  // quadratic coefficients
  float a = dot(look, look);
  float b = 2 * dot(eye - center, look);
  float3 to_center = eye - center;
  float c = dot(to_center, to_center) - radius*radius;

  // discriminant
  float d = b*b - 4*a*c;

  if (d < 0) {
    return HUGE_VALF;
  }

  float s1 = (sqrt(d) - b) / (2 * a);
  if (s1 <= 0) {
    s1 = HUGE_VALF;
  }

  float s2 = (-sqrt(d) - b) / (2 * a);
  if (s1 < s2 || s2 <= 0) {
    return s1;
  }

  return s2;
}

typedef struct {
  float4 row1;
  float4 row2;
  float4 row3;
  float4 row4;
} mat4;

float4 vmult(mat4 m, float4 v) {
  return
    (float4)(
      dot(m.row1, v),
      dot(m.row2, v),
      dot(m.row3, v),
      dot(m.row4, v)
    );
}

mat4 screen_to_view(uint w, uint h, float fovy) {
  float aspect = (float)w / (float)h;

  float b = tan(fovy / 2);
  float a = 2 * b / h;
  b = -b;

  mat4 r;
  // We divide both x and y by h, to take aspect ratio into account.
  // An increase in image width will cause wider rays to be shot.
  r.row1 = (float4)(a, 0, 0, b * aspect);
  r.row2 = (float4)(0, a, 0, b);
  r.row3 = (float4)(0, 0, 1, 0);
  r.row4 = (float4)(0, 0, 0, 1);

  return r;
}

mat4 view_to_world(float3 eye, float3 look, float3 up) {
  float4 x = (float4)(cross(look, up), 0);
  float4 y = (float4)(up, 0);
  float4 z = (float4)(look, 0);
  float4 w = (float4)(eye, 1);

  // Change of basis

  mat4 r;
  r.row1.x = x.x;
  r.row2.x = x.y;
  r.row3.x = x.z;
  r.row4.x = x.w;

  r.row1.y = y.x;
  r.row2.y = y.y;
  r.row3.y = y.z;
  r.row4.y = y.w;

  r.row1.z = z.x;
  r.row2.z = z.y;
  r.row3.z = z.z;
  r.row4.z = z.w;

  r.row1.w = w.x;
  r.row2.w = w.y;
  r.row3.w = w.z;
  r.row4.w = w.w;

  return r;
}

typedef struct {
  float r;
  float g;
  float b;
} RGB;

RGB rgb(float3 xyz) {
  RGB r;
  r.r = xyz.x;
  r.g = xyz.y;
  r.b = xyz.z;
  return r;
}

typedef struct { float x, y, z } float3_parse;

float3 pack_float3(float3_parse f) {
  return (float3)(f.x, f.y, f.z);
}

typedef struct {
  float3_parse center;
  float radius;
  float3_parse color;
  float emittance;
} Object;

float parse_float(__global const float** data) {
  float r = **data;
  ++*data;
  return r;
}

float3_parse parse_float3(__global const float** data) {
  float3_parse r;
  r.x = parse_float(data);
  r.y = parse_float(data);
  r.z = parse_float(data);
  return r;
}

Object parse_object(__global const float* data) {
  Object r;
  r.center    = parse_float3(&data);
  r.radius    = parse_float(&data);
  r.color     = parse_float3(&data);
  r.emittance = parse_float(&data);
  return r;
}

typedef struct {
  float3 origin;
  float3 direction;
} Ray;

void raycast(
  Ray ray,

  __global const float* objects,
  const uint num_objects,

  float* toi,
  Object* collision
) {
  *toi = HUGE_VALF;

  for (uint i = 0; i < num_objects; ++i) {
    Object object = parse_object((Object*)objects + i);
    float this_toi = sphere_toi(ray.origin, ray.direction, pack_float3(object.center), object.radius);

    if (this_toi >= *toi) {
      continue;
    }

    *toi = this_toi;
    *collision = object;
  }
}

float rand(mwc64x_state_t* rand_state) {
  return (float)MWC64X_NextUint(rand_state) / (float)UINT_MAX;
}

float3 perturb(mwc64x_state_t* rand_state, float3 x, float3 y, float3 z) {
  float3 coeffs;
  coeffs.y = 2 * (rand(rand_state) - 0.5);
  const float xz = sqrt(1 - coeffs.y * coeffs.y);
  float azimuth = rand(rand_state ) * 2 * 3.14;
  coeffs.x = cos(azimuth);
  coeffs.z = sin(azimuth);
  coeffs.x *= xz;
  coeffs.z *= xz;
  return coeffs.x*x + coeffs.y*y + coeffs.z*z;
}

float3 pathtrace(
  Ray ray,
  uint max_depth,
  float3 ambient,
  mwc64x_state_t* rand_state,
  __global const float* objects,
  const uint num_objects
) {
  if (max_depth == 0) {
    return ambient;
  }

  float toi;
  Object collided_object;

  raycast(ray, objects, num_objects, &toi, &collided_object);

  if (toi == HUGE_VALF) {
    return ambient;
  }

  float3 emitted = (float3)(collided_object.emittance);

  float3 collision_point = ray.origin + toi*ray.direction;
  float3 normal = (collision_point - pack_float3(collided_object.center)) / collided_object.radius;

  // TODO: cos_theta < 0?
  // TODO: loop + accumulators
  Ray reflected_ray;
  {
    float cos_theta = dot(ray.direction, normal);
    const float3 reflected = 2*cos_theta*normal - ray.direction;

    const float3 y = reflected;
    // TODO: find z/x better when normal ~= reflected
    const float3 z = normalize(cross(normal, y));
    const float3 x = normalize(cross(z, y));
    do {
      reflected_ray.direction = perturb(rand_state, x, y, z);
    }
    while (dot(reflected_ray.direction, normal) < 0);

    reflected_ray.origin = collision_point + 0.1f * reflected_ray.direction;
  }

  float3 reflected = pathtrace(reflected_ray, max_depth - 1, ambient, rand_state, objects, num_objects);

  return pack_float3(collided_object.color) * (ambient + emitted + reflected);
}

mwc64x_state_t init_rand_state(ulong random_seed) {
  mwc64x_state_t rand_state;
  rand_state.x = (uint)(random_seed & 0xFFFFFFFF);
  rand_state.c = (random_seed & 0xFFFFFFFF00000000) >> 32;
  return rand_state;
}

__kernel void render(
  const uint image_width,
  const uint image_height,

  const float fovy,
  float3 eye,
  const float3 look,
  const float3 up,

  ulong random_seed,
  float3 ambient_light,

  __global const float* objects,
  const uint num_objects,

  __global RGB * output)
{
  int id = get_global_id(0);

  const int x_pix = id % image_width;
  const int y_pix = id / image_width;

  float4 world_pos =
    vmult(view_to_world(eye, look, up),
    vmult(screen_to_view(image_width, image_height, fovy),
    (float4)(x_pix, y_pix, 1, 1)
  ));

  Ray ray;
  ray.origin = eye;
  ray.direction = normalize((world_pos / world_pos.w).xyz - eye);

  random_seed = random_seed * id * id;
  mwc64x_state_t rand_state = init_rand_state(random_seed);
  MWC64X_Skip(&rand_state, 20);

  output[id] = rgb(pathtrace(ray, 2, ambient_light, &rand_state, objects, num_objects));
}

#include "cl/mwc64x/cl/mwc64x.cl"

// Doesn't return 0 so that rays can bounce without bumping.
float sphere_toi(
  const float3 eye,
  const float3 look,

  const float3 center,
  const float radius)
{
  // quadratic coefficients
  float a = dot(look, look);
  float b = 2 * dot(eye - center, look);
  float3 to_center = eye - center;
  float c = dot(to_center, to_center) - radius*radius;

  // discriminant
  float d = b*b - 4*a*c;

  if (d < 0) {
    return HUGE_VALF;
  }

  float s1 = (sqrt(d) - b) / (2 * a);
  if (s1 <= 0) {
    s1 = HUGE_VALF;
  }

  float s2 = (-sqrt(d) - b) / (2 * a);
  if (s1 < s2 || s2 <= 0) {
    return s1;
  }

  return s2;
}

typedef struct {
  float4 row1;
  float4 row2;
  float4 row3;
  float4 row4;
} mat4;

float4 vmult(mat4 m, float4 v) {
  return
    (float4)(
      dot(m.row1, v),
      dot(m.row2, v),
      dot(m.row3, v),
      dot(m.row4, v)
    );
}

mat4 screen_to_view(uint w, uint h, float fovy) {
  float aspect = (float)w / (float)h;

  float b = tan(fovy / 2);
  float a = 2 * b / h;
  b = -b;

  mat4 r;
  // We divide both x and y by h, to take aspect ratio into account.
  // An increase in image width will cause wider rays to be shot.
  r.row1 = (float4)(a, 0, 0, b * aspect);
  r.row2 = (float4)(0, a, 0, b);
  r.row3 = (float4)(0, 0, 1, 0);
  r.row4 = (float4)(0, 0, 0, 1);

  return r;
}

mat4 view_to_world(float3 eye, float3 look, float3 up) {
  float4 x = (float4)(cross(look, up), 0);
  float4 y = (float4)(up, 0);
  float4 z = (float4)(look, 0);
  float4 w = (float4)(eye, 1);

  // Change of basis

  mat4 r;
  r.row1.x = x.x;
  r.row2.x = x.y;
  r.row3.x = x.z;
  r.row4.x = x.w;

  r.row1.y = y.x;
  r.row2.y = y.y;
  r.row3.y = y.z;
  r.row4.y = y.w;

  r.row1.z = z.x;
  r.row2.z = z.y;
  r.row3.z = z.z;
  r.row4.z = z.w;

  r.row1.w = w.x;
  r.row2.w = w.y;
  r.row3.w = w.z;
  r.row4.w = w.w;

  return r;
}

typedef struct {
  float r;
  float g;
  float b;
} RGB;

RGB rgb(float3 xyz) {
  RGB r;
  r.r = xyz.x;
  r.g = xyz.y;
  r.b = xyz.z;
  return r;
}

typedef struct { float x, y, z } float3_parse;

float3 pack_float3(float3_parse f) {
  return (float3)(f.x, f.y, f.z);
}

typedef struct {
  float3_parse center;
  float radius;
  float3_parse color;
  float emittance;
} Object;

float parse_float(__global const float** data) {
  float r = **data;
  ++*data;
  return r;
}

float3_parse parse_float3(__global const float** data) {
  float3_parse r;
  r.x = parse_float(data);
  r.y = parse_float(data);
  r.z = parse_float(data);
  return r;
}

Object parse_object(__global const float* data) {
  Object r;
  r.center    = parse_float3(&data);
  r.radius    = parse_float(&data);
  r.color     = parse_float3(&data);
  r.emittance = parse_float(&data);
  return r;
}

typedef struct {
  float3 origin;
  float3 direction;
} Ray;

void raycast(
  Ray ray,

  __global const float* objects,
  const uint num_objects,

  float* toi,
  Object* collision
) {
  *toi = HUGE_VALF;

  for (uint i = 0; i < num_objects; ++i) {
    Object object = parse_object((Object*)objects + i);
    float this_toi = sphere_toi(ray.origin, ray.direction, pack_float3(object.center), object.radius);

    if (this_toi >= *toi) {
      continue;
    }

    *toi = this_toi;
    *collision = object;
  }
}

float3 from_euler(float3 x, float3 y, float3 z, float azimuth, float altitude) {
  const float ky = sin(altitude);
  const float kxz = cos(altitude);
  const float kz = sin(azimuth)*kxz;
  const float kx = cos(azimuth)*kxz;
  return kx*x + ky*y + kz*z;
}

float rand(mwc64x_state_t* rand_state) {
  return (float)MWC64X_NextUint(rand_state) / (float)UINT_MAX;
}

float3 perturb(mwc64x_state_t* rand_state, float3 x, float3 y, float3 z) {
  const float azimuth = rand(rand_state) * 2 * 3.14;
  const float altitude = 3.14 * (0.5 - rand(rand_state));
  return from_euler(x, y, z, azimuth, altitude);
}

float3 pathtrace(
  Ray ray,
  uint max_depth,
  float3 ambient,
  mwc64x_state_t* rand_state,
  __global const float* objects,
  const uint num_objects
) {
  if (max_depth == 0) {
    return ambient;
  }

  float toi;
  Object collided_object;

  raycast(ray, objects, num_objects, &toi, &collided_object);

  if (toi == HUGE_VALF) {
    return ambient;
  }

  float3 emitted = (float3)(collided_object.emittance);

  float3 collision_point = ray.origin + toi*ray.direction;
  float3 normal = (collision_point - pack_float3(collided_object.center)) / collided_object.radius;

  // TODO: cos_theta < 0?
  // TODO: loop + accumulators
  Ray reflected_ray;
  {
    float cos_theta = dot(ray.direction, normal);
    const float3 reflected = ray.direction - 2*cos_theta*normal;

    const float3 y = (float3)(0, 1, 0);//reflected;
    // TODO: find z/x better when normal ~= reflected
    const float3 z = normalize(cross(normal, y));
    const float3 x = normalize(cross(z, y));
    do {
      reflected_ray.direction = perturb(rand_state, x, y, z);
    }
    while (dot(reflected_ray.direction, normal) < 0);

    reflected_ray.origin = collision_point + 0.1f * reflected_ray.direction;
  }

  float3 reflected = pathtrace(reflected_ray, max_depth - 1, ambient, rand_state, objects, num_objects);

  return pack_float3(collided_object.color) * (ambient + emitted + reflected);
}

mwc64x_state_t init_rand_state(ulong random_seed) {
  mwc64x_state_t rand_state;
  rand_state.x = (uint)(random_seed & 0xFFFFFFFF);
  rand_state.c = (random_seed & 0xFFFFFFFF00000000) >> 32;
  return rand_state;
}

__kernel void render(
  const uint image_width,
  const uint image_height,

  const float fovy,
  float3 eye,
  const float3 look,
  const float3 up,

  ulong random_seed,
  float3 ambient_light,

  __global const float* objects,
  const uint num_objects,

  __global RGB * output)
{
  int id = get_global_id(0);

  const int x_pix = id % image_width;
  const int y_pix = id / image_width;

  float4 world_pos =
    vmult(view_to_world(eye, look, up),
    vmult(screen_to_view(image_width, image_height, fovy),
    (float4)(x_pix, y_pix, 1, 1)
  ));

  Ray ray;
  ray.origin = eye;
  ray.direction = normalize((world_pos / world_pos.w).xyz - eye);

  random_seed = random_seed * id * id;
  mwc64x_state_t rand_state = init_rand_state(random_seed);
  MWC64X_Skip(&rand_state, 20);

  output[id] = rgb(pathtrace(ray, 2, ambient_light, &rand_state, objects, num_objects));
}