batched

batched

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// number of filter taps in the processing wk_item
//#define MLO_WEI_WKITEM 5

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_HORIZ_PIX_SZ + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_OUT_WEI_SCAN_BLK (MLO_OUT_WEI_EXT_SCAN_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_HORIZ_PIX_SZ + MLO_OUT_WEI_SCAN_BLK - 1)/MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ *MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)
#if ((MLO_OUT_HEIGHT/MLO_N_ALIGNED_OUT_SCAN_BLK)*MLO_N_ALIGNED_OUT_SCAN_BLK == MLO_OUT_HEIGHT)
#define MLO_BLK_ALIGNED 1
#else
#define MLO_BLK_ALIGNED 0
#endif

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#define MLO_N_IN_HORIZ_READS ((((MLO_OUT_HORIZ_PIX_SZ-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#if (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm
// data layout.
// registers:
// per wk_item:
// MLO_READ_UNIT x MLO_N_LCL_OUT_MAPS of dy
// group:
// MLO_N_OUT_BLK_GRP blocks of MLO_N_OUT_BLK_GRP_PIX_SZ of dy
//
// LDS:
// data buffer
// MLO_N_LCL_IN_MAPS blocks of MLO_IN_LCL_SZ of x

// dw local buffer keeps 1 set of dweghts calculated at each iteration
MLO_TEMP_WEI_BUFFER_SZ

//
// inputs:
// read into LDS
// MLO_N_IN_BLK_GRP groups
// output
// fetch
// MLO_N_OUT_BLK_GRP groups

// alg
// for each batch
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for k in filter size1
//				for l in filter size 0
//					at for each out/in pixel pair op,ip in wk_item
//                  calculate the contribution to a dweights by formula ip.x (input pixel.x) = (op.x * stride.x - pad .x + l) % (filter size 0), ip.y = (op.y * stride.y - pad.y + k) % filter size1,
//					wei[k,l] = in[ip.y, ip.x]*op[op.y, op.x]
//                  lcl_wei[MLO_OUT_BLK_GRP_PIX_SZ * out.grp + [op.y, op.x]] = we[k,l]
//                  accumulate the weights in the wk_item with wei_accum +=  lcl_wei[accum_id], accum_id = MLO_OUT_BLK_GRP*MLO_N_LCL_OUT_MAPS*map_id + k*(filter size 0) + l
//	write out accululate weights

// group layout
// 0 - input channel index
// 1 - output channel index
// 2 - 1

// loop over batches
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
	// input/output tiles

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	int lcl_id = get_local_id(0);

	int w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	int w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
	int w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
	int w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	for (int i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);
	// zero out pvt accum
	// over all batches

#if 0
	if (lcl_id == 0)
	{
		printf("K:i: %d %d %d %d %d\n",
			ib,
			gbl_in_off,
			gbl_out_off,
			MLO_N_BATCH_LOOPS,
			MLO_N_LCL_BATCHS
		);
	}
#endif

#if 1

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;
		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1
		// LDS UPDATE !!!!
		// input scan
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;
		for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
		{
			int c_scan = 0;

			for (int p4 = lcl_id;  p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1) && in_y + c_scan < MLO_IN_HEIGHT;
				//				c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS),
				p4 += MLO_GRP_SZ)
			{

				c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS);
				int c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);
				// TODO :: multiply on right mask if needed
				*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[lb*MLO_IN_BATCH_STRIDE + gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
				// TO DO:: CHECK HEIGHT
				// TO DO:: CHECK PADDING
				//			*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					lcl_bot[lb*MLO_IN_LCL_SZ + (c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
				}

#if 0
				if (c_idx == 0 && o_idx == 0 && (c_pix4 == 0 || c_pix4 == 1))
				{
					printf("K:i: %d %d %d %d %d %d %d     %f %f %f %f\n",
						lcl_id,
						lb,
						p4,
						c_scan,
						in_y + c_scan,
						lb*MLO_IN_BATCH_STRIDE + gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
						lb*MLO_IN_LCL_SZ + (c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT,
						in_rd_data[0],
						in_rd_data[1],
						in_rd_data[2],
						in_rd_data[3]
					);
				}
#endif

			}
		}

		in_y += MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1;

		// TO DO: HANDLE PADDING
		// over all out blocks
		for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{
#if MLO_BLK_ALIGNED == 0
			for (int i = lcl_id; ob == MLO_N_OUT_BLK - 1 && i < MLO_TOTAL_OUT_LCL_SZ; i += MLO_GRP_SZ)
			{
				lcl_top[i] = 0;
			}
#endif

			barrier(CLK_LOCAL_MEM_FENCE);

			// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
			// TODO:: HANDLE multiple INPUTS
			// overshot i shadled by out put
			gbl_in_scan_off = gbl_in_off + mul24(in_y, MLO_IN_STRIDE);
			for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
			{
				int c_scan = 0;
				for (int p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS * (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1) && in_y + c_scan < MLO_IN_HEIGHT;
					p4 += MLO_GRP_SZ)
				{
					c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS);
					int c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[lb*MLO_IN_BATCH_STRIDE + gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
					//					*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_bot[lb*MLO_IN_LCL_SZ + (c_scan + MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
					}

#if 0
					if (c_idx == 1 && o_idx == 0 && in_y + c_scan == 0)
					{
						printf("K:i: %d %d %f %f\n",
							lcl_id,
							gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
							in_rd_data[0],
							in_rd_data[2]
						);
					}
#endif

				}

			}

			gbl_out_scan_off = gbl_out_off + mul24(out_y, MLO_OUT_STRIDE);

			// over all outputs groups
			int gbl_out_scan_off1 = gbl_out_scan_off;
			for (int og = 0; og < MLO_N_OUT_BLK_GRP
				; ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS*MLO_OUT_CHANNEL_STRIDE)
			{
				//				barrier(CLK_LOCAL_MEM_FENCE);

				// fetch output
				// ouput scan
				__private _FLOAT out_rd_data[MLO_READ_UNIT];

				for (int ob = 0; ob < MLO_N_LCL_BATCHS; ++ob)
				{
					for (int oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS);
						oo_p4 += MLO_GRP_SZ)
					{
						int o = iDiv(oo_p4, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
						int o_pX4 = iMod(oo_p4, o, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
						int o_scan = iDiv(o_pX4, MLO_N_OUT_HORIZ_READS);
						int o_pix4 = iMod(o_pX4, o_scan, MLO_N_OUT_HORIZ_READS);
						if (out_y + o_scan < MLO_OUT_HEIGHT
							&& o_idx + og *MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS
							)
						{
							*(MLO_READ_TYPE*)out_rd_data
								= *(MLO_READ_TYPE*)&top_df[ob*MLO_OUT_BATCH_STRIDE + gbl_out_scan_off1 + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT];
#if MLO_OUT_SCAN_NOT_DIVBY4
							if (o_pix4 == (MLO_N_OUT_HORIZ_READS - 1))
							{
								for (int i = MLO_READ_UNIT - 1; i >= MLO_READ_UNIT - MLO_OUT_N_PIXS_OFF; --i)
								{
									out_rd_data[i] = 0;
								}
							}
#endif
							// write with MLO_OUT_HORIZ_PIX_EXT_SZ stride
							//						*(MLO_READ_TYPE*)&lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)out_rd_data;
							//
							lcl_top[(ob*MLO_N_LCL_OUT_MAPS + o) * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 0] = out_rd_data[0];
							lcl_top[(ob*MLO_N_LCL_OUT_MAPS + o) * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 1] = out_rd_data[1];
							lcl_top[(ob*MLO_N_LCL_OUT_MAPS + o) * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 2] = out_rd_data[2];
							lcl_top[(ob*MLO_N_LCL_OUT_MAPS + o) * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 3] = out_rd_data[3];
							//						for (int i = 0; i < MLO_READ_UNIT; ++i)
							//						{
							//							lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + i] = out_rd_data[o*MLO_READ_UNIT + i];
							//						}

#if 0
							if (c_idx == 0 && o_idx == 0 && og == 0 && o == 0/* && out_y + o_scan == 0*/)
							{
								printf("K:o: %d %d %d %f %f %f\n",
									lcl_id,
									gbl_out_scan_off + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT,
									o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT,
									lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 0],
									lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 1],
									lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 2]
								);
							}
#endif

						}
					}
				}
				barrier(CLK_LOCAL_MEM_FENCE);

				// process
				// algorithm

#if 1
				//				if (w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
				{
					for (int j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
					{

						_FLOAT i_vals[MLO_N_LCL_BATCHS*MLO_WEI_WKITEM];

						for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
						{

							for (int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
							{
								int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
								int i_off = lb*MLO_IN_LCL_SZ + (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
								_FLOAT i_val = lcl_bot[i_off];

								i_vals[lb*MLO_WEI_WKITEM + w] = i_val;

							}
						}

						// if we overshoot the scanline
						// out data will be 0 by initial setting
						for (int i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
						{

							//						for(int w = 0; w < MLO_WEI_WKITEM; ++w)

							for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
							{
								int w_x = w_x0 + (MLO_WEI_WKITEM - 1)* MLO_WEI_BLK_SZ0;
								int i_off = lb*MLO_IN_LCL_SZ + (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
								_FLOAT i_val = lcl_bot[i_off];

								i_vals[lb*MLO_WEI_WKITEM + (MLO_WEI_WKITEM - 1)] = i_val;

							}

							for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
							{
								// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
								for (int ob = 0; ob < MLO_N_LCL_BATCHS; ++ob)
								{
									_FLOAT o_val
										= lcl_top[(ob*MLO_N_LCL_OUT_MAPS + o)*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

									int w = 0;
									_FLOAT i_val;
									for (/*int w = 0*/; w < MLO_WEI_WKITEM; ++w)
									{

										for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
										{
											i_val = i_vals[lb*MLO_WEI_WKITEM + w];
#if 0
											pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = fma(i_val, o_val, pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w]);
#else
											pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;
#endif

#if 0
											int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
											if (c_idx == 0 && o_idx == 0 && og == 0 && o == 0 && w_y == 0 && w_x == 0 && w_blk_idx < MLO_MAX_WEI_BLK_LOOP /* && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM*/)
											{
												int i_off = lb*MLO_IN_LCL_SZ + (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
												printf("K:s: %d %d %d %d %d %d %d %f %f %f\n",
													lcl_id,
													lb,
													j,
													i,
													w,
													i_off,
													(ob*MLO_N_LCL_OUT_MAPS + o)*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i),
													pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w],
													i_val,
													o_val
												);
											}
#endif
										}
									} // for (/*int w = 0*/; w < MLO_WEI_WKITEM; ++w)
								}
							} // for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)

							for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
							{

								for (int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
								{
									i_vals[lb*MLO_WEI_WKITEM + w] = i_vals[lb*MLO_WEI_WKITEM + w + 1];

								}
							}

						}
					}
				}
#else
				for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
				{
					for (int w = 0; w < MLO_WEI_WKITEM; ++w)
					{
						pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = lcl_top[(og * MLO_N_LCL_OUT_MAPS + o)] * lcl_bot[w];
					}
				}
#endif

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

			barrier(CLK_LOCAL_MEM_FENCE);

			// move data up
			for (int lb = 0; lb < MLO_N_LCL_BATCHS; ++lb)
			{
				int c_scan = 0;
				for (int p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1);
					p4 += MLO_GRP_SZ, c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS))
				{
					int c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&lcl_bot[lb*MLO_IN_LCL_SZ + (c_scan + (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1))*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[lb*MLO_IN_LCL_SZ + c_scan*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

				}
			}

		} // for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (int b = 0;

#endif

	int oo = iDiv(lcl_id, MLO_WEI_CHANNEL_STRIDE);
	int wei_i = iMod(lcl_id, oo, MLO_WEI_CHANNEL_STRIDE);
	int wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
	int wei_i_x = iMod(wei_i, wei_i_y, (MLO_FILTER_SIZE0));
	// send it out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);

	int wei_lcl_off = 0;
	_FLOAT final_sum = 0;

	// save in lcl and orgnize in a proper order
	// outputs
	//	filter size1
	//	  filter size0
	// TO DO DEPENDING ON THE GROUP SIZE
	for (int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for (int o = 0; w_blk_idx < MLO_MAX_WEI_BLK_LOOP/* && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM*/ && o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for (; w < MLO_WEI_WKITEM; ++w)
			{
				// save "virtual" filter table
				int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
				wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1 + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
#if 0
				// wei_lcl_off == 1020 || wei_lcl_off == 1080 || wei_lcl_off == 1140)
				if (c_idx == 0 && o_idx == 0 && og == 0 && o == 0 && w_y == 1 && w_x == 1/* && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM*/)
				{
					printf("K:u: %d %d %d %f %f\n",
						lcl_id,
						w_blk_idx,
						wei_lcl_off,
						lcl_bot[wei_lcl_off],
						pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
#if 1
		// read into real filter table

		if (lcl_id < (MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
		{
			final_sum = 0;
			for (int i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
			{
				final_sum += lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x];
#if 0
				if (c_idx == 0 && o_idx == 0 && og == 0 && oo == 0 && wei_i_y == 0 && wei_i_x == 0)
				{
					printf("K:fs: %d %d %d  %f %f\n",
						lcl_id,
						i,
						((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x, final_sum,
						lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x]
					);
				}
#endif
				}

#if 1

			weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //

#if 0
			if (og == 0 && oo == 0 && wei_i == 0 && c_idx == 0)
			{

				printf("K:o: %d %d %d %d %d %d %f\n",
					ib,
					o_idx,
					c_idx,
					(og *  MLO_N_LCL_OUT_MAPS + oo),
					wei_df_off,
					wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i,
					weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i]
				);
			}
#endif
#else
			_FLOAT t_accum = 0;

			for (int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
			{
				for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
				{
					for (int w = 0; w < MLO_WEI_WKITEM; ++w)
					{
						t_accum += pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
					}
				}
			}

			weights_df[lcl_id] = t_accum;
#endif

	}
#endif
		barrier(CLK_LOCAL_MEM_FENCE);

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

}

__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT] = { 0,0 };

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
#if 0
		if (wei_blk_idx == 0 && wei_idx == 0)
		{
			printf("k:a: %d  %f\n",
				(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx,
				pvt_accum_wei[0]
				);
		}
#endif

	}
	*(MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

 // number of filter taps in the processing wk_item
//#define MLO_WEI_WKITEM 5

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_HORIZ_PIX_SZ + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_OUT_WEI_SCAN_BLK (MLO_OUT_WEI_EXT_SCAN_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_HORIZ_PIX_SZ + MLO_OUT_WEI_SCAN_BLK - 1)/MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ *MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)
#if ((MLO_OUT_HEIGHT/MLO_N_ALIGNED_OUT_SCAN_BLK)*MLO_N_ALIGNED_OUT_SCAN_BLK == MLO_OUT_HEIGHT)
#define MLO_BLK_ALIGNED 1
#else
#define MLO_BLK_ALIGNED 0
#endif

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#define MLO_N_IN_HORIZ_READS ((((MLO_OUT_HORIZ_PIX_SZ-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#if (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u,(int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm
// data layout.
// registers:
// per wk_item:
// MLO_READ_UNIT x MLO_N_LCL_OUT_MAPS of dy
// group:
// MLO_N_OUT_BLK_GRP blocks of MLO_N_OUT_BLK_GRP_PIX_SZ of dy
//
// LDS:
// data buffer
// MLO_N_LCL_IN_MAPS blocks of MLO_IN_LCL_SZ of x

// dw local buffer keeps 1 set of dweghts calculated at each iteration
	MLO_TEMP_WEI_BUFFER_SZ

//
// inputs:
// read into LDS
// MLO_N_IN_BLK_GRP groups
// output
// fetch
// MLO_N_OUT_BLK_GRP groups

// alg
// for each batch
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for k in filter size1
//				for l in filter size 0
//					at for each out/in pixel pair op,ip in wk_item
//                  calculate the contribution to a dweights by formula ip.x (input pixel.x) = (op.x * stride.x - pad .x + l) % (filter size 0), ip.y = (op.y * stride.y - pad.y + k) % filter size1,
//					wei[k,l] = in[ip.y, ip.x]*op[op.y, op.x]
//                  lcl_wei[MLO_OUT_BLK_GRP_PIX_SZ * out.grp + [op.y, op.x]] = we[k,l]
//                  accumulate the weights in the wk_item with wei_accum +=  lcl_wei[accum_id], accum_id = MLO_OUT_BLK_GRP*MLO_N_LCL_OUT_MAPS*map_id + k*(filter size 0) + l
//	write out accululate weights

// group layout
// 0 - input channel index
// 1 - output channel index
// 2 - 1

// loop over batches
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
// input/output tiles

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2),(MLO_BATCH_SZ/(MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	int lcl_id = get_local_id(0);

	int w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	int w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
	int w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
	int w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	for (int i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

// zero out LDS
	for (int i = lcl_id; i < ( MLO_LCL_SZ ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);
// zero out pvt accum
	// over all batches

#if 0
	if (lcl_id == 0)
	{
		printf("K:i: %d %d %d %d\n",
			ib,
			gbl_in_off,
			gbl_out_off
		);
	}
#endif

#if 1
//	gbl_in_off += MLO_IN_BATCH_STRIDE;
//	gbl_out_off += MLO_OUT_BATCH_STRIDE;

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;
// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1
// LDS UPDATE !!!!
// input scan
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		int c_scan = 0;
		for(int p4 = lcl_id;  p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1) && in_y + c_scan < MLO_IN_HEIGHT;
//			c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS),
			p4 += MLO_GRP_SZ)
		{
			c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS);
			int c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);
// TODO :: multiply on right mask if needed
			*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
// TO DO:: CHECK HEIGHT
// TO DO:: CHECK PADDING
//			*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
			}

#if 0
			if (c_idx == 0 && o_idx == 0 && (c_pix4 == 0 || c_pix4 == 1))
			{
				printf("K:i: %d %d %d %d %d %d %d %d %f %f\n",
					lcl_id,
					p4,
					c_scan,
					MLO_N_IN_HORIZ_READS,
					(MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1),
					in_y + c_scan,
					MLO_IN_HEIGHT,
					gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
					in_rd_data[0],
					in_rd_data[2]
				);
			}
#endif

		}

		in_y +=  MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1;

// TO DO: HANDLE PADDING
// over all out blocks
		for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{
#if MLO_BLK_ALIGNED == 0
			for (int i = lcl_id; ob == MLO_N_OUT_BLK-1 && i < MLO_TOTAL_OUT_LCL_SZ; i+= MLO_GRP_SZ)
			{
				lcl_top[i] = 0;
			}
#endif

			barrier(CLK_LOCAL_MEM_FENCE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
               // TODO:: HANDLE multiple INPUTS
			   // overshot i shadled by out put
			gbl_in_scan_off = gbl_in_off + mul24(in_y, MLO_IN_STRIDE);
			c_scan = 0;
			for (int p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS * (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1) && in_y + c_scan < MLO_IN_HEIGHT;
					p4 += MLO_GRP_SZ)
			{
					c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS);
					int c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
//					*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_bot[(c_scan + MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
					}

#if 0
					if (c_idx == 1 && o_idx == 0 && in_y + c_scan == 0)
					{
						printf("K:i: %d %d %f %f\n",
							lcl_id,
							gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
							in_rd_data[0],
							in_rd_data[2]
						);
					}
#endif

			}

			gbl_out_scan_off = gbl_out_off + mul24(out_y, MLO_OUT_STRIDE);

// over all outputs groups

			int gbl_out_scan_off1 = gbl_out_scan_off;
			for(int og = 0; og < MLO_N_OUT_BLK_GRP
				; ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS*MLO_OUT_CHANNEL_STRIDE )
			{
//				barrier(CLK_LOCAL_MEM_FENCE);

// fetch output
// ouput scan
				__private _FLOAT out_rd_data[MLO_READ_UNIT];

				for(int oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS);
						oo_p4 += MLO_GRP_SZ)
				{
					int o = iDiv(oo_p4, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
					int o_pX4 = iMod(oo_p4, o, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
					int o_scan = iDiv(o_pX4, MLO_N_OUT_HORIZ_READS);
					int o_pix4 = iMod(o_pX4, o_scan, MLO_N_OUT_HORIZ_READS);
					if (out_y + o_scan < MLO_OUT_HEIGHT
						&& o_idx + og *MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS
						)
					{
						*(MLO_READ_TYPE*)out_rd_data
							= *(MLO_READ_TYPE*)&top_df[gbl_out_scan_off1 + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT];
#if MLO_OUT_SCAN_NOT_DIVBY4
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS - 1))
						{
							for (int i = MLO_READ_UNIT - 1; i >= MLO_READ_UNIT - MLO_OUT_N_PIXS_OFF; --i)
							{
								out_rd_data[i] = 0;
							}
						}
#endif
						// write with MLO_OUT_HORIZ_PIX_EXT_SZ stride
						//						*(MLO_READ_TYPE*)&lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)out_rd_data;
						//
						lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 0] = out_rd_data[0];
						lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 1] = out_rd_data[1];
						lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 2] = out_rd_data[2];
						lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 3] = out_rd_data[3];
						//						for (int i = 0; i < MLO_READ_UNIT; ++i)
						//						{
						//							lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + i] = out_rd_data[o*MLO_READ_UNIT + i];
						//						}

#if 0
						if (c_idx == 0 && o_idx == 0 && og == 0 && o == 0/* && out_y + o_scan == 0*/)
						{
							printf("K:o: %d %d %d %f %f %f\n",
								lcl_id,
								gbl_out_scan_off + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT,
								o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT,
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 0],
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 1],
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 2]
							);
						}
#endif

					}
				}

				barrier(CLK_LOCAL_MEM_FENCE);

// process
// algorithm

#if 1
//				if (w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
				{
					for (int j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
					{

						_FLOAT i_vals[MLO_WEI_WKITEM];
						for (int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
						{
							int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

						}

						// if we overshoot the scanline
						// out data will be 0 by initial setting
						for (int i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
						{

							//						for(int w = 0; w < MLO_WEI_WKITEM; ++w)

							{
								int w_x = w_x0 + (MLO_WEI_WKITEM - 1)* MLO_WEI_BLK_SZ0;
								int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
								_FLOAT i_val = lcl_bot[i_off];

								i_vals[(MLO_WEI_WKITEM - 1)] = i_val;

							}

							for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
							{
								// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
								_FLOAT o_val
									= lcl_top[o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

								int w = 0;
								_FLOAT i_val;
								for (/*int w = 0*/; w < MLO_WEI_WKITEM; ++w)
								{

									i_val = i_vals[w];
#if 0
									pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = fma(i_val, o_val, pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w]);
#else
									pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;
#endif

#if 0
									int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
									if (c_idx == 0 && o_idx == 0 && og == 0 && o == 0 && w_y == 0 && w_x == 6 && w_blk_idx < MLO_MAX_WEI_BLK_LOOP /* && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM*/)
									{
										int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
										printf("K:s: %d %d %d %d %d %d %f %f %f\n",
											lcl_id,
											j,
											i,
											w,
											i_off,
											o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i),
											pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w],
											i_val,
											o_val
										);
									}
#endif
								} // for (/*int w = 0*/; w < MLO_WEI_WKITEM; ++w)

							} // for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)

							for (int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
							{
								i_vals[w] = i_vals[w + 1];

							}

						}
					}
				}
#else
						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
							for(int w = 0; w < MLO_WEI_WKITEM; ++w)
							{
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = lcl_top[(og * MLO_N_LCL_OUT_MAPS + o)] * lcl_bot[w];
							}
						}
#endif

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

			barrier(CLK_LOCAL_MEM_FENCE);

// move data up
			c_scan = 0;
			for (int p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1);
									p4 += MLO_GRP_SZ, c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS))
			{
				int c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);

				*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&lcl_bot[(c_scan + (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1))*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT];
				*(MLO_READ_TYPE*)&lcl_bot[c_scan*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

		} // for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (int b = 0;

#endif

	int oo = iDiv(lcl_id, MLO_WEI_CHANNEL_STRIDE);
	int wei_i = iMod(lcl_id, oo, MLO_WEI_CHANNEL_STRIDE);
	int wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
	int wei_i_x = iMod(wei_i,wei_i_y, (MLO_FILTER_SIZE0));
	// send it out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
				+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);

	int wei_lcl_off = 0;
	_FLOAT final_sum = 0;

// save in lcl and orgnize in a proper order
// outputs
//	filter size1
//	  filter size0
// TO DO DEPENDING ON THE GROUP SIZE
	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0;  w_blk_idx < MLO_MAX_WEI_BLK_LOOP/* && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM*/ && o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
// save "virtual" filter table
				int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
				wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1  + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
#if 0
				// wei_lcl_off == 1020 || wei_lcl_off == 1080 || wei_lcl_off == 1140)
				if (c_idx == 0 && o_idx == 0 && og == 0 && o == 0 && w_y == 1 && w_x == 1/* && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM*/)
				{
					printf("K:u: %d %d %d %f %f\n",
						lcl_id,
						w_blk_idx,
						wei_lcl_off,
						lcl_bot[wei_lcl_off],
						pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
#if 1
// read into real filter table

		if (lcl_id < ( MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
		{
			final_sum = 0;
			for(int i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
			{
				final_sum += lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x];
#if 0
				if (c_idx == 0 && o_idx == 0  && og == 0 && oo  == 0 && wei_i_y == 1 && wei_i_x == 1)
				{
					printf("K:fs: %d %d %d  %f %f\n",
						lcl_id,
						i,
						((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x,						final_sum,
						lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x]
					);
				}
#endif
			}

#if 1

			weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //

#if 0
			if (og == 0 && oo == 0 && wei_i == 0 && c_idx == 0)
			{

				printf("K:o: %d %d %d %d %d %d %f\n",
					ib,
					o_idx,
					c_idx,
					(og *  MLO_N_LCL_OUT_MAPS + oo),
					wei_df_off,
					wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i,
					weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i]
					);
			}
#endif
#else
			_FLOAT t_accum = 0;

			for (int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
			{
				for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
				{
					for (int w = 0; w < MLO_WEI_WKITEM; ++w)
					{
						t_accum += pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
					}
				}
			}

			weights_df[lcl_id] = t_accum;
#endif

		}
#endif
		barrier(CLK_LOCAL_MEM_FENCE);

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

}

__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT] = { 0,0 };

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
#if 0
		if (wei_blk_idx == 0 && wei_idx == 0)
		{
			printf("k:a: %d  %f\n",
				(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx,
				pvt_accum_wei[0]
				);
		}
#endif

	}
	*(MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}


Fix for Nvidia CL implementation.

Fix for Nvidia CL implementation.

Evidently kernels aren't supposed to have size_t arguments... which AMD
allows in its implementation.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant int bilinear_filters[8][2] = {
    { 128, 0},
    { 112, 16},
    { 96, 32},
    { 80, 48},
    { 64, 64},
    { 48, 80},
    { 32, 96},
    { 16, 112}
};

__constant short sub_pel_filters[8][8] = {
    //Note that these were originally 8x6, but are padded for vector ops
    { 0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0, 0, 0},
    { 2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0, 0, 0},
    { 3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0, 0, 0},
    { 1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0, 0, 0},
};

void vp8_filter_block2d_first_pass(
    __global unsigned char *src_ptr,
    __private int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        for (i=0; i < output_width*output_height; i++){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

#if 1
                int8 t8, s, f;
                int4 t4;
                int2 t2;

                f = convert_int8(vload8(0,vp8_filter));
                s = convert_int8(vload8(0,&src_ptr[src_offset-2]));

                t8 = s * f;

                //Collapse 8-element vector to single int and round
                //t4 = t8.s0123 + t8.s4567;
                //t2 = t4.xy + t4.zw;
                t2 = t8.s01 + t8.s23 + t8.s45;
                Temp = t2.x + t2.y + (VP8_FILTER_WEIGHT >> 1);
#else
                Temp = (int)(src_ptr[src_offset - PS2]      * vp8_filter[0]) +
                   (int)(src_ptr[src_offset - (int)pixel_step] * vp8_filter[1]) +
                   (int)(src_ptr[src_offset]                * vp8_filter[2]) +
                   (int)(src_ptr[src_offset + pixel_step]   * vp8_filter[3]) +
                   (int)(src_ptr[src_offset + PS2]          * vp8_filter[4]) +
                   (int)(src_ptr[src_offset + PS3]          * vp8_filter[5]) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */
#endif
            /* Normalize back to 0-255 */
            Temp = Temp >> VP8_FILTER_SHIFT;

            //Temp = (int)src_ptr[2];
            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
        }
    }

    //Add a fence so that no 2nd pass stuff starts before 1st pass writes are done.
    write_mem_fence(CLK_GLOBAL_MEM_FENCE);
}

void vp8_filter_block2d_second_pass
(
    __private int *src_ptr,
    __global unsigned char *output_ptr,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    int out_offset;
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

__kernel void vp8_block_variation_kernel
(
    __global unsigned char  *src_ptr,
    int   src_pixels_per_line,
    __global int *HVar,
    __global int *VVar
)
{
    int i, j;
    __global unsigned char *Ptr = src_ptr;

    for (i = 0; i < 4; i++)
    {
        for (j = 0; j < 4; j++)
        {
            *HVar += abs((int)Ptr[j] - (int)Ptr[j+1]);
            *VVar += abs((int)Ptr[j] - (int)Ptr[j+src_pixels_per_line]);
        }

        Ptr += src_pixels_per_line;
    }
}

__kernel void vp8_sixtap_predict_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
        ) {

    __private int FData[9*4]; /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 9, 4, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[8], dst_ptr, dst_pitch, 4, 4, 4, 4, yoffset);
}

__kernel void vp8_sixtap_predict8x8_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data bufffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 13, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_pitch, 8, 8, 8, 8, yoffset);

}

__kernel void vp8_sixtap_predict8x4_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 9, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_pitch, 8, 8, 4, 8, yoffset);

}

__kernel void vp8_sixtap_predict16x16_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
)
{
    __private int FData[21*24];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 21, 16, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[32], dst_ptr, dst_pitch, 16, 16, 16, 16, yoffset);

    return;
}

void vp8_filter_block2d_bil_first_pass(
    __global unsigned char *src_ptr,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);

    unsigned int i, j;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    unsigned int out_row,out_offset;
    unsigned int src_offset;
    int src_increment = src_pixels_per_line - output_width;

    for (i = 0; i < output_height; i++)
    {
        src_offset = i*(output_width+src_increment);
        out_row = output_width * i;
        for (j = 0; j < output_width; j++)
        {
            out_offset = out_row + j;

            /* Apply bilinear filter */
            output_ptr[out_offset] = (((int)src_ptr[src_offset]          * vp8_filter[0]) +
                             ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                             (VP8_FILTER_WEIGHT / 2)) >> VP8_FILTER_SHIFT;
            //src_ptr++;
            src_offset++;
        }

        /* Next row... */
        //src_ptr    += src_pixels_per_line - output_width;
        //output_ptr += output_width;
    }

    //write_mem_fence(CLK_GLOBAL_MEM_FENCE);
    //barrier(CLK_GLOBAL_MEM_FENCE);
}

void vp8_filter_block2d_bil_second_pass
(
    __global int *src_ptr,
    __global unsigned char *output_ptr,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {
    uint tid = get_global_id(0);

    unsigned int i, j;
    int Temp;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    int out_offset;
    int src_offset;
    int src_increment = src_pixels_per_line - output_width;

    if (tid < output_width*output_height){
        for (i = 0; i < output_height; i++)
        {
            src_offset = i*(output_width+src_increment);
            out_offset = i*output_pitch;
            for (j = 0; j < output_width; j++)
            {
                /* Apply filter */
                Temp = ((int)src_ptr[src_offset]         * vp8_filter[0]) +
                       ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                       (VP8_FILTER_WEIGHT / 2);
                output_ptr[out_offset++] = (unsigned int)(Temp >> VP8_FILTER_SHIFT);
                //printf("output[%d]=%d\n",out_offset-1, Temp>>VP8_FILTER_SHIFT);
                src_offset++;
            }
        }
    }
}

void vp8_filter_block2d_bil
(
    __global unsigned char *src_ptr,
    __global unsigned char *output_ptr,
    unsigned int   src_pixels_per_line,
    unsigned int   dst_pitch,
    int      xoffset,
    int      yoffset,
    int            Width,
    int            Height,
    __global int *FData
)
{

    //__private unsigned short FData[17*16];    /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass(src_ptr, FData, src_pixels_per_line, 1, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass(FData, output_ptr, dst_pitch, Width, Width, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict4x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
 ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 4, 4, FData);
}

__kernel void vp8_bilinear_predict8x8_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 8, FData);

}

__kernel void vp8_bilinear_predict8x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 4, FData);
}

__kernel void vp8_bilinear_predict16x16_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 16, 16, FData);
}

kernel void vp8_memcpy_kernel(
    global unsigned char *src,
    int src_stride,
    global unsigned char *dst,
    int dst_stride,
    int num_bytes,
    int num_iter
){

    int i,r;
    int src_offset, dst_offset;

    r = get_global_id(1);
    if (r < get_global_size(1)){
        i = get_global_id(0);
        if (i < get_global_size(0)){
            src_offset = r*src_stride + i;
            dst_offset = r*dst_stride + i;
            dst[dst_offset] = src[src_offset];
        }
    }
}

void vp8_memset_short(
    global short *mem,
    int offset,
    short newval,
    size_t size
)
{
    int tid = get_global_id(0);

    if (tid < (size/2)){
        mem[offset+tid/2] = newval;
    }
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant int bilinear_filters[8][2] = {
    { 128, 0},
    { 112, 16},
    { 96, 32},
    { 80, 48},
    { 64, 64},
    { 48, 80},
    { 32, 96},
    { 16, 112}
};

__constant short sub_pel_filters[8][8] = {
    //Note that these were originally 8x6, but are padded for vector ops
    { 0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0, 0, 0},
    { 2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0, 0, 0},
    { 3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0, 0, 0},
    { 1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0, 0, 0},
};

void vp8_filter_block2d_first_pass(
    __global unsigned char *src_ptr,
    __private int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        for (i=0; i < output_width*output_height; i++){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

#if 1
                int8 t8, s, f;
                int4 t4;
                int2 t2;

                f = convert_int8(vload8(0,vp8_filter));
                s = convert_int8(vload8(0,&src_ptr[src_offset-2]));

                t8 = s * f;

                //Collapse 8-element vector to single int and round
                //t4 = t8.s0123 + t8.s4567;
                //t2 = t4.xy + t4.zw;
                t2 = t8.s01 + t8.s23 + t8.s45;
                Temp = t2.x + t2.y + (VP8_FILTER_WEIGHT >> 1);
#else
                Temp = (int)(src_ptr[src_offset - PS2]      * vp8_filter[0]) +
                   (int)(src_ptr[src_offset - (int)pixel_step] * vp8_filter[1]) +
                   (int)(src_ptr[src_offset]                * vp8_filter[2]) +
                   (int)(src_ptr[src_offset + pixel_step]   * vp8_filter[3]) +
                   (int)(src_ptr[src_offset + PS2]          * vp8_filter[4]) +
                   (int)(src_ptr[src_offset + PS3]          * vp8_filter[5]) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */
#endif
            /* Normalize back to 0-255 */
            Temp = Temp >> VP8_FILTER_SHIFT;

            //Temp = (int)src_ptr[2];
            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
        }
    }

    //Add a fence so that no 2nd pass stuff starts before 1st pass writes are done.
    write_mem_fence(CLK_GLOBAL_MEM_FENCE);
}

void vp8_filter_block2d_second_pass
(
    __private int *src_ptr,
    __global unsigned char *output_ptr,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    int out_offset;
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

__kernel void vp8_block_variation_kernel
(
    __global unsigned char  *src_ptr,
    int   src_pixels_per_line,
    __global int *HVar,
    __global int *VVar
)
{
    int i, j;
    __global unsigned char *Ptr = src_ptr;

    for (i = 0; i < 4; i++)
    {
        for (j = 0; j < 4; j++)
        {
            *HVar += abs((int)Ptr[j] - (int)Ptr[j+1]);
            *VVar += abs((int)Ptr[j] - (int)Ptr[j+src_pixels_per_line]);
        }

        Ptr += src_pixels_per_line;
    }
}

__kernel void vp8_sixtap_predict_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
        ) {

    __private int FData[9*4]; /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 9, 4, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[8], dst_ptr, dst_pitch, 4, 4, 4, 4, yoffset);
}

__kernel void vp8_sixtap_predict8x8_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data bufffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 13, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_pitch, 8, 8, 8, 8, yoffset);

}

__kernel void vp8_sixtap_predict8x4_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 9, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_pitch, 8, 8, 4, 8, yoffset);

}

__kernel void vp8_sixtap_predict16x16_kernel
(
    __global unsigned char  *src_ptr,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int  dst_pitch
)
{
    __private int FData[21*24];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, FData, src_pixels_per_line, 1, 21, 16, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[32], dst_ptr, dst_pitch, 16, 16, 16, 16, yoffset);

    return;
}

void vp8_filter_block2d_bil_first_pass(
    __global unsigned char *src_ptr,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);

    unsigned int i, j;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    unsigned int out_row,out_offset;
    unsigned int src_offset;
    int src_increment = src_pixels_per_line - output_width;

    for (i = 0; i < output_height; i++)
    {
        src_offset = i*(output_width+src_increment);
        out_row = output_width * i;
        for (j = 0; j < output_width; j++)
        {
            out_offset = out_row + j;

            /* Apply bilinear filter */
            output_ptr[out_offset] = (((int)src_ptr[src_offset]          * vp8_filter[0]) +
                             ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                             (VP8_FILTER_WEIGHT / 2)) >> VP8_FILTER_SHIFT;
            //src_ptr++;
            src_offset++;
        }

        /* Next row... */
        //src_ptr    += src_pixels_per_line - output_width;
        //output_ptr += output_width;
    }

    //write_mem_fence(CLK_GLOBAL_MEM_FENCE);
    //barrier(CLK_GLOBAL_MEM_FENCE);
}

void vp8_filter_block2d_bil_second_pass
(
    __global int *src_ptr,
    __global unsigned char *output_ptr,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {
    uint tid = get_global_id(0);

    unsigned int i, j;
    int Temp;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    int out_offset;
    int src_offset;
    int src_increment = src_pixels_per_line - output_width;

    if (tid < output_width*output_height){
        for (i = 0; i < output_height; i++)
        {
            src_offset = i*(output_width+src_increment);
            out_offset = i*output_pitch;
            for (j = 0; j < output_width; j++)
            {
                /* Apply filter */
                Temp = ((int)src_ptr[src_offset]         * vp8_filter[0]) +
                       ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                       (VP8_FILTER_WEIGHT / 2);
                output_ptr[out_offset++] = (unsigned int)(Temp >> VP8_FILTER_SHIFT);
                //printf("output[%d]=%d\n",out_offset-1, Temp>>VP8_FILTER_SHIFT);
                src_offset++;
            }
        }
    }
}

void vp8_filter_block2d_bil
(
    __global unsigned char *src_ptr,
    __global unsigned char *output_ptr,
    unsigned int   src_pixels_per_line,
    unsigned int   dst_pitch,
    int      xoffset,
    int      yoffset,
    int            Width,
    int            Height,
    __global int *FData
)
{

    //__private unsigned short FData[17*16];    /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass(src_ptr, FData, src_pixels_per_line, 1, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass(FData, output_ptr, dst_pitch, Width, Width, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict4x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
 ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 4, 4, FData);
}

__kernel void vp8_bilinear_predict8x8_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 8, FData);

}

__kernel void vp8_bilinear_predict8x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 4, FData);
}

__kernel void vp8_bilinear_predict16x16_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_ptr,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, dst_ptr, src_pixels_per_line, dst_pitch, xoffset, yoffset, 16, 16, FData);
}

kernel void vp8_memcpy_kernel(
    global unsigned char *src,
    int src_stride,
    global unsigned char *dst,
    int dst_stride,
    int num_bytes,
    int num_iter
){

    int i,r;
    int src_offset, dst_offset;

    r = get_global_id(1);
    if (r < get_global_size(1)){
        i = get_global_id(0);
        if (i < get_global_size(0)){
            src_offset = r*src_stride + i;
            dst_offset = r*dst_stride + i;
            dst[dst_offset] = src[src_offset];
        }
    }
}

kernel void vp8_memset_short_kernel(
    global short *mem,
    int offset,
    short newval,
    size_t size
)
{
    int tid = get_global_id(0);

    if (tid < (size/2)){
        mem[offset+tid/2] = newval;
    }
}


Minor formatting changes.

Minor formatting changes.


Minor changes

Minor changes

#define BLOCK_SIZE 16

__attribute((reqd_work_group_size(BLOCK_SIZE,BLOCK_SIZE,1)))
__attribute((num_simd_work_items(4)))

__kernel void gbc(__global float* restrict matrix, // input-data, e.g. 80000x192
		  __global float* restrict gbcresult,    // output-data, 32x80000
		  int width,                             // 192
		  int startseed                          // 0, 32, 64, ...
		  )
{
  // For each working id, calculate the correlation between
  // global_row id and global_col+startseed
  // such that:
  //    A = global_col+startseed; (the seeds)
  //    B = global_row (the whole image)

  __local float A_local[BLOCK_SIZE*BLOCK_SIZE];
  __local float B_local[BLOCK_SIZE*BLOCK_SIZE];

  int global_row = get_global_id(0); // 0 .. 79999
  int global_col = get_global_id(1); // 0 .. 31

  int local_row = get_local_id(0); // rows
  int local_col = get_local_id(1); // columns

  //  float old = gbcresult[global_row*width+global_col];
  float running_sum = 0;
  int a, b;

  for(a=0;a<width;a+=BLOCK_SIZE) {
    A_local[local_col*BLOCK_SIZE+local_row] = matrix[(startseed+global_col)*width+a+local_row];
    B_local[local_row*BLOCK_SIZE+local_col] = matrix[global_row*width+local_col+a];
    barrier(CLK_LOCAL_MEM_FENCE);
        #pragma unroll
    for(int k=0;k<BLOCK_SIZE;++k) {
      running_sum += A_local[local_col*BLOCK_SIZE+k]*B_local[local_row*BLOCK_SIZE+k];
    }
    barrier(CLK_LOCAL_MEM_FENCE);
  }

  if(running_sum<0)
    running_sum = 0;

  gbcresult[global_row*BLOCK_SIZE+global_col] += running_sum;
}

#define BLOCK_SIZE 16

//__attribute__((num_compute_units(8)))

__attribute((reqd_work_group_size(BLOCK_SIZE,BLOCK_SIZE,1)))
__attribute((num_simd_work_items(4)))

__kernel void gbc(__global float* restrict matrix, // input-data, e.g. 80000x192
		  __global float* restrict gbcresult,    // output-data, 32x80000
		  int width,                             // 192
		  int startseed                          // 0, 32, 64, ...
		  )
{
  // For each working id, calculate the correlation between
  // global_row id and global_col+startseed
  // such that:
  //    A = global_col+startseed; (the seeds)
  //    B = global_row (the whole image)

  __local float A_local[BLOCK_SIZE*BLOCK_SIZE];
  __local float B_local[BLOCK_SIZE*BLOCK_SIZE];

  int global_row = get_global_id(0); // 0 .. 79999
  int global_col = get_global_id(1); // 0 .. 31

  int local_row = get_local_id(0); // rows
  int local_col = get_local_id(1); // columns

  //  float old = gbcresult[global_row*width+global_col];
  float running_sum = 0;
  int a, b;

  for(a=0;a<width;a+=BLOCK_SIZE) {
    A_local[local_col*BLOCK_SIZE+local_row] = matrix[(startseed+global_col)*width+a+local_row];
    B_local[local_row*BLOCK_SIZE+local_col] = matrix[global_row*width+local_col+a];
    barrier(CLK_LOCAL_MEM_FENCE);
        #pragma unroll
    for(int k=0;k<BLOCK_SIZE;++k) {
      running_sum += A_local[local_col*BLOCK_SIZE+k]*B_local[local_row*BLOCK_SIZE+k];
    }
    barrier(CLK_LOCAL_MEM_FENCE);
  }

  if(running_sum<0)
    running_sum = 0;

  gbcresult[global_row*BLOCK_SIZE+global_col] += running_sum;
}


fix pf_cl

fix pf_cl

//============================================================================
// vSMC/example/pf/pf_cl.cl
//----------------------------------------------------------------------------
//                         vSMC: Scalable Monte Carlo
//----------------------------------------------------------------------------
// Copyright (c) 2013-2015, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#include <vsmc/rngc/rngc.h>

typedef struct {
    float pos_x;
    float pos_y;
    float vel_x;
    float vel_y;
} cv;

typedef struct {
    float pos_x;
    float pos_y;
} cv_pos;

VSMC_STATIC_INLINE float4 normal01(vsmc_philox4x32 *rng)
{
    vsmc_philox4x32_gen(&rng->ctr, &rng->key, &rng->state);
    float u1 = sqrt(-2 * log(vsmc_u01_open_closed_u32_f32(rng->state.v[0])));
    float u2 = sqrt(-2 * log(vsmc_u01_open_closed_u32_f32(rng->state.v[1])));
    float v1 = 6.283185 * vsmc_u01_open_closed_u32_f32(rng->state.v[2]);
    float v2 = 6.283185 * vsmc_u01_open_closed_u32_f32(rng->state.v[3]);
    float4 r;
    r.x = u1 * cos(v1);
    r.y = u1 * sin(v1);
    r.z = u2 * cos(v2);
    r.w = u2 * sin(v2);

    return r;
}

VSMC_STATIC_INLINE float log_likelihood(const cv *sp, float obs_x, float obs_y)
{
    const float scale = 10;
    const float nu = 10;

    float llh_x = scale * (sp->pos_x - obs_x);
    float llh_y = scale * (sp->pos_y - obs_y);

    llh_x = log1p(llh_x * llh_x / nu);
    llh_y = log1p(llh_y * llh_y / nu);

    return -0.5F * (nu + 1) * (llh_x + llh_y);
}

__kernel void cv_init(__global cv *state, __global ulong *accept,
    __global float *log_weight, __global float *obs_x, __global float *obs_y)
{
    ulong i = get_global_id(0);
    if (i >= SIZE)
        return;

    const float sd_pos0 = 2;
    const float sd_vel0 = 1;
    cv sp = state[i];

    vsmc_philox4x32 rng;
    vsmc_philox4x32_init(&rng, SEED + i);
    rng.ctr.v[1] = i;

    float4 r = normal01(&rng);
    sp.pos_x = r.x * sd_pos0;
    sp.pos_y = r.y * sd_pos0;
    sp.vel_x = r.z * sd_vel0;
    sp.vel_y = r.w * sd_vel0;

    state[i] = sp;
    accept[i] = 1;
    log_weight[i] = log_likelihood(&sp, obs_x[0], obs_y[0]);
}

__kernel void cv_move(ulong iter, __global cv *state, __global ulong *accept,
    __global float *inc_weight, __global float *obs_x, __global float *obs_y)
{
    ulong i = get_global_id(0);
    if (i >= SIZE)
        return;

    const float var_pos = 0.02F;
    const float var_vel = 0.001F;
    const float sd_pos = sqrt(var_pos);
    const float sd_vel = sqrt(var_vel);
    const float delta = 0.1F;
    cv sp = state[i];

    vsmc_philox4x32 rng;
    vsmc_philox4x32_init(&rng, SEED + i);
    rng.ctr.v[1] = i;

    float4 r = normal01(&rng);
    sp.pos_x += r.x * sd_pos + delta * sp.vel_x;
    sp.pos_y += r.y * sd_pos + delta * sp.vel_y;
    sp.vel_x += r.z * sd_vel;
    sp.vel_y += r.w * sd_vel;

    state[i] = sp;
    accept[i] = 1;
    inc_weight[i] = log_likelihood(&sp, obs_x[iter], obs_y[iter]);
}

__kernel void cv_est(
    ulong iter, ulong dim, __global cv *state, __global cv_pos *est)
{
    ulong i = get_global_id(0);
    if (i >= SIZE)
        return;

    est[i].pos_x = state[i].pos_x;
    est[i].pos_y = state[i].pos_y;
}

//============================================================================
// vSMC/example/pf/pf_cl.cl
//----------------------------------------------------------------------------
//                         vSMC: Scalable Monte Carlo
//----------------------------------------------------------------------------
// Copyright (c) 2013-2015, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#include <vsmc/rngc/rngc.h>

typedef struct {
    float pos_x;
    float pos_y;
    float vel_x;
    float vel_y;
} cv;

typedef struct {
    float pos_x;
    float pos_y;
} cv_pos;

VSMC_STATIC_INLINE float4 normal01(vsmc_philox4x32 *rng)
{
    vsmc_philox4x32_gen(&rng->ctr, &rng->key, &rng->state);
    float u1 = sqrt(-2 * vsmc_u01_open_closed_u32_f32(rng->state.v[0]));
    float u2 = sqrt(-2 * vsmc_u01_open_closed_u32_f32(rng->state.v[1]));
    float v1 = 6.283185 *vsmc_u01_open_closed_u32_f32(rng->state.v[2]);
    float v2 = 6.283185 *vsmc_u01_open_closed_u32_f32(rng->state.v[3]);
    float4 r;
    r.x = u1 * cos(v1);
    r.y = u1 * sin(v1);
    r.z = u2 * cos(v2);
    r.w = u2 * sin(v2);

    return r;
}

VSMC_STATIC_INLINE float log_likelihood(
        const cv *sp, float obs_x, float obs_y)
{
    const float scale = 10;
    const float nu = 10;

    float llh_x = scale * (sp->pos_x - obs_x);
    float llh_y = scale * (sp->pos_y - obs_y);

    llh_x = log1p(llh_x * llh_x / nu);
    llh_y = log1p(llh_y * llh_y / nu);

    return -0.5F * (nu + 1) * (llh_x + llh_y);
}

__kernel void cv_init(__global cv *state, __global ulong *accept,
        __global float *log_weight, __global float *obs_x,
        __global float *obs_y)
{
    ulong i = get_global_id(0);
    if (i >= SIZE)
        return;

    const float sd_pos0 = 2;
    const float sd_vel0 = 1;
    cv sp = state[i];

    vsmc_philox4x32 rng;
    vsmc_philox4x32_init(&rng, SEED + i);
    rng.ctr.v[1] = i;

    float4 r = normal01(&rng);
    sp.pos_x = r.x * sd_pos0;
    sp.pos_y = r.y * sd_pos0;
    sp.vel_x = r.z * sd_vel0;
    sp.vel_y = r.w * sd_vel0;

    state[i] = sp;
    accept[i] = 1;
    log_weight[i] = log_likelihood(&sp, obs_x[0], obs_y[0]);
}

__kernel void cv_move(ulong iter, __global cv *state, __global ulong *accept,
        __global float *inc_weight, __global float *obs_x,
        __global float *obs_y)
{
    ulong i = get_global_id(0);
    if (i >= SIZE)
        return;

    const float var_pos = 0.02F;
    const float var_vel = 0.001F;
    const float sd_pos = sqrt(var_pos);
    const float sd_vel = sqrt(var_vel);
    const float delta = 0.1F;
    cv sp = state[i];

    vsmc_philox4x32 rng;
    vsmc_philox4x32_init(&rng, SEED + i);
    rng.ctr.v[1] = i;

    float4 r = normal01(&rng);
    sp.pos_x += r.x * sd_pos + delta * sp.vel_x;
    sp.pos_y += r.y * sd_pos + delta * sp.vel_y;
    sp.vel_x += r.z * sd_vel;
    sp.vel_y += r.w * sd_vel;

    state[i] = sp;
    accept[i] = 1;
    inc_weight[i] = log_likelihood(&sp, obs_x[iter], obs_y[iter]);
}

__kernel void cv_est(
        ulong iter, ulong dim, __global cv *state, __global cv_pos *est)
{
    ulong i = get_global_id(0);
    if (i >= SIZE)
        return;

    est[i].pos_x = state[i].pos_x;
    est[i].pos_y = state[i].pos_y;
}


R600: Map address spaces for atomic_xchg

Signed-off-by: Aaron Watry <awatry@gmail.com>
Reviewed-by: Tom Stellard <thomas.stellard@amd.com>

git-svn-id: e7f1ab7c2a259259587475a3e7520e757882f167@217924 91177308-0d34-0410-b5e6-96231b3b80d8

#include <clc/clc.h>

#define ATOMIC_FUNC_DEFINE(RET_SIGN, ARG_SIGN, TYPE, CL_FUNCTION, CLC_FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE) \
_CLC_OVERLOAD _CLC_DEF RET_SIGN TYPE CL_FUNCTION (volatile CL_ADDRSPACE RET_SIGN TYPE *p, RET_SIGN TYPE val) { \
	return (RET_SIGN TYPE)__clc_##CLC_FUNCTION##_addr##LLVM_ADDRSPACE((volatile CL_ADDRSPACE ARG_SIGN TYPE*)p, (ARG_SIGN TYPE)val); \
}

/* For atomic functions that don't need different bitcode dependending on argument signedness */
#define ATOMIC_FUNC_SIGN(TYPE, FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE) \
	_CLC_DECL signed TYPE __clc_##FUNCTION##_addr##LLVM_ADDRSPACE(volatile CL_ADDRSPACE signed TYPE*, signed TYPE); \
	ATOMIC_FUNC_DEFINE(signed, signed, TYPE, FUNCTION, FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE) \
	ATOMIC_FUNC_DEFINE(unsigned, signed, TYPE, FUNCTION, FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE)

#define ATOMIC_FUNC_ADDRSPACE(TYPE, FUNCTION) \
	ATOMIC_FUNC_SIGN(TYPE, FUNCTION, global, 1) \
	ATOMIC_FUNC_SIGN(TYPE, FUNCTION, local, 3)

#define ATOMIC_FUNC(FUNCTION) \
	ATOMIC_FUNC_ADDRSPACE(int, FUNCTION)

ATOMIC_FUNC(atomic_add)
ATOMIC_FUNC(atomic_and)
ATOMIC_FUNC(atomic_or)
ATOMIC_FUNC(atomic_sub)
ATOMIC_FUNC(atomic_xchg)
ATOMIC_FUNC(atomic_xor)

_CLC_DECL signed int __clc_atomic_max_addr1(volatile global signed int*, signed int);
_CLC_DECL signed int __clc_atomic_max_addr3(volatile local signed int*, signed int);
_CLC_DECL uint __clc_atomic_umax_addr1(volatile global uint*, uint);
_CLC_DECL uint __clc_atomic_umax_addr3(volatile local uint*, uint);

ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_max, atomic_max, global, 1)
ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_max, atomic_max, local, 3)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_max, atomic_umax, global, 1)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_max, atomic_umax, local, 3)

_CLC_DECL signed int __clc_atomic_min_addr1(volatile global signed int*, signed int);
_CLC_DECL signed int __clc_atomic_min_addr3(volatile local signed int*, signed int);
_CLC_DECL uint __clc_atomic_umin_addr1(volatile global uint*, uint);
_CLC_DECL uint __clc_atomic_umin_addr3(volatile local uint*, uint);

ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_min, atomic_min, global, 1)
ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_min, atomic_min, local, 3)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_min, atomic_umin, global, 1)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_min, atomic_umin, local, 3)

#include <clc/clc.h>

#define ATOMIC_FUNC_DEFINE(RET_SIGN, ARG_SIGN, TYPE, CL_FUNCTION, CLC_FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE) \
_CLC_OVERLOAD _CLC_DEF RET_SIGN TYPE CL_FUNCTION (volatile CL_ADDRSPACE RET_SIGN TYPE *p, RET_SIGN TYPE val) { \
	return (RET_SIGN TYPE)__clc_##CLC_FUNCTION##_addr##LLVM_ADDRSPACE((volatile CL_ADDRSPACE ARG_SIGN TYPE*)p, (ARG_SIGN TYPE)val); \
}

/* For atomic functions that don't need different bitcode dependending on argument signedness */
#define ATOMIC_FUNC_SIGN(TYPE, FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE) \
	_CLC_DECL signed TYPE __clc_##FUNCTION##_addr##LLVM_ADDRSPACE(volatile CL_ADDRSPACE signed TYPE*, signed TYPE); \
	ATOMIC_FUNC_DEFINE(signed, signed, TYPE, FUNCTION, FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE) \
	ATOMIC_FUNC_DEFINE(unsigned, signed, TYPE, FUNCTION, FUNCTION, CL_ADDRSPACE, LLVM_ADDRSPACE)

#define ATOMIC_FUNC_ADDRSPACE(TYPE, FUNCTION) \
	ATOMIC_FUNC_SIGN(TYPE, FUNCTION, global, 1) \
	ATOMIC_FUNC_SIGN(TYPE, FUNCTION, local, 3)

#define ATOMIC_FUNC(FUNCTION) \
	ATOMIC_FUNC_ADDRSPACE(int, FUNCTION)

ATOMIC_FUNC(atomic_add)
ATOMIC_FUNC(atomic_and)
ATOMIC_FUNC(atomic_or)
ATOMIC_FUNC(atomic_sub)
ATOMIC_FUNC(atomic_xor)

_CLC_DECL signed int __clc_atomic_max_addr1(volatile global signed int*, signed int);
_CLC_DECL signed int __clc_atomic_max_addr3(volatile local signed int*, signed int);
_CLC_DECL uint __clc_atomic_umax_addr1(volatile global uint*, uint);
_CLC_DECL uint __clc_atomic_umax_addr3(volatile local uint*, uint);

ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_max, atomic_max, global, 1)
ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_max, atomic_max, local, 3)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_max, atomic_umax, global, 1)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_max, atomic_umax, local, 3)

_CLC_DECL signed int __clc_atomic_min_addr1(volatile global signed int*, signed int);
_CLC_DECL signed int __clc_atomic_min_addr3(volatile local signed int*, signed int);
_CLC_DECL uint __clc_atomic_umin_addr1(volatile global uint*, uint);
_CLC_DECL uint __clc_atomic_umin_addr3(volatile local uint*, uint);

ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_min, atomic_min, global, 1)
ATOMIC_FUNC_DEFINE(signed, signed, int, atomic_min, atomic_min, local, 3)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_min, atomic_umin, global, 1)
ATOMIC_FUNC_DEFINE(unsigned, unsigned, int, atomic_min, atomic_umin, local, 3)


Fix `cond` macro, usable now

Fix `cond` macro, usable now

;; @file macros.cl
;;
;; @breif Examples using macros
;;

; `progn` implementation with lambda function
(defmacro progn1 (&rest body)
  `(funcall (lambda () ,@body)))
; `when` macro
(defmacro when1 (condition &rest body)
  `(if ,condition (progn1 ,@body)))
; `unless` macro
(defmacro unless1 (condition &rest body)
  `(if (not ,condition) (progn1 ,@body)))
; `cond1` macro
(defmacro cond1 ((condition body))
  (let ((condition-value condition))
    (when condition-value `(progn ,body ,condition-value))))
; `cond` macro
(defmacro cond-a (&body branches)
  (dolist (branch branches)
    (let ((condition (pop branch)) (body branch))
      (when condition (return `(progn ,@body))))))
(defmacro cond-b (&body branches)
  (with-gensyms (branch condition body)
    (dolist (branch branches)
      (let ((condition (pop branch)) (body branch))
        (when condition (return `(progn ,@body)))))))

;; @file macros.cl
;;
;; @breif Examples using macros
;;

; `progn` implementation with lambda function
(defmacro progn1 (&rest body)
  `(funcall (lambda () ,@body)))
; `when` macro
(defmacro when1 (condition &rest body)
  `(if ,condition (progn1 ,@body)))
; `unless` macro
(defmacro unless1 (condition &rest body)
  `(if (not ,condition) (progn1 ,@body)))
; `cond` macro
(defmacro cond1 (&rest branches)
  `(dolist (branch ,branches)
    (let ((condition (pop branch)) (body branch))
      (if condition (progn @body) break))))