update

update

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

// TO DO: remove f_s and c from offest calculation
static inline void fetchWeights(int c, int f_s, int lcl_id, int wei_read, int gbl_wei_off, __local _FLOAT * wei_mem, const __global _FLOAT * weights)
{
	// read weights by stride
	for (int w = lcl_id; w < wei_read* MLO_N_LCL_OUT_MAPS; w += MLO_GRP_SZ)
	{
		int k = iDiv(w, wei_read);
		int t0 = iMod(w, k, wei_read);
		int j = iDiv(t0, MLO_FILTER_SIZE0);
		int i = iMod(t0, j, MLO_FILTER_SIZE0);
		int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

		if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
			if (ob == 0 && k == 1)
			{
				printf("G:w: %d %d %d %d   %f %f\n",
					//										lcl_id,
					//										w,
					//										f_s,
					//										j,
					//										i,
					//										k_idx,
					k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
					gbl_wei_off,
					wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
					weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
					wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
				);
			}

#endif

		}
		else
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
		}
	}
}

static inline void  fetchData(int f_s, int lcl_id, int lcl_scan, int n_reads, int in_y, int gbl_in_scan_off, __local _FLOAT * bot_mem, const __global _FLOAT * bot)
{
	__private _FLOAT in_rd_data[MLO_READ_UNIT];

	for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
		p4 += MLO_GRP_SZ)
	{
		int b = 0;
		int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
		b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
		t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
		c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
		int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
		int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

		for (int i = 0; i < MLO_READ_UNIT; ++i)
		{
			in_rd_data[i] = 0;
		}

		if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
		{

			int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
			// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				int i = 0;
				for (; i < MLO_IN_N_PIXS_OFF; ++i)
				{
					in_rd_data[i] = bot[gbl_off + i];
				}
				//								for (; i < MLO_READ_UNIT; ++i)
				//								{
				//									in_rd_data[i] = 0;
				//								}

			}
			else
#endif
			{

				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = bot[gbl_off + i];
				}
			}

		}
		int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
		for (int i = 0; i < MLO_READ_UNIT; ++i)
		{
			bot_mem[lcl_off + i] = in_rd_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;
	int out_y = ob*MLO_OUT_EXTENT1;
	int in_y = out_y*MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;
	gbl_in_off += in_y * MLO_IN_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				int f_s = 0;
				for (; f_s < MLO_FILTER_STRIDE1 - 1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

// get set of taps
					fetchWeights(c, f_s, lcl_id, MLO_WEI_SZ, gbl_wei_off, wei_mem, weights);

					int n_reads = MLO_IN_LCL_HEIGHT; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride

#if 1
					fetchData(f_s, lcl_id, lcl_scan, n_reads, in_y, gbl_in_scan_off, bot_mem, bot);
#else
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
//								for (; i < MLO_READ_UNIT; ++i)
//								{
//									in_rd_data[i] = 0;
//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

#endif
					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l <  MLO_FILTER_STRIDE0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
// 3d
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0-1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

// last f_s
				{

					barrier(CLK_LOCAL_MEM_FENCE);
#define MLO_WEI_READ ((MLO_N_FILTER_SPLITS1 - 1)*MLO_WEI_LCL_WIDTH)
// fetch set of weight vertical taps

					fetchWeights(c, f_s, lcl_id, MLO_WEI_READ, gbl_wei_off, wei_mem, weights);

					int n_reads = MLO_IN_LCL_HEIGHT - 1; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

									  // fetch input by stride

#if 1
					fetchData(f_s, lcl_id, lcl_scan, n_reads, in_y, gbl_in_scan_off, bot_mem, bot);
#else

					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								//								for (; i < MLO_READ_UNIT; ++i)
								//								{
								//									in_rd_data[i] = 0;
								//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

#endif

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1 - 1; ++m)
					{
						// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l < MLO_FILTER_STRIDE0 - 1; ++l)
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
						  // 3d
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS && ex_row < MLO_OUT_EXTENT1 && (out_y + ex_row) < MLO_OUT_HEIGHT; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS && (k_idx + k) < MLO_N_OUTPUTS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

// TO DO: remove f_s and c from offest calculation
static inline fetchWeights(int c, int f_s, int lcl_id, int wei_read, int gbl_wei_off, __local _FLOAT * wei_mem, const __global _FLOAT * weights)
{
	// read weights by stride
	for (int w = lcl_id; w < wei_read* MLO_N_LCL_OUT_MAPS; w += MLO_GRP_SZ)
	{
		int k = iDiv(w, wei_read);
		int t0 = iMod(w, k, wei_read);
		int j = iDiv(t0, MLO_FILTER_SIZE0);
		int i = iMod(t0, j, MLO_FILTER_SIZE0);
		int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

		if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
			if (ob == 0 && k == 1)
			{
				printf("G:w: %d %d %d %d   %f %f\n",
					//										lcl_id,
					//										w,
					//										f_s,
					//										j,
					//										i,
					//										k_idx,
					k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
					gbl_wei_off,
					wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
					weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
					wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
				);
			}

#endif

		}
		else
		{
			wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
		}
	}
}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;
	int out_y = ob*MLO_OUT_EXTENT1;
	int in_y = out_y*MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;
	gbl_in_off += in_y * MLO_IN_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				int f_s = 0;
				for (; f_s < MLO_FILTER_STRIDE1 - 1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

// get set of taps
					fetchWeights(c, f_s, lcl_id, MLO_WEI_SZ, gbl_wei_off, wei_mem, weights);

					int n_reads = MLO_IN_LCL_HEIGHT; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
//								for (; i < MLO_READ_UNIT; ++i)
//								{
//									in_rd_data[i] = 0;
//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l <  MLO_FILTER_STRIDE0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
// 3d
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0-1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i <  MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

// last f_s
				{

					barrier(CLK_LOCAL_MEM_FENCE);
#define MLO_WEI_READ ((MLO_N_FILTER_SPLITS1 - 1)*MLO_WEI_LCL_WIDTH)

					fetchWeights(c, f_s, lcl_id, MLO_WEI_READ, gbl_wei_off, wei_mem, weights);

					int n_reads = MLO_IN_LCL_HEIGHT - 1; // ((ob == 0 && (f_s < MLO_FILTER_PAD1)) || (ob == get_local_size(0) - 1 && (MLO_FILTER_STRIDE1 - f_s) < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = 0; // (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

									  // fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = 0;
						int t0 = p4;
#if MLO_N_LCL_BATCHS > 1
						b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
#endif
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s;

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = 0;
						}

						if (0 <= in_y + in_scan && in_y + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								//								for (; i < MLO_READ_UNIT; ++i)
								//								{
								//									in_rd_data[i] = 0;
								//								}

							}
							else
#endif
							{

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}

						}
						int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							bot_mem[lcl_off + i] = in_rd_data[i];
						}

					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1 - 1; ++m)
					{
						// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_N_FILTER_SPLITS0];
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1)];

						// first 2 splits
						int l;
						for (l = 0; l < MLO_FILTER_STRIDE0 - 1; ++l)
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
						  // 3d
						{
							// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
								{
									wei_vals[k*MLO_N_FILTER_SPLITS0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i*MLO_FILTER_STRIDE0 + l];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int i = 0; i < (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1); ++i)
								{
									in_vals[bb*(MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + i]
										= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i*MLO_FILTER_STRIDE0 + l];
								}

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_N_FILTER_SPLITS0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* (MLO_OUT_PIX_TILE0 + MLO_N_FILTER_SPLITS0 - 1) + n + i];
											_FLOAT wei_val = wei_vals[k*MLO_N_FILTER_SPLITS0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 1 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													n,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS && ex_row < MLO_OUT_EXTENT1 && (out_y + ex_row) < MLO_OUT_HEIGHT; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS && (k_idx + k) < MLO_N_OUTPUTS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}


Update SpGEMM_ESC_2heap_kernels.cl

Update SpGEMM_ESC_2heap_kernels.cl

/* ************************************************************************
* The MIT License (MIT)
* Copyright 2014-2015 weifengliu

*  Permission is hereby granted, free of charge, to any person obtaining a copy
*  of this software and associated documentation files (the "Software"), to deal
*  in the Software without restriction, including without limitation the rights
*  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
*  copies of the Software, and to permit persons to whom the Software is
*  furnished to do so, subject to the following conditions:

*  The above copyright notice and this permission notice shall be included in
*  all copies or substantial portions of the Software.

*  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
*  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
*  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
*  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
*  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
*  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
*  THE SOFTWARE.
* ************************************************************************ */
//////////////////////////////////////////////////////////////////////////
// < A CUDA/OpenCL General Sparse Matrix-Matrix Multiplication Program >
//
// < See paper:
// Weifeng Liu and Brian Vinter, "An Efficient GPU General Sparse
// Matrix-Matrix Multiplication for Irregular Data," Parallel and
// Distributed Processing Symposium, 2014 IEEE 28th International
// (IPDPS '14), pp.370-381, 19-23 May 2014
// for details. >
//////////////////////////////////////////////////////////////////////////

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

#define TUPLE_QUEUE 6
typedef double   vT;

inline
void siftDown(__local int   *s_key,
              __local vT *s_val,
              const int start,
              const int stop,
              const int local_id,
              const int local_size)
{
    int root = start;
    int child, swap;

    int temp_swap_key;
    vT temp_swap_val;

    while (root * 2 + 1 <= stop)
    {
        child = root * 2 + 1;
        swap = root;

        if (s_key[swap * local_size + local_id] < s_key[child * local_size + local_id])
            swap = child;

        if (child + 1 <= stop && s_key[swap * local_size + local_id] < s_key[(child + 1) * local_size + local_id])
            swap = child + 1;

        if (swap != root)
        {
            const int index1 = root * local_size + local_id;
            const int index2 = swap * local_size + local_id;

            //swap root and swap
            temp_swap_key = s_key[index1];
            s_key[index1] = s_key[index2];
            s_key[index2] = temp_swap_key;

            temp_swap_val = s_val[index1];
            s_val[index1] = s_val[index2];
            s_val[index2] = temp_swap_val;

            root = swap;
        }
        else
            return;
    }
}

inline
int heapsort(__local int   *s_key,
             __local vT *s_val,
             const int segment_size,
             const int local_id,
             const int local_size)
{
    // heapsort - heapify max-heap
    int start = (segment_size - 1) / 2;
    int stop  = segment_size - 1;

    int index1, index2;

    while (start >= 0)
    {
        siftDown(s_key, s_val, start, stop, local_id, local_size);
        start--;
    }

    // inject root element to the end

    int temp_swap_key;
    vT temp_swap_val;

    index1 = stop * local_size + local_id;

    temp_swap_key = s_key[local_id];
    s_key[local_id] = s_key[index1];
    s_key[index1] = temp_swap_key;

    temp_swap_val = s_val[local_id];
    s_val[local_id] = s_val[index1];
    s_val[index1] = temp_swap_val;

    stop--;

    siftDown(s_key, s_val, 0, stop, local_id, local_size);

    // this start is compressed list's start
    start = segment_size - 1;

    // heapsort - remove-max and compress
    while (stop >= 0)
    {
        index2 = stop * local_size + local_id;

        if (s_key[local_id] == s_key[start * local_size + local_id])
        {
            s_val[start * local_size + local_id] += s_val[local_id];

            s_key[local_id] = s_key[index2];
            s_val[local_id] = s_val[index2];
        }
        else
        {
            start--;

            index1 = start * local_size + local_id;

            if (stop == start)
            {
                temp_swap_key = s_key[local_id];
                s_key[local_id] = s_key[index2];
                s_key[index2] = temp_swap_key;

                temp_swap_val = s_val[local_id];
                s_val[local_id] = s_val[index2];
                s_val[index2] = temp_swap_val;
            }
            else
            {
                s_key[index1] = s_key[local_id];
                s_val[index1] = s_val[local_id];

                s_key[local_id] = s_key[index2];
                s_val[local_id] = s_val[index2];
            }
        }

        stop--;

        siftDown(s_key, s_val, 0, stop, local_id, local_size);
    }

    return start;
}

__kernel
void ESC_2heap_noncoalesced_local(__global const int   *d_queue,
                    __global const int   *d_csrRowPtrA,
                    __global const int   *d_csrColIndA,
                    __global const vT *d_csrValA,
                    __global const int   *d_csrRowPtrB,
                    __global const int   *d_csrColIndB,
                    __global const vT *d_csrValB,
                    __global int         *d_csrRowPtrC,
                    __global const int   *d_csrRowPtrCt,
                    __global int         *d_csrColIndCt,
                    __global vT       *d_csrValCt,
                    __local  int         *s_key,          // SEGMENTSIZE * LOCALSIZE
                    __local  vT       *s_val,          // SEGMENTSIZE * LOCALSIZE
                    const int queue_size,
                    const int queue_offset)
{
    const int local_id = get_local_id(0);
    const int group_id = get_group_id(0);
    const int global_id = get_global_id(0);
    const int local_size = get_local_size(0);
    int index = 0;

    if (global_id < queue_size)
    {
        int i, counter = 0;
        int start_col_index_A, stop_col_index_A;
        int rowidB, start_col_index_B, stop_col_index_B;
        vT value_A;

        int rowidC = d_queue[TUPLE_QUEUE * (queue_offset + global_id)];

        start_col_index_A = d_csrRowPtrA[rowidC];
        stop_col_index_A  = d_csrRowPtrA[rowidC + 1];

        // i is both col index of A and row index of B
        for (i = start_col_index_A; i < stop_col_index_A; i++)
        {
            rowidB = d_csrColIndA[i];
            value_A  = d_csrValA[i];

            start_col_index_B = d_csrRowPtrB[rowidB];
            stop_col_index_B  = d_csrRowPtrB[rowidB + 1];

            for (int j = start_col_index_B; j < stop_col_index_B; j++)
            {
                index = counter * local_size + local_id;
                s_key[index] = d_csrColIndB[j];
                s_val[index] = d_csrValB[j] * value_A;

                counter++;
            }
        }

        // heapsort in each work-item
        int local_start = heapsort(s_key, s_val, counter, local_id, local_size);

        counter -= local_start;
        d_csrRowPtrC[rowidC] = counter;

        const int base_index = d_queue[TUPLE_QUEUE * (queue_offset + group_id * local_size + local_id) + 1];;
        for (int i = 0; i < counter; i++)
        {
            d_csrColIndCt[base_index + i] = s_key[(local_start+i) * local_size + local_id];
            d_csrValCt[base_index + i] = s_val[(local_start+i) * local_size + local_id];
        }
    }
}


Fixed a bug

Fixed a bug

#define __CLKERNEL

#ifdef __CLKERNEL
#else
#include <stdio.h>
#define uint unsigned int
#endif

#define OFFSET_BASIS 2166136261
#define FNV_PRIME 16777619
#define FLOAT_T float
#define HASH_T uint
#define N 5

typedef float4 Point;
typedef int4 IntPoint;

// FNV hash: http://isthe.com/chongo/tech/comp/fnv/#FNV-source
HASH_T hash(HASH_T i, HASH_T j, HASH_T k) {
  return (HASH_T)((((((OFFSET_BASIS ^ (HASH_T)i) * FNV_PRIME) ^ (HASH_T)j) * FNV_PRIME) ^ (HASH_T)k) * FNV_PRIME);
}

//~ FLOAT_T our_distance(Point p1, Point p2) {
    //~ Point d = fabs(p1-p2);
    //~ if(d.x > d.y && d.x > d.z)
	//~ return d.x;
    //~ if(d.y > d.z)
	//~ return d.y;
    //~ return d.z;
//~ }

// Return the square of the distance between points p1 and p2
FLOAT_T our_distance(Point p1, Point p2) {
  return (p1.x-p2.x)*(p1.x-p2.x) + (p1.y-p2.y)*(p1.y-p2.y) + (p1.z-p2.z)*(p1.z-p2.z);
}

void insert(FLOAT_T *arr, FLOAT_T value) {
  // Ugly hack to prevent duplicate values
  for(int i=0; i < N; ++i)
    if(arr[i] == value)
  	return;

  float temp;
  for(int i=N-1; i>=0; i--) {
    if(value > arr[i]) break;
    temp = arr[i];
    arr[i] = value;
    if(i+1<N) arr[i+1] = temp;
  }
}

// LCG Random Number Generator
uint rng(uint last) {
  return ((1103515245 * last + 12345) % 0x100000000);
}

// Generated with "AccountingForm[N[Table[CDF[PoissonDistribution[4], i], {i, 1, 9}], 20]*2^32]" //"N[Table[CDF[PoissonDistribution[4], i], {i, 1, 9}], 20]"
uint prob_lookup(uint value)
{
    if(value < 393325350) return 1;
    if(value < 1022645910) return 2;
    if(value < 1861739990) return 3;
    if(value < 2700834071) return 4;
    if(value < 3372109335) return 5;
    if(value < 3819626178) return 6;
    if(value < 4075350088) return 7;
    if(value < 4203212043) return 8;
    return 9;
}

 //~ if(value < 0.091578194443670901469) return 1;
 //~ else if(value < 0.23810330555354434382) return 2;
 //~ else if(value < 0.43347012036670893362) return 3;
 //~ else if(value < 0.62883693517987352342) return 4;
 //~ else if(value < 0.78513038703040519526) return 5;
 //~ else if(value < 0.88932602159742630982) return 6;
 //~ else if(value < 0.94886638420715266099) return 7;
 //~ else if(value < 0.97863656551201583658) return 8;
 //~ else return 9;
//~ }

void findDistancesForCube(FLOAT_T *distanceArray, Point p, IntPoint c) {
  uint rngLast = rng( hash(c.x, c.y, c.z) );
  uint rng1,rng2,rng3;
  Point randomDiff,featurePoint;
  uint numFPoints = prob_lookup( rngLast );

  for(uint i = 0; i < numFPoints; ++i) {
	rng1 = rng(rngLast);
	rng2 = rng(rng1);
	rng3 = rng(rng2);
	rngLast = rng3;

	randomDiff.x = (float)rng1 / 0x100000000;
	randomDiff.y = (float)rng2 / 0x100000000;
	randomDiff.z = (float)rng3 / 0x100000000;
	featurePoint = randomDiff + convert_float4(c);

	insert(distanceArray, our_distance(p,featurePoint));
  }
}

void forAll(FLOAT_T *distanceArray, Point p) {
    for(int i=-1; i < 2; ++i) {
	for(int j=-1; j < 2; ++j) {
	    for(int k=-1; k < 2; ++k) {
		IntPoint cube;
		cube.x = p.x + i;
		cube.y = p.y + j;
		cube.z = p.z + k;
		findDistancesForCube(distanceArray, p, cube);
	    }
	}
    }
}

__kernel void WorleyNoise(__global FLOAT_T *arrX, __global FLOAT_T *arrY, __global FLOAT_T *arrZ, __global FLOAT_T *output) {//, int width, int height, int depth) {
    uint idX = get_global_id(0);
    uint idY = get_global_id(1);
    uint idZ = get_global_id(2);
    uint width = get_global_size(0);
    uint height = get_global_size(1);
    uint depth = get_global_size(2);

  // Shall we do work?
  if(idX < width && idY < height && idZ < depth) {
	// Initalize array
	FLOAT_T darr[N];
	for(int i=0; i<N; ++i)
	  darr[i] = 888;

	Point p;
	p.x = arrX[idX];
	p.y = arrY[idY];
	p.z = arrZ[idZ];
	forAll(darr,p);
	//findDistancesForCube(darr, p, convert_int4(p));

	output[depth*height*idX + depth*idY + idZ] = darr[1] - darr[0];
  }
}

#define __CLKERNEL

#ifdef __CLKERNEL
#else
#include <stdio.h>
#define uint unsigned int
#endif

#define OFFSET_BASIS 2166136261
#define FNV_PRIME 16777619
#define FLOAT_T float
#define HASH_T uint
#define N 5

typedef float4 Point;
typedef int4 IntPoint;

// FNV hash: http://isthe.com/chongo/tech/comp/fnv/#FNV-source
HASH_T hash(HASH_T i, HASH_T j, HASH_T k) {
  return (HASH_T)((((((OFFSET_BASIS ^ (HASH_T)i) * FNV_PRIME) ^ (HASH_T)j) * FNV_PRIME) ^ (HASH_T)k) * FNV_PRIME);
}

//~ FLOAT_T our_distance(Point p1, Point p2) {
    //~ Point d = fabs(p1-p2);
    //~ if(d.x > d.y && d.x > d.z)
	//~ return d.x;
    //~ if(d.y > d.z)
	//~ return d.y;
    //~ return d.z;
//~ }

// Return the square of the distance between points p1 and p2
FLOAT_T our_distance(Point p1, Point p2) {
  return (p1.x-p2.x)*(p1.x-p2.x) + (p1.y-p2.y)*(p1.y-p2.y) + (p1.z-p2.z)*(p1.z-p2.z);
}

void insert(FLOAT_T *arr, FLOAT_T value) {
  // Ugly hack to prevent duplicate values
  for(int i=0; i < N; ++i)
    if(arr[i] == value)
	return;

  float temp;
  for(int i=N-1; i>=0; i--) {
    if(value > arr[i]) break;
    temp = arr[i];
    arr[i] = value;
    if(i+1<N) arr[i+1] = temp;
  }
}

// LCG Random Number Generator
uint rng(uint last) {
  return ((1103515245 * last + 12345) % 0x100000000);
}

// Generated with "AccountingForm[N[Table[CDF[PoissonDistribution[4], i], {i, 1, 9}], 20]*2^32]" //"N[Table[CDF[PoissonDistribution[4], i], {i, 1, 9}], 20]"
uint prob_lookup(uint value)
{
    if(value < 393325350) return 1;
    if(value < 1022645910) return 2;
    if(value < 1861739990) return 3;
    if(value < 2700834071) return 4;
    if(value < 3372109335) return 5;
    if(value < 3819626178) return 6;
    if(value < 4075350088) return 7;
    if(value < 4203212043) return 8;
    return 9;
}

 //~ if(value < 0.091578194443670901469) return 1;
 //~ else if(value < 0.23810330555354434382) return 2;
 //~ else if(value < 0.43347012036670893362) return 3;
 //~ else if(value < 0.62883693517987352342) return 4;
 //~ else if(value < 0.78513038703040519526) return 5;
 //~ else if(value < 0.88932602159742630982) return 6;
 //~ else if(value < 0.94886638420715266099) return 7;
 //~ else if(value < 0.97863656551201583658) return 8;
 //~ else return 9;
//~ }

void findDistancesForCube(FLOAT_T *distanceArray, Point p, IntPoint c) {
  uint rngLast = rng( hash(c.x, c.y, c.z) );
  uint rng1,rng2,rng3;
  Point randomDiff,featurePoint;
  uint numFPoints = prob_lookup( rngLast );

  for(uint i = 0; i < numFPoints; ++i) {
	rng1 = rng(rngLast);
	rng2 = rng(rng1);
	rng3 = rng(rng2);

	randomDiff.x = (float)rng1 / 0x100000000;
	randomDiff.y = (float)rng2 / 0x100000000;
	randomDiff.z = (float)rng3 / 0x100000000;
	featurePoint = randomDiff + convert_float4(c);

	insert(distanceArray, our_distance(p,featurePoint));
  }
}

void forAll(FLOAT_T *distanceArray, Point p) {
    for(int i=-1; i < 2; ++i) {
	for(int j=-1; j < 2; ++j) {
	    for(int k=-1; k < 2; ++k) {
		IntPoint cube;
		cube.x = p.x + i;
		cube.y = p.y + j;
		cube.z = p.z + k;
		findDistancesForCube(distanceArray, p, cube);
	    }
	}
    }
}

__kernel void WorleyNoise(__global FLOAT_T *arrX, __global FLOAT_T *arrY, __global FLOAT_T *arrZ, __global FLOAT_T *output) {//, int width, int height, int depth) {
    uint idX = get_global_id(0);
    uint idY = get_global_id(1);
    uint idZ = get_global_id(2);
    uint width = get_global_size(0);
    uint height = get_global_size(1);
    uint depth = get_global_size(2);

  // Shall we do work?
  if(idX < width && idY < height && idZ < depth) {
	// Initalize array
	FLOAT_T darr[N];
	for(int i=0; i<N; ++i)
	  darr[i] = 888;

	Point p;
	p.x = arrX[idX];
	p.y = arrY[idY];
	p.z = arrZ[idZ];
	forAll(darr,p);
	//findDistancesForCube(darr, p, convert_int4(p));

	output[depth*height*idX + depth*idY + idZ] = darr[1] - darr[0];
  }
}


debayering: fix divide for reading 12 bit values (#24546)

#ifdef HALF_AS_FLOAT
#define half float
#define half2 float2
#define half3 float3
#define half4 float4
#else
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#endif

#define UV_WIDTH RGB_WIDTH / 2
#define UV_HEIGHT RGB_HEIGHT / 2
#define U_OFFSET RGB_WIDTH * RGB_HEIGHT
#define V_OFFSET RGB_WIDTH * RGB_HEIGHT + UV_WIDTH * UV_HEIGHT

#define RGB_TO_Y(r, g, b) ((((mul24(b, 13) + mul24(g, 65) + mul24(r, 33)) + 64) >> 7) + 16)
#define RGB_TO_U(r, g, b) ((mul24(b, 56) - mul24(g, 37) - mul24(r, 19) + 0x8080) >> 8)
#define RGB_TO_V(r, g, b) ((mul24(r, 56) - mul24(g, 47) - mul24(b, 9) + 0x8080) >> 8)
#define AVERAGE(x, y, z, w) ((convert_ushort(x) + convert_ushort(y) + convert_ushort(z) + convert_ushort(w) + 1) >> 1)

// post wb CCM
const __constant half3 color_correction_0 = (half3)(1.82717181, -0.31231438, 0.07307673);
const __constant half3 color_correction_1 = (half3)(-0.5743977, 1.36858544, -0.53183455);
const __constant half3 color_correction_2 = (half3)(-0.25277411, -0.05627105, 1.45875782);

// tone mapping params
const half gamma_k = 0.75;
const half gamma_b = 0.125;
const half mp = 0.01; // ideally midpoint should be adaptive
const half rk = 9 - 100*mp;

inline half3 gamma_apply(half3 x) {
  // poly approximation for s curve
  return (x > mp) ?
    ((rk * (x-mp) * (1-(gamma_k*mp+gamma_b)) * (1+1/(rk*(1-mp))) / (1+rk*(x-mp))) + gamma_k*mp + gamma_b) :
    ((rk * (x-mp) * (gamma_k*mp+gamma_b) * (1+1/(rk*mp)) / (1-rk*(x-mp))) + gamma_k*mp + gamma_b);
}

inline half3 color_correct(half3 rgb) {
  half3 ret = (half)rgb.x * color_correction_0;
  ret += (half)rgb.y * color_correction_1;
  ret += (half)rgb.z * color_correction_2;
  return gamma_apply(ret);
}

inline half get_vignetting_s(float r) {
  if (r < 62500) {
    return (half)(1.0f + 0.0000008f*r);
  } else if (r < 490000) {
    return (half)(0.9625f + 0.0000014f*r);
  } else if (r < 1102500) {
    return (half)(1.26434f + 0.0000000000016f*r*r);
  } else {
    return (half)(0.53503625f + 0.0000000000022f*r*r);
  }
}

inline half val_from_10(const uchar * source, int gx, int gy, half black_level) {
  // parse 12bit
  int start = gy * FRAME_STRIDE + (3 * (gx / 2)) + (FRAME_STRIDE * FRAME_OFFSET);
  int offset = gx % 2;
  uint major = (uint)source[start + offset] << 4;
  uint minor = (source[start + 2] >> (4 * offset)) & 0xf;
  half pv = ((half)(major + minor)) / 4.0;

  // normalize
  pv = max((half)0.0, pv - black_level);
  pv /= (1024.0 - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    pv *= get_vignetting_s(gx*gx + gy*gy);
  }

  pv = clamp(pv, (half)0.0, (half)1.0);
  return pv;
}

inline half get_k(half a, half b, half c, half d) {
  return 2.0 - (fabs(a - b) + fabs(c - d));
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        float black_level
                       )
{
  const int gid_x = get_global_id(0);
  const int gid_y = get_global_id(1);

  const int lid_x = get_local_id(0);
  const int lid_y = get_local_id(1);

  const int localRowLen = mad24(get_local_size(0), 2, 2); // 2 padding
  const int localColLen = mad24(get_local_size(1), 2, 2);

  const int x_global = mul24(gid_x, 2);
  const int y_global = mul24(gid_y, 2);

  const int x_local = mad24(lid_x, 2, 1);
  const int y_local = mad24(lid_y, 2, 1);

  const int x_global_mod = (gid_x == 0 || gid_x == get_global_size(0) - 1) ? -1: 1;
  const int y_global_mod = (gid_y == 0 || gid_y == get_global_size(1) - 1) ? -1: 1;

  int localColOffset = 0;
  int globalColOffset;

  cached[mad24(y_local + 0, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + 0, black_level);
  cached[mad24(y_local + 0, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + 0, black_level);
  cached[mad24(y_local + 1, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + 1, black_level);
  cached[mad24(y_local + 1, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + 1, black_level);

  if (lid_x == 0) {  // left edge
    localColOffset = -1;
    globalColOffset = -x_global_mod;
    cached[mad24(y_local + 0, localRowLen, x_local - 1)] = val_from_10(in, x_global - x_global_mod, y_global + 0, black_level);
    cached[mad24(y_local + 1, localRowLen, x_local - 1)] = val_from_10(in, x_global - x_global_mod, y_global + 1, black_level);
  } else if (lid_x == get_local_size(0) - 1) {  // right edge
    localColOffset = 2;
    globalColOffset = x_global_mod + 1;
    cached[mad24(y_local + 0, localRowLen, x_local + 2)] = val_from_10(in, x_global + x_global_mod + 1, y_global + 0, black_level);
    cached[mad24(y_local + 1, localRowLen, x_local + 2)] = val_from_10(in, x_global + x_global_mod + 1, y_global + 1, black_level);
  }

  if (lid_y == 0) {  // top row
    cached[mad24(y_local - 1, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global - y_global_mod, black_level);
    cached[mad24(y_local - 1, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global - y_global_mod, black_level);
    if (localColOffset != 0) {  // cache corners
      cached[mad24(y_local - 1, localRowLen, x_local + localColOffset)] = val_from_10(in, x_global + globalColOffset, y_global - y_global_mod, black_level);
    }
  } else if (lid_y == get_local_size(1) - 1) {  // bottom row
    cached[mad24(y_local + 2, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + y_global_mod + 1, black_level);
    cached[mad24(y_local + 2, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + y_global_mod + 1, black_level);
    if (localColOffset != 0) {  // cache corners
      cached[mad24(y_local + 2, localRowLen, x_local + localColOffset)] = val_from_10(in, x_global + globalColOffset, y_global + y_global_mod + 1, black_level);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half3 rgb;
  uchar3 rgb_out[4];

  const half4 va = vload4(0, cached + mad24(lid_y * 2 + 0, localRowLen, lid_x * 2));
  const half4 vb = vload4(0, cached + mad24(lid_y * 2 + 1, localRowLen, lid_x * 2));
  const half4 vc = vload4(0, cached + mad24(lid_y * 2 + 2, localRowLen, lid_x * 2));
  const half4 vd = vload4(0, cached + mad24(lid_y * 2 + 3, localRowLen, lid_x * 2));

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  const half k01 = get_k(va.s0, vb.s1, va.s2, vb.s1);
  const half k02 = get_k(va.s2, vb.s1, vc.s2, vb.s1);
  const half k03 = get_k(vc.s0, vb.s1, vc.s2, vb.s1);
  const half k04 = get_k(va.s0, vb.s1, vc.s0, vb.s1);
  rgb.x = (k02*vb.s2+k04*vb.s0)/(k02+k04); // R_G1
  rgb.y = vb.s1; // G1(R)
  rgb.z = (k01*va.s1+k03*vc.s1)/(k01+k03); // B_G1
  rgb_out[0] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k11 = get_k(va.s1, vc.s1, va.s3, vc.s3);
  const half k12 = get_k(va.s2, vb.s1, vb.s3, vc.s2);
  const half k13 = get_k(va.s1, va.s3, vc.s1, vc.s3);
  const half k14 = get_k(va.s2, vb.s3, vc.s2, vb.s1);
  rgb.x = vb.s2; // R
  rgb.y = (k11*(va.s2+vc.s2)*0.5+k13*(vb.s3+vb.s1)*0.5)/(k11+k13); // G_R
  rgb.z = (k12*(va.s3+vc.s1)*0.5+k14*(va.s1+vc.s3)*0.5)/(k12+k14); // B_R
  rgb_out[1] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k21 = get_k(vb.s0, vd.s0, vb.s2, vd.s2);
  const half k22 = get_k(vb.s1, vc.s0, vc.s2, vd.s1);
  const half k23 = get_k(vb.s0, vb.s2, vd.s0, vd.s2);
  const half k24 = get_k(vb.s1, vc.s2, vd.s1, vc.s0);
  rgb.x = (k22*(vb.s2+vd.s0)*0.5+k24*(vb.s0+vd.s2)*0.5)/(k22+k24); // R_B
  rgb.y = (k21*(vb.s1+vd.s1)*0.5+k23*(vc.s2+vc.s0)*0.5)/(k21+k23); // G_B
  rgb.z = vc.s1; // B
  rgb_out[2] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k31 = get_k(vb.s1, vc.s2, vb.s3, vc.s2);
  const half k32 = get_k(vb.s3, vc.s2, vd.s3, vc.s2);
  const half k33 = get_k(vd.s1, vc.s2, vd.s3, vc.s2);
  const half k34 = get_k(vb.s1, vc.s2, vd.s1, vc.s2);
  rgb.x = (k31*vb.s2+k33*vd.s2)/(k31+k33); // R_G2
  rgb.y = vc.s2; // G2(B)
  rgb.z = (k32*vc.s3+k34*vc.s1)/(k32+k34); // B_G2
  rgb_out[3] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  // write ys
  uchar2 yy = (uchar2)(
    RGB_TO_Y(rgb_out[0].s0, rgb_out[0].s1, rgb_out[0].s2),
    RGB_TO_Y(rgb_out[1].s0, rgb_out[1].s1, rgb_out[1].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2, RGB_WIDTH, gid_x * 2));
  yy = (uchar2)(
    RGB_TO_Y(rgb_out[2].s0, rgb_out[2].s1, rgb_out[2].s2),
    RGB_TO_Y(rgb_out[3].s0, rgb_out[3].s1, rgb_out[3].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2 + 1, RGB_WIDTH, gid_x * 2));

  // write uvs
  const short ar = AVERAGE(rgb_out[0].s0, rgb_out[1].s0, rgb_out[2].s0, rgb_out[3].s0);
  const short ag = AVERAGE(rgb_out[0].s1, rgb_out[1].s1, rgb_out[2].s1, rgb_out[3].s1);
  const short ab = AVERAGE(rgb_out[0].s2, rgb_out[1].s2, rgb_out[2].s2, rgb_out[3].s2);
  out[U_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_U(ar, ag, ab);
  out[V_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_V(ar, ag, ab);
}

#ifdef HALF_AS_FLOAT
#define half float
#define half2 float2
#define half3 float3
#define half4 float4
#else
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#endif

#define UV_WIDTH RGB_WIDTH / 2
#define UV_HEIGHT RGB_HEIGHT / 2
#define U_OFFSET RGB_WIDTH * RGB_HEIGHT
#define V_OFFSET RGB_WIDTH * RGB_HEIGHT + UV_WIDTH * UV_HEIGHT

#define RGB_TO_Y(r, g, b) ((((mul24(b, 13) + mul24(g, 65) + mul24(r, 33)) + 64) >> 7) + 16)
#define RGB_TO_U(r, g, b) ((mul24(b, 56) - mul24(g, 37) - mul24(r, 19) + 0x8080) >> 8)
#define RGB_TO_V(r, g, b) ((mul24(r, 56) - mul24(g, 47) - mul24(b, 9) + 0x8080) >> 8)
#define AVERAGE(x, y, z, w) ((convert_ushort(x) + convert_ushort(y) + convert_ushort(z) + convert_ushort(w) + 1) >> 1)

// post wb CCM
const __constant half3 color_correction_0 = (half3)(1.82717181, -0.31231438, 0.07307673);
const __constant half3 color_correction_1 = (half3)(-0.5743977, 1.36858544, -0.53183455);
const __constant half3 color_correction_2 = (half3)(-0.25277411, -0.05627105, 1.45875782);

// tone mapping params
const half gamma_k = 0.75;
const half gamma_b = 0.125;
const half mp = 0.01; // ideally midpoint should be adaptive
const half rk = 9 - 100*mp;

inline half3 gamma_apply(half3 x) {
  // poly approximation for s curve
  return (x > mp) ?
    ((rk * (x-mp) * (1-(gamma_k*mp+gamma_b)) * (1+1/(rk*(1-mp))) / (1+rk*(x-mp))) + gamma_k*mp + gamma_b) :
    ((rk * (x-mp) * (gamma_k*mp+gamma_b) * (1+1/(rk*mp)) / (1-rk*(x-mp))) + gamma_k*mp + gamma_b);
}

inline half3 color_correct(half3 rgb) {
  half3 ret = (half)rgb.x * color_correction_0;
  ret += (half)rgb.y * color_correction_1;
  ret += (half)rgb.z * color_correction_2;
  return gamma_apply(ret);
}

inline half get_vignetting_s(float r) {
  if (r < 62500) {
    return (half)(1.0f + 0.0000008f*r);
  } else if (r < 490000) {
    return (half)(0.9625f + 0.0000014f*r);
  } else if (r < 1102500) {
    return (half)(1.26434f + 0.0000000000016f*r*r);
  } else {
    return (half)(0.53503625f + 0.0000000000022f*r*r);
  }
}

inline half val_from_10(const uchar * source, int gx, int gy, half black_level) {
  // parse 12bit
  int start = gy * FRAME_STRIDE + (3 * (gx / 2)) + (FRAME_STRIDE * FRAME_OFFSET);
  int offset = gx % 2;
  uint major = (uint)source[start + offset] << 4;
  uint minor = (source[start + 2] >> (4 * offset)) & 0xf;
  half pv = (half)((major + minor)/4);

  // normalize
  pv = max((half)0.0, pv - black_level);
  pv /= (1024.0 - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    pv *= get_vignetting_s(gx*gx + gy*gy);
  }

  pv = clamp(pv, (half)0.0, (half)1.0);
  return pv;
}

inline half get_k(half a, half b, half c, half d) {
  return 2.0 - (fabs(a - b) + fabs(c - d));
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        float black_level
                       )
{
  const int gid_x = get_global_id(0);
  const int gid_y = get_global_id(1);

  const int lid_x = get_local_id(0);
  const int lid_y = get_local_id(1);

  const int localRowLen = mad24(get_local_size(0), 2, 2); // 2 padding
  const int localColLen = mad24(get_local_size(1), 2, 2);

  const int x_global = mul24(gid_x, 2);
  const int y_global = mul24(gid_y, 2);

  const int x_local = mad24(lid_x, 2, 1);
  const int y_local = mad24(lid_y, 2, 1);

  const int x_global_mod = (gid_x == 0 || gid_x == get_global_size(0) - 1) ? -1: 1;
  const int y_global_mod = (gid_y == 0 || gid_y == get_global_size(1) - 1) ? -1: 1;

  int localColOffset = 0;
  int globalColOffset;

  cached[mad24(y_local + 0, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + 0, black_level);
  cached[mad24(y_local + 0, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + 0, black_level);
  cached[mad24(y_local + 1, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + 1, black_level);
  cached[mad24(y_local + 1, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + 1, black_level);

  if (lid_x == 0) {  // left edge
    localColOffset = -1;
    globalColOffset = -x_global_mod;
    cached[mad24(y_local + 0, localRowLen, x_local - 1)] = val_from_10(in, x_global - x_global_mod, y_global + 0, black_level);
    cached[mad24(y_local + 1, localRowLen, x_local - 1)] = val_from_10(in, x_global - x_global_mod, y_global + 1, black_level);
  } else if (lid_x == get_local_size(0) - 1) {  // right edge
    localColOffset = 2;
    globalColOffset = x_global_mod + 1;
    cached[mad24(y_local + 0, localRowLen, x_local + 2)] = val_from_10(in, x_global + x_global_mod + 1, y_global + 0, black_level);
    cached[mad24(y_local + 1, localRowLen, x_local + 2)] = val_from_10(in, x_global + x_global_mod + 1, y_global + 1, black_level);
  }

  if (lid_y == 0) {  // top row
    cached[mad24(y_local - 1, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global - y_global_mod, black_level);
    cached[mad24(y_local - 1, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global - y_global_mod, black_level);
    if (localColOffset != 0) {  // cache corners
      cached[mad24(y_local - 1, localRowLen, x_local + localColOffset)] = val_from_10(in, x_global + globalColOffset, y_global - y_global_mod, black_level);
    }
  } else if (lid_y == get_local_size(1) - 1) {  // bottom row
    cached[mad24(y_local + 2, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + y_global_mod + 1, black_level);
    cached[mad24(y_local + 2, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + y_global_mod + 1, black_level);
    if (localColOffset != 0) {  // cache corners
      cached[mad24(y_local + 2, localRowLen, x_local + localColOffset)] = val_from_10(in, x_global + globalColOffset, y_global + y_global_mod + 1, black_level);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half3 rgb;
  uchar3 rgb_out[4];

  const half4 va = vload4(0, cached + mad24(lid_y * 2 + 0, localRowLen, lid_x * 2));
  const half4 vb = vload4(0, cached + mad24(lid_y * 2 + 1, localRowLen, lid_x * 2));
  const half4 vc = vload4(0, cached + mad24(lid_y * 2 + 2, localRowLen, lid_x * 2));
  const half4 vd = vload4(0, cached + mad24(lid_y * 2 + 3, localRowLen, lid_x * 2));

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  const half k01 = get_k(va.s0, vb.s1, va.s2, vb.s1);
  const half k02 = get_k(va.s2, vb.s1, vc.s2, vb.s1);
  const half k03 = get_k(vc.s0, vb.s1, vc.s2, vb.s1);
  const half k04 = get_k(va.s0, vb.s1, vc.s0, vb.s1);
  rgb.x = (k02*vb.s2+k04*vb.s0)/(k02+k04); // R_G1
  rgb.y = vb.s1; // G1(R)
  rgb.z = (k01*va.s1+k03*vc.s1)/(k01+k03); // B_G1
  rgb_out[0] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k11 = get_k(va.s1, vc.s1, va.s3, vc.s3);
  const half k12 = get_k(va.s2, vb.s1, vb.s3, vc.s2);
  const half k13 = get_k(va.s1, va.s3, vc.s1, vc.s3);
  const half k14 = get_k(va.s2, vb.s3, vc.s2, vb.s1);
  rgb.x = vb.s2; // R
  rgb.y = (k11*(va.s2+vc.s2)*0.5+k13*(vb.s3+vb.s1)*0.5)/(k11+k13); // G_R
  rgb.z = (k12*(va.s3+vc.s1)*0.5+k14*(va.s1+vc.s3)*0.5)/(k12+k14); // B_R
  rgb_out[1] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k21 = get_k(vb.s0, vd.s0, vb.s2, vd.s2);
  const half k22 = get_k(vb.s1, vc.s0, vc.s2, vd.s1);
  const half k23 = get_k(vb.s0, vb.s2, vd.s0, vd.s2);
  const half k24 = get_k(vb.s1, vc.s2, vd.s1, vc.s0);
  rgb.x = (k22*(vb.s2+vd.s0)*0.5+k24*(vb.s0+vd.s2)*0.5)/(k22+k24); // R_B
  rgb.y = (k21*(vb.s1+vd.s1)*0.5+k23*(vc.s2+vc.s0)*0.5)/(k21+k23); // G_B
  rgb.z = vc.s1; // B
  rgb_out[2] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k31 = get_k(vb.s1, vc.s2, vb.s3, vc.s2);
  const half k32 = get_k(vb.s3, vc.s2, vd.s3, vc.s2);
  const half k33 = get_k(vd.s1, vc.s2, vd.s3, vc.s2);
  const half k34 = get_k(vb.s1, vc.s2, vd.s1, vc.s2);
  rgb.x = (k31*vb.s2+k33*vd.s2)/(k31+k33); // R_G2
  rgb.y = vc.s2; // G2(B)
  rgb.z = (k32*vc.s3+k34*vc.s1)/(k32+k34); // B_G2
  rgb_out[3] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  // write ys
  uchar2 yy = (uchar2)(
    RGB_TO_Y(rgb_out[0].s0, rgb_out[0].s1, rgb_out[0].s2),
    RGB_TO_Y(rgb_out[1].s0, rgb_out[1].s1, rgb_out[1].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2, RGB_WIDTH, gid_x * 2));
  yy = (uchar2)(
    RGB_TO_Y(rgb_out[2].s0, rgb_out[2].s1, rgb_out[2].s2),
    RGB_TO_Y(rgb_out[3].s0, rgb_out[3].s1, rgb_out[3].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2 + 1, RGB_WIDTH, gid_x * 2));

  // write uvs
  const short ar = AVERAGE(rgb_out[0].s0, rgb_out[1].s0, rgb_out[2].s0, rgb_out[3].s0);
  const short ag = AVERAGE(rgb_out[0].s1, rgb_out[1].s1, rgb_out[2].s1, rgb_out[3].s1);
  const short ab = AVERAGE(rgb_out[0].s2, rgb_out[1].s2, rgb_out[2].s2, rgb_out[3].s2);
  out[U_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_U(ar, ag, ab);
  out[V_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_V(ar, ag, ab);
}