Fix structure in the final loop of MLOpenCDFGen[4]

Having early exit from loop prevents it from being unrolled, because
LLVM does not know its trip count. If the loop is not unrolled, then
private array which addressing depends on the loop induction variable
if not eliminated, thus we have performance hit.

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

//MLO_GRP_SZ0              group size in dim 0
//MLO_GRP_SZ1				group size in dim 1
//MLO_GRP_SZ2				group size in dim 2
//MLO_GRP_SZ               n of wk-item in the group
//MLO_N_IN_CHNLS			total number of input channels
//MLO_LCL_N_IN_CHNLS		n of localy kept input channels
//MLO_IN_WIDTH				input width in NCHW layout
//MLO_IN_HEIGHT			input height stride in NCHW layout
//MLO_IN_STRIDE			input stride in NCHW layout
//MLO_IN_CHNL_STRIDE       input channel stride in NCHW layout
//MLO_IN_BATCH_STRIDE      input batch stride in NCHW layout
//MLO_BATCH_SZ		        batch szie
//MLO_FLTR_SZ0             filter 0 dim size
//MLO_FLTR_PAD_SZ0				filter 0 dim pad
//MLO_FLTR_STRIDE0			filter 0 dim stride
//MLO_FLTR_SZ1             filter 1 dim size
//MLO_FLTR_PAD_SZ1				filter 1 dim pad
//MLO_FLTR_STRIDE1			filter 1 dim stride
//MLO_IN_CHNL_LOOP         main input channel loop
//MLO_OUT_WIDTH			output width in NCHW layout
//MLO_OUT_HEIGHT			output height stride in NCHW layout
//MLO_OUT_STRIDE			output stride in NCHW layout
//MLO_N_OUT_PIX_SZ0        n output pixel per wk item in 0 dim
//MLO_N_OUT_PIX_SZ1		n output pexels per wk item in 1 dim
//MLO_N_IN_PIX_SZ0        n input pixels per wk item in 0 dim
//MLO_N_IN_PIX_SZ1		n input pexels per wk item in 1 dim
//MLO_N_STACKS           n of separate data stacks
//MLO_N_PROCS1           n of processors per stack 1 dim
//MLO_N_PROCS0           n of processors per stack 0 dim
//MLO_IN_SZ0			horizontal read dim 0
//MLO_IN_SZ1			vertical read dim 1

// inputs are taken from different stacks of batches - to use the same filters
#define MLO_LCL_IMG_WIDTH (MLO_IN_SZ0 + MLO_FLTR_SZ0 - 1)
#define MLO_LCL_IMG_HEIGHT  (MLO_IN_SZ1 + MLO_FLTR_SZ1 - 1)
#define MLO_LCL_IMG_SIZE (MLO_LCL_IMG_WIDTH * MLO_LCL_IMG_HEIGHT)
#define MLO_PVT_COL_STG_HEIGHT ((MLO_N_OUT_PIX_SZ1 - 1) * MLO_FLTR_STRIDE1 + 1)
#define MLO_PVT_COL_STG_WIDTH ((MLO_N_OUT_PIX_SZ0 -1) * MLO_FLTR_STRIDE0 +  MLO_FLTR_SZ0)
#define MLO_LCL_WEI_SIZE (MLO_LCL_N_OUT_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1)

#define MLO_PVT_OUT_DATA_HEIGHT MLO_N_OUT_PIX_SZ1 * MLO_LCL_N_OUT_CHNLS
#define MLO_PVT_OUT_DATA_SZ MLO_PVT_OUT_DATA_HEIGHT * MLO_N_OUT_PIX_SZ0

static inline void calculateXYPos(int linPos, int width, int *x, int *y)
{
	(*y) = (int)((float)linPos/width + 0.0001f);
	(*x) = linPos - (*y) * width;
}

static inline int calculateOffset(int stride, int x, int y)
{
	int ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data, int linPos, int gbl_width, int gbl_stride, int gbl_base, int lcl_stride, int lcl_base)
{
	int x, y;
	calculateXYPos(linPos, gbl_width, &x, &y);
	int gbl_off = calculateOffset(gbl_stride, x, y);
	gbl_off += gbl_base;
	int lcl_off = calculateOffset(lcl_stride, x, y);
	lcl_off += lcl_base;
	lcl_data[lcl_off] = gbl_data[gbl_off];
}

// split the group into several input vector processors
// each processor reads its own input channel
static inline void readData(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data, int lcl_p_id, int lcl_p_stride, int size, int gbl_width, int gbl_stride, int gbl_base, int lcl_stride, int lcl_base)
{

	for(int i = lcl_p_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(lcl_data, gbl_data, i, gbl_width, gbl_stride, gbl_base, lcl_stride, lcl_base);
	}

}

static inline void readDataTile(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data,
						int tile_y, int tile_x,
						int gbl_stride, int gbl_base,
						int lcl_stride, int lcl_base,
						int gbl_height, int gbl_width,
						int lcl_height, int lcl_width,
						int lcl_id1, int lcl_id0,
						int lcl_grp_sz1, int lcl_grp_sz0,
						int fltr_pad1, int fltr_pad0,
						_FLOAT padding_val)
{
			for( int j = lcl_id1; j < lcl_height; j += lcl_grp_sz1)
			{
				int y_act = (j - fltr_pad1);
				bool invisibleY = (tile_y + y_act < 0) || (tile_y + y_act >= gbl_height);

				int y_gbl_off = y_act * gbl_stride + gbl_base;

				int y_lcl_off = j * lcl_stride + lcl_base;

				for(int i = lcl_id0; i < lcl_width; i += lcl_grp_sz0)
				{
					int x_act = (i - fltr_pad0);
					bool invisibleX = (tile_x + x_act < 0) || (tile_x + x_act >= gbl_width);

					bool invis = invisibleX || invisibleY;

					int g_off = (invis) ? 0 : y_gbl_off + x_act;

					_FLOAT val = gbl_data[g_off];

					val = (invis)? padding_val : val;

					lcl_data[y_lcl_off + i] = val;
				}
			}
}

static inline void readWeights(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data, int p_id, int p_stride, int size, int lcl_stride, int gbl_stride)
{
	for(int i = p_id; i < size; i+= p_stride)
	{
		int lcl_out = i/lcl_stride;
		int lcl_we = i - mul24(lcl_out, lcl_stride);

		lcl_data[i] = gbl_data[mad24(lcl_out,gbl_stride,lcl_we)];
	}
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0,MLO_GRP_SZ1,MLO_GRP_SZ2)))
__kernel void MLOpenCDFGen(
       const __global _FLOAT * bot,
		const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
       const __global _FLOAT * bias,
#endif
	  __global _FLOAT *top,
	   _FLOAT padding_val
//	   int in_main_loop
	   )
{
	__local _FLOAT lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS];
	__local _FLOAT lcl_wei[MLO_LCL_WEI_SIZE];

	_FLOAT pvt_bot_dat[MLO_PVT_COL_STG_HEIGHT * MLO_PVT_COL_STG_WIDTH];
	_FLOAT pvt_top_dat[MLO_PVT_OUT_DATA_SZ];
	_FLOAT pvt_wei_dat[MLO_FLTR_SZ0];

	int x_out_grp = get_group_id(0) * MLO_N_PROCS0 *  MLO_N_OUT_PIX_SZ0;
	int y_out_grp = get_group_id(1) * MLO_N_PROCS1 * MLO_N_OUT_PIX_SZ1;
	int y_in_grp = y_out_grp * MLO_FLTR_STRIDE1;
	int x_in_grp = x_out_grp * MLO_FLTR_STRIDE0;

	int lcl_id = mad24(get_local_id(1),(uint)MLO_GRP_SZ0, get_local_id(0));
	int lcl_proc = lcl_id/(MLO_N_PROCS0*MLO_N_PROCS1);  // input id from diff stack
	int lcl_in_proc_id = -mad24(lcl_proc, (MLO_N_PROCS0*MLO_N_PROCS1), -lcl_id);  // wk item id for the input to make a coalesed read
	int lcl_proc_id1 = lcl_in_proc_id/ MLO_N_PROCS0;  //
	int lcl_proc_id0 = -mad24(lcl_proc_id1, MLO_N_PROCS0, -lcl_in_proc_id);  //
	int x_out_lcl = mul24(lcl_proc_id0, MLO_N_OUT_PIX_SZ0);
	int y_out_lcl = mul24(lcl_proc_id1, MLO_N_OUT_PIX_SZ1);
	int x_out = x_out_grp + x_out_lcl;
	int y_out = y_out_grp + y_out_lcl;

	int y_in_lcl = y_out * MLO_FLTR_STRIDE1 - y_in_grp;
	int x_in_lcl = x_out * MLO_FLTR_STRIDE0 - x_in_grp;

	int ob = get_global_id(2);
	int o_id = ob / MLO_N_STACKS;  // block of outputs
	int b_id = -mad24(o_id, MLO_N_STACKS, -ob);         // block of batchs
// my batch
	int b = b_id* MLO_LCL_N_IN_CHNLS + lcl_proc;

	int in_off = b * MLO_IN_BATCH_STRIDE;
	int wei_off = mul24(o_id, MLO_LCL_N_OUT_CHNLS * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1);
	int lcl_off = mul24(lcl_proc, MLO_LCL_IMG_SIZE) + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl;

#if MLO_BIG == 0
	for(int i = lcl_id; i < MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS;  i += MLO_GRP_SZ)
	{
		lcl_img[i] = 0;
	}
#endif
	for(int i = 0; i < MLO_PVT_OUT_DATA_SZ; ++i)
	{
		pvt_top_dat[i] = 0;
	}

	for(int c = 0; c < MLO_N_IN_CHNLS; c++, in_off += MLO_IN_CHNL_STRIDE, wei_off += MLO_FLTR_SZ0 * MLO_FLTR_SZ1)
	{

		barrier(CLK_LOCAL_MEM_FENCE);
// put weights for all our outputs for this input into LDS
		for(int i = lcl_id; i < MLO_LCL_N_OUT_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1; i += MLO_GRP_SZ)
		{
			int lcl_o = (int)((float)i / (MLO_FLTR_SZ0 * MLO_FLTR_SZ1) + 0.00001f);
			int lcl_o_i = i - lcl_o * (MLO_FLTR_SZ0 * MLO_FLTR_SZ1);

			lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1 + lcl_o_i];
		}

		readDataTile(lcl_img,
				bot,
				y_in_grp,
				x_in_grp,
				MLO_IN_STRIDE,
				(in_off + y_in_grp * MLO_IN_STRIDE + x_in_grp),
				MLO_LCL_IMG_WIDTH,
				0,
				MLO_IN_HEIGHT,
				MLO_IN_WIDTH,
				MLO_LCL_IMG_HEIGHT,
				MLO_LCL_IMG_WIDTH,
				lcl_proc_id1,
				lcl_proc_id0,
				MLO_N_PROCS1,
				MLO_N_PROCS0,
				MLO_FLTR_PAD_SZ1,
				MLO_FLTR_PAD_SZ0,
				padding_val);

		barrier(CLK_LOCAL_MEM_FENCE);

// get first MLO_N_OUT_PIX_SZ1 lines

		int j = 0;

		int lcl_off2 = lcl_off;

		for(; j < MLO_PVT_COL_STG_HEIGHT - 1; ++j, lcl_off2 += MLO_LCL_IMG_WIDTH)
		{

// read input data

			for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[j * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];
			}

		}

// convolve with the filter
		int lcl_wei_off = 0;
		for(int k = 0; k < MLO_FLTR_SZ1; ++k, ++j, lcl_off2 += MLO_LCL_IMG_WIDTH,  lcl_wei_off+= MLO_FLTR_SZ0)
		{

// read input data
			for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[(MLO_PVT_COL_STG_HEIGHT - 1) * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];

			}

// convolve over all outputs
			int lcl_wei_off2 = lcl_wei_off;

			for(int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, lcl_wei_off2 += MLO_FLTR_SZ1 * MLO_FLTR_SZ0)
			{
// read weights
				for(int w = 0; w < MLO_FLTR_SZ0; ++w)
				{
					pvt_wei_dat[w] = lcl_wei[lcl_wei_off2 + w];
				}

// convolve over the tile
				for( int pj = 0; pj < MLO_N_OUT_PIX_SZ1; ++pj)
				{
					for(int pi = 0; pi < MLO_N_OUT_PIX_SZ0; ++pi)
					{

						for(int m = 0; m < MLO_FLTR_SZ0; ++m)
						{
								pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi] += pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m]
																												* pvt_wei_dat[m];

#if 0
								if (y_out + pj == 0 && x_out + pi == 14)
								{

									printf("K:cnv: %f %f %f\n",
										pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi],
										pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m],
										pvt_wei_dat[m]
										);
								}

#endif

						}
					}
				}

			}

// move up

			for(int jj = 0; jj < MLO_PVT_COL_STG_HEIGHT - 1; ++jj)
			{

				for(int ii = 0; ii < MLO_PVT_COL_STG_WIDTH; ++ii)
				{
					pvt_bot_dat[jj * MLO_PVT_COL_STG_WIDTH + ii] = pvt_bot_dat[(jj+1) * MLO_PVT_COL_STG_WIDTH + ii];
				}

			}

		}

	}

// write into all output feature maps
	int top_off = b * MLO_OUT_BATCH_STRIDE + o_id * MLO_LCL_N_OUT_CHNLS * MLO_OUT_CHNL_STRIDE +  y_out *MLO_OUT_STRIDE + x_out;

	for(int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, top_off += MLO_OUT_CHNL_STRIDE)
	{

#if MLO_OUT_ALIGNED == 0
		if ( o_id * MLO_LCL_N_OUT_CHNLS + o < MLO_N_OUT_CHNLS)
#endif
		{
			_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS == 1

			bias_val = bias[o_id * MLO_LCL_N_OUT_CHNLS + o];
#endif

			int top_off2 = top_off;

			for(int j = 0; j < MLO_N_OUT_PIX_SZ1; ++j, top_off2 += MLO_OUT_STRIDE)
			{
				for(int i = 0; i < MLO_N_OUT_PIX_SZ0; ++i)
				{
#if MLO_ALIGNED == 0
					if ( y_out + j < MLO_OUT_HEIGHT && x_out + i < MLO_OUT_WIDTH)
#endif
						top[top_off2 + i] = pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i] + bias_val;
#if 0
					if (y_out + j == 0 && x_out + i == 14)
					{

						printf("K:out: %d %f %f\n",
							top_off2 + i,
							pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i],
							top[top_off2 + i]
							);
					}

#endif
				}
			}
		}
	}

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCDFGen4(
const __global _FLOAT * bot,
const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
const __global _FLOAT * bias,
#endif
__global _FLOAT *top,
_FLOAT padding_val
//	   int in_main_loop
)
{
	__local _FLOAT lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS * MLO_IN_STACKS];
	__local _FLOAT lcl_wei[MLO_LCL_WEI_SIZE * MLO_OUT_STACKS];

	_FLOAT pvt_bot_dat[MLO_PVT_COL_STG_HEIGHT * MLO_PVT_COL_STG_WIDTH];
	_FLOAT pvt_top_dat[MLO_PVT_OUT_DATA_SZ];
	_FLOAT pvt_wei_dat[MLO_FLTR_SZ0];

	int x_out_grp = get_group_id(0) * MLO_N_PROCS0 *  MLO_N_OUT_PIX_SZ0;
	int y_out_grp = get_group_id(1) * MLO_N_PROCS1 * MLO_N_OUT_PIX_SZ1;
	int y_in_grp = y_out_grp * MLO_FLTR_STRIDE1;
	int x_in_grp = x_out_grp * MLO_FLTR_STRIDE0;

	int lcl_id = mad24(get_local_id(1), (uint)MLO_GRP_SZ0, get_local_id(0));
	int proc_tile1 = (get_local_id(1) >> MLO_LG2N_PROC_TILE1);
	int lcl_proc_id1 = get_local_id(1) - (proc_tile1 << MLO_LG2N_PROC_TILE1);  //
	int lcl_proc_id0 = get_local_id(0);  //
	int x_out_lcl = mul24(lcl_proc_id0, MLO_N_OUT_PIX_SZ0);
	int y_out_lcl = mul24(lcl_proc_id1, MLO_N_OUT_PIX_SZ1);
	int x_out = x_out_grp + x_out_lcl;
	int y_out = y_out_grp + y_out_lcl;

	int y_in_lcl = y_out * MLO_FLTR_STRIDE1 - y_in_grp;
	int x_in_lcl = x_out * MLO_FLTR_STRIDE0 - x_in_grp;

	int ob = get_global_id(2);
	int o_id = ob / MLO_N_STACKS;  // block of outputs
	int b_id = ob - mul24(o_id, (int)MLO_N_STACKS);         // block of batchs
	// my batch
	int b = b_id* MLO_LCL_N_IN_CHNLS*MLO_IN_STACKS + proc_tile1;

	int in_off = b * MLO_IN_BATCH_STRIDE;
	int wei_off = mul24(o_id, (int)(MLO_LCL_N_OUT_CHNLS * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1));
	int lcl_off = MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS * proc_tile1 + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl;

#if MLO_BIG == 0
	for (int i = lcl_id; i < MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS; i += MLO_GRP_SZ)
	{
		lcl_img[i] = 0;
	}
#endif
	for (int i = 0; i < MLO_PVT_OUT_DATA_SZ; ++i)
	{
		pvt_top_dat[i] = 0;
	}

	for (int c = 0; c < MLO_N_IN_CHNLS; c++, in_off += MLO_IN_CHNL_STRIDE, wei_off += MLO_FLTR_SZ0 * MLO_FLTR_SZ1)
	{

		barrier(CLK_LOCAL_MEM_FENCE);
		// put weights for all our outputs for this input into LDS
		for (int i = lcl_id; i < MLO_LCL_N_OUT_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1; i += MLO_GRP_SZ)
		{
			int lcl_o = (int)((float)i / (MLO_FLTR_SZ0 * MLO_FLTR_SZ1) + 0.00001f);
			int lcl_o_i = i - lcl_o * (MLO_FLTR_SZ0 * MLO_FLTR_SZ1);

			lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1 + lcl_o_i];
		}

		readDataTile(&lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS * proc_tile1],
			bot,
			y_in_grp,
			x_in_grp,
			MLO_IN_STRIDE,
			(in_off + y_in_grp * MLO_IN_STRIDE + x_in_grp),
			MLO_LCL_IMG_WIDTH,
			0,
			MLO_IN_HEIGHT,
			MLO_IN_WIDTH,
			MLO_LCL_IMG_HEIGHT,
			MLO_LCL_IMG_WIDTH,
			lcl_proc_id1,
			lcl_proc_id0,
			MLO_N_PROCS1,
			MLO_N_PROCS0,
			MLO_FLTR_PAD_SZ1,
			MLO_FLTR_PAD_SZ0,
			padding_val);

		barrier(CLK_LOCAL_MEM_FENCE);

		// get first MLO_N_OUT_PIX_SZ1 lines

		int j = 0;

		int lcl_off2 = lcl_off;

		for (; j < MLO_PVT_COL_STG_HEIGHT - 1; ++j, lcl_off2 += MLO_LCL_IMG_WIDTH)
		{

			// read input data

			for (int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[j * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];
			}

		}

		// convolve with the filter
		int lcl_wei_off = 0;
		for (int k = 0; k < MLO_FLTR_SZ1; ++k, ++j, lcl_off2 += MLO_LCL_IMG_WIDTH, lcl_wei_off += MLO_FLTR_SZ0)
		{

			// read input data
			for (int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[(MLO_PVT_COL_STG_HEIGHT - 1) * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];

			}

			// convolve over all outputs
			int lcl_wei_off2 = lcl_wei_off;

			for (int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, lcl_wei_off2 += MLO_FLTR_SZ1 * MLO_FLTR_SZ0)
			{
				// read weights
				for (int w = 0; w < MLO_FLTR_SZ0; ++w)
				{
					pvt_wei_dat[w] = lcl_wei[lcl_wei_off2 + w];
				}

				// convolve over the tile
				for (int pj = 0; pj < MLO_N_OUT_PIX_SZ1; ++pj)
				{
					for (int pi = 0; pi < MLO_N_OUT_PIX_SZ0; ++pi)
					{

						for (int m = 0; m < MLO_FLTR_SZ0; ++m)
						{
							pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi] += pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m]
								* pvt_wei_dat[m];

#if 0
							if (get_group_id(0) == 0 && get_group_id(1) == 0 && get_group_id(2) == 0 && proc_tile1 == 0 && o == 0 && y_out + pj == 2 && x_out + pi == 0)
							{

								printf("K:cnv: %d %d %d %d %f %f %f\n",
									get_local_id(1),
									get_local_id(0),
									y_out,
									x_out,
									pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi],
									pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m],
									pvt_wei_dat[m]
									);
							}

#endif

						}
					}
				}

			}

			// move up

			for (int jj = 0; jj < MLO_PVT_COL_STG_HEIGHT - 1; ++jj)
			{

				for (int ii = 0; ii < MLO_PVT_COL_STG_WIDTH; ++ii)
				{
					pvt_bot_dat[jj * MLO_PVT_COL_STG_WIDTH + ii] = pvt_bot_dat[(jj + 1) * MLO_PVT_COL_STG_WIDTH + ii];
				}

			}

		}

	}

	// write into all output feature maps
	int top_off = b * MLO_OUT_BATCH_STRIDE + o_id * MLO_LCL_N_OUT_CHNLS * MLO_OUT_CHNL_STRIDE + y_out *MLO_OUT_STRIDE + x_out;

	for (int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, top_off += MLO_OUT_CHNL_STRIDE)
	{

#if MLO_OUT_ALIGNED == 0
		if (o_id * MLO_LCL_N_OUT_CHNLS + o < MLO_N_OUT_CHNLS)
#endif
		{
			_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS == 1

			bias_val = bias[o_id * MLO_LCL_N_OUT_CHNLS + o];
#endif

			int top_off2 = top_off;

			for (int j = 0; j < MLO_N_OUT_PIX_SZ1; ++j, top_off2 += MLO_OUT_STRIDE)
			{
				for (int i = 0; i < MLO_N_OUT_PIX_SZ0; ++i)
				{
#if MLO_ALIGNED == 0
					if (y_out + j < MLO_OUT_HEIGHT && x_out + i < MLO_OUT_WIDTH)
#endif
						top[top_off2 + i] = pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i] + bias_val;
#if 0
					if (get_group_id(0) == 0 && get_group_id(1) == 0 && get_group_id(2) == 0 && proc_tile1 == 0 && o == 0 && y_out + j == 2 && x_out + i == 0)
					{

						printf("K:out: %d %f %f\n",
							top_off2 + i,
							pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i],
							top[top_off2 + i]
							);
					}

#endif
				}
			}
		}
	}

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0,MLO_GRP_SZ1,MLO_GRP_SZ2)))
__kernel void aDNNConv_img2col(
       const __global _FLOAT * img,
	  __global _FLOAT *col,
	   _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS];
	_FLOAT col_stg[MLO_PVT_COL_STG_HEIGHT * MLO_PVT_COL_STG_WIDTH];
//	_FLOAT col_out[MLO_PVT_COL_OUT_HEIGHT * MLO_PVT_COL_OUT_WIDTH];

	int x_out_grp = get_group_id(0) * MLO_N_PROCS0 *  MLO_N_OUT_PIX_SZ0;
	int y_out_grp = get_group_id(1) * MLO_N_PROCS1 * MLO_N_OUT_PIX_SZ1;
	int y_in_grp = y_out_grp * MLO_FLTR_STRIDE1;
	int x_in_grp = x_out_grp * MLO_FLTR_STRIDE0;

	int lcl_id = mad24(get_local_id(1),(uint)MLO_GRP_SZ0, get_local_id(0));
	int lcl_proc = (int)((float)lcl_id/(MLO_N_PROCS0*MLO_N_PROCS1));  // input id from diff stack
	int lcl_in_proc_id = -mad24(lcl_proc, (MLO_N_PROCS0*MLO_N_PROCS1), -lcl_id);  // wk item id for the input to make a coalesed read
	int lcl_proc_id1 = (int)((float)lcl_in_proc_id/ MLO_N_PROCS0);  //
	int lcl_proc_id0 = -mad24(lcl_proc_id1, MLO_N_PROCS0, -lcl_in_proc_id);  //
	int x_out_lcl = mul24(lcl_proc_id0, MLO_N_OUT_PIX_SZ0);
	int y_out_lcl = mul24(lcl_proc_id1, MLO_N_OUT_PIX_SZ1);

	int cb = get_global_id(2);
	int b_id = (int)((float)cb / (float)MLO_N_IN_CHNLS);  // batch block
	int c = -mad24(b_id, MLO_N_IN_CHNLS, -cb);         // channel
// my batch
	int b = b_id* MLO_LCL_N_IN_CHNLS + lcl_proc;

	int x_out = x_out_grp + x_out_lcl;
	int y_out = y_out_grp + y_out_lcl;

	int in_off = b * MLO_IN_BATCH_STRIDE + c*MLO_IN_CHNL_STRIDE;

	for(int i = lcl_id; i < MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS;  i += MLO_GRP_SZ)
	{
		lcl_img[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_BIG

	readDataTile(lcl_img,
				img,
				y_in_grp,
				x_in_grp,
				MLO_IN_STRIDE,
				(in_off + y_in_grp * MLO_IN_STRIDE + x_in_grp),
				MLO_LCL_IMG_WIDTH,
				0,
				MLO_IN_HEIGHT,
				MLO_IN_WIDTH,
				MLO_LCL_IMG_HEIGHT,
				MLO_LCL_IMG_WIDTH,
				lcl_proc_id1,
				lcl_proc_id0,
				MLO_N_PROCS1,
				MLO_N_PROCS0,
				MLO_FLTR_PAD_SZ1,
				MLO_FLTR_PAD_SZ0,
				padding_val);

#else
			int lcl_base = MLO_LCL_IMG_WIDTH * MLO_FLTR_PAD_SZ1 + MLO_FLTR_PAD_SZ0;
			readData(&lcl_img[lcl_proc * MLO_LCL_IMG_SIZE + lcl_base],
				img,
				lcl_in_proc_id,
				(MLO_N_PROCS0*MLO_N_PROCS1),
				(MLO_IN_WIDTH*MLO_IN_HEIGHT),
				MLO_IN_WIDTH,
				MLO_IN_STRIDE,
				in_off,
				MLO_LCL_IMG_WIDTH,
				0);

#endif
		barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_BATCH_ALIGNED == 0
	if ( b >= MLO_BATCH_SZ)
	{
		return;
	}
#endif

// concatinate
	int col_off = c * MLO_OUT_STRIDE*MLO_FLTR_SZ1*MLO_FLTR_SZ0 + (b * MLO_OUT_HEIGHT + y_out) * MLO_OUT_WIDTH + x_out;

// get first MLO_N_OUT_PIX_SZ1 lines

	int j = 0;
	int y_in_lcl = y_out * MLO_FLTR_STRIDE1 - y_in_grp;
	int x_in_lcl = x_out * MLO_FLTR_STRIDE0 - x_in_grp;

	int lcl_off = lcl_proc * MLO_LCL_IMG_SIZE + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl;

	for(; j < MLO_PVT_COL_STG_HEIGHT - 1; ++j, lcl_off += MLO_LCL_IMG_WIDTH)
	{

// read input data

		for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
		{
			col_stg[j * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off + i];

		}

	}

	for(int k = 0; k < MLO_FLTR_SZ1; ++k, ++j, lcl_off += MLO_LCL_IMG_WIDTH)
	{

// read input data
		for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
		{
			col_stg[(MLO_PVT_COL_STG_HEIGHT - 1) * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off + i];

#if 0
			if( b==0 && c==0 && x_out==0 && y_out==0)
			{
				printf("k: j=%d i=%d y_l=%d x_l=%d l_of=%d v00=%f v10=%f  v01=%f v11=%f\n",
				j,
				i,
				y_in_lcl,
				x_in_lcl,
				lcl_off + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl,
				lcl_img[lcl_off + (y_in_lcl-1) * MLO_LCL_IMG_WIDTH + x_in_lcl - 1],
				lcl_img[lcl_off + (y_in_lcl-1) * MLO_LCL_IMG_WIDTH + x_in_lcl],
				lcl_img[lcl_off + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl - 1],
				lcl_img[lcl_off + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl]
				);
			}
#endif

		}

// transpose
		int out_offsets[MLO_N_OUT_PIX_SZ1 * MLO_N_OUT_PIX_SZ0];
		_FLOAT out_vals[MLO_N_OUT_PIX_SZ1 * MLO_N_OUT_PIX_SZ0];

		for(int l = 0; l < MLO_FLTR_SZ0; ++l, col_off += MLO_OUT_STRIDE)
		{
			for(int ii = 0; ii < MLO_N_OUT_PIX_SZ0; ++ii)
			{

				for(int jj = 0, col_off3 = 0; jj < MLO_N_OUT_PIX_SZ1; ++jj, col_off3 += MLO_OUT_WIDTH)
				{
					out_vals[jj*MLO_N_OUT_PIX_SZ0 + ii] = col_stg[jj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + ii* MLO_FLTR_STRIDE0 + l];
					out_offsets[jj*MLO_N_OUT_PIX_SZ0 + ii] = col_off + col_off3 + ii;
#if MLO_ALIGNED == 0
					if ( y_out + jj >= MLO_OUT_HEIGHT || x_out + ii >= MLO_OUT_WIDTH)
					{
						out_vals[jj*MLO_N_OUT_PIX_SZ0 + ii] = 0;
						out_offsets[jj*MLO_N_OUT_PIX_SZ0 + ii] = 0;
					}
#endif

				}
			}

			for(int ii = 0; ii < MLO_N_OUT_PIX_SZ0; ++ii)
			{

				for(int jj = 0, col_off3 = 0; jj < MLO_N_OUT_PIX_SZ1; ++jj, col_off3 += MLO_OUT_WIDTH)
				{
						col[out_offsets[jj*MLO_N_OUT_PIX_SZ0 + ii]]
							= out_vals[jj*MLO_N_OUT_PIX_SZ0 + ii];

				}
			}
		}

// move up

		for(int jj = 0; jj < MLO_PVT_COL_STG_HEIGHT - 1; ++jj)
		{

			for(int ii = 0; ii < MLO_PVT_COL_STG_WIDTH; ++ii)
			{
				col_stg[jj * MLO_PVT_COL_STG_WIDTH + ii] = col_stg[(jj+1) * MLO_PVT_COL_STG_WIDTH + ii];
			}

		}

	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

//MLO_GRP_SZ0              group size in dim 0
//MLO_GRP_SZ1				group size in dim 1
//MLO_GRP_SZ2				group size in dim 2
//MLO_GRP_SZ               n of wk-item in the group
//MLO_N_IN_CHNLS			total number of input channels
//MLO_LCL_N_IN_CHNLS		n of localy kept input channels
//MLO_IN_WIDTH				input width in NCHW layout
//MLO_IN_HEIGHT			input height stride in NCHW layout
//MLO_IN_STRIDE			input stride in NCHW layout
//MLO_IN_CHNL_STRIDE       input channel stride in NCHW layout
//MLO_IN_BATCH_STRIDE      input batch stride in NCHW layout
//MLO_BATCH_SZ		        batch szie
//MLO_FLTR_SZ0             filter 0 dim size
//MLO_FLTR_PAD_SZ0				filter 0 dim pad
//MLO_FLTR_STRIDE0			filter 0 dim stride
//MLO_FLTR_SZ1             filter 1 dim size
//MLO_FLTR_PAD_SZ1				filter 1 dim pad
//MLO_FLTR_STRIDE1			filter 1 dim stride
//MLO_IN_CHNL_LOOP         main input channel loop
//MLO_OUT_WIDTH			output width in NCHW layout
//MLO_OUT_HEIGHT			output height stride in NCHW layout
//MLO_OUT_STRIDE			output stride in NCHW layout
//MLO_N_OUT_PIX_SZ0        n output pixel per wk item in 0 dim
//MLO_N_OUT_PIX_SZ1		n output pexels per wk item in 1 dim
//MLO_N_IN_PIX_SZ0        n input pixels per wk item in 0 dim
//MLO_N_IN_PIX_SZ1		n input pexels per wk item in 1 dim
//MLO_N_STACKS           n of separate data stacks
//MLO_N_PROCS1           n of processors per stack 1 dim
//MLO_N_PROCS0           n of processors per stack 0 dim
//MLO_IN_SZ0			horizontal read dim 0
//MLO_IN_SZ1			vertical read dim 1

// inputs are taken from different stacks of batches - to use the same filters
#define MLO_LCL_IMG_WIDTH (MLO_IN_SZ0 + MLO_FLTR_SZ0 - 1)
#define MLO_LCL_IMG_HEIGHT  (MLO_IN_SZ1 + MLO_FLTR_SZ1 - 1)
#define MLO_LCL_IMG_SIZE (MLO_LCL_IMG_WIDTH * MLO_LCL_IMG_HEIGHT)
#define MLO_PVT_COL_STG_HEIGHT ((MLO_N_OUT_PIX_SZ1 - 1) * MLO_FLTR_STRIDE1 + 1)
#define MLO_PVT_COL_STG_WIDTH ((MLO_N_OUT_PIX_SZ0 -1) * MLO_FLTR_STRIDE0 +  MLO_FLTR_SZ0)
#define MLO_LCL_WEI_SIZE (MLO_LCL_N_OUT_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1)

#define MLO_PVT_OUT_DATA_HEIGHT MLO_N_OUT_PIX_SZ1 * MLO_LCL_N_OUT_CHNLS
#define MLO_PVT_OUT_DATA_SZ MLO_PVT_OUT_DATA_HEIGHT * MLO_N_OUT_PIX_SZ0

static inline void calculateXYPos(int linPos, int width, int *x, int *y)
{
	(*y) = (int)((float)linPos/width + 0.0001f);
	(*x) = linPos - (*y) * width;
}

static inline int calculateOffset(int stride, int x, int y)
{
	int ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data, int linPos, int gbl_width, int gbl_stride, int gbl_base, int lcl_stride, int lcl_base)
{
	int x, y;
	calculateXYPos(linPos, gbl_width, &x, &y);
	int gbl_off = calculateOffset(gbl_stride, x, y);
	gbl_off += gbl_base;
	int lcl_off = calculateOffset(lcl_stride, x, y);
	lcl_off += lcl_base;
	lcl_data[lcl_off] = gbl_data[gbl_off];
}

// split the group into several input vector processors
// each processor reads its own input channel
static inline void readData(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data, int lcl_p_id, int lcl_p_stride, int size, int gbl_width, int gbl_stride, int gbl_base, int lcl_stride, int lcl_base)
{

	for(int i = lcl_p_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(lcl_data, gbl_data, i, gbl_width, gbl_stride, gbl_base, lcl_stride, lcl_base);
	}

}

static inline void readDataTile(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data,
						int tile_y, int tile_x,
						int gbl_stride, int gbl_base,
						int lcl_stride, int lcl_base,
						int gbl_height, int gbl_width,
						int lcl_height, int lcl_width,
						int lcl_id1, int lcl_id0,
						int lcl_grp_sz1, int lcl_grp_sz0,
						int fltr_pad1, int fltr_pad0,
						_FLOAT padding_val)
{
			for( int j = lcl_id1; j < lcl_height; j += lcl_grp_sz1)
			{
				int y_act = (j - fltr_pad1);
				bool invisibleY = (tile_y + y_act < 0) || (tile_y + y_act >= gbl_height);

				int y_gbl_off = y_act * gbl_stride + gbl_base;

				int y_lcl_off = j * lcl_stride + lcl_base;

				for(int i = lcl_id0; i < lcl_width; i += lcl_grp_sz0)
				{
					int x_act = (i - fltr_pad0);
					bool invisibleX = (tile_x + x_act < 0) || (tile_x + x_act >= gbl_width);

					bool invis = invisibleX || invisibleY;

					int g_off = (invis) ? 0 : y_gbl_off + x_act;

					_FLOAT val = gbl_data[g_off];

					val = (invis)? padding_val : val;

					lcl_data[y_lcl_off + i] = val;
				}
			}
}

static inline void readWeights(__local _FLOAT *lcl_data, const __global _FLOAT * gbl_data, int p_id, int p_stride, int size, int lcl_stride, int gbl_stride)
{
	for(int i = p_id; i < size; i+= p_stride)
	{
		int lcl_out = i/lcl_stride;
		int lcl_we = i - mul24(lcl_out, lcl_stride);

		lcl_data[i] = gbl_data[mad24(lcl_out,gbl_stride,lcl_we)];
	}
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0,MLO_GRP_SZ1,MLO_GRP_SZ2)))
__kernel void MLOpenCDFGen(
       const __global _FLOAT * bot,
		const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
       const __global _FLOAT * bias,
#endif
	  __global _FLOAT *top,
	   _FLOAT padding_val
//	   int in_main_loop
	   )
{
	__local _FLOAT lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS];
	__local _FLOAT lcl_wei[MLO_LCL_WEI_SIZE];

	_FLOAT pvt_bot_dat[MLO_PVT_COL_STG_HEIGHT * MLO_PVT_COL_STG_WIDTH];
	_FLOAT pvt_top_dat[MLO_PVT_OUT_DATA_SZ];
	_FLOAT pvt_wei_dat[MLO_FLTR_SZ0];

	int x_out_grp = get_group_id(0) * MLO_N_PROCS0 *  MLO_N_OUT_PIX_SZ0;
	int y_out_grp = get_group_id(1) * MLO_N_PROCS1 * MLO_N_OUT_PIX_SZ1;
	int y_in_grp = y_out_grp * MLO_FLTR_STRIDE1;
	int x_in_grp = x_out_grp * MLO_FLTR_STRIDE0;

	int lcl_id = mad24(get_local_id(1),(uint)MLO_GRP_SZ0, get_local_id(0));
	int lcl_proc = lcl_id/(MLO_N_PROCS0*MLO_N_PROCS1);  // input id from diff stack
	int lcl_in_proc_id = -mad24(lcl_proc, (MLO_N_PROCS0*MLO_N_PROCS1), -lcl_id);  // wk item id for the input to make a coalesed read
	int lcl_proc_id1 = lcl_in_proc_id/ MLO_N_PROCS0;  //
	int lcl_proc_id0 = -mad24(lcl_proc_id1, MLO_N_PROCS0, -lcl_in_proc_id);  //
	int x_out_lcl = mul24(lcl_proc_id0, MLO_N_OUT_PIX_SZ0);
	int y_out_lcl = mul24(lcl_proc_id1, MLO_N_OUT_PIX_SZ1);
	int x_out = x_out_grp + x_out_lcl;
	int y_out = y_out_grp + y_out_lcl;

	int y_in_lcl = y_out * MLO_FLTR_STRIDE1 - y_in_grp;
	int x_in_lcl = x_out * MLO_FLTR_STRIDE0 - x_in_grp;

	int ob = get_global_id(2);
	int o_id = ob / MLO_N_STACKS;  // block of outputs
	int b_id = -mad24(o_id, MLO_N_STACKS, -ob);         // block of batchs
// my batch
	int b = b_id* MLO_LCL_N_IN_CHNLS + lcl_proc;

	int in_off = b * MLO_IN_BATCH_STRIDE;
	int wei_off = mul24(o_id, MLO_LCL_N_OUT_CHNLS * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1);
	int lcl_off = mul24(lcl_proc, MLO_LCL_IMG_SIZE) + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl;

#if MLO_BIG == 0
	for(int i = lcl_id; i < MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS;  i += MLO_GRP_SZ)
	{
		lcl_img[i] = 0;
	}
#endif
	for(int i = 0; i < MLO_PVT_OUT_DATA_SZ; ++i)
	{
		pvt_top_dat[i] = 0;
	}

	for(int c = 0; c < MLO_N_IN_CHNLS; c++, in_off += MLO_IN_CHNL_STRIDE, wei_off += MLO_FLTR_SZ0 * MLO_FLTR_SZ1)
	{

		barrier(CLK_LOCAL_MEM_FENCE);
// put weights for all our outputs for this input into LDS
		for(int i = lcl_id; i < MLO_LCL_N_OUT_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1; i += MLO_GRP_SZ)
		{
			int lcl_o = (int)((float)i / (MLO_FLTR_SZ0 * MLO_FLTR_SZ1) + 0.00001f);
			int lcl_o_i = i - lcl_o * (MLO_FLTR_SZ0 * MLO_FLTR_SZ1);

			lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1 + lcl_o_i];
		}

		readDataTile(lcl_img,
				bot,
				y_in_grp,
				x_in_grp,
				MLO_IN_STRIDE,
				(in_off + y_in_grp * MLO_IN_STRIDE + x_in_grp),
				MLO_LCL_IMG_WIDTH,
				0,
				MLO_IN_HEIGHT,
				MLO_IN_WIDTH,
				MLO_LCL_IMG_HEIGHT,
				MLO_LCL_IMG_WIDTH,
				lcl_proc_id1,
				lcl_proc_id0,
				MLO_N_PROCS1,
				MLO_N_PROCS0,
				MLO_FLTR_PAD_SZ1,
				MLO_FLTR_PAD_SZ0,
				padding_val);

		barrier(CLK_LOCAL_MEM_FENCE);

// get first MLO_N_OUT_PIX_SZ1 lines

		int j = 0;

		int lcl_off2 = lcl_off;

		for(; j < MLO_PVT_COL_STG_HEIGHT - 1; ++j, lcl_off2 += MLO_LCL_IMG_WIDTH)
		{

// read input data

			for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[j * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];
			}

		}

// convolve with the filter
		int lcl_wei_off = 0;
		for(int k = 0; k < MLO_FLTR_SZ1; ++k, ++j, lcl_off2 += MLO_LCL_IMG_WIDTH,  lcl_wei_off+= MLO_FLTR_SZ0)
		{

// read input data
			for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[(MLO_PVT_COL_STG_HEIGHT - 1) * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];

			}

// convolve over all outputs
			int lcl_wei_off2 = lcl_wei_off;

			for(int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, lcl_wei_off2 += MLO_FLTR_SZ1 * MLO_FLTR_SZ0)
			{
// read weights
				for(int w = 0; w < MLO_FLTR_SZ0; ++w)
				{
					pvt_wei_dat[w] = lcl_wei[lcl_wei_off2 + w];
				}

// convolve over the tile
				for( int pj = 0; pj < MLO_N_OUT_PIX_SZ1; ++pj)
				{
					for(int pi = 0; pi < MLO_N_OUT_PIX_SZ0; ++pi)
					{

						for(int m = 0; m < MLO_FLTR_SZ0; ++m)
						{
								pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi] += pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m]
																												* pvt_wei_dat[m];

#if 0
								if (y_out + pj == 0 && x_out + pi == 14)
								{

									printf("K:cnv: %f %f %f\n",
										pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi],
										pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m],
										pvt_wei_dat[m]
										);
								}

#endif

						}
					}
				}

			}

// move up

			for(int jj = 0; jj < MLO_PVT_COL_STG_HEIGHT - 1; ++jj)
			{

				for(int ii = 0; ii < MLO_PVT_COL_STG_WIDTH; ++ii)
				{
					pvt_bot_dat[jj * MLO_PVT_COL_STG_WIDTH + ii] = pvt_bot_dat[(jj+1) * MLO_PVT_COL_STG_WIDTH + ii];
				}

			}

		}

	}

// write into all output feature maps
	int top_off = b * MLO_OUT_BATCH_STRIDE + o_id * MLO_LCL_N_OUT_CHNLS * MLO_OUT_CHNL_STRIDE +  y_out *MLO_OUT_STRIDE + x_out;

	for(int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, top_off += MLO_OUT_CHNL_STRIDE)
	{

#if MLO_OUT_ALIGNED == 0
		if ( o_id * MLO_LCL_N_OUT_CHNLS + o >= MLO_N_OUT_CHNLS)
		{
			return;
		}
#endif
		_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS == 1

		bias_val = bias[o_id * MLO_LCL_N_OUT_CHNLS + o];
#endif

		int top_off2 = top_off;

		for(int j = 0; j < MLO_N_OUT_PIX_SZ1; ++j, top_off2 += MLO_OUT_STRIDE)
		{
			for(int i = 0; i < MLO_N_OUT_PIX_SZ0; ++i)
			{
#if MLO_ALIGNED == 0
				if ( y_out + j < MLO_OUT_HEIGHT && x_out + i < MLO_OUT_WIDTH)
#endif
					top[top_off2 + i] = pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i] + bias_val;
#if 0
				if (y_out + j == 0 && x_out + i == 14)
				{

					printf("K:out: %d %f %f\n",
						top_off2 + i,
						pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i],
						top[top_off2 + i]
						);
				}

#endif
			}
		}
	}

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCDFGen4(
const __global _FLOAT * bot,
const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
const __global _FLOAT * bias,
#endif
__global _FLOAT *top,
_FLOAT padding_val
//	   int in_main_loop
)
{
	__local _FLOAT lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS * MLO_IN_STACKS];
	__local _FLOAT lcl_wei[MLO_LCL_WEI_SIZE * MLO_OUT_STACKS];

	_FLOAT pvt_bot_dat[MLO_PVT_COL_STG_HEIGHT * MLO_PVT_COL_STG_WIDTH];
	_FLOAT pvt_top_dat[MLO_PVT_OUT_DATA_SZ];
	_FLOAT pvt_wei_dat[MLO_FLTR_SZ0];

	int x_out_grp = get_group_id(0) * MLO_N_PROCS0 *  MLO_N_OUT_PIX_SZ0;
	int y_out_grp = get_group_id(1) * MLO_N_PROCS1 * MLO_N_OUT_PIX_SZ1;
	int y_in_grp = y_out_grp * MLO_FLTR_STRIDE1;
	int x_in_grp = x_out_grp * MLO_FLTR_STRIDE0;

	int lcl_id = mad24(get_local_id(1), (uint)MLO_GRP_SZ0, get_local_id(0));
	int proc_tile1 = (get_local_id(1) >> MLO_LG2N_PROC_TILE1);
	int lcl_proc_id1 = get_local_id(1) - (proc_tile1 << MLO_LG2N_PROC_TILE1);  //
	int lcl_proc_id0 = get_local_id(0);  //
	int x_out_lcl = mul24(lcl_proc_id0, MLO_N_OUT_PIX_SZ0);
	int y_out_lcl = mul24(lcl_proc_id1, MLO_N_OUT_PIX_SZ1);
	int x_out = x_out_grp + x_out_lcl;
	int y_out = y_out_grp + y_out_lcl;

	int y_in_lcl = y_out * MLO_FLTR_STRIDE1 - y_in_grp;
	int x_in_lcl = x_out * MLO_FLTR_STRIDE0 - x_in_grp;

	int ob = get_global_id(2);
	int o_id = ob / MLO_N_STACKS;  // block of outputs
	int b_id = ob - mul24(o_id, (int)MLO_N_STACKS);         // block of batchs
	// my batch
	int b = b_id* MLO_LCL_N_IN_CHNLS*MLO_IN_STACKS + proc_tile1;

	int in_off = b * MLO_IN_BATCH_STRIDE;
	int wei_off = mul24(o_id, (int)(MLO_LCL_N_OUT_CHNLS * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1));
	int lcl_off = MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS * proc_tile1 + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl;

#if MLO_BIG == 0
	for (int i = lcl_id; i < MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS; i += MLO_GRP_SZ)
	{
		lcl_img[i] = 0;
	}
#endif
	for (int i = 0; i < MLO_PVT_OUT_DATA_SZ; ++i)
	{
		pvt_top_dat[i] = 0;
	}

	for (int c = 0; c < MLO_N_IN_CHNLS; c++, in_off += MLO_IN_CHNL_STRIDE, wei_off += MLO_FLTR_SZ0 * MLO_FLTR_SZ1)
	{

		barrier(CLK_LOCAL_MEM_FENCE);
		// put weights for all our outputs for this input into LDS
		for (int i = lcl_id; i < MLO_LCL_N_OUT_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1; i += MLO_GRP_SZ)
		{
			int lcl_o = (int)((float)i / (MLO_FLTR_SZ0 * MLO_FLTR_SZ1) + 0.00001f);
			int lcl_o_i = i - lcl_o * (MLO_FLTR_SZ0 * MLO_FLTR_SZ1);

			lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_IN_CHNLS * MLO_FLTR_SZ0 * MLO_FLTR_SZ1 + lcl_o_i];
		}

		readDataTile(&lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS * proc_tile1],
			bot,
			y_in_grp,
			x_in_grp,
			MLO_IN_STRIDE,
			(in_off + y_in_grp * MLO_IN_STRIDE + x_in_grp),
			MLO_LCL_IMG_WIDTH,
			0,
			MLO_IN_HEIGHT,
			MLO_IN_WIDTH,
			MLO_LCL_IMG_HEIGHT,
			MLO_LCL_IMG_WIDTH,
			lcl_proc_id1,
			lcl_proc_id0,
			MLO_N_PROCS1,
			MLO_N_PROCS0,
			MLO_FLTR_PAD_SZ1,
			MLO_FLTR_PAD_SZ0,
			padding_val);

		barrier(CLK_LOCAL_MEM_FENCE);

		// get first MLO_N_OUT_PIX_SZ1 lines

		int j = 0;

		int lcl_off2 = lcl_off;

		for (; j < MLO_PVT_COL_STG_HEIGHT - 1; ++j, lcl_off2 += MLO_LCL_IMG_WIDTH)
		{

			// read input data

			for (int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[j * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];
			}

		}

		// convolve with the filter
		int lcl_wei_off = 0;
		for (int k = 0; k < MLO_FLTR_SZ1; ++k, ++j, lcl_off2 += MLO_LCL_IMG_WIDTH, lcl_wei_off += MLO_FLTR_SZ0)
		{

			// read input data
			for (int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
			{
				pvt_bot_dat[(MLO_PVT_COL_STG_HEIGHT - 1) * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off2 + i];

			}

			// convolve over all outputs
			int lcl_wei_off2 = lcl_wei_off;

			for (int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, lcl_wei_off2 += MLO_FLTR_SZ1 * MLO_FLTR_SZ0)
			{
				// read weights
				for (int w = 0; w < MLO_FLTR_SZ0; ++w)
				{
					pvt_wei_dat[w] = lcl_wei[lcl_wei_off2 + w];
				}

				// convolve over the tile
				for (int pj = 0; pj < MLO_N_OUT_PIX_SZ1; ++pj)
				{
					for (int pi = 0; pi < MLO_N_OUT_PIX_SZ0; ++pi)
					{

						for (int m = 0; m < MLO_FLTR_SZ0; ++m)
						{
							pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi] += pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m]
								* pvt_wei_dat[m];

#if 0
							if (get_group_id(0) == 0 && get_group_id(1) == 0 && get_group_id(2) == 0 && proc_tile1 == 0 && o == 0 && y_out + pj == 2 && x_out + pi == 0)
							{

								printf("K:cnv: %d %d %d %d %f %f %f\n",
									get_local_id(1),
									get_local_id(0),
									y_out,
									x_out,
									pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + pj) * MLO_N_OUT_PIX_SZ0 + pi],
									pvt_bot_dat[pj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + pi* MLO_FLTR_STRIDE0 + m],
									pvt_wei_dat[m]
									);
							}

#endif

						}
					}
				}

			}

			// move up

			for (int jj = 0; jj < MLO_PVT_COL_STG_HEIGHT - 1; ++jj)
			{

				for (int ii = 0; ii < MLO_PVT_COL_STG_WIDTH; ++ii)
				{
					pvt_bot_dat[jj * MLO_PVT_COL_STG_WIDTH + ii] = pvt_bot_dat[(jj + 1) * MLO_PVT_COL_STG_WIDTH + ii];
				}

			}

		}

	}

	// write into all output feature maps
	int top_off = b * MLO_OUT_BATCH_STRIDE + o_id * MLO_LCL_N_OUT_CHNLS * MLO_OUT_CHNL_STRIDE + y_out *MLO_OUT_STRIDE + x_out;

	for (int o = 0; o < MLO_LCL_N_OUT_CHNLS; ++o, top_off += MLO_OUT_CHNL_STRIDE)
	{

#if MLO_OUT_ALIGNED == 0
		if (o_id * MLO_LCL_N_OUT_CHNLS + o >= MLO_N_OUT_CHNLS)
		{
			return;
		}
#endif
		_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS == 1

		bias_val = bias[o_id * MLO_LCL_N_OUT_CHNLS + o];
#endif

		int top_off2 = top_off;

		for (int j = 0; j < MLO_N_OUT_PIX_SZ1; ++j, top_off2 += MLO_OUT_STRIDE)
		{
			for (int i = 0; i < MLO_N_OUT_PIX_SZ0; ++i)
			{
#if MLO_ALIGNED == 0
				if (y_out + j < MLO_OUT_HEIGHT && x_out + i < MLO_OUT_WIDTH)
#endif
					top[top_off2 + i] = pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i] + bias_val;
#if 0
				if (get_group_id(0) == 0 && get_group_id(1) == 0 && get_group_id(2) == 0 && proc_tile1 == 0 && o == 0 && y_out + j == 2 && x_out + i == 0)
				{

					printf("K:out: %d %f %f\n",
						top_off2 + i,
						pvt_top_dat[(o * MLO_N_OUT_PIX_SZ1 + j) * MLO_N_OUT_PIX_SZ0 + i],
						top[top_off2 + i]
						);
				}

#endif
			}
		}
	}

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0,MLO_GRP_SZ1,MLO_GRP_SZ2)))
__kernel void aDNNConv_img2col(
       const __global _FLOAT * img,
	  __global _FLOAT *col,
	   _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_img[MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS];
	_FLOAT col_stg[MLO_PVT_COL_STG_HEIGHT * MLO_PVT_COL_STG_WIDTH];
//	_FLOAT col_out[MLO_PVT_COL_OUT_HEIGHT * MLO_PVT_COL_OUT_WIDTH];

	int x_out_grp = get_group_id(0) * MLO_N_PROCS0 *  MLO_N_OUT_PIX_SZ0;
	int y_out_grp = get_group_id(1) * MLO_N_PROCS1 * MLO_N_OUT_PIX_SZ1;
	int y_in_grp = y_out_grp * MLO_FLTR_STRIDE1;
	int x_in_grp = x_out_grp * MLO_FLTR_STRIDE0;

	int lcl_id = mad24(get_local_id(1),(uint)MLO_GRP_SZ0, get_local_id(0));
	int lcl_proc = (int)((float)lcl_id/(MLO_N_PROCS0*MLO_N_PROCS1));  // input id from diff stack
	int lcl_in_proc_id = -mad24(lcl_proc, (MLO_N_PROCS0*MLO_N_PROCS1), -lcl_id);  // wk item id for the input to make a coalesed read
	int lcl_proc_id1 = (int)((float)lcl_in_proc_id/ MLO_N_PROCS0);  //
	int lcl_proc_id0 = -mad24(lcl_proc_id1, MLO_N_PROCS0, -lcl_in_proc_id);  //
	int x_out_lcl = mul24(lcl_proc_id0, MLO_N_OUT_PIX_SZ0);
	int y_out_lcl = mul24(lcl_proc_id1, MLO_N_OUT_PIX_SZ1);

	int cb = get_global_id(2);
	int b_id = (int)((float)cb / (float)MLO_N_IN_CHNLS);  // batch block
	int c = -mad24(b_id, MLO_N_IN_CHNLS, -cb);         // channel
// my batch
	int b = b_id* MLO_LCL_N_IN_CHNLS + lcl_proc;

	int x_out = x_out_grp + x_out_lcl;
	int y_out = y_out_grp + y_out_lcl;

	int in_off = b * MLO_IN_BATCH_STRIDE + c*MLO_IN_CHNL_STRIDE;

	for(int i = lcl_id; i < MLO_LCL_IMG_SIZE * MLO_LCL_N_IN_CHNLS;  i += MLO_GRP_SZ)
	{
		lcl_img[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_BIG

	readDataTile(lcl_img,
				img,
				y_in_grp,
				x_in_grp,
				MLO_IN_STRIDE,
				(in_off + y_in_grp * MLO_IN_STRIDE + x_in_grp),
				MLO_LCL_IMG_WIDTH,
				0,
				MLO_IN_HEIGHT,
				MLO_IN_WIDTH,
				MLO_LCL_IMG_HEIGHT,
				MLO_LCL_IMG_WIDTH,
				lcl_proc_id1,
				lcl_proc_id0,
				MLO_N_PROCS1,
				MLO_N_PROCS0,
				MLO_FLTR_PAD_SZ1,
				MLO_FLTR_PAD_SZ0,
				padding_val);

#else
			int lcl_base = MLO_LCL_IMG_WIDTH * MLO_FLTR_PAD_SZ1 + MLO_FLTR_PAD_SZ0;
			readData(&lcl_img[lcl_proc * MLO_LCL_IMG_SIZE + lcl_base],
				img,
				lcl_in_proc_id,
				(MLO_N_PROCS0*MLO_N_PROCS1),
				(MLO_IN_WIDTH*MLO_IN_HEIGHT),
				MLO_IN_WIDTH,
				MLO_IN_STRIDE,
				in_off,
				MLO_LCL_IMG_WIDTH,
				0);

#endif
		barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_BATCH_ALIGNED == 0
	if ( b >= MLO_BATCH_SZ)
	{
		return;
	}
#endif

// concatinate
	int col_off = c * MLO_OUT_STRIDE*MLO_FLTR_SZ1*MLO_FLTR_SZ0 + (b * MLO_OUT_HEIGHT + y_out) * MLO_OUT_WIDTH + x_out;

// get first MLO_N_OUT_PIX_SZ1 lines

	int j = 0;
	int y_in_lcl = y_out * MLO_FLTR_STRIDE1 - y_in_grp;
	int x_in_lcl = x_out * MLO_FLTR_STRIDE0 - x_in_grp;

	int lcl_off = lcl_proc * MLO_LCL_IMG_SIZE + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl;

	for(; j < MLO_PVT_COL_STG_HEIGHT - 1; ++j, lcl_off += MLO_LCL_IMG_WIDTH)
	{

// read input data

		for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
		{
			col_stg[j * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off + i];

		}

	}

	for(int k = 0; k < MLO_FLTR_SZ1; ++k, ++j, lcl_off += MLO_LCL_IMG_WIDTH)
	{

// read input data
		for(int i = 0; i < MLO_PVT_COL_STG_WIDTH; ++i)
		{
			col_stg[(MLO_PVT_COL_STG_HEIGHT - 1) * MLO_PVT_COL_STG_WIDTH + i] = lcl_img[lcl_off + i];

#if 0
			if( b==0 && c==0 && x_out==0 && y_out==0)
			{
				printf("k: j=%d i=%d y_l=%d x_l=%d l_of=%d v00=%f v10=%f  v01=%f v11=%f\n",
				j,
				i,
				y_in_lcl,
				x_in_lcl,
				lcl_off + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl,
				lcl_img[lcl_off + (y_in_lcl-1) * MLO_LCL_IMG_WIDTH + x_in_lcl - 1],
				lcl_img[lcl_off + (y_in_lcl-1) * MLO_LCL_IMG_WIDTH + x_in_lcl],
				lcl_img[lcl_off + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl - 1],
				lcl_img[lcl_off + y_in_lcl * MLO_LCL_IMG_WIDTH + x_in_lcl]
				);
			}
#endif

		}

// transpose
		int out_offsets[MLO_N_OUT_PIX_SZ1 * MLO_N_OUT_PIX_SZ0];
		_FLOAT out_vals[MLO_N_OUT_PIX_SZ1 * MLO_N_OUT_PIX_SZ0];

		for(int l = 0; l < MLO_FLTR_SZ0; ++l, col_off += MLO_OUT_STRIDE)
		{
			for(int ii = 0; ii < MLO_N_OUT_PIX_SZ0; ++ii)
			{

				for(int jj = 0, col_off3 = 0; jj < MLO_N_OUT_PIX_SZ1; ++jj, col_off3 += MLO_OUT_WIDTH)
				{
					out_vals[jj*MLO_N_OUT_PIX_SZ0 + ii] = col_stg[jj * MLO_FLTR_STRIDE1 * MLO_PVT_COL_STG_WIDTH + ii* MLO_FLTR_STRIDE0 + l];
					out_offsets[jj*MLO_N_OUT_PIX_SZ0 + ii] = col_off + col_off3 + ii;
#if MLO_ALIGNED == 0
					if ( y_out + jj >= MLO_OUT_HEIGHT || x_out + ii >= MLO_OUT_WIDTH)
					{
						out_vals[jj*MLO_N_OUT_PIX_SZ0 + ii] = 0;
						out_offsets[jj*MLO_N_OUT_PIX_SZ0 + ii] = 0;
					}
#endif

				}
			}

			for(int ii = 0; ii < MLO_N_OUT_PIX_SZ0; ++ii)
			{

				for(int jj = 0, col_off3 = 0; jj < MLO_N_OUT_PIX_SZ1; ++jj, col_off3 += MLO_OUT_WIDTH)
				{
						col[out_offsets[jj*MLO_N_OUT_PIX_SZ0 + ii]]
							= out_vals[jj*MLO_N_OUT_PIX_SZ0 + ii];

				}
			}
		}

// move up

		for(int jj = 0; jj < MLO_PVT_COL_STG_HEIGHT - 1; ++jj)
		{

			for(int ii = 0; ii < MLO_PVT_COL_STG_WIDTH; ++ii)
			{
				col_stg[jj * MLO_PVT_COL_STG_WIDTH + ii] = col_stg[(jj+1) * MLO_PVT_COL_STG_WIDTH + ii];
			}

		}

	}

}


cl-bayer: change sum from binary to quarternary

 * quarternay sum brings better perf than binary sum. this can
   save barrier waiting time
 * verified-by: Wind Yuan <d3b25194a13defcddf619d66a03bde7d96f8d487@intel.com>

Signed-off-by: Wind Yuan <d3b25194a13defcddf619d66a03bde7d96f8d487@intel.com>

/*
 * function: kernel_bayer_pipe
 * params:
 *   input:    image2d_t as read only
 *   output:   image2d_t as write only
 *   blc_config: black level correction configuration
 *   wb_config: whitebalance configuration
 *   gamma_table: RGGB table
 *   stats_output: 3a stats output
 */

#define GRID_X_SIZE 2
#define GRID_Y_SIZE 2

#define SHARED_PIXEL_WIDTH 16
#define SHARED_PIXEL_HEIGHT 16
#define SHARED_PIXEL_X_OFFSET 2
#define SHARED_PIXEL_Y_OFFSET 2
#define SHARED_PIXEL_X_SIZE  (SHARED_PIXEL_WIDTH + SHARED_PIXEL_X_OFFSET * 2)
#define SHARED_PIXEL_Y_SIZE  (SHARED_PIXEL_HEIGHT + SHARED_PIXEL_Y_OFFSET * 2)

#define SHARED_GRID_WIDTH (SHARED_PIXEL_WIDTH/GRID_X_SIZE)
#define SHARED_GRID_HEIGHT (SHARED_PIXEL_HEIGHT/GRID_Y_SIZE)
#define SHARED_GRID_X_OFFSET (SHARED_PIXEL_X_OFFSET/GRID_X_SIZE)
#define SHARED_GRID_Y_OFFSET (SHARED_PIXEL_Y_OFFSET/GRID_Y_SIZE)
#define SHARED_GRID_X_SIZE (SHARED_PIXEL_X_SIZE/GRID_X_SIZE)
#define SHARED_GRID_Y_SIZE (SHARED_PIXEL_Y_SIZE/GRID_Y_SIZE)

#define WORK_ITEM_X_SIZE GRID_X_SIZE
#define WORK_ITEM_Y_SIZE GRID_Y_SIZE

#define STATS_3A_GRID_SIZE (16/GRID_X_SIZE)

#define X 0
#define Y 1
#define Z 2
#define W 3

typedef struct  {
    float  level_gr;  /* Black level for GR pixels */
    float  level_r;   /* Black level for R pixels */
    float  level_b;   /* Black level for B pixels */
    float  level_gb;  /* Black level for GB pixels */
    uint   color_bits;
} CLBLCConfig;

typedef struct
{
    float r_gain;
    float gr_gain;
    float gb_gain;
    float b_gain;
} CLWBConfig;

typedef struct
{
    unsigned int avg_y;

    unsigned int avg_r;
    unsigned int avg_gr;
    unsigned int avg_gb;
    unsigned int avg_b;
    unsigned int valid_wb_count;

    unsigned int f_value1;
    unsigned int f_value2;
} XCamGridStat;

/* BA10=> GRBG  */
inline void blc (float4 *in_out, CLBLCConfig *blc_config)
{
    float multiplier = (float)(1 << (16 - blc_config->color_bits));
    in_out->x = in_out->x * multiplier - blc_config->level_gr;
    in_out->y = in_out->y * multiplier - blc_config->level_r;
    in_out->z = in_out->z * multiplier - blc_config->level_b;
    in_out->w = in_out->w * multiplier - blc_config->level_gb;
}

inline void wb (float4 *in_out, CLWBConfig *wbconfig)
{
    in_out->x *= wbconfig->gr_gain;
    in_out->y *= wbconfig->r_gain;
    in_out->z *= wbconfig->b_gain;
    in_out->w *= wbconfig->gb_gain;
}

inline void gamma_correct(float4 *in_out, __global float *table)
{
    in_out->x = table[clamp(convert_int(in_out->x * 255.0f), 0, 255)];
    in_out->y = table[clamp(convert_int(in_out->y * 255.0f), 0, 255)];
    in_out->z = table[clamp(convert_int(in_out->z * 255.0f), 0, 255)];
    in_out->w = table[clamp(convert_int(in_out->w * 255.0f), 0, 255)];
}

inline int get_shared_pos_x (int i)
{
    return i % SHARED_GRID_X_SIZE;
}

inline int get_shared_pos_y (int i)
{
    return i / SHARED_GRID_X_SIZE;
}

inline int shared_pos (int x, int y)
{
    return mad24(y, SHARED_GRID_X_SIZE, x);
}

/* BA10=> GRBG  */
inline float4 simple_calculate (
    __local float *px, __local float *py, __local float *pz, __local float *pw,
    int index, __read_only image2d_t input, sampler_t sampler, int x_start, int y_start,
    __local float4 *stats_cache,
    CLBLCConfig *blc_config,
    CLWBConfig *wb_config,
    uint enable_gamma,
    __global float *gamma_table)
{
    float4 data;
    int x0 = get_shared_pos_x (index) * WORK_ITEM_X_SIZE + x_start;
    int y0 = get_shared_pos_y (index) * WORK_ITEM_Y_SIZE + y_start;
    //Gr
    data.x = read_imagef (input, sampler, (int2)(x0, y0)).x;
    //R
    data.y = read_imagef (input, sampler, (int2)(x0 + 1, y0)).x;
    //B
    data.z = read_imagef (input, sampler, (int2)(x0, y0 + 1)).x;
    //Gb
    data.w = read_imagef (input, sampler, (int2)(x0 + 1, y0 + 1)).x;

    blc (&data, blc_config);

    /* write back for 3a stats calculation R, G, B, Y */
    stats_cache[index] = data;

    wb (&data, wb_config);
    if (enable_gamma)
        gamma_correct (&data, gamma_table);

    px[index] = data.x;
    py[index] = data.y;
    pz[index] = data.z;
    pw[index] = data.w;
}

#define MAX_DELTA_COFF 5.0f
#define MIN_DELTA_COFF 1.0f
#define DEFAULT_DELTA_COFF 4.0f

inline float delta_coff (float delta)
{
    float coff = MAX_DELTA_COFF - 20.0f * fabs(delta);
    return fmax (1.0f, coff);
}

inline float4
demosaic_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    out_data.y = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
                  in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = in_y[shared_pos(x, y)];
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
                  in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;
    return out_data;
}

inline float4
demosaic_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
                  in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    out_data.z = in_z[shared_pos(x, y)];
    return out_data;
}

inline float4
demosaic_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    out_data.y = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_denoise_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_y[shared_pos(x, y - 1)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
             in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    coff[1] = delta_coff(in_x[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_x[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y - 1)] * coff[1] +
                  in_x[shared_pos(x - 1, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x - 1, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = in_y[shared_pos(x, y)];
    coff[1] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x, y - 1)] * coff[1] +
                  in_y[shared_pos(x - 1, y)] * coff[2] +
                  in_y[shared_pos(x + 1, y)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y - 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
             in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;

    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
             in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_w[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = in_z[shared_pos(x, y)];
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x - 1, y)] * coff[2] +
                  in_z[shared_pos(x + 1, y)] * coff[3] +
                  in_z[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;
    return out_data;
};

inline float4
demosaic_denoise_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x + 1, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x + 1, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    coff[1] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y + 1)] - value);
    out_data.y = (in_w[shared_pos(x, y - 1)] * coff[1] +
                  in_w[shared_pos(x - 1, y)] * coff[2] +
                  in_w[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    value = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y + 1)] * coff[3] +
                  in_z[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

void shared_demosaic (
    __local float *x_data_in, __local float *y_data_in, __local float *z_data_in, __local float *w_data_in,
    int in_x, int in_y,
    __write_only image2d_t out, int out_x, int out_y,
    uint has_denoise)
{
    float4 out_data;

    if (has_denoise) {
        out_data = demosaic_denoise_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_denoise_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_denoise_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_denoise_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    } else {
        out_data = demosaic_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    }

}

inline void stats_3a_calculate (
    __local float4 * input,
    __global XCamGridStat * stats_output,
    CLWBConfig *wb_config)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE / 4;

    for (; count > 0; count /= 4) {
        if ((l_id_x % STATS_3A_GRID_SIZE) + (l_id_y % STATS_3A_GRID_SIZE)* STATS_3A_GRID_SIZE < count) {
            int index1 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET);
            int index2 = shared_pos (SHARED_GRID_X_OFFSET + ((l_id_x + count) % STATS_3A_GRID_SIZE),
                                     SHARED_GRID_Y_OFFSET + l_id_y + (l_id_x + count) / STATS_3A_GRID_SIZE);
            int index3 = shared_pos (SHARED_GRID_X_OFFSET + ((l_id_x + count * 2) % STATS_3A_GRID_SIZE),
                                     SHARED_GRID_Y_OFFSET + l_id_y + (l_id_x + count * 2) / STATS_3A_GRID_SIZE);
            int index4 = shared_pos (SHARED_GRID_X_OFFSET + ((l_id_x + count * 3) % STATS_3A_GRID_SIZE),
                                     SHARED_GRID_Y_OFFSET + l_id_y + (l_id_x + count * 3) / STATS_3A_GRID_SIZE);
            input[index1] = (input[index1] + input[index2] + input[index3] + input[index4]) / 4.0f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    if (l_id_x % STATS_3A_GRID_SIZE == 0 && l_id_y % STATS_3A_GRID_SIZE == 0) {
        float4 tmp_data;
        int out_index = mad24(g_id_y / STATS_3A_GRID_SIZE,  g_size_x / STATS_3A_GRID_SIZE, g_id_x / STATS_3A_GRID_SIZE);
        tmp_data = input[shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET)];
        stats_output[out_index].avg_gr = convert_uchar_sat(tmp_data.x * 255.0f);
        stats_output[out_index].avg_r = convert_uchar_sat(tmp_data.y * 255.0f);
        stats_output[out_index].avg_b = convert_uchar_sat(tmp_data.z * 255.0f);
        stats_output[out_index].avg_gb = convert_uchar_sat(tmp_data.w * 255.0f);
        stats_output[out_index].valid_wb_count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE;
        stats_output[out_index].avg_y =
            convert_uchar_sat(((tmp_data.x * wb_config->gr_gain + tmp_data.w * wb_config->gb_gain) * 0.2935f +
                               tmp_data.y * wb_config->r_gain * 0.299f + tmp_data.z * wb_config->b_gain * 0.114f) * 255.0f);
        stats_output[out_index].f_value1 = 0;
        stats_output[out_index].f_value2 = 0;
    }
}

__kernel void kernel_bayer_pipe (__read_only image2d_t input,
                                 __write_only image2d_t output,
                                 CLBLCConfig blc_config,
                                 CLWBConfig wb_config,
                                 uint has_denoise,
                                 uint enable_gamma,
                                 __global float * gamma_table,
                                 __global XCamGridStat * stats_output)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int l_size_x = get_local_size (0);
    int l_size_y = get_local_size (1);

    sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

    __local float p1_x[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_y[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_z[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_w[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 p2[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 *stats_cache = p2;

    int out_x_start, out_y_start;
    int x_start = (g_id_x - l_id_x) * WORK_ITEM_X_SIZE - SHARED_PIXEL_X_OFFSET;
    int y_start = (g_id_y - l_id_y) * WORK_ITEM_Y_SIZE - SHARED_PIXEL_Y_OFFSET;
    int i = l_id_x + l_id_y * l_size_x;

    for (; i < SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE; i += l_size_x * l_size_y) {
        simple_calculate (p1_x, p1_y, p1_z, p1_w, i,
                          input, sampler, x_start, y_start,
                          stats_cache,
                          &blc_config,
                          &wb_config,
                          enable_gamma,
                          gamma_table);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    stats_3a_calculate (stats_cache, stats_output, &wb_config);

    shared_demosaic (
        p1_x, p1_y, p1_z, p1_w, l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET,
        output, g_id_x * WORK_ITEM_X_SIZE, g_id_y * WORK_ITEM_Y_SIZE, has_denoise);
}

/*
 * function: kernel_bayer_pipe
 * params:
 *   input:    image2d_t as read only
 *   output:   image2d_t as write only
 *   blc_config: black level correction configuration
 *   wb_config: whitebalance configuration
 *   gamma_table: RGGB table
 *   stats_output: 3a stats output
 */

#define GRID_X_SIZE 2
#define GRID_Y_SIZE 2

#define SHARED_PIXEL_WIDTH 16
#define SHARED_PIXEL_HEIGHT 16
#define SHARED_PIXEL_X_OFFSET 2
#define SHARED_PIXEL_Y_OFFSET 2
#define SHARED_PIXEL_X_SIZE  (SHARED_PIXEL_WIDTH + SHARED_PIXEL_X_OFFSET * 2)
#define SHARED_PIXEL_Y_SIZE  (SHARED_PIXEL_HEIGHT + SHARED_PIXEL_Y_OFFSET * 2)

#define SHARED_GRID_WIDTH (SHARED_PIXEL_WIDTH/GRID_X_SIZE)
#define SHARED_GRID_HEIGHT (SHARED_PIXEL_HEIGHT/GRID_Y_SIZE)
#define SHARED_GRID_X_OFFSET (SHARED_PIXEL_X_OFFSET/GRID_X_SIZE)
#define SHARED_GRID_Y_OFFSET (SHARED_PIXEL_Y_OFFSET/GRID_Y_SIZE)
#define SHARED_GRID_X_SIZE (SHARED_PIXEL_X_SIZE/GRID_X_SIZE)
#define SHARED_GRID_Y_SIZE (SHARED_PIXEL_Y_SIZE/GRID_Y_SIZE)

#define WORK_ITEM_X_SIZE GRID_X_SIZE
#define WORK_ITEM_Y_SIZE GRID_Y_SIZE

#define STATS_3A_GRID_SIZE (16/GRID_X_SIZE)

#define X 0
#define Y 1
#define Z 2
#define W 3

typedef struct  {
    float  level_gr;  /* Black level for GR pixels */
    float  level_r;   /* Black level for R pixels */
    float  level_b;   /* Black level for B pixels */
    float  level_gb;  /* Black level for GB pixels */
    uint   color_bits;
} CLBLCConfig;

typedef struct
{
    float r_gain;
    float gr_gain;
    float gb_gain;
    float b_gain;
} CLWBConfig;

typedef struct
{
    unsigned int avg_y;

    unsigned int avg_r;
    unsigned int avg_gr;
    unsigned int avg_gb;
    unsigned int avg_b;
    unsigned int valid_wb_count;

    unsigned int f_value1;
    unsigned int f_value2;
} XCamGridStat;

/* BA10=> GRBG  */
inline void blc (float4 *in_out, CLBLCConfig *blc_config)
{
    float multiplier = (float)(1 << (16 - blc_config->color_bits));
    in_out->x = in_out->x * multiplier - blc_config->level_gr;
    in_out->y = in_out->y * multiplier - blc_config->level_r;
    in_out->z = in_out->z * multiplier - blc_config->level_b;
    in_out->w = in_out->w * multiplier - blc_config->level_gb;
}

inline void wb (float4 *in_out, CLWBConfig *wbconfig)
{
    in_out->x *= wbconfig->gr_gain;
    in_out->y *= wbconfig->r_gain;
    in_out->z *= wbconfig->b_gain;
    in_out->w *= wbconfig->gb_gain;
}

inline void gamma_correct(float4 *in_out, __global float *table)
{
    in_out->x = table[clamp(convert_int(in_out->x * 255.0f), 0, 255)];
    in_out->y = table[clamp(convert_int(in_out->y * 255.0f), 0, 255)];
    in_out->z = table[clamp(convert_int(in_out->z * 255.0f), 0, 255)];
    in_out->w = table[clamp(convert_int(in_out->w * 255.0f), 0, 255)];
}

inline int get_shared_pos_x (int i)
{
    return i % SHARED_GRID_X_SIZE;
}

inline int get_shared_pos_y (int i)
{
    return i / SHARED_GRID_X_SIZE;
}

inline int shared_pos (int x, int y)
{
    return mad24(y, SHARED_GRID_X_SIZE, x);
}

/* BA10=> GRBG  */
inline float4 simple_calculate (
    __local float *px, __local float *py, __local float *pz, __local float *pw,
    int index, __read_only image2d_t input, sampler_t sampler, int x_start, int y_start,
    __local float4 *stats_cache,
    CLBLCConfig *blc_config,
    CLWBConfig *wb_config,
    uint enable_gamma,
    __global float *gamma_table)
{
    float4 data;
    int x0 = get_shared_pos_x (index) * WORK_ITEM_X_SIZE + x_start;
    int y0 = get_shared_pos_y (index) * WORK_ITEM_Y_SIZE + y_start;
    //Gr
    data.x = read_imagef (input, sampler, (int2)(x0, y0)).x;
    //R
    data.y = read_imagef (input, sampler, (int2)(x0 + 1, y0)).x;
    //B
    data.z = read_imagef (input, sampler, (int2)(x0, y0 + 1)).x;
    //Gb
    data.w = read_imagef (input, sampler, (int2)(x0 + 1, y0 + 1)).x;

    blc (&data, blc_config);

    /* write back for 3a stats calculation R, G, B, Y */
    stats_cache[index] = data;

    wb (&data, wb_config);
    if (enable_gamma)
        gamma_correct (&data, gamma_table);

    px[index] = data.x;
    py[index] = data.y;
    pz[index] = data.z;
    pw[index] = data.w;
}

#define MAX_DELTA_COFF 5.0f
#define MIN_DELTA_COFF 1.0f
#define DEFAULT_DELTA_COFF 4.0f

inline float delta_coff (float delta)
{
    float coff = MAX_DELTA_COFF - 20.0f * fabs(delta);
    return fmax (1.0f, coff);
}

inline float4
demosaic_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    out_data.y = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
                  in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = in_y[shared_pos(x, y)];
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
                  in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;
    return out_data;
}

inline float4
demosaic_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
                  in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    out_data.z = in_z[shared_pos(x, y)];
    return out_data;
}

inline float4
demosaic_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    out_data.y = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_denoise_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_y[shared_pos(x, y - 1)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
             in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    coff[1] = delta_coff(in_x[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_x[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y - 1)] * coff[1] +
                  in_x[shared_pos(x - 1, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x - 1, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = in_y[shared_pos(x, y)];
    coff[1] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x, y - 1)] * coff[1] +
                  in_y[shared_pos(x - 1, y)] * coff[2] +
                  in_y[shared_pos(x + 1, y)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y - 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
             in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;

    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
             in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_w[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = in_z[shared_pos(x, y)];
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x - 1, y)] * coff[2] +
                  in_z[shared_pos(x + 1, y)] * coff[3] +
                  in_z[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;
    return out_data;
};

inline float4
demosaic_denoise_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x + 1, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x + 1, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    coff[1] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y + 1)] - value);
    out_data.y = (in_w[shared_pos(x, y - 1)] * coff[1] +
                  in_w[shared_pos(x - 1, y)] * coff[2] +
                  in_w[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    value = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y + 1)] * coff[3] +
                  in_z[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

void shared_demosaic (
    __local float *x_data_in, __local float *y_data_in, __local float *z_data_in, __local float *w_data_in,
    int in_x, int in_y,
    __write_only image2d_t out, int out_x, int out_y,
    uint has_denoise)
{
    float4 out_data;

    if (has_denoise) {
        out_data = demosaic_denoise_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_denoise_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_denoise_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_denoise_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    } else {
        out_data = demosaic_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    }

}

inline void stats_3a_calculate (
    __local float4 * input,
    __global XCamGridStat * stats_output,
    CLWBConfig *wb_config)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE / 2;

    for (; count > 0; count /= 2) {
        if ((l_id_x % STATS_3A_GRID_SIZE) + (l_id_y % STATS_3A_GRID_SIZE)* STATS_3A_GRID_SIZE < count) {
            int index1 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET);
            int index2 = shared_pos (SHARED_GRID_X_OFFSET + ((l_id_x + count) % STATS_3A_GRID_SIZE),
                                     SHARED_GRID_Y_OFFSET + l_id_y + (l_id_x + count) / STATS_3A_GRID_SIZE);
            input[index1] = (input[index1] + input[index2]) / 2.0f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    if (l_id_x % STATS_3A_GRID_SIZE == 0 && l_id_y % STATS_3A_GRID_SIZE == 0) {
        float4 tmp_data;
        int out_index = mad24(g_id_y / STATS_3A_GRID_SIZE,  g_size_x / STATS_3A_GRID_SIZE, g_id_x / STATS_3A_GRID_SIZE);
        tmp_data = input[shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET)];
        stats_output[out_index].avg_gr = convert_uchar_sat(tmp_data.x * 255.0f);
        stats_output[out_index].avg_r = convert_uchar_sat(tmp_data.y * 255.0f);
        stats_output[out_index].avg_b = convert_uchar_sat(tmp_data.z * 255.0f);
        stats_output[out_index].avg_gb = convert_uchar_sat(tmp_data.w * 255.0f);
        stats_output[out_index].valid_wb_count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE;
        stats_output[out_index].avg_y =
            convert_uchar_sat(((tmp_data.x * wb_config->gr_gain + tmp_data.w * wb_config->gb_gain) * 0.2935f +
                               tmp_data.y * wb_config->r_gain * 0.299f + tmp_data.z * wb_config->b_gain * 0.114f) * 255.0f);
        stats_output[out_index].f_value1 = 0;
        stats_output[out_index].f_value2 = 0;
    }
}

__kernel void kernel_bayer_pipe (__read_only image2d_t input,
                                 __write_only image2d_t output,
                                 CLBLCConfig blc_config,
                                 CLWBConfig wb_config,
                                 uint has_denoise,
                                 uint enable_gamma,
                                 __global float * gamma_table,
                                 __global XCamGridStat * stats_output)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int l_size_x = get_local_size (0);
    int l_size_y = get_local_size (1);

    sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

    __local float p1_x[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_y[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_z[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_w[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 p2[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 *stats_cache = p2;

    int out_x_start, out_y_start;
    int x_start = (g_id_x - l_id_x) * WORK_ITEM_X_SIZE - SHARED_PIXEL_X_OFFSET;
    int y_start = (g_id_y - l_id_y) * WORK_ITEM_Y_SIZE - SHARED_PIXEL_Y_OFFSET;
    int i = l_id_x + l_id_y * l_size_x;

    for (; i < SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE; i += l_size_x * l_size_y) {
        simple_calculate (p1_x, p1_y, p1_z, p1_w, i,
                          input, sampler, x_start, y_start,
                          stats_cache,
                          &blc_config,
                          &wb_config,
                          enable_gamma,
                          gamma_table);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    stats_3a_calculate (stats_cache, stats_output, &wb_config);

    shared_demosaic (
        p1_x, p1_y, p1_z, p1_w, l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET,
        output, g_id_x * WORK_ITEM_X_SIZE, g_id_y * WORK_ITEM_Y_SIZE, has_denoise);
}