another "TGTBT"

another "TGTBT"

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	int out_y = ob*MLO_OUT_EXTENT1;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				for (int f_s = 0; f_s < MLO_FILTER_STRIDE1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_OUT_STACKS);
						int t0 = iMod(w, k, MLO_OUT_STACKS);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}

					}

					int n_reads = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						int t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = c_scan* MLO_FILTER_STRIDE1 + f_s;

						if (out_y*MLO_FILTER_STRIDE1 + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + (out_y*MLO_FILTER_STRIDE1 + in_scan) * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								for (; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = 0;
								}

							}
							else
#endif
							{
								//					*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}
							int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
							for (int i = 0; i < MLO_READ_UNIT; ++i)
							{
								bot_mem[lcl_off + i] = in_rd_data[i];
							}

						}
					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
						// select all vertical scans that matches the vertical filter tap
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)];
						// read input values for this filter phase
						for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
						{
							for (int i = 0; i < ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0); ++i)
							{
								in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + i]
									= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row* MLO_N_FILTER_SPLITS1 + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i];
							}

						}
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_FILTER_STRIDE0];
						// first 2 splits
						int l;
						for (l = 0; l < MLO_N_FILTER_SPLITS0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + l*MLO_FILTER_STRIDE0 + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l
// 3d
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && (out_y + ex_row) < MLO_OUT_HEIGHT && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{

				for (int f_s = 0; f_s < MLO_FILTER_STRIDE1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_OUT_STACKS);
						int t0 = iMod(w, k, MLO_OUT_STACKS);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}

					}

					int n_reads = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						int t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = c_scan* MLO_FILTER_STRIDE1 + f_s;

						if (ob*MLO_OUT_EXTENT1*MLO_FILTER_STRIDE1 + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								for (; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = 0;
								}

							}
							else
#endif
							{
								//					*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}
							int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
							for (int i = 0; i < MLO_READ_UNIT; ++i)
							{
								bot_mem[lcl_off + i] = in_rd_data[i];
							}

						}
					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
						// select all vertical scans that matches the vertical filter tap
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)];
						// read input values for this filter phase
						for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
						{
							for (int i = 0; i < ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0); ++i)
							{
								in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + i]
									= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row* MLO_N_FILTER_SPLITS1 + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i];
							}

						}
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_FILTER_STRIDE0];
						// first 2 splits
						int l;
						for (l = 0; l < MLO_N_FILTER_SPLITS0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + l*MLO_FILTER_STRIDE0 + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l
// 3d
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);
			int out_y = ob*MLO_OUT_EXTENT1;

			for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && (out_y + ex_row) < MLO_OUT_HEIGHT && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}


Cleaning up opencl/kernel/random.cl

Cleaning up opencl/kernel/random.cl


Added new barrier for testing for opencl gpu code

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@8902 7c0444ca-5550-0410-9008-faecdbe78eb5

//Defining a compiler directive to tell it the kernel needs the file
/* Data type for weights and activity */
#define pvwdata_t float
#define pvadata_t float
#define pvdata_t float
#define RESTRICT

//#define CL_KERNEL_INCLUDE
//#include "../layers/accumulate_functions.h"
//#undef CL_KERNEL_INCLUDE

#ifndef PV_USE_OPENCL
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_CONST
#  define CL_MEM_LOCAL
#else  /* compiling with OpenCL */
#  define CL_KERNEL       __kernel
#  define CL_MEM_GLOBAL   __global
#  define CL_MEM_CONST    __constant
#  define CL_MEM_LOCAL    __local

//TODO see if you can include conversions.h
static inline int kxPos(int k, int nx, int ny, int nf)
{
   return (k/nf) % nx;
}

static inline int kyPos(int k, int nx, int ny, int nf)
{
   return k / (nx*nf);
}

static inline int featureIndex(int k, int nx, int ny, int nf)
{
   return k % nf;
}

static inline int kIndex(int kx, int ky, int kf, int nx, int ny, int nf)
{
   return kf + (kx + ky * nx) * nf;
}

static inline int kIndexExtended(int k, int nx, int ny, int nf, int lt, int rt, int dn, int up)
{
   const int kx_ex = lt + kxPos(k, nx, ny, nf);
   const int ky_ex = up + kyPos(k, nx, ny, nf);
   const int kf = featureIndex(k, nx, ny, nf);
   return kIndex(kx_ex, ky_ex, kf, nx + lt + rt, ny + dn + up, nf);
}

int pvpatch_accumulate_from_post(int nk, CL_MEM_LOCAL float * v, CL_MEM_LOCAL float * a, CL_MEM_LOCAL float * w, float dt_factor, void * auxPtr) {
   //See what size nk is, and see if you can split it into vector multiplication
   int k = 0;
   while(k < nk){
      //GPUs can do 2, 4, 8, and 16 vector multiplications
      if(nk - k > 16){
         CL_MEM_LOCAL float16 * activity16 = (CL_MEM_LOCAL float16*) (a+k);
         CL_MEM_LOCAL float16 * weight16   = (CL_MEM_LOCAL float16*) (w+k);
         float16 ans16 = *activity16 * *weight16;
         *v += (ans16.s0 + ans16.s1 + ans16.s2 + ans16.s3 + ans16.s4 + ans16.s5 + ans16.s6 + ans16.s7 +
                ans16.s8 + ans16.s9 + ans16.sa + ans16.sb + ans16.sc + ans16.sd + ans16.se + ans16.sf) * dt_factor;
         k += 16;
      }
      else if(nk - k > 8){
         CL_MEM_LOCAL float8 * activity8 = (CL_MEM_LOCAL float8*) (a+k);
         CL_MEM_LOCAL float8 * weight8   = (CL_MEM_LOCAL float8*) (w+k);
         float8 ans8 = *activity8 * *weight8;
         *v += (ans8.s0 + ans8.s1 + ans8.s2 + ans8.s3 + ans8.s4 + ans8.s5 + ans8.s6 + ans8.s7) * dt_factor;
         k += 8;
      }
      else if(nk - k > 4){
         CL_MEM_LOCAL float4 * activity4 = (CL_MEM_LOCAL float4*) (a+k);
         CL_MEM_LOCAL float4 * weight4   = (CL_MEM_LOCAL float4*) (w+k);
         float4 ans4 = *activity4 * *weight4;
         *v += (ans4.x + ans4.y + ans4.z + ans4.w) * dt_factor;
         k += 4;
      }
      else if(nk - k > 2){
         CL_MEM_LOCAL float2 * activity2 = (CL_MEM_LOCAL float2*) (a+k);
         CL_MEM_LOCAL float2 * weight2   = (CL_MEM_LOCAL float2*) (w+k);
         float2 ans2 = *activity2 * *weight2;
         *v += (ans2.x + ans2.y) * dt_factor;
         k += 2;
      }
      else{
         *v += a[k] * w[k] * dt_factor;
         k+= 1;
      }
   }
   return 0;
}
#endif

CL_KERNEL
void HyPerLayer_recv_post(
      const int nxRes, //num post neurons
      const int nyRes,
      const int nf,

      const int nblt, //Border of orig
      const int nbrt, //Border of orig
      const int nbdn, //Border of orig
      const int nbup, //Border of orig

      const int nxp,
      const int nyp,
      const int nfp,

      const int localBufSizeX,
      const int localBufSizeY,
      const float preToPostScaleX,
      const float preToPostScaleY,

      const int sy,
      const int syp,
      const int numPerStride,
      const float dt_factor,
      const int sharedWeights,

      CL_MEM_GLOBAL long* startSourceExtBuf,
      CL_MEM_GLOBAL float* preData,
      CL_MEM_GLOBAL float* weights,
      CL_MEM_GLOBAL float* postGsyn,
      CL_MEM_GLOBAL int* patch2datalookuptable,

      CL_MEM_LOCAL float* preBuffer,
      CL_MEM_LOCAL float* postBuffer,
      CL_MEM_LOCAL float* weightsBuffer
){
#ifdef PV_USE_OPENCL
      int fTargetRes = get_global_id(0);
      int xTargetRes = get_global_id(1);
      int yTargetRes = get_global_id(2);

      int localF = get_local_size(0);
      int localX = get_local_size(1);
      int localY = get_local_size(2);

      int localFIndex = get_local_id(0);
      int localXIndex = get_local_id(1);
      int localYIndex = get_local_id(2);

      //Calculate kTargetRes based on x, y, and f
      int kTargetRes = kIndex(xTargetRes, yTargetRes, fTargetRes, nxRes, nyRes, nf);
      //Change restricted to extended post neuron
      int kTargetExt = kIndexExtended(kTargetRes, nxRes, nyRes, nf, nblt, nbrt, nbdn, nbup);

      //TODO this won't work with HyPerConns
      CL_MEM_LOCAL int wIdx;
      if(localXIndex == 0 && localYIndex == 0){
         int kernelIndex;
         if(sharedWeights == 1){
            kernelIndex = patch2datalookuptable[kTargetExt];
         }
         else{
            kernelIndex = kTargetExt;
         }
         wIdx = kernelIndex * nxp * nyp * nfp;
      }

      //Get top left most neuron in the group
      CL_MEM_LOCAL long localStartSourceExt;
      if(localXIndex == 0 && localYIndex == 0 && localFIndex == 0){
         localStartSourceExt = startSourceExtBuf[kTargetRes];
      }

      int localIndex = kIndex(localXIndex, localYIndex, localFIndex, localX, localY, localF);

      postBuffer[localIndex] = 0;

      int numXfBuffer = localBufSizeX * nfp;

      //Match preBuffer to indPreData, need to find x and y offsets
      int xOffset = localXIndex * preToPostScaleX;
      //int yOffset = localYIndex * preToPostScaleY;

      CL_MEM_LOCAL float* postAddr = postBuffer + localIndex;

      //All processes in workgroup needs that local index
      barrier(CLK_LOCAL_MEM_FENCE);

      for(int ky = 0; ky < nyp; ky++){
         event_t event1, event2;
         //Copy global to local, do this with all threads
         //Pre buffer
         async_work_group_copy(preBuffer, &(preData[localStartSourceExt + ky * sy]), numXfBuffer, event1);
         //for(int i = localIndex; i < numXfBuffer; i+=numThreads){
         //   preBuffer[i] = preData[localStartSourceExt + ky * sy + i];
         //}

         //Weights
         async_work_group_copy(weightsBuffer, &weights[wIdx+ky * syp], nxp*nfp, event2);
         //for(int i = localIndex; i < nxp*nfp; i+= numThreads){
         //   weightsBuffer[i] = weights[wIdx + ky * syp + i];
         //}

         wait_group_events(1, event1);
         wait_group_events(1, event2);
         barrier(CLK_LOCAL_MEM_FENCE);

         CL_MEM_LOCAL float * activityY = &(preBuffer[xOffset * nfp]);
         //CL_MEM_GLOBAL float * activityY = &(preData[startSourceExt + ky * sy]);
         //CL_MEM_GLOBAL float * weightY = &(weights[wIdx + ky*syp]);
         //Summing into post buffer indexed by localIndex
         //pvpatch_accumulate_from_post(numPerStride, postAddr, activityY, weightY, dt_factor, (void*)0);
         pvpatch_accumulate_from_post(numPerStride, postAddr, activityY, weightsBuffer, dt_factor, (void*)0);
         barrier(CLK_LOCAL_MEM_FENCE);
      }

      //Barrier to make sure the work group's postbuffer is set
      barrier(CLK_LOCAL_MEM_FENCE);

      //Sum into global memory
      postGsyn[kTargetRes] += postBuffer[localIndex];
#endif
}

//Defining a compiler directive to tell it the kernel needs the file
/* Data type for weights and activity */
#define pvwdata_t float
#define pvadata_t float
#define pvdata_t float
#define RESTRICT

//#define CL_KERNEL_INCLUDE
//#include "../layers/accumulate_functions.h"
//#undef CL_KERNEL_INCLUDE

#ifndef PV_USE_OPENCL
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_CONST
#  define CL_MEM_LOCAL
#else  /* compiling with OpenCL */
#  define CL_KERNEL       __kernel
#  define CL_MEM_GLOBAL   __global
#  define CL_MEM_CONST    __constant
#  define CL_MEM_LOCAL    __local

//TODO see if you can include conversions.h
static inline int kxPos(int k, int nx, int ny, int nf)
{
   return (k/nf) % nx;
}

static inline int kyPos(int k, int nx, int ny, int nf)
{
   return k / (nx*nf);
}

static inline int featureIndex(int k, int nx, int ny, int nf)
{
   return k % nf;
}

static inline int kIndex(int kx, int ky, int kf, int nx, int ny, int nf)
{
   return kf + (kx + ky * nx) * nf;
}

static inline int kIndexExtended(int k, int nx, int ny, int nf, int lt, int rt, int dn, int up)
{
   const int kx_ex = lt + kxPos(k, nx, ny, nf);
   const int ky_ex = up + kyPos(k, nx, ny, nf);
   const int kf = featureIndex(k, nx, ny, nf);
   return kIndex(kx_ex, ky_ex, kf, nx + lt + rt, ny + dn + up, nf);
}

int pvpatch_accumulate_from_post(int nk, CL_MEM_LOCAL float * v, CL_MEM_LOCAL float * a, CL_MEM_LOCAL float * w, float dt_factor, void * auxPtr) {
   //See what size nk is, and see if you can split it into vector multiplication
   int k = 0;
   while(k < nk){
      //GPUs can do 2, 4, 8, and 16 vector multiplications
      if(nk - k > 16){
         CL_MEM_LOCAL float16 * activity16 = (CL_MEM_LOCAL float16*) (a+k);
         CL_MEM_LOCAL float16 * weight16   = (CL_MEM_LOCAL float16*) (w+k);
         float16 ans16 = *activity16 * *weight16;
         *v += (ans16.s0 + ans16.s1 + ans16.s2 + ans16.s3 + ans16.s4 + ans16.s5 + ans16.s6 + ans16.s7 +
                ans16.s8 + ans16.s9 + ans16.sa + ans16.sb + ans16.sc + ans16.sd + ans16.se + ans16.sf) * dt_factor;
         k += 16;
      }
      else if(nk - k > 8){
         CL_MEM_LOCAL float8 * activity8 = (CL_MEM_LOCAL float8*) (a+k);
         CL_MEM_LOCAL float8 * weight8   = (CL_MEM_LOCAL float8*) (w+k);
         float8 ans8 = *activity8 * *weight8;
         *v += (ans8.s0 + ans8.s1 + ans8.s2 + ans8.s3 + ans8.s4 + ans8.s5 + ans8.s6 + ans8.s7) * dt_factor;
         k += 8;
      }
      else if(nk - k > 4){
         CL_MEM_LOCAL float4 * activity4 = (CL_MEM_LOCAL float4*) (a+k);
         CL_MEM_LOCAL float4 * weight4   = (CL_MEM_LOCAL float4*) (w+k);
         float4 ans4 = *activity4 * *weight4;
         *v += (ans4.x + ans4.y + ans4.z + ans4.w) * dt_factor;
         k += 4;
      }
      else if(nk - k > 2){
         CL_MEM_LOCAL float2 * activity2 = (CL_MEM_LOCAL float2*) (a+k);
         CL_MEM_LOCAL float2 * weight2   = (CL_MEM_LOCAL float2*) (w+k);
         float2 ans2 = *activity2 * *weight2;
         *v += (ans2.x + ans2.y) * dt_factor;
         k += 2;
      }
      else{
         *v += a[k] * w[k] * dt_factor;
         k+= 1;
      }
   }
   return 0;
}
#endif

CL_KERNEL
void HyPerLayer_recv_post(
      const int nxRes, //num post neurons
      const int nyRes,
      const int nf,

      const int nblt, //Border of orig
      const int nbrt, //Border of orig
      const int nbdn, //Border of orig
      const int nbup, //Border of orig

      const int nxp,
      const int nyp,
      const int nfp,

      const int localBufSizeX,
      const int localBufSizeY,
      const float preToPostScaleX,
      const float preToPostScaleY,

      const int sy,
      const int syp,
      const int numPerStride,
      const float dt_factor,
      const int sharedWeights,

      CL_MEM_GLOBAL long* startSourceExtBuf,
      CL_MEM_GLOBAL float* preData,
      CL_MEM_GLOBAL float* weights,
      CL_MEM_GLOBAL float* postGsyn,
      CL_MEM_GLOBAL int* patch2datalookuptable,

      CL_MEM_LOCAL float* preBuffer,
      CL_MEM_LOCAL float* postBuffer,
      CL_MEM_LOCAL float* weightsBuffer
){
#ifdef PV_USE_OPENCL
      int fTargetRes = get_global_id(0);
      int xTargetRes = get_global_id(1);
      int yTargetRes = get_global_id(2);

      int localF = get_local_size(0);
      int localX = get_local_size(1);
      int localY = get_local_size(2);

      int localFIndex = get_local_id(0);
      int localXIndex = get_local_id(1);
      int localYIndex = get_local_id(2);

      //Calculate kTargetRes based on x, y, and f
      int kTargetRes = kIndex(xTargetRes, yTargetRes, fTargetRes, nxRes, nyRes, nf);
      //Change restricted to extended post neuron
      int kTargetExt = kIndexExtended(kTargetRes, nxRes, nyRes, nf, nblt, nbrt, nbdn, nbup);

      //TODO this won't work with HyPerConns
      CL_MEM_LOCAL int wIdx;
      if(localXIndex == 0 && localYIndex == 0){
         int kernelIndex;
         if(sharedWeights == 1){
            kernelIndex = patch2datalookuptable[kTargetExt];
         }
         else{
            kernelIndex = kTargetExt;
         }
         wIdx = kernelIndex * nxp * nyp * nfp;
      }

      //Get top left most neuron in the group
      CL_MEM_LOCAL long localStartSourceExt;
      if(localXIndex == 0 && localYIndex == 0 && localFIndex == 0){
         localStartSourceExt = startSourceExtBuf[kTargetRes];
      }

      int localIndex = kIndex(localXIndex, localYIndex, localFIndex, localX, localY, localF);

      postBuffer[localIndex] = 0;

      int numXfBuffer = localBufSizeX * nfp;

      //Match preBuffer to indPreData, need to find x and y offsets
      int xOffset = localXIndex * preToPostScaleX;
      //int yOffset = localYIndex * preToPostScaleY;

      int numThreads = localX * localY * localF;

      CL_MEM_LOCAL float* postAddr = postBuffer + localIndex;

      //All processes in workgroup needs that local index
      barrier(CLK_LOCAL_MEM_FENCE);

      for(int ky = 0; ky < nyp; ky++){
         //Copy global to local, do this with all threads
         //Pre buffer
         async_work_group_copy(preBuffer, &(preData[localStartSourceExt + ky * sy]), numXfBuffer, 0);
         //for(int i = localIndex; i < numXfBuffer; i+=numThreads){
         //   preBuffer[i] = preData[localStartSourceExt + ky * sy + i];
         //}
         //Weights
         async_work_group_copy(weightsBuffer, &weights[wIdx+ky * syp], nxp*nfp, 0);
         //for(int i = localIndex; i < nxp*nfp; i+= numThreads){
         //   weightsBuffer[i] = weights[wIdx + ky * syp + i];
         //}
         barrier(CLK_LOCAL_MEM_FENCE);

         CL_MEM_LOCAL float * activityY = &(preBuffer[xOffset * nfp]);
         //CL_MEM_GLOBAL float * activityY = &(preData[startSourceExt + ky * sy]);
         //CL_MEM_GLOBAL float * weightY = &(weights[wIdx + ky*syp]);
         //Summing into post buffer indexed by localIndex
         //pvpatch_accumulate_from_post(numPerStride, postAddr, activityY, weightY, dt_factor, (void*)0);
         pvpatch_accumulate_from_post(numPerStride, postAddr, activityY, weightsBuffer, dt_factor, (void*)0);
         barrier(CLK_LOCAL_MEM_FENCE);
      }

      //Barrier to make sure the work group's postbuffer is set
      barrier(CLK_LOCAL_MEM_FENCE);

      //Sum into global memory
      postGsyn[kTargetRes] += postBuffer[localIndex];
#endif
}


Another try at flitering.

Another try at flitering.

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define THREE 3
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT3 PPCAT(_FLOAT, THREE)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define DBG_OUT 0

#define UNUSED __attribute__((__unused__))

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LEFT_PAD0 (((MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / MLO_READ_UNIT) * MLO_READ_UNIT)
#define MLO_LRN_RIGHT_SIDE                                                               \
    (((MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / \
      MLO_READ_UNIT) *                                                                   \
     MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_LEFT_PAD0 + MLO_LRN_RIGHT_SIDE)
#define MLO_LCL_READ4 (MLO_LRN_LCL_DATA_WIDTH / MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

struct LRNForwardParam
{
    _FLOAT alphaoverarea;
    _FLOAT alpha;
    _FLOAT beta;
    _FLOAT K;
};

static inline int iDiv(int v, int d)
{
    int r = (int)((float)v / d + 0.00001f);
    return (r);
}

static inline int iMod(int v, int u, int d)
{
    int r = v - mul24((int)u, (int)d);
    return (r);
}
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannel_PS(const __global _FLOAT* bot,
                          __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                          __global _FLOAT* scale,
#endif
                          _FLOAT alphaoverarea,
                          UNUSED _FLOAT alpha,
                          _FLOAT beta,
                          _FLOAT K)
{
    // IT's taken from POOLING AVE with stride = 1'
    __local _FLOAT bot_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x       = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y       = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0 = get_local_id(0);
    int lcl_id1 = get_local_id(1);
    int ob      = get_global_id(2); // output * batch_sz
    int o       = iDiv(ob, MLO_LRN_BATCH_SZ);
    int b       = iMod(ob, o, MLO_LRN_BATCH_SZ);
    int bot_x   = x;
    int bot_y   = y;
    int bot_off = b * MLO_LRN_BOT_BATCH_STRIDE + o * MLO_LRN_BOT_CHANNEL_STRIDE;

    // load tile
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int bot_y_act = bot_y + b_j - MLO_LRN_PAD1;

        bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_LRN_BOT_HEIGHT);

        int bot_y_off = bot_y_act * MLO_LRN_BOT_STRIDE;

        int lcl_off_v = mul24(b_j, (int)MLO_LRN_LCL_DATA_WIDTH);

        for(int b_i = lcl_id0; b_i < MLO_LCL_READ4; b_i += MLO_LRN_GROUP_SZ0)
        {

            int bot_x_act = bot_x + (b_i * MLO_READ_UNIT) - MLO_LRN_LEFT_PAD0;

            bool invisibleX;
            for(int i = 0; i < MLO_READ_UNIT; ++i)
            {

                int bot_off_x = bot_off + bot_y_off + bot_x_act + i;

                invisibleX = (bot_x_act + i < 0) || (bot_x_act + i >= MLO_LRN_BOT_WIDTH);

                bot_off_x = (invisibleX || invisibleY) ? 0 : bot_off_x;

                _FLOAT bot_val = bot[bot_off_x];

                bot_val = (invisibleX || invisibleY) ? 0 : bot_val;

                bot_data[lcl_off_v + (b_i * MLO_READ_UNIT) + i] = bot_val;
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
    _FLOAT partial_sum_x[MLO_LRN_N_HORIZ_OUT_PIX - 1]; // horizontal partial sum
#endif
#if MLO_LRN_N_VERT_OUT_PIX > 1
    _FLOAT partial_sum_xy[MLO_LRN_N_VERT_OUT_PIX - 1]
                         [MLO_LRN_N_HORIZ_OUT_PIX]; // horizontal-vertical partial sums.
#endif
    _FLOAT accum[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; // accumulator

    int top_y = mad24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX, y);
    int top_x = mad24(lcl_id0, (int)MLO_LRN_N_HORIZ_OUT_PIX, x);

    int lcl_y = mul24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX);
    int lcl_x =
        mad24(lcl_id0, (int)(MLO_LRN_N_HORIZ_OUT_PIX), (int)(MLO_LRN_LEFT_PAD0 - MLO_LRN_PAD0));
    int lcl_off = mad24(lcl_y, MLO_LRN_LCL_DATA_WIDTH, lcl_x);

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[j][i] = 0;
        }
    }
#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX - 1; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            partial_sum_xy[j][i] = 0;
        }
    }
#endif

    // running window  summation
    _FLOAT mov_accum;
    int jj = 0;
    int ii = 0;

// first to get vertica partial sums

#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(; jj < (int)(MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];

            _FLOAT accum_tmp = bot_val * bot_val;

#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            // save horizontal partial sums
            partial_sum_x[ii] = accum_tmp;
#endif
            // accumulate in vert-horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            // accumulate in vert horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            // calculate all vertical-horizontal partial sums
            partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0] +
                (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                 -
                 partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                 );
        }

        // put into accumulator[0][i]
        // whatever has been accumulated so far
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[0][i] += partial_sum_xy[jj][i];
        }
    }
#endif

    // calculate row 0 accumulators
    for(; jj < (int)MLO_LRN_KERNEL_SZ1; ++jj)
    {
        mov_accum = 0;

        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            mov_accum += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            mov_accum += accum_tmp;
        }

        accum[0][0] += mov_accum;
        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            // running horizontal window
            mov_accum += (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                          -
                          partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                          );
            accum[0][ii - MLO_LRN_KERNEL_SZ0 + 1] += mov_accum;
        }
    }

    // accumulate all other rows besides 0
    for(; jj < (int)(MLO_LRN_KERNEL_SZ1 + MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        // first running horizontal winodw as before
        mov_accum = 0;
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        // running horizontal window

        int ii1 = ii;
        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            //
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0] + accum_tmp;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_x[ii - MLO_LRN_KERNEL_SZ0];
#endif
        }

        // finally running vertical window

        for(ii = ii1; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            // finish horizontal summation
            // add/substarct vertical patial sum
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] +=
                accum[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#if MLO_LRN_N_VERT_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#endif
        }
#if MLO_LRN_N_VERT_OUT_PIX > 1
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] -= partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][0];
#endif
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum[jj - MLO_LRN_KERNEL_SZ1][0];
    }

    // normalization
    _FLOAT prv_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    _FLOAT adj_alphaoverarea = alphaoverarea;
    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++)
    {

        //			int hstart = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX  + k -
        // MLO_LRN_PAD1;
        //			int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_HEIGHT +
        // MLO_LRN_PAD1);

        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++)
        {

            //				int wstart = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + l -
            // MLO_LRN_PAD0;
            //				int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_WIDTH
            //+
            // MLO_LRN_PAD0);
            //				int adj_area_size = (hend - hstart) * (wend - wstart);
            //				adj_alphaoverarea = alpha / adj_area_size;

            prv_scale[k][l] = K + accum[k][l] * adj_alphaoverarea;
        }
    }

    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  top_y * MLO_LRN_TOP_STRIDE + top_x;
#if MLO_LRN_DO_SCALE
    int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    top_y * MLO_LRN_SCALE_STRIDE + top_x;
#endif

    // final output

    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX
#if MLO_OUT_VERT_ALIGNED == 0
                   &&
                   (top_y + k < MLO_LRN_TOP_HEIGHT)
#endif
            ;
        k++)
    {
        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX
#if MLO_OUT_HORIZ_ALIGNED == 0
                       &&
                       (top_x + l < MLO_LRN_TOP_WIDTH)
#endif
                ;
            l++)
        {
            _FLOAT s;
            s = exp((_FLOAT)-beta * log(prv_scale[k][l]));
            //					s = pow(prv_scale[k][l], -beta);
            _FLOAT bot_val = bot_data[lcl_off + mad24((k + MLO_LRN_PAD1),
                                                      (int)MLO_LRN_LCL_DATA_WIDTH,
                                                      (l + MLO_LRN_PAD0))];
#if MLO_LRN_DO_SCALE
            scale[scale_off + k * MLO_LRN_SCALE_STRIDE + l] = prv_scale[k][l];
#endif
            top[top_off + k * MLO_LRN_TOP_STRIDE + l] = bot_val * s;
        }
    }
}

#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannels4(const __global _FLOAT* bottom,
                         __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                         __global _FLOAT* scale,
#endif
                         _FLOAT alphaoverarea,
                         UNUSED _FLOAT alpha,
                         _FLOAT beta,
                         _FLOAT K)
{
    int pix_id          = get_global_id(0); //
    int b               = get_global_id(2); // batch
    MLO_READ_TYPE accum = 0;
    MLO_READ_TYPE bot_in2[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    for(int i = 0; i < MLO_LRN_KERNEL_SZ; ++i)
    {
        bot_in2[i] = 0;
    }

    int top_off   = 0;
    int scale_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD; c_i++)
    {
        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_i < MLO_LRN_N_INPUTS)
#endif
        {
#if MLO_C1x1_PIXLEFT > 0
            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {

                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    ((_FLOAT*)&prv_in)[j] =
                        bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                               (pix_id * MLO_READ_UNIT) + j];
                }
            }
            else
#endif
            {
                prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                           MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                           (pix_id * MLO_READ_UNIT)];
            }
        }

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];
        //				fma(bot_in[c_i + MLO_LRN_PAD], bot_in[c_i + MLO_LRN_PAD],
        // accum);
    }

    for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
    {
        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_i < MLO_LRN_N_INPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0
            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {

                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    ((_FLOAT*)&prv_in)[j] =
                        bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                               (pix_id * MLO_READ_UNIT) + j];
                }
            }
            else
#endif
            {
                prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                           MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                           (pix_id * MLO_READ_UNIT)];
            }
        }

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, (_FLOAT)1.f);

        MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //	MLO_READ_TYPE prv_out = sqrt(bot_in2[c_o]);
        MLO_READ_TYPE prv_out = bot_in2[c_o];
        prv_out               = sqrt(prv_out);
        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
                    printf("K:o0: %d %f %f %f %f %f\n",
                           top_off + j,
                           top[top_off + j],
                           ((_FLOAT*)&prv_out)[j],
                           ((_FLOAT*)&exp_scale)[j],
                           ((_FLOAT*)&prv_scale)[j],
                           ((_FLOAT*)&accum)[j]);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++)
    {

        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }

        MLO_READ_TYPE prv_bot_in2 = prv_in * prv_in;
        accum                     = accum + prv_bot_in2;

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        bot_in2[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in2;

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, (_FLOAT)1.f);

        MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = sqrt(prv_out);
#if(MLO_C1x1_PIXLEFT <= 0 || MIOPEN_USE_FP16 == 0)
        // \todo DLOWELL: adding this preprocesser check to avoid unused variable
        MLO_READ_TYPE out_val = prv_out * exp_scale;
#endif
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
                    printf("K:o1: %d %f %f %f\n",
                           top_off + j,
                           top[top_off + j],
                           ((_FLOAT*)&prv_out)[j],
                           ((_FLOAT*)&exp_scale)[j]);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {
#if(MIOPEN_USE_FP32 == 1)
//*((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
// \todo DLOWELL: this needs a proper fix. This hack is only to get PRs through.
#else
                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#endif
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, (_FLOAT)1.f);

        MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = sqrt(prv_out);

        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
                    printf("K:o2: %d %f %f %f\n",
                           top_off + j,
                           top[top_off + j],
                           ((_FLOAT*)&prv_out)[j],
                           ((_FLOAT*)&exp_scale)[j]);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

//  *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define THREE 3
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT3 PPCAT(_FLOAT, THREE)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define DBG_OUT 0

#define UNUSED __attribute__((__unused__))

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LEFT_PAD0 (((MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / MLO_READ_UNIT) * MLO_READ_UNIT)
#define MLO_LRN_RIGHT_SIDE                                                               \
    (((MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / \
      MLO_READ_UNIT) *                                                                   \
     MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_LEFT_PAD0 + MLO_LRN_RIGHT_SIDE)
#define MLO_LCL_READ4 (MLO_LRN_LCL_DATA_WIDTH / MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

struct LRNForwardParam
{
    _FLOAT alphaoverarea;
    _FLOAT alpha;
    _FLOAT beta;
    _FLOAT K;
};

static inline int iDiv(int v, int d)
{
    int r = (int)((float)v / d + 0.00001f);
    return (r);
}

static inline int iMod(int v, int u, int d)
{
    int r = v - mul24((int)u, (int)d);
    return (r);
}
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannel_PS(const __global _FLOAT* bot,
                          __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                          __global _FLOAT* scale,
#endif
                          _FLOAT alphaoverarea,
                          UNUSED _FLOAT alpha,
                          _FLOAT beta,
                          _FLOAT K)
{
    // IT's taken from POOLING AVE with stride = 1'
    __local _FLOAT bot_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x       = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y       = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0 = get_local_id(0);
    int lcl_id1 = get_local_id(1);
    int ob      = get_global_id(2); // output * batch_sz
    int o       = iDiv(ob, MLO_LRN_BATCH_SZ);
    int b       = iMod(ob, o, MLO_LRN_BATCH_SZ);
    int bot_x   = x;
    int bot_y   = y;
    int bot_off = b * MLO_LRN_BOT_BATCH_STRIDE + o * MLO_LRN_BOT_CHANNEL_STRIDE;

    // load tile
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int bot_y_act = bot_y + b_j - MLO_LRN_PAD1;

        bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_LRN_BOT_HEIGHT);

        int bot_y_off = bot_y_act * MLO_LRN_BOT_STRIDE;

        int lcl_off_v = mul24(b_j, (int)MLO_LRN_LCL_DATA_WIDTH);

        for(int b_i = lcl_id0; b_i < MLO_LCL_READ4; b_i += MLO_LRN_GROUP_SZ0)
        {

            int bot_x_act = bot_x + (b_i * MLO_READ_UNIT) - MLO_LRN_LEFT_PAD0;

            bool invisibleX;
            for(int i = 0; i < MLO_READ_UNIT; ++i)
            {

                int bot_off_x = bot_off + bot_y_off + bot_x_act + i;

                invisibleX = (bot_x_act + i < 0) || (bot_x_act + i >= MLO_LRN_BOT_WIDTH);

                bot_off_x = (invisibleX || invisibleY) ? 0 : bot_off_x;

                _FLOAT bot_val = bot[bot_off_x];

                bot_val = (invisibleX || invisibleY) ? 0 : bot_val;

                bot_data[lcl_off_v + (b_i * MLO_READ_UNIT) + i] = bot_val;
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
    _FLOAT partial_sum_x[MLO_LRN_N_HORIZ_OUT_PIX - 1]; // horizontal partial sum
#endif
#if MLO_LRN_N_VERT_OUT_PIX > 1
    _FLOAT partial_sum_xy[MLO_LRN_N_VERT_OUT_PIX - 1]
                         [MLO_LRN_N_HORIZ_OUT_PIX]; // horizontal-vertical partial sums.
#endif
    _FLOAT accum[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; // accumulator

    int top_y = mad24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX, y);
    int top_x = mad24(lcl_id0, (int)MLO_LRN_N_HORIZ_OUT_PIX, x);

    int lcl_y = mul24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX);
    int lcl_x =
        mad24(lcl_id0, (int)(MLO_LRN_N_HORIZ_OUT_PIX), (int)(MLO_LRN_LEFT_PAD0 - MLO_LRN_PAD0));
    int lcl_off = mad24(lcl_y, MLO_LRN_LCL_DATA_WIDTH, lcl_x);

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[j][i] = 0;
        }
    }
#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX - 1; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            partial_sum_xy[j][i] = 0;
        }
    }
#endif

    // running window  summation
    _FLOAT mov_accum;
    int jj = 0;
    int ii = 0;

// first to get vertica partial sums

#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(; jj < (int)(MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];

            _FLOAT accum_tmp = bot_val * bot_val;

#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            // save horizontal partial sums
            partial_sum_x[ii] = accum_tmp;
#endif
            // accumulate in vert-horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            // accumulate in vert horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            // calculate all vertical-horizontal partial sums
            partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0] +
                (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                 -
                 partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                 );
        }

        // put into accumulator[0][i]
        // whatever has been accumulated so far
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[0][i] += partial_sum_xy[jj][i];
        }
    }
#endif

    // calculate row 0 accumulators
    for(; jj < (int)MLO_LRN_KERNEL_SZ1; ++jj)
    {
        mov_accum = 0;

        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            mov_accum += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            mov_accum += accum_tmp;
        }

        accum[0][0] += mov_accum;
        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            // running horizontal window
            mov_accum += (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                          -
                          partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                          );
            accum[0][ii - MLO_LRN_KERNEL_SZ0 + 1] += mov_accum;
        }
    }

    // accumulate all other rows besides 0
    for(; jj < (int)(MLO_LRN_KERNEL_SZ1 + MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        // first running horizontal winodw as before
        mov_accum = 0;
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        // running horizontal window

        int ii1 = ii;
        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val * bot_val;
            //
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0] + accum_tmp;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_x[ii - MLO_LRN_KERNEL_SZ0];
#endif
        }

        // finally running vertical window

        for(ii = ii1; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            // finish horizontal summation
            // add/substarct vertical patial sum
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] +=
                accum[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#if MLO_LRN_N_VERT_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#endif
        }
#if MLO_LRN_N_VERT_OUT_PIX > 1
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] -= partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][0];
#endif
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum[jj - MLO_LRN_KERNEL_SZ1][0];
    }

    // normalization
    _FLOAT prv_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    _FLOAT adj_alphaoverarea = alphaoverarea;
    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++)
    {

        //			int hstart = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX  + k -
        // MLO_LRN_PAD1;
        //			int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_HEIGHT +
        // MLO_LRN_PAD1);

        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++)
        {

            //				int wstart = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + l -
            // MLO_LRN_PAD0;
            //				int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_WIDTH
            //+
            // MLO_LRN_PAD0);
            //				int adj_area_size = (hend - hstart) * (wend - wstart);
            //				adj_alphaoverarea = alpha / adj_area_size;

            prv_scale[k][l] = K + accum[k][l] * adj_alphaoverarea;
        }
    }

    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  top_y * MLO_LRN_TOP_STRIDE + top_x;
#if MLO_LRN_DO_SCALE
    int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    top_y * MLO_LRN_SCALE_STRIDE + top_x;
#endif

    // final output

    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX
#if MLO_OUT_VERT_ALIGNED == 0
                   &&
                   (top_y + k < MLO_LRN_TOP_HEIGHT)
#endif
            ;
        k++)
    {
        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX
#if MLO_OUT_HORIZ_ALIGNED == 0
                       &&
                       (top_x + l < MLO_LRN_TOP_WIDTH)
#endif
                ;
            l++)
        {
            _FLOAT s;
            s = exp((_FLOAT)-beta * log(prv_scale[k][l]));
            //					s = pow(prv_scale[k][l], -beta);
            _FLOAT bot_val = bot_data[lcl_off + mad24((k + MLO_LRN_PAD1),
                                                      (int)MLO_LRN_LCL_DATA_WIDTH,
                                                      (l + MLO_LRN_PAD0))];
#if MLO_LRN_DO_SCALE
            scale[scale_off + k * MLO_LRN_SCALE_STRIDE + l] = prv_scale[k][l];
#endif
            top[top_off + k * MLO_LRN_TOP_STRIDE + l] = bot_val * s;
        }
    }
}

#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannels4(const __global _FLOAT* bottom,
                         __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                         __global _FLOAT* scale,
#endif
                         _FLOAT alphaoverarea,
                         UNUSED _FLOAT alpha,
                         _FLOAT beta,
                         _FLOAT K)
{
    int pix_id          = get_global_id(0); //
    int b               = get_global_id(2); // batch
    MLO_READ_TYPE accum = 0;
    MLO_READ_TYPE bot_in2[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    for(int i = 0; i < MLO_LRN_KERNEL_SZ; ++i)
    {
        bot_in2[i] = 0;
    }

    int top_off   = 0;
    int scale_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD; c_i++)
    {
        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_i < MLO_LRN_N_INPUTS)
#endif
        {
#if MLO_C1x1_PIXLEFT > 0
            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {

                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    ((_FLOAT*)&prv_in)[j] =
                        bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                               (pix_id * MLO_READ_UNIT) + j];
                }
            }
            else
#endif
            {
                prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                           MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                           (pix_id * MLO_READ_UNIT)];
            }
        }

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];
        //				fma(bot_in[c_i + MLO_LRN_PAD], bot_in[c_i + MLO_LRN_PAD],
        // accum);
    }

    for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
    {
        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_i < MLO_LRN_N_INPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0
            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {

                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    ((_FLOAT*)&prv_in)[j] =
                        bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                               (pix_id * MLO_READ_UNIT) + j];
                }
            }
            else
#endif
            {
                prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                           MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                           (pix_id * MLO_READ_UNIT)];
            }
        }

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, (_FLOAT)1.f);

        MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //	MLO_READ_TYPE prv_out = sqrt(bot_in2[c_o]);
        MLO_READ_TYPE prv_out = bot_in2[c_o];
        prv_out               = sqrt(prv_out);
        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
                    printf("K:o0: %d %f %f %f %f %f\n",
                           top_off + j,
                           top[top_off + j],
                           ((_FLOAT*)&prv_out)[j],
                           ((_FLOAT*)&exp_scale)[j],
                           ((_FLOAT*)&prv_scale)[j],
                           ((_FLOAT*)&accum)[j]);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++)
    {

        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }

        MLO_READ_TYPE prv_bot_in2 = prv_in * prv_in;
        accum                     = accum + prv_bot_in2;

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        bot_in2[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in2;

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, (_FLOAT)1.f);

        MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = sqrt(prv_out);
#if(MLO_C1x1_PIXLEFT <= 0 || MIOPEN_USE_FP16 == 0)
        // \todo DLOWELL: adding this preprocesser check to avoid unused variable
        MLO_READ_TYPE out_val = prv_out * exp_scale;
#endif
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
                    printf("K:o1: %d %f %f %f\n",
                           top_off + j,
                           top[top_off + j],
                           ((_FLOAT*)&prv_out)[j],
                           ((_FLOAT*)&exp_scale)[j]);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {
#if(MIOPEN_USE_FP16 == 1)
//*((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
// \todo DLOWELL: this needs a proper fix. This hack is only to get PRs through.
#else
                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#endif
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, (_FLOAT)1.f);

        MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = sqrt(prv_out);

        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
                    printf("K:o2: %d %f %f %f\n",
                           top_off + j,
                           top[top_off + j],
                           ((_FLOAT*)&prv_out)[j],
                           ((_FLOAT*)&exp_scale)[j]);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

//  *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }
}


Fixed clang formatting

Fixed clang formatting

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

static inline int iDiv(int v, int d)
{
    int r = v / d;
    return (r);
}

static inline int iMod(int v, int u, int d)
{
    int r = v - mul24((int)u, (int)d);
    return (r);
}

static inline void ReduceKernel(__local float* lcl_mem, int sum_stride, int unit_id, int unit_len)
{
    float sum      = 0.0f;
    int lcl_offset = unit_id * unit_len;
    for(int i = 0; i < unit_len; i += sum_stride)
    {
        sum += lcl_mem[lcl_offset + i];
    }
    lcl_mem[lcl_offset] = sum;
}

__attribute__((reqd_work_group_size(MLO_CONVBWD_GROUP_SZ0, MLO_CONVBWD_GROUP_SZ1, 1))) __kernel void
MIOpenConvBwdB(const __global _FLOAT* top_df, __global _FLOAT* bias_df)
{
    int lid        = (int)get_local_id(0);
    int output_map = get_group_id(1);

    __local float lcl_sum[MLO_CONVBWDB_LCL_MEMSZ];
    float sum = 0.0f;

    for(int j = lid; j < MLO_WK_SIZE * MLO_OUT_BATCH_SZ; j += MLO_CONVBWD_GROUP_SZ0)
    {
        int map_id            = iDiv(j, MLO_WK_SIZE);
        int read_id           = iMod(j, map_id, MLO_WK_SIZE);
        int glb_top_df_offset = output_map * MLO_OUT_CHANNEL_STRIDE +
                                (map_id * MLO_OUT_BATCH_STRIDE) + (read_id * MLO_CONVBWDB_UNITSIZE);
#if MLO_N_PIX_OFF
        if(read_id == MLO_WK_SIZE - 1)
        {
            for(int k = 0; k < MLO_N_PIX_OFF; k++)
                sum += (float)(top_df[glb_top_df_offset + k]);
        }
        else
        {
            for(int k = 0; k < MLO_CONVBWDB_UNITSIZE; k++)
                sum += (float)(top_df[glb_top_df_offset + k]);
        }
#else
        for(int k = 0; k < MLO_CONVBWDB_UNITSIZE; k++)
            sum += (float)(top_df[glb_top_df_offset + k]);
#endif
    }
    lcl_sum[lid] = sum;

    barrier(CLK_LOCAL_MEM_FENCE);

    // Reduction over a work-grp: 256 -> 64 -> 16 -> 1
    if(lid < (MLO_CONVBWD_GROUP_SZ0 >> 2))
    {
        ReduceKernel(lcl_sum, 1, lid, 4);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(lid < (MLO_CONVBWD_GROUP_SZ0 >> 4))
    {
        ReduceKernel(lcl_sum, 4, lid, 16);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(lid == 0)
    {
        ReduceKernel(lcl_sum, 16, lid, 256);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    bias_df[output_map] = (_FLOAT)(lcl_sum[0]);
}