fixed some opencl bugs in image gradient on amd platform

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

#ifdef TYPE_UINT
#define READ_IMAGE (float)read_imageui
#elif TYPE_INT
#define READ_IMAGE (float)read_imagei
#elif TYPE_FLOAT
#define READ_IMAGE read_imagef
#endif

__kernel void gradient2D(
        __read_only image2d_t input,
        __write_only image2d_t output
        ) {
    const int2 pos = {get_global_id(0), get_global_id(1)};

    // TODO take pixel spacing into account
    float4 gradient = {
            (READ_IMAGE(input, sampler, pos+(int2)(1,0)).x + READ_IMAGE(input, sampler, pos-(int2)(1,0)).x)/2.0f,
            (READ_IMAGE(input, sampler, pos+(int2)(0,1)).x + READ_IMAGE(input, sampler, pos-(int2)(0,1)).x)/2.0f,
            0,0
    };
    //printf("gradient: %f %f\n", READ_IMAGE(input, sampler, pos+(int2)(0,1)).x, gradient.y);
    write_imagef(output, pos, gradient);
}

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

#ifdef TYPE_UINT
#define READ_IMAGE read_imageui
#elif TYPE_INT
#define READ_IMAGE read_imageui
#elif TYPE_FLOAT
#define READ_IMAGE read_imagef
#endif

__kernel void gradient2D(
        __read_only image2d_t input,
        __write_only image2d_t output
        ) {
    const int2 pos = {get_global_id(0), get_global_id(1)};

    // TODO take pixel spacing into account
    float4 gradient = {
            0.5f*(READ_IMAGE(input, sampler, pos+(int2)(1,0)).x + READ_IMAGE(input, sampler, pos-(int2)(1,0)).x),
            0.5f*(READ_IMAGE(input, sampler, pos+(int2)(0,1)).x + READ_IMAGE(input, sampler, pos-(int2)(0,1)).x),
            0,0
    };

    write_imagef(output, pos, gradient);
}


another fix

another fix

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	int out_y = ob*MLO_OUT_EXTENT1;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				for (int f_s = 0; f_s < MLO_FILTER_STRIDE1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_WEI_SZ);
						int t0 = iMod(w, k, MLO_WEI_SZ);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
								if (ob==0)
								{
									printf("G:w: %d %d %d   %f %f\n",
//										lcl_id,
//										w,
//										f_s,
//										j,
//										i,
//										k_idx,
//										k,
										k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
										gbl_wei_off,
										wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
										weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
										wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
									);
								}

#endif

						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}
					}

					int n_reads = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						int t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1;

						if (0 <= out_y*MLO_FILTER_STRIDE1 + in_scan && out_y*MLO_FILTER_STRIDE1 + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + (out_y*MLO_FILTER_STRIDE1 + in_scan) * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								for (; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = 0;
								}

							}
							else
#endif
							{
								//					*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}
							int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
							for (int i = 0; i < MLO_READ_UNIT; ++i)
							{
								bot_mem[lcl_off + i] = in_rd_data[i];
							}

						}
					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
						// select all vertical scans that matches the vertical filter tap
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)];
						// read input values for this filter phase
						for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
						{
							for (int i = 0; i < ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0); ++i)
							{
								in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + i]
									= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i];
							}

						}
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_FILTER_STRIDE0];
						// first 2 splits
						int l;
						for (l = 0; l < MLO_N_FILTER_SPLITS0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + l*MLO_FILTER_STRIDE0 + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 0 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
// 3d
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + l*MLO_FILTER_STRIDE0 + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 0 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1,
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif
										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && (out_y + ex_row) < MLO_OUT_HEIGHT && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	int out_y = ob*MLO_OUT_EXTENT1;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{
				for (int f_s = 0; f_s < MLO_FILTER_STRIDE1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_WEI_SZ);
						int t0 = iMod(w, k, MLO_WEI_SZ);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
#if 0
								if (ob==0)
								{
									printf("G:w: %d %d %d   %f %f\n",
//										lcl_id,
//										w,
//										f_s,
//										j,
//										i,
//										k_idx,
//										k,
										k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i,
										gbl_wei_off,
										wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i,
										weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i],
										wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i]
									);
								}

#endif

						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}
					}

					int n_reads = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						int t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = (c_scan + lcl_scan) * MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1;

						if (0 <= out_y*MLO_FILTER_STRIDE1 + in_scan && out_y*MLO_FILTER_STRIDE1 + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + (out_y*MLO_FILTER_STRIDE1 + in_scan) * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								for (; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = 0;
								}

							}
							else
#endif
							{
								//					*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}
							int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
							for (int i = 0; i < MLO_READ_UNIT; ++i)
							{
								bot_mem[lcl_off + i] = in_rd_data[i];
							}

						}
					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
						// select all vertical scans that matches the vertical filter tap
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)];
						// read input values for this filter phase
						for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
						{
							for (int i = 0; i < ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0); ++i)
							{
								in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + i]
									= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i];
							}

						}
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_FILTER_STRIDE0];
						// first 2 splits
						int l;
						for (l = 0; l < MLO_N_FILTER_SPLITS0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + l*MLO_FILTER_STRIDE0 + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib+b+bb == 0 && k_idx+k == 0 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1, // actual input vertical position
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif

										}
									}
								}
							} // b
						} // l
// 3d
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
#if 0
											if (wei_val * in_val != 0 && ib + b + bb == 0 && k_idx + k == 0 && out_y + ex_row == 0 && ex_pix + n == 0)
											{
												printf("G:c: %d %d %d %d %d %d %d %d %d %d %d  %f %f %f %f\n",
													f_s,
													out_y,
													ex_row,
													ex_pix,
													m,
													l,
													i,
													(out_y + ex_row)*MLO_FILTER_STRIDE1 + m*MLO_FILTER_STRIDE1 + f_s - MLO_FILTER_PAD1,
													(ex_pix + n)*MLO_FILTER_STRIDE0 + l*MLO_FILTER_STRIDE0 + i - MLO_FILTER_PAD0, // actual input horiz pos (assuming full scan is inside LDS)
													m*MLO_FILTER_STRIDE1 + f_s, // actual filter vet pos
													l*MLO_FILTER_STRIDE0 + i, // actual filter horiz pos
													pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n],
													wei_val * in_val,
													wei_val,
													in_val
												);
											}

#endif
										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && (out_y + ex_row) < MLO_OUT_HEIGHT && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}


Remove testing stuff

Remove testing stuff

(defun poly+ (poly1 poly2)
  (format t "Poly One: ~d~%" poly1)
  (format t "Poly Two: ~d~%" poly2))

(defun find-variable-symbol (single-poly)
  (car (car (car single-poly))))

(defun find-exponent (single-poly)
  (cdr (car (car single-poly))))

(defun find-multiplier (single-poly)
  (cdr (car single-poly)))

(poly+ '(((x 2) 3)) '(((y 2) 4)))
(poly+ '(((x 2) 3) ((y 2) 3)) '(((y 2) 4)))

(defun poly+ (poly1 poly2)
  (format t "Poly One: ~d~%" poly1)
  (format t "Poly Two: ~d~%" poly2)
  (if (equal (length poly1) 1)
    (format t "Poly 1 has exactly one term: ~a Ex: ~a Mul: ~a~%"
            (find-variable-symbol poly1)
            (find-exponent        poly1)
            (find-multiplier      poly1))
    (format t "Poly 1 has more than one term~%")))

(defun collect-like-terms (poly)
  (format t "collecting terms"))

(defun find-variable-symbol (single-poly)
  (car (car (car single-poly))))

(defun find-exponent (single-poly)
  (cdr (car (car single-poly))))

(defun find-multiplier (single-poly)
  (cdr (car single-poly)))

(poly+ '(((x 2) 3)) '(((y 2) 4)))
(poly+ '(((x 2) 3) ((y 2) 3)) '(((y 2) 4)))


Performance improvements for MIOpenConvBwdWrW_GxL_1x1.cl

Also contains perf w/a to pre-calculate g_data pointer with
offset to help vectorization.

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#ifndef MLO_N_PIXS_OFF
#define MLO_N_PIXS_OFF 0
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, uint lcl_id, uint scan_lcl, uint sum_stride, uint unit_len, UNUSED bool debug)
{

	for (uint j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (uint i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT * __restrict sum, __private _FLOAT * __restrict c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readData(uint n, int gbl_data_off, uint gbl_data_stride, uint map_stride, int map_base,  int map_limit, const __global _FLOAT * __restrict g_data, __private _FLOAT * __restrict p_data, bool last_pixel)
{
	for (uint j = 0; j < n; ++j)
	{
		int gbl_data_off0 = (j*map_stride + map_base < map_limit) ? gbl_data_off + j*map_stride*gbl_data_stride : 0;
		const __global _FLOAT * g_data_p = &g_data[gbl_data_off0];

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[j*MLO_READ_UNIT + i] = g_data_p[i];
			}

			for (uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = 0;
			}

		}
		else
#else
		(void)last_pixel;
#endif
		{
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = g_data_p[i];
			}
		}

	} // for (int j = 0; j < n; ++j)

}

__attribute__((always_inline))
void readDataFlex(uint n, uint gbl_data_off, uint gbl_data_stride, uint map_stride, uint map_base, uint map_limit, const __global _FLOAT * __restrict g_data, __local _FLOAT * __restrict l_data)
{

	for( uint l = get_local_id(0); l < n*map_stride* MLO_MAP_WK_SZ; l += MLO_GRP_SZ)
	{

		uint r = 0;
		uint k0 = l;
#if MLO_N_LCL_OUT_MAPS > 1
		r = iDiv(l, map_stride* MLO_MAP_WK_SZ);  // maps row
		k0 = iMod(l, r, map_stride* MLO_MAP_WK_SZ);
#endif

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint k = iDiv(k0, MLO_MAP_WK_SZ);  // map column
		uint p4 = iMod(k0, k, MLO_MAP_WK_SZ); // pixel block
#else
		uint k = (k0 / MLO_MAP_WK_SZ);  // map column
		uint p4 = (k0 & (MLO_MAP_WK_SZ-1)); // pixel block

#endif

		uint gbl_data_off0 = (r*map_stride + k + map_base < map_limit) ? gbl_data_off + (r*map_stride + k)*gbl_data_stride + p4*MLO_READ_UNIT : 0;
                const __global _FLOAT * g_data_p = &g_data[gbl_data_off0];
		__private _FLOAT p_data[MLO_READ_UNIT];

#if MLO_N_PIXS_OFF > 0

		bool last_pixel = (p4 == MLO_MAP_WK_SZ -1);
		if (last_pixel)
		{
			for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[i] = g_data_p[i];
			}

			for (uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = 0;
			}

		}
		else
#endif
		{
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = g_data_p[i];
			}
		}

		for(uint i = 0; i < MLO_READ_UNIT; ++i)
		{
			l_data[(k*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i] = p_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_IN_STACKS per group, MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// loop over MLO_OUT_STACKS of output

// convolve with MLO_N_LCL_IN_MAPS per wk-item

// reduce with transform

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
#if MLO_MAP_WK_SZ > 8
	__local _FLOAT * red_mem = lcl_mem;
#endif
	__local _FLOAT * proc_mem = lcl_mem;

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_IN_STACKS * MLO_N_LCL_IN_MAPS); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

#if MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1)
	// map id inside group
	uint m_idx = iDiv(lcl_id, MLO_MAP_WK_SZ);
	// read pixel inside the map
	uint p4 = iMod(lcl_id, m_idx, MLO_MAP_WK_SZ);
#else
	uint m_idx = lcl_id / MLO_MAP_WK_SZ;
	uint p4 = lcl_id & (MLO_MAP_WK_SZ - 1);
#endif

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

	gbl_in_off += m_idx * MLO_IN_CHANNEL_STRIDE + p4*MLO_READ_UNIT;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS * MLO_OUT_STACKS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}
	// over all batches

	for (uint b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		uint gbl_in_scan_off = gbl_in_off;
		uint gbl_out_scan_off = gbl_out_off;

		// read input maps
		readData(MLO_N_LCL_IN_MAPS, gbl_in_scan_off, MLO_IN_CHANNEL_STRIDE, MLO_IN_STACKS, (m_idx + c_idx), MLO_N_INPUTS, bot, bot_dat, last_pixel);

		// read output maps into LDS
		readDataFlex(MLO_N_LCL_OUT_MAPS, gbl_out_scan_off, MLO_OUT_CHANNEL_STRIDE, MLO_OUT_STACKS, (k_idx), MLO_N_OUTPUTS, top_df, proc_mem);

		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			/*
			core processing loop
			bot - input
			top - output diff

			do convolution with all available input maps

			*/

			for (uint n = 0; n < MLO_OUT_STACKS; ++n)
			{
				__private _FLOAT pvt_top[MLO_READ_UNIT];
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_top[i] = proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i]; //top_df[gbl_out_scan_off + (k*MLO_OUT_STACKS + n0)*MLO_IN_CHANNEL_STRIDE + i]; //proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
				}

				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c]
							+= bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i];
#if 0

						if (k_idx + k*MLO_OUT_STACKS + n == 0 && c_idx + c * MLO_IN_STACKS + m_idx == 16 && m_idx < MLO_IN_STACKS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS)
						{
							printf("K:s: %d %d %d %d %d %d %d %d %f %f %f %f\n",
								MLO_OUT_STACKS,
								n,
								m_idx,
								gbl_out_scan_off,
								get_group_id(1),
								get_local_id(0),
								k,
								c,
								pvt_accum[(n* MLO_N_LCL_OUT_MAPS + k)*MLO_N_LCL_IN_MAPS + c],
								bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i],
								bot_dat[c*MLO_READ_UNIT + i],
								pvt_top[i]
							);
						}
#endif

					}

				}
			}
		}

	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_MAP_WK_SZ > 8
// transpose
	uint red_base_off = m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;
	for (uint l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		// write data
		red_mem[red_base_off + p4* MLO_ACCUM_SZ + l] = pvt_accum[l];

	}
	barrier(CLK_LOCAL_MEM_FENCE);

	// do final summation

	__private _FLOAT final_sum[1] = { 0 };

#if MLO_ACCUM_SZ & (MLO_ACCUM_SZ-1)
	uint new_m_idx = iDiv(lcl_id, MLO_ACCUM_SZ);
	uint new_p4 = iMod(lcl_id, new_m_idx, MLO_ACCUM_SZ);
#else
	uint new_m_idx = (lcl_id / MLO_ACCUM_SZ);
	uint new_p4 = (lcl_id & (MLO_ACCUM_SZ-1));

#endif

	uint new_red_base_off = new_m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;

	for (uint s = 0; s < MLO_MAP_WK_SZ; ++s)
	{
		final_sum[0] += red_mem[new_red_base_off + s*MLO_ACCUM_SZ + new_p4];
	}

	// write out
	// inputs are outputs
	uint wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (uint)MLO_WEI_BATCH_STRIDE) + ((c_idx + new_m_idx) * MLO_WEI_CHANNEL_STRIDE);

#if MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS-1)

	uint n = iDiv(new_p4, MLO_N_LCL_IN_MAPS);
	uint c = iMod(new_p4, n, MLO_N_LCL_IN_MAPS);
#else

	uint n = (new_p4 / MLO_N_LCL_IN_MAPS);
	uint c = (new_p4 & (MLO_N_LCL_IN_MAPS-1));
#endif

	uint k = 0;

	if (new_m_idx < MLO_IN_STACKS &&k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + new_m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
	{
		weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = final_sum[0];
	}
#else

#if 0 //MLO_IN_WIDTH > 24
// logar reduction
	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		red_mem[i] = 0;
	}

	int red_base_off = (m_idx >= MLO_OUT_STACKS) ? MLO_LCL_MEM_SZ : m_idx * MLO_POW2_MAP_WK_SZ;
	// final summation over each filter row
	for (uint l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// write data
		red_mem[red_base_off + p4] = pvt_accum[l];

		// barrier inside
		ReduceKernel(&red_mem[red_base_off], &pvt_accum[l], p4, p4, MLO_POW2_MAP_WK_SZ, 1, false);

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#else
	// direct reduction
	for (uint l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		lcl_mem[lcl_id] = pvt_accum[l];
		barrier(CLK_LOCAL_MEM_FENCE);
		if (p4 == 0)
		{
			for (uint i = 1; i < MLO_MAP_WK_SZ; ++i)
			{
				pvt_accum[l] += lcl_mem[lcl_id + i];
			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

#endif

	// write out
	// inputs are outputs
	uint wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + m_idx) * MLO_WEI_CHANNEL_STRIDE);

	if (p4 == 0)
	{
		for (uint n = 0; n < MLO_OUT_STACKS  ; ++n)
		{
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if (m_idx < MLO_IN_STACKS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
					{
						weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}
	}

#endif
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * __restrict weight_df_tmp,
	__global _FLOAT * __restrict weights_df
)
{
	uint gbl_id = get_global_id(0);
	uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
	uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	uint wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
	uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
	uint wei_idx = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	uint batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (uint i = 0; i < batch_loop; ++i)
	{
		for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#ifndef MLO_N_PIXS_OFF
#define MLO_N_PIXS_OFF 0
#endif

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, UNUSED bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readData(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * __restrict g_data, __private _FLOAT * __restrict p_data, bool last_pixel)
{
	for (int j = 0; j < n; ++j)
	{
		int gbl_data_off0 = (j*map_stride + map_base < map_limit) ? gbl_data_off + j*map_stride*gbl_data_stride : 0;

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = 0;
			}

		}
		else
#else
		(void)last_pixel;
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}
		}

	} // for (int j = 0; j < n; ++j)

}

__attribute__((always_inline))
void readDataFlex(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * __restrict g_data, __local _FLOAT * __restrict l_data)
{

	for( int l = get_local_id(0); l < n*map_stride* MLO_MAP_WK_SZ; l += MLO_GRP_SZ)
	{

	    int r = 0;
		int k0 = l;
#if MLO_N_LCL_OUT_MAPS > 1
#if (map_stride* MLO_MAP_WK_SZ) &  (map_stride*MLO_MAP_WK_SZ - 1)
		r = iDiv(l, map_stride* MLO_MAP_WK_SZ);  // maps row
		k0 = iMod(l, r, map_stride* MLO_MAP_WK_SZ);
#else
		r = ((uint)l / (map_stride* MLO_MAP_WK_SZ));  // maps row
		k0 = ((uint)l & (map_stride* MLO_MAP_WK_SZ-1));

#endif
#endif

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		int k = iDiv(k0, MLO_MAP_WK_SZ);  // map column
		int p4 = iMod(k0, k, MLO_MAP_WK_SZ); // pixel block
#else
		int k = ((uint)k0 / MLO_MAP_WK_SZ);  // map column
		int p4 = ((uint)k0 & (MLO_MAP_WK_SZ-1)); // pixel block

#endif

		int gbl_data_off0 = (r*map_stride + k + map_base < map_limit) ? gbl_data_off + (r*map_stride + k)*gbl_data_stride + p4*MLO_READ_UNIT : 0;
		__private _FLOAT p_data[MLO_READ_UNIT];

#if MLO_N_PIXS_OFF > 0

		bool last_pixel = (p4 == MLO_MAP_WK_SZ -1);
		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = 0;
			}

		}
		else
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = g_data[gbl_data_off0 + i];
			}
		}

		for(int i = 0; i < MLO_READ_UNIT; ++i)
		{
			l_data[(k*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i] = p_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_IN_STACKS per group, MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// loop over MLO_OUT_STACKS of output

// convolve with MLO_N_LCL_IN_MAPS per wk-item

// reduce with transform

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
#if MLO_MAP_WK_SZ > 8
	__local _FLOAT * red_mem = lcl_mem;
#endif
	__local _FLOAT * proc_mem = lcl_mem;

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_IN_STACKS * MLO_N_LCL_IN_MAPS); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	// map id inside group
	int m_idx = iDiv(lcl_id, MLO_MAP_WK_SZ);
	// read pixel inside the map
	int p4 = iMod(lcl_id, m_idx, MLO_MAP_WK_SZ);

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

	gbl_in_off += m_idx * MLO_IN_CHANNEL_STRIDE + p4*MLO_READ_UNIT;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS * MLO_OUT_STACKS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}
	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input maps
		readData(MLO_N_LCL_IN_MAPS, gbl_in_scan_off, MLO_IN_CHANNEL_STRIDE, MLO_IN_STACKS, (m_idx + c_idx), MLO_N_INPUTS, bot, bot_dat, last_pixel);

		// read output maps into LDS
		readDataFlex(MLO_N_LCL_OUT_MAPS, gbl_out_scan_off, MLO_OUT_CHANNEL_STRIDE, MLO_OUT_STACKS, (k_idx), MLO_N_OUTPUTS, top_df, proc_mem);

		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			/*
			core processing loop
			bot - input
			top - output diff

			do convolution with all available input maps

			*/

			for (int n = 0; n < MLO_OUT_STACKS; ++n)
			{
				__private _FLOAT pvt_top[MLO_READ_UNIT];
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_top[i] = proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i]; //top_df[gbl_out_scan_off + (k*MLO_OUT_STACKS + n0)*MLO_IN_CHANNEL_STRIDE + i]; //proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
				}

				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c]
							+= bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i];
#if 0

						if (k_idx + k*MLO_OUT_STACKS + n == 0 && c_idx + c * MLO_IN_STACKS + m_idx == 16 && m_idx < MLO_IN_STACKS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS)
						{
							printf("K:s: %d %d %d %d %d %d %d %d %f %f %f %f\n",
								MLO_OUT_STACKS,
								n,
								m_idx,
								gbl_out_scan_off,
								get_group_id(1),
								get_local_id(0),
								k,
								c,
								pvt_accum[(n* MLO_N_LCL_OUT_MAPS + k)*MLO_N_LCL_IN_MAPS + c],
								bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i],
								bot_dat[c*MLO_READ_UNIT + i],
								pvt_top[i]
							);
						}
#endif

					}

				}
			}
		}

	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_MAP_WK_SZ > 8
// transpose
	int red_base_off = m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		// write data
		red_mem[red_base_off + p4* MLO_ACCUM_SZ + l] = pvt_accum[l];

	}
	barrier(CLK_LOCAL_MEM_FENCE);

	// do final summation

	__private _FLOAT final_sum[1] = { 0 };

#if MLO_ACCUM_SZ & (MLO_ACCUM_SZ-1)
	int new_m_idx = iDiv(lcl_id, MLO_ACCUM_SZ);
	int new_p4 = iMod(lcl_id, new_m_idx, MLO_ACCUM_SZ);
#else
	int new_m_idx = ((uint)lcl_id / MLO_ACCUM_SZ);
	int new_p4 = ((uint)lcl_id & (MLO_ACCUM_SZ-1));

#endif

	int new_red_base_off = new_m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;

	for (int s = 0; s < MLO_MAP_WK_SZ; ++s)
	{
		final_sum[0] += red_mem[new_red_base_off + s*MLO_ACCUM_SZ + new_p4];
	}

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + new_m_idx) * MLO_WEI_CHANNEL_STRIDE);

#if MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS-1)

	int n = iDiv(new_p4, MLO_N_LCL_IN_MAPS);
	int c = iMod(new_p4, n, MLO_N_LCL_IN_MAPS);
#else

	int n = ((uint)new_p4 / MLO_N_LCL_IN_MAPS);
	int c = ((uint)new_p4 & (MLO_N_LCL_IN_MAPS-1));
#endif

	int k = 0;

	if (new_m_idx < MLO_IN_STACKS &&k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + new_m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
	{
		weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = final_sum[0];
	}
#else

#if 0 //MLO_IN_WIDTH > 24
// logar reduction
	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		red_mem[i] = 0;
	}

	int red_base_off = (m_idx >= MLO_OUT_STACKS) ? MLO_LCL_MEM_SZ : m_idx * MLO_POW2_MAP_WK_SZ;
	// final summation over each filter row
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// write data
		red_mem[red_base_off + p4] = pvt_accum[l];

		// barrier inside
		ReduceKernel(&red_mem[red_base_off], &pvt_accum[l], p4, p4, MLO_POW2_MAP_WK_SZ, 1, false);

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#else
	// direct reduction
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		lcl_mem[lcl_id] = pvt_accum[l];
		barrier(CLK_LOCAL_MEM_FENCE);
		if (p4 == 0)
		{
			for (int i = 1; i < MLO_MAP_WK_SZ; ++i)
			{
				pvt_accum[l] += lcl_mem[lcl_id + i];
			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

#endif

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + m_idx) * MLO_WEI_CHANNEL_STRIDE);

	if (p4 == 0)
	{
		for (int n = 0; n < MLO_OUT_STACKS  ; ++n)
		{
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if (m_idx < MLO_IN_STACKS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS)
					{
						weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}
	}

#endif
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		for (int j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (int j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}


removed some thread sync points

removed some thread sync points

#if __OPENCL_VERSION__ == CL_VERSION_1_1
    #pragma OPENCL EXTENSION cl_khr_fp64 : enable
#endif
#define NVIDIA_WARP_SIZE 32
#define MAPPING  __local double mapping[4]; mapping[0]=0.0; mapping[1]=9.0; mapping[2]=1.0; mapping[3]=2.0;

__kernel void reset_x(
	__global double * x,
	const int n,
	const double val
) {
	int i = get_global_id(0);
	if (i < n){
		x[i] = val;
	}
	return;
}

__kernel void compute_xt_times_vector(
	const int observations,
	const int snps,
	const int subject_chunks,
	const int packedstride_snpmajor,
	const int max_work_group_size,
	__global const char * packedgeno_snpmajor,
	__global double * Xt_vec_chunks,
	__global const double * vec,
	__global const double * means,
	__global const double * precisions,
	__global const long * mask_n,
	__local  double * local_floatgeno
){
    // initialize lookup table
    MAPPING

    // extract thread information
    int subject_chunk = get_group_id(0);
    int snp           = get_group_id(1);
    int threadindex   = get_local_id(0);

    // pull current snp mean, prec from input
    double mean      = means[snp];
    double precision = precisions[snp];

    // initialize values of floating point and compressed genotype arrays
    local_floatgeno[threadindex] = 0.0;

    // synchronize threads in local workgroup
    // barrier(CLK_LOCAL_MEM_FENCE);

    // index of current subject?
    int subject_index = subject_chunk * max_work_group_size + threadindex;

    // now query current value of vector y
    // also save the bitmask value
    double y = vec[subject_index]
    int mask = mask_n[subject_index]

    // ensure that subject is in set of observations
    // this is a standard bounds check with OpenCL kernels
    if (subject_index < observations){

        // decompress genotypes into local floating point arrays
        // decompressed genotypes are in local_floatgeno
        int row = subject_chunk * max_work_group_size + threadindex;
        int k = 2*(row & 3);
        char genotype_block = packedgeno_snpmajor[snp * packedstride_snpmajor + (row >> 2)];
        int val  = (((int)genotype_block)>>k) & 3;
        //local_floatgeno[threadindex] = mapping[val];
        double geno = mapping[val];

        // synchronize threads in local workgroup
        // barrier(CLK_LOCAL_MEM_FENCE);

        // now translate and standardize the decompressed genotype to the correct floating point genotype
        // if missing then set to 0.0, otherwise standardize on fly using mean and precision
        // in this process, will multiply against value at correct index of vector
        // in essence, this is the X'*y part
        // local_floatgeno[threadindex] = (local_floatgeno[threadindex] == 9.0 || mask_n[subject_index] == 0) ? 0.0 : (local_floatgeno[threadindex] - mean) * precision * vec[subject_index];
        // try storing this as a local variable too
        local_floatgeno[threadindex] = (geno == 9.0 || mask == 0) ? 0.0 : (geno - mean) * precision * y;

        // synchronize threads in local workgroup
        barrier(CLK_LOCAL_MEM_FENCE);

        // reduce across local workgroup
        // GPUs that are compliant with OpenCL 2.0 can replace this loop with work_group_reduce(local_floatgeno)
        for(int s = max_work_group_size / 2; s > 0; s >>= 1) {
            if (threadindex < s) {
                local_floatgeno[threadindex] += local_floatgeno[threadindex + s];
            }

            // synchronize threads in local workgroup
            barrier(CLK_LOCAL_MEM_FENCE);
        }

        if(threadindex == 0){
            Xt_vec_chunks[snp * subject_chunks + subject_chunk] += local_floatgeno[0];
        }
    }
    return;
}

__kernel void reduce_xt_vec_chunks(
    const int observations,
    const int subject_chunks,
    const int chunk_clusters,
    const int max_work_group_size,
    __global double * Xt_vec_chunks,
    __global double * Xt_vec,
    __local  double * local_xt
){
    // get SNP and thread indices
    int snp = get_group_id(1);
    int threadindex = get_local_id(0);

    // initialize local memory array with components equal to 0.0
    local_xt[threadindex] = 0.0;

    // synchronize threads in local workgroup
    //barrier(CLK_LOCAL_MEM_FENCE);

    // spawn computations for each chunk
    for (int chunk_cluster = 0; chunk_cluster < chunk_clusters; ++chunk_cluster){

        int subject_chunk = chunk_cluster * max_work_group_size + threadindex;

        if (subject_chunk < subject_chunks){

            // this accumuates chunks of X' times vector
            local_xt[threadindex] += Xt_vec_chunks[snp * subject_chunks + subject_chunk];
        }

        // synchronize threads in local workgroup
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    // now reduce across the workgroup
    // in OpenCL 2.0, can use work_group_reduce(local_xt)
    for (int s = max_work_group_size / 2; s > 0; s >>= 1) {
        if (threadindex < s) {
            local_xt[threadindex] += local_xt[threadindex + s];
        }

        // synchronize threads in local workgroup
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    // topmost thread assigns reduction across workgroup to correct entry in output vector
    if ( threadindex == 0){
        Xt_vec[snp] = local_xt[0];
    }
    return;
}

#if __OPENCL_VERSION__ == CL_VERSION_1_1
    #pragma OPENCL EXTENSION cl_khr_fp64 : enable
#endif
#define NVIDIA_WARP_SIZE 32
#define MAPPING  __local double mapping[4]; mapping[0]=0.0; mapping[1]=9.0; mapping[2]=1.0; mapping[3]=2.0;

__kernel void reset_x(
	__global double * x,
	const int n,
	const double val
) {
	int i = get_global_id(0);
	if (i < n){
		x[i] = val;
	}
	return;
}

__kernel void compute_xt_times_vector(
	const int observations,
	const int snps,
	const int subject_chunks,
	const int packedstride_snpmajor,
	const int max_work_group_size,
	__global const char * packedgeno_snpmajor,
	__global double * Xt_vec_chunks,
	__global const double * vec,
	__global const double * means,
	__global const double * precisions,
	__global const long * mask_n,
	__local  double * local_floatgeno
){
	// initialize lookup table
    MAPPING

	// extract thread information
    int subject_chunk = get_group_id(0);
    int snp           = get_group_id(1);
    int threadindex   = get_local_id(0);

	// pull current snp mean, prec from input
	double mean      = means[snp];
	double precision = precisions[snp];

	// initialize values of floating point and compressed genotype arrays
    local_floatgeno[threadindex] = 0.0;

    // synchronize threads in local workgroup
	barrier(CLK_LOCAL_MEM_FENCE);

	// index of current subject?
	int subject_index = subject_chunk * max_work_group_size + threadindex;

    // ensure that subject is in set of observations
	// this is a standard bounds check with OpenCL kernels
    if (subject_index < observations){

        // decompress genotypes into local floating point arrays
        // decompressed genotypes are in local_floatgeno
		int row = subject_chunk * max_work_group_size + threadindex;
		int k = 2*(row & 3);
		char genotype_block = packedgeno_snpmajor[snp * packedstride_snpmajor + (row >> 2)];
		int val  = (((int)genotype_block)>>k) & 3;
		local_floatgeno[threadindex] = mapping[val];

        // synchronize threads in local workgroup
        barrier(CLK_LOCAL_MEM_FENCE);

        // now translate and standardize the decompressed genotype to the correct floating point genotype
        // if missing then set to 0.0, otherwise standardize on fly using mean and precision
		// in this process, will multiply against value at correct index of vector
		// in essence, this is the X'*y part
        local_floatgeno[threadindex] = (local_floatgeno[threadindex] == 9.0 || mask_n[subject_index] == 0) ? 0.0 : (local_floatgeno[threadindex] - mean) * precision * vec[subject_index];

        // synchronize threads in local workgroup
        barrier(CLK_LOCAL_MEM_FENCE);

        // reduce across local workgroup
//        for(unsigned int s = max_work_group_size / 2; s > 0; s >>= 1) {
        for(int s = max_work_group_size / 2; s > 0; s >>= 1) {
            if (threadindex < s) {
                local_floatgeno[threadindex] += local_floatgeno[threadindex + s];
            }

            // synchronize threads in local workgroup
            barrier(CLK_LOCAL_MEM_FENCE);
        }

        if(threadindex == 0){
            Xt_vec_chunks[snp * subject_chunks + subject_chunk] += local_floatgeno[0];
        }
    }
    return;
}

__kernel void reduce_xt_vec_chunks(
    const int observations,
    const int subject_chunks,
    const int chunk_clusters,
	const int max_work_group_size,
    __global double * Xt_vec_chunks,
    __global double * Xt_vec,
    __local  double * local_xt
){
	// get SNP and thread indices
    int snp = get_group_id(1);
    int threadindex = get_local_id(0);

	// initialize local memory array with components equal to 0.0
    local_xt[threadindex] = 0.0;

    // synchronize threads in local workgroup
    barrier(CLK_LOCAL_MEM_FENCE);

    // spawn computations for each chunk
    for (int chunk_cluster = 0; chunk_cluster < chunk_clusters; ++chunk_cluster){

        int subject_chunk = chunk_cluster * max_work_group_size + threadindex;

        if (subject_chunk < subject_chunks){

            // this accumuates chunks of X' times vector
            local_xt[threadindex] += Xt_vec_chunks[snp * subject_chunks + subject_chunk];
        }

		// synchronize threads in local workgroup
		barrier(CLK_LOCAL_MEM_FENCE);
    }

    // now reduce across the workgroup
//    for (unsigned int s = max_work_group_size / 2; s > 0; s >>= 1) {
    for (int s = max_work_group_size / 2; s > 0; s >>= 1) {
        if (threadindex < s) {
            local_xt[threadindex] += local_xt[threadindex + s];
        }

        // synchronize threads in local workgroup
        barrier(CLK_LOCAL_MEM_FENCE);
    }

	// topmost thread assigns reduction across workgroup to correct entry in output vector
    if ( threadindex == 0){
        Xt_vec[snp] = local_xt[0];
    }
	return;
}


Moved to kernel directory.

Moved to kernel directory.

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@3029 7c0444ca-5550-0410-9008-faecdbe78eb5

//#include <OpenCL/opencl.h>

#include <stdio.h>

//
// update the state of an LIF layer
//

//
//// 4. change if () guard so that it uses kx and ky

__kernel void update_state (
    __global float * V,
    __global float * G_E,
    __global float * G_I,
    __global float * G_IB,
    __global float * phi,
    __global float * activity,
    const uint nx,
    const uint ny,
    const uint nf,
    const uint nPad)
{
   float tauInf, VmemInf;
   float expExcite, expInhib, expInhibB;
   float dt, tau;
   float Vrest, Vexc, Vinh, VinhB;

   //ADD DT, HARD CODE IN VARIABLES, FIX stuff, implement and check by hand, and check boundaries

   const uint kx = get_global_id(0);
   const uint ky = get_global_id(1);
   const uint k  = kx + nx*ky;

   float one = 1.1f;

   VmemInf = 20.0f;
   //tauInf = 3.141f;

   //Hard Coded Variables
   tau = 20.0;
   Vrest = -70.0;
   Vexc = 0.0;
   Vinh = -75.0;
   VinhB = -90.0;

   dt = 1.0f;
   tau = 1.0f;
   Vrest = 2.0f;
   Vexc = 2.0f;
   Vinh = 2.0f;
   VinhB = 2.0f;

   expExcite = 1.0f;
   expInhib = 2.0f;
   expInhibB = 2.0f;

   if (kx < nx && ky < ny) {
      //printf ("%f %f %f\n", kx, ky, k);
      G_E[k]  = 1 + G_E[k] ;phiExc[k] + G_E[k]  * expExcite;
      G_I[k]  = phiInh[k]; //phiInh[k] + G_I[k]  * expInhib;
      G_IB[k] = phiInhB[k] + G_IB[k] * expInhibB;
      tauInf  = (dt / tau) * (1 + G_E[k] + G_I[k] + G_IB[k]);
      VmemInf = ( Vrest + G_E[k] * Vexc + G_I[k] * Vinh + G_IB[k] * VinhB )
                / (1.0 + G_E[k] + G_I[k] + G_IB[k]);
      V[k] = VmemInf + (V[k] - VmemInf) * exp(-tauInf);

      //printf("G_I[%d]=%f\n", k, G_I[k]);
      //printf("kx=%f\n", kx);
      //printf("nx=%f\n", nx);

//      V[k] = 8.0f;
   }
}