2x4 works

2x4 works

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS (MLO_IN_WIDTH)
#define MLO_PER_WAVE_READ ((MLO_IN_WIDTH + MLO_N_WAVES - 1) / MLO_N_WAVES)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_PER_WAVE_READ * MLO_N_WAVES)
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)

#define MLO_IN_VERT_READS (1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_READS*MLO_READ_UNIT - MLO_N_IN_HORIZ_PIX_READS)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ ((MLO_N_WAVES - 1) * MLO_HW_WAVE_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void ReduceKernel64(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
	// add second half
	// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

inline void readInput(int lcl_id, int c_idx, int sc, int gbl_in_scan_off, __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int c_scan = sc;

		int c = iDiv(p4, (MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS));
		int c_pix4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS*MLO_IN_VERT_READS));

		if (c_idx + c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_READ_UNIT - 1; i >= MLO_READ_UNIT - MLO_IN_N_PIXS_OFF; --i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_pix4*MLO_READ_UNIT];
			}

// stack of inputs, each has 1 line
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
#if 0
				if (c == 0 && c_scan ==1 && c_pix4 == 0)
				{
					printf("K:g: %d %d %d %d %f\n",
						lcl_id,
						c_scan,
						i,
						lcl_in_off,
						lcl_bot[lcl_in_off]
					);
				}
#endif
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output line line on number of spans by number of waves
// read MLO_FILTER_SIZE1 number of such spans into SGPS (for example 5 *7 = 35)
// read 1 input line for 64 maps into LDS
//
// alg
//
// accumulate 1 scan of data per wave per each input map per wk-item

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private int top_wave_base[MLO_N_WAVES];

	for (int i = 0; i < MLO_N_WAVES; ++i)
	{
		top_wave_base[i] = i * MLO_PER_WAVE_READ;
	}

	__private _FLOAT bot_dat[MLO_PER_WAVE_READ*MLO_FILTER_SIZE1];

	for (int i = 0; i < MLO_PER_WAVE_READ*MLO_FILTER_SIZE1; ++i)
	{
		bot_dat[i] = 0;
	}

	__private _FLOAT pvt_accum[(MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)];

	for (int i = 0; i < (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

//	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	if (wei_tl_idx == 4 && o_idx == 0 && c_idx == 0 && (dat_tl_idx == 0 || dat_tl_idx == 1))
	{
		printf("G:p: %d %d %d %d %d\n",
			lcl_id,
			wei_tl_idx1,
			wei_tl_idx0,
			wei_tl1,
			wei_tl0
		);
	}

#endif

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;
		// over all out blocks
		// processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

//		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch output
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j, ++out_y)
		{
			int top_df_off = (out_y < MLO_OUT_HEIGHT) ? gbl_out_scan_off + j * MLO_IN_STRIDE + top_wave_base[wave_id] : 0;
			_FLOAT mask = (out_y < MLO_OUT_HEIGHT) ? 1 : 0;
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				_FLOAT top_val = top_df[top_df_off + i] * mask;
				bot_dat[j*MLO_PER_WAVE_READ + i] = top_val;
			}
		}

		// prolog
		// handling padding

		int sc = 0;
// pad0
		for (; sc < 1; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);
// processing
			for (int l = MLO_FILTER_PAD1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:ap0: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
									* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
								);
						}
#endif

					}

				}

			}

		}

// pad1
		for (; sc < 2 && sc < MLO_OUT_HEIGHT; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_PAD1 + 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:ap1: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			}

		}

// generic

		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE, ++out_y)
		{

			int top_df_off = (out_y < MLO_OUT_HEIGHT) ? gbl_out_scan_off + (sc + MLO_FILTER_PAD1) * MLO_IN_STRIDE + top_wave_base[wave_id] : 0;
			_FLOAT mask = (out_y < MLO_OUT_HEIGHT) ? 1 : 0;
			// move in the last output scan
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				bot_dat[(MLO_FILTER_SIZE1 - 1) *MLO_PER_WAVE_READ + i] = top_df[top_df_off + i] * mask;
			}

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}
// move up output
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
				{
					bot_dat[j*MLO_PER_WAVE_READ + i] = bot_dat[(j+1)*MLO_PER_WAVE_READ + i];
				}
			}

			barrier(CLK_LOCAL_MEM_FENCE);
		}

// epilog
// handling padding
// pad1
		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a0p: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			} // for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

// pad0
		for (; sc < MLO_OUT_HEIGHT; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			// processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a1p: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

// final summation
	for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
	{
		for (int n = 0; n < MLO_FILTER_SIZE0 && wave_id > 0; ++n)
		{
			lcl[((wave_id - 1) * MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n] =
				pvt_accum[l*MLO_FILTER_SIZE0 + n];
#if 0
			if (lcl_wv_id == 0 && l == 2 && n == 3)
			{
				printf("G:s1: %f\n",
					pvt_accum[l*MLO_FILTER_SIZE0 + n]
				);
			}
#endif

		}

		barrier(CLK_LOCAL_MEM_FENCE);

		for(int w = 0; w < MLO_N_WAVES-1 && wave_id == 0; ++w)
		{

			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				pvt_accum[l*MLO_FILTER_SIZE0 + n]
					+= lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n];
#if 0
				if (lcl_wv_id == 0 && l == 2 && n == 3)
				{
					printf("G:s2: %f %f\n",
						pvt_accum[l*MLO_FILTER_SIZE0 + n],
						lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
	}

// output
// inputs are outputs
	int c = lcl_wv_id;

	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24((c_idx + c), (int)MLO_WEI_CHANNEL_STRIDE);
	if (wave_id == 0)
	{
		for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
		{
			weights_df[wei_df_off + i] = pvt_accum[i];
		}

	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS (MLO_IN_WIDTH)
#define MLO_PER_WAVE_READ ((MLO_IN_WIDTH + MLO_N_WAVES - 1) / MLO_N_WAVES)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_PER_WAVE_READ * MLO_N_WAVES)
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)

#define MLO_IN_VERT_READS (1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_READS*MLO_READ_UNIT - MLO_N_IN_HORIZ_PIX_READS)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ ((MLO_N_WAVES - 1) * MLO_HW_WAVE_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void ReduceKernel64(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
	// add second half
	// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

inline void readInput(int lcl_id, int c_idx, int sc, int gbl_in_scan_off, __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int c_scan = sc;

		int c = iDiv(p4, (MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS));
		int c_pix4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS*MLO_IN_VERT_READS));

		if (c_idx + c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_READ_UNIT - 1; i >= MLO_READ_UNIT - MLO_IN_N_PIXS_OFF; --i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
			}

			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
#if 1
				if (c == 38 && c_pix4 == 0)
				{
					printf("K:g: %d %d %d %d %f\n",
						lcl_id,
						c_scan,
						i,
						lcl_in_off,
						lcl_bot[lcl_in_off]
					);
				}
#endif
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output line line on number of spans by number of waves
// read MLO_FILTER_SIZE1 number of such spans into SGPS (for example 5 *7 = 35)
// read 1 input line for 64 maps into LDS
//
// alg
//
// accumulate 1 scan of data per wave per each input map per wk-item

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private int top_wave_base[MLO_N_WAVES];

	for (int i = 0; i < MLO_N_WAVES; ++i)
	{
		top_wave_base[i] = i * MLO_PER_WAVE_READ;
	}

	__private _FLOAT bot_dat[MLO_PER_WAVE_READ*MLO_FILTER_SIZE1];

	for (int i = 0; i < MLO_PER_WAVE_READ*MLO_FILTER_SIZE1; ++i)
	{
		bot_dat[i] = 0;
	}

	__private _FLOAT pvt_accum[(MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)];

	for (int i = 0; i < (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

//	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	if (wei_tl_idx == 4 && o_idx == 0 && c_idx == 0 && (dat_tl_idx == 0 || dat_tl_idx == 1))
	{
		printf("G:p: %d %d %d %d %d\n",
			lcl_id,
			wei_tl_idx1,
			wei_tl_idx0,
			wei_tl1,
			wei_tl0
		);
	}

#endif

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;
		// over all out blocks
		// processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

//		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch output
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j, ++out_y)
		{
			int top_df_off = (out_y < MLO_OUT_HEIGHT) ? gbl_out_scan_off + j * MLO_IN_STRIDE + top_wave_base[wave_id] : 0;
			_FLOAT mask = (out_y < MLO_OUT_HEIGHT) ? 1 : 0;
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				_FLOAT top_val = top_df[top_df_off + i] * mask;
				bot_dat[j*MLO_PER_WAVE_READ + i] = top_val;
			}
		}

		// prolog
		// handling padding

		int sc = 0;
// pad0
		for (; sc < 1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);
// processing
			for (int l = MLO_FILTER_PAD1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
							* bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m];
#if 1
						if (lcl_wv_id == 38 && l == 2 && n == 2)
						{
							printf("G:ap0: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
									* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
								);
						}
#endif

					}

				}

			}

		}

// pad1
		for (; sc < 2 && sc < MLO_OUT_HEIGHT; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_PAD1 + 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
							* bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:ap1: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			}

		}

// generic

		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE, ++out_y)
		{

			int top_df_off = (out_y < MLO_OUT_HEIGHT) ? gbl_out_scan_off + (sc + MLO_FILTER_PAD1) * MLO_IN_STRIDE + top_wave_base[wave_id] : 0;
			_FLOAT mask = (out_y < MLO_OUT_HEIGHT) ? 1 : 0;
			// move in the last output scan
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				bot_dat[(MLO_FILTER_SIZE1 - 1) *MLO_PER_WAVE_READ + i] = top_df[top_df_off + i] * mask;
			}

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
							* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}
// move up output
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
				{
					bot_dat[j*MLO_PER_WAVE_READ + i] = bot_dat[(j+1)*MLO_PER_WAVE_READ + i];
				}
			}

			barrier(CLK_LOCAL_MEM_FENCE);

		}

// epilog
// handling padding
// pad1
		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
							* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a0p: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			} // for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

// pad0
		for (; sc < MLO_OUT_HEIGHT; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			// processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
							* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a1p: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

// final summation
	for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
	{
		for (int n = 0; n < MLO_FILTER_SIZE0 && wave_id > 0; ++n)
		{
			lcl[((wave_id - 1) * MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n] =
				pvt_accum[l*MLO_FILTER_SIZE0 + n];
#if 1
			if (lcl_wv_id == 38 && l == 2 && n == 2)
			{
				printf("G:s1: %f\n",
					pvt_accum[l*MLO_FILTER_SIZE0 + n]
				);
			}
#endif

		}

		barrier(CLK_LOCAL_MEM_FENCE);

		for(int w = 0; w < MLO_N_WAVES-1 && wave_id == 0; ++w)
		{

			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				pvt_accum[l*MLO_FILTER_SIZE0 + n]
					+= lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n];
#if 1
				if (lcl_wv_id == 38 && l == 2 && n == 2)
				{
					printf("G:s2: %f %f\n",
						pvt_accum[l*MLO_FILTER_SIZE0 + n],
						lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
	}

// output
// inputs are outputs
	int c = lcl_wv_id;

	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24((c_idx + c), (int)MLO_WEI_CHANNEL_STRIDE);
	if (wave_id == 0)
	{
		for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
		{
			weights_df[wei_df_off + i] = pvt_accum[i];
		}

	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}


Properly remove this test file, that I copied over to test/SemaOpenCL in r173352.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@173353 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -x cl -O1 -emit-llvm  %s -o - -triple x86_64-linux-gnu | FileCheck %s
// OpenCL essentially reduces all shift amounts to the last word-size bits before evaluating.
// Test this both for variables and constants evaluated in the front-end.

//CHECK: @array0 = common global [256 x i8]
char array0[((int)1)<<40];
//CHECK: @array1 = common global [256 x i8]
char array1[((int)1)<<(-24)];

//CHECK: @negativeShift32
int negativeShift32(int a,int b) {
  //CHECK: ret i32 65536
  return ((int)1)<<(-16);
}


Fix issue #621 - kernel/test_fabs ldexp fails on i386

Applied fix-test_fabs-ldexp.patch from Andreas Beckmann <anbe@debian.org>

// TESTING: copysign
// TESTING: fabs
// TESTING: isfinite
// TESTING: isinf
// TESTING: isnan
// TESTING: isnormal
// TESTING: signbit
// TESTING: ilogb
// TESTING: ldexp

typedef constant char* string;

#ifdef cl_khr_fp64
#pragma OPENCL EXTENSION cl_khr_fp64 : enable
#endif

#define IMPLEMENT_BODY_V(NAME, BODY, SIZE,                              \
                         VTYPE, STYPE, ITYPE, SITYPE, JTYPE, SJTYPE)    \
  void NAME##_##VTYPE()                                                 \
  {                                                                     \
    typedef VTYPE vtype;                                                \
    typedef STYPE stype;                                                \
    typedef ITYPE itype;                                                \
    typedef SITYPE sitype;                                              \
    typedef JTYPE jtype;                                                \
    typedef SJTYPE sjtype;                                              \
    string const typename = #VTYPE;                                     \
    const int vecsize = SIZE;                                           \
    BODY;                                                               \
  }
#define DEFINE_BODY_V(NAME, EXPR)                                       \
  IMPLEMENT_BODY_V(NAME, EXPR,  1, float   , float , int  , int, int   , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  2, float2  , float , int2 , int, int2  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  3, float3  , float , int3 , int, int3  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  4, float4  , float , int4 , int, int4  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  8, float8  , float , int8 , int, int8  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR, 16, float16 , float , int16, int, int16 , int ) \
  __IF_FP64(                                                            \
  IMPLEMENT_BODY_V(NAME, EXPR,  1, double  , double, int  , int, long  , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  2, double2 , double, int2 , int, long2 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  3, double3 , double, int3 , int, long3 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  4, double4 , double, int4 , int, long4 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  8, double8 , double, int8 , int, long8 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR, 16, double16, double, int16, int, long16, long))

#define CALL_FUNC_V(NAME)                       \
  NAME##_float   ();                            \
  NAME##_float2  ();                            \
  NAME##_float3  ();                            \
  NAME##_float4  ();                            \
  NAME##_float8  ();                            \
  NAME##_float16 ();                            \
  __IF_FP64(                                    \
  NAME##_double  ();                            \
  NAME##_double2 ();                            \
  NAME##_double3 ();                            \
  NAME##_double4 ();                            \
  NAME##_double8 ();                            \
  NAME##_double16();)

#if __has_extension(c_generic_selections) && (__clang_major__ < 6)
 #ifdef cl_khr_fp64
 # define is_floating(T) _Generic((T)0, float: 1, double: 1, default: 0)
 #else
 # define is_floating(T) _Generic((T)0, float: 1, default: 0)
 #endif
#else
# define is_floating(T) 1
#endif
#define is_signed(T)   ((T)-1 < (T)+1)
#define count_bits(T)  (CHAR_BIT * sizeof(T))

#define ISNAN(x) (isnan(x) || as_int((float)(x)) == as_int((float)NAN))
#define ISEQ(x, y) (ISNAN(x) == ISNAN(y) && (ISNAN(x) || (x)==(y)))

DEFINE_BODY_V
(test_fabs,
 ({
   _CL_STATIC_ASSERT(stype, is_floating(stype));
   float const values[] = {
     0.0f,
     0.1f,
     0.9f,
     1.0f,
     1.1f,
     10.0f,
     1000000.0f,
     1000000000000.0f,
     MAXFLOAT,
     HUGE_VALF,
     INFINITY,
     NAN,
     FLT_MAX,
     FLT_MIN,
     FLT_EPSILON,
   };
   int const nvalues = sizeof(values) / sizeof(*values);
   int ninputs = 1;
#ifdef cl_khr_fp64
   double const dvalues[] = {
     0.0,
     0.1,
     0.9,
     1.0,
     1.1,
     10.0,
     1000000.0,
     1000000000000.0,
     1000000000000000000000000.0,
     HUGE_VAL,
     INFINITY,
     NAN,
     DBL_MAX,
     DBL_MIN,
     DBL_EPSILON,
   };
   int const ndvalues = sizeof(dvalues) / sizeof(*dvalues);
   ++ninputs;
#endif

   for (int input=0; input<ninputs; ++input) {
     for (int iter=0; iter<nvalues; ++iter) {
       for (int sign=-1; sign<=+1; sign+=2) {
         typedef union {
           vtype v;
           volatile stype s[16];
           /* s is declared as volatile to force all accesses through memory,
            * avoiding excess precision from x87 fpu registers
            * see https://github.com/pocl/pocl/issues/621, https://gcc.gnu.org/wiki/x87note
            */
         } Tvec;
         Tvec val, good, val2;
         for (int n=0; n<vecsize; ++n) {
           if (input==0) {
             val.s[n]  = sign * values[(iter+n) % nvalues];
             good.s[n] =        values[(iter+n) % nvalues];
             val2.s[n] =        values[(iter+n+1) % nvalues];
           } else {
#ifdef cl_khr_fp64
             val.s[n]  = sign * dvalues[(iter+n) % ndvalues];
             good.s[n] =        dvalues[(iter+n) % ndvalues];
             val2.s[n] =        dvalues[(iter+n+1) % ndvalues];
#endif
           }
         }
         Tvec res;
         bool equal;
         typedef union {
           stype  s;
           sjtype sj;
         } S;
         typedef union {
           itype  v;
           sitype s[16];
         } Ivec;
         typedef union {
           jtype  v;
           sjtype s[16];
         } Jvec;
         /* fabs */
         res.v = fabs(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           S r, g;
           r.s = res.s[n];
           g.s = good.s[n];
           equal = equal && (ISNAN(val.s[n]) || r.sj == g.sj);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: fabs type=%s val=%.17g res=%.17g good=%.17g\n",
                    typename, val.s[n], res.s[n], good.s[n]);
           }
           return;
         }
         /* signbit */
         Jvec jres;
         jres.v = signbit(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             (ISNAN(val.s[n]) ||
              jres.s[n] == (sign>0 ? 0 : vecsize==1 ? +1 : -1));
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: signbit type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (sign>0 ? 0 : vecsize==1 ? +1 : -1));
           }
           return;
         }
         /* isfinite */
         jres.v = isfinite(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isfinite(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isfinite type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isfinite(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* isinf */
         jres.v = isinf(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isinf(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isinf type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isinf(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* isnan */
         jres.v = isnan(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isnan(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isnan type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isnan(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* isnormal */
         jres.v = isnormal(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isnormal(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isnormal type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isnormal(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* copysign */
         for (int sign2=-1; sign2<=+1; sign2+=2) {
           res.v = copysign(val.v, (stype)sign2*val2.v);
           equal = true;
           for (int n=0; n<vecsize; ++n) {
             S r, g;
             r.s = res.s[n];
             g.s = sign2*good.s[n];
             equal = equal &&
               (ISNAN(val.s[n]) || ISNAN(val2.s[n]) || r.sj == g.sj);
           }
           if (!equal) {
             for (int n=0; n<vecsize; ++n) {
               printf("FAIL: copysign type=%s val=%.17g sign=%.17g res=%.17g good=%.17g\n",
                      typename, val.s[n], sign2*val2.s[n], res.s[n], good.s[n]);
             }
             return;
           }
         }
         /* ilogb */
         Ivec ires;
         ires.v = ilogb(val.v);
         Ivec igoodres;
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           if (ISNAN(val.s[n])) {
             igoodres.s[n] = FP_ILOGBNAN;
           } else if (val.s[n] == (stype)0) {
             igoodres.s[n] = FP_ILOGB0;
           } else if (isinf(val.s[n])) {
             igoodres.s[n] = INT_MAX;
           } else {
             // We round down to "correct" for inaccuracies in log2
             // We divide by 2 since log2 is wrong for large inputs
             igoodres.s[n] = 1+rint(floor(0.999999f*log2(0.5*fabs(val.s[n]))));
           }
           equal = equal && ires.s[n] == igoodres.s[n];
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: ilogb type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)ires.s[n], (int)igoodres.s[n]);
           }
           return;
         }
         /* ldexp */
         Ivec ival2;
         ival2.v = (itype)4;
         res.v = ldexp(val.v, ival2.v);
         Tvec goodres;
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           if (ISNAN(val.s[n])) {
             goodres.s[n] = NAN;
           } else if (val.s[n] == (stype)0) {
             goodres.s[n] = val.s[n];
           } else if (isinf(val.s[n])) {
             goodres.s[n] = val.s[n];
           } else {
             goodres.s[n] = val.s[n] * (stype)(16.0);
           }
           equal = equal && ISEQ(res.s[n], goodres.s[n]);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: ldexp type=%s val=%.17g val2=%d res=%.17g good=%.17g\n",
                    typename, val.s[n], (int)ival2.s[n], res.s[n], goodres.s[n]);
           }
           return;
         }
       }
     }
   }
 })
 )

kernel void test_fabs()
{
  CALL_FUNC_V(test_fabs)
}

// TESTING: copysign
// TESTING: fabs
// TESTING: isfinite
// TESTING: isinf
// TESTING: isnan
// TESTING: isnormal
// TESTING: signbit
// TESTING: ilogb
// TESTING: ldexp

typedef constant char* string;

#ifdef cl_khr_fp64
#pragma OPENCL EXTENSION cl_khr_fp64 : enable
#endif

#define IMPLEMENT_BODY_V(NAME, BODY, SIZE,                              \
                         VTYPE, STYPE, ITYPE, SITYPE, JTYPE, SJTYPE)    \
  void NAME##_##VTYPE()                                                 \
  {                                                                     \
    typedef VTYPE vtype;                                                \
    typedef STYPE stype;                                                \
    typedef ITYPE itype;                                                \
    typedef SITYPE sitype;                                              \
    typedef JTYPE jtype;                                                \
    typedef SJTYPE sjtype;                                              \
    string const typename = #VTYPE;                                     \
    const int vecsize = SIZE;                                           \
    BODY;                                                               \
  }
#define DEFINE_BODY_V(NAME, EXPR)                                       \
  IMPLEMENT_BODY_V(NAME, EXPR,  1, float   , float , int  , int, int   , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  2, float2  , float , int2 , int, int2  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  3, float3  , float , int3 , int, int3  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  4, float4  , float , int4 , int, int4  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR,  8, float8  , float , int8 , int, int8  , int ) \
  IMPLEMENT_BODY_V(NAME, EXPR, 16, float16 , float , int16, int, int16 , int ) \
  __IF_FP64(                                                            \
  IMPLEMENT_BODY_V(NAME, EXPR,  1, double  , double, int  , int, long  , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  2, double2 , double, int2 , int, long2 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  3, double3 , double, int3 , int, long3 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  4, double4 , double, int4 , int, long4 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR,  8, double8 , double, int8 , int, long8 , long) \
  IMPLEMENT_BODY_V(NAME, EXPR, 16, double16, double, int16, int, long16, long))

#define CALL_FUNC_V(NAME)                       \
  NAME##_float   ();                            \
  NAME##_float2  ();                            \
  NAME##_float3  ();                            \
  NAME##_float4  ();                            \
  NAME##_float8  ();                            \
  NAME##_float16 ();                            \
  __IF_FP64(                                    \
  NAME##_double  ();                            \
  NAME##_double2 ();                            \
  NAME##_double3 ();                            \
  NAME##_double4 ();                            \
  NAME##_double8 ();                            \
  NAME##_double16();)

#if __has_extension(c_generic_selections) && (__clang_major__ < 6)
 #ifdef cl_khr_fp64
 # define is_floating(T) _Generic((T)0, float: 1, double: 1, default: 0)
 #else
 # define is_floating(T) _Generic((T)0, float: 1, default: 0)
 #endif
#else
# define is_floating(T) 1
#endif
#define is_signed(T)   ((T)-1 < (T)+1)
#define count_bits(T)  (CHAR_BIT * sizeof(T))

#define ISNAN(x) (isnan(x) || as_int((float)(x)) == as_int((float)NAN))
#define ISEQ(x, y) (ISNAN(x) == ISNAN(y) && (ISNAN(x) || (x)==(y)))

DEFINE_BODY_V
(test_fabs,
 ({
   _CL_STATIC_ASSERT(stype, is_floating(stype));
   float const values[] = {
     0.0f,
     0.1f,
     0.9f,
     1.0f,
     1.1f,
     10.0f,
     1000000.0f,
     1000000000000.0f,
     MAXFLOAT,
     HUGE_VALF,
     INFINITY,
     NAN,
     FLT_MAX,
     FLT_MIN,
     FLT_EPSILON,
   };
   int const nvalues = sizeof(values) / sizeof(*values);
   int ninputs = 1;
#ifdef cl_khr_fp64
   double const dvalues[] = {
     0.0,
     0.1,
     0.9,
     1.0,
     1.1,
     10.0,
     1000000.0,
     1000000000000.0,
     1000000000000000000000000.0,
     HUGE_VAL,
     INFINITY,
     NAN,
     DBL_MAX,
     DBL_MIN,
     DBL_EPSILON,
   };
   int const ndvalues = sizeof(dvalues) / sizeof(*dvalues);
   ++ninputs;
#endif

   for (int input=0; input<ninputs; ++input) {
     for (int iter=0; iter<nvalues; ++iter) {
       for (int sign=-1; sign<=+1; sign+=2) {
         typedef union {
           vtype v;
           stype s[16];
         } Tvec;
         Tvec val, good, val2;
         for (int n=0; n<vecsize; ++n) {
           if (input==0) {
             val.s[n]  = sign * values[(iter+n) % nvalues];
             good.s[n] =        values[(iter+n) % nvalues];
             val2.s[n] =        values[(iter+n+1) % nvalues];
           } else {
#ifdef cl_khr_fp64
             val.s[n]  = sign * dvalues[(iter+n) % ndvalues];
             good.s[n] =        dvalues[(iter+n) % ndvalues];
             val2.s[n] =        dvalues[(iter+n+1) % ndvalues];
#endif
           }
         }
         Tvec res;
         bool equal;
         typedef union {
           stype  s;
           sjtype sj;
         } S;
         typedef union {
           itype  v;
           sitype s[16];
         } Ivec;
         typedef union {
           jtype  v;
           sjtype s[16];
         } Jvec;
         /* fabs */
         res.v = fabs(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           S r, g;
           r.s = res.s[n];
           g.s = good.s[n];
           equal = equal && (ISNAN(val.s[n]) || r.sj == g.sj);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: fabs type=%s val=%.17g res=%.17g good=%.17g\n",
                    typename, val.s[n], res.s[n], good.s[n]);
           }
           return;
         }
         /* signbit */
         Jvec jres;
         jres.v = signbit(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             (ISNAN(val.s[n]) ||
              jres.s[n] == (sign>0 ? 0 : vecsize==1 ? +1 : -1));
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: signbit type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (sign>0 ? 0 : vecsize==1 ? +1 : -1));
           }
           return;
         }
         /* isfinite */
         jres.v = isfinite(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isfinite(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isfinite type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isfinite(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* isinf */
         jres.v = isinf(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isinf(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isinf type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isinf(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* isnan */
         jres.v = isnan(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isnan(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isnan type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isnan(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* isnormal */
         jres.v = isnormal(val.v);
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           equal = equal &&
             jres.s[n] == (isnormal(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: isnormal type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)jres.s[n],
                    (isnormal(val.s[n]) ? (vecsize==1 ? +1 : -1) : 0));
           }
           return;
         }
         /* copysign */
         for (int sign2=-1; sign2<=+1; sign2+=2) {
           res.v = copysign(val.v, (stype)sign2*val2.v);
           equal = true;
           for (int n=0; n<vecsize; ++n) {
             S r, g;
             r.s = res.s[n];
             g.s = sign2*good.s[n];
             equal = equal &&
               (ISNAN(val.s[n]) || ISNAN(val2.s[n]) || r.sj == g.sj);
           }
           if (!equal) {
             for (int n=0; n<vecsize; ++n) {
               printf("FAIL: copysign type=%s val=%.17g sign=%.17g res=%.17g good=%.17g\n",
                      typename, val.s[n], sign2*val2.s[n], res.s[n], good.s[n]);
             }
             return;
           }
         }
         /* ilogb */
         Ivec ires;
         ires.v = ilogb(val.v);
         Ivec igoodres;
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           if (ISNAN(val.s[n])) {
             igoodres.s[n] = FP_ILOGBNAN;
           } else if (val.s[n] == (stype)0) {
             igoodres.s[n] = FP_ILOGB0;
           } else if (isinf(val.s[n])) {
             igoodres.s[n] = INT_MAX;
           } else {
             // We round down to "correct" for inaccuracies in log2
             // We divide by 2 since log2 is wrong for large inputs
             igoodres.s[n] = 1+rint(floor(0.999999f*log2(0.5*fabs(val.s[n]))));
           }
           equal = equal && ires.s[n] == igoodres.s[n];
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: ilogb type=%s val=%.17g res=%d good=%d\n",
                    typename, val.s[n], (int)ires.s[n], (int)igoodres.s[n]);
           }
           return;
         }
         /* ldexp */
         Ivec ival2;
         ival2.v = (itype)4;
         res.v = ldexp(val.v, ival2.v);
         Tvec goodres;
         equal = true;
         for (int n=0; n<vecsize; ++n) {
           if (ISNAN(val.s[n])) {
             goodres.s[n] = NAN;
           } else if (val.s[n] == (stype)0) {
             goodres.s[n] = val.s[n];
           } else if (isinf(val.s[n])) {
             goodres.s[n] = val.s[n];
           } else {
             goodres.s[n] = val.s[n] * (stype)(16.0);
           }
           equal = equal && ISEQ(res.s[n], goodres.s[n]);
         }
         if (!equal) {
           for (int n=0; n<vecsize; ++n) {
             printf("FAIL: ldexp type=%s val=%.17g val2=%d res=%.17g good=%.17g\n",
                    typename, val.s[n], (int)ival2.s[n], res.s[n], goodres.s[n]);
           }
           return;
         }
       }
     }
   }
 })
 )

kernel void test_fabs()
{
  CALL_FUNC_V(test_fabs)
}