first correct run

first correct run

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// number of filter taps in the processing wk_item
#define MLO_WEI_WKITEM 4
// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
#define MLO_OUT_WEI_SCAN_BLK ((MLO_OUT_WIDTH + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_WIDTH + MLO_OUT_WEI_SCAN_BLK - 1)/MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ *MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_HORIZ_PIX_SZ + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)
#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_N_WEI_BLK)
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#define MLO_N_IN_HORIZ_READS ((((MLO_OUT_HORIZ_PIX_SZ-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#if (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u,(int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm
// data layout.
// registers:
// per wk_item:
// MLO_READ_UNIT x MLO_N_LCL_OUT_MAPS of dy
// group:
// MLO_N_OUT_BLK_GRP blocks of MLO_N_OUT_BLK_GRP_PIX_SZ of dy
//
// LDS:
// data buffer
// MLO_N_LCL_IN_MAPS blocks of MLO_IN_LCL_SZ of x

// dw local buffer keeps 1 set of dweghts calculated at each iteration
	MLO_TEMP_WEI_BUFFER_SZ

//
// inputs:
// read into LDS
// MLO_N_IN_BLK_GRP groups
// output
// fetch
// MLO_N_OUT_BLK_GRP groups

// alg
// for each batch
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for k in filter size1
//				for l in filter size 0
//					at for each out/in pixel pair op,ip in wk_item
//                  calculate the contribution to a dweights by formula ip.x (input pixel.x) = (op.x * stride.x - pad .x + l) % (filter size 0), ip.y = (op.y * stride.y - pad.y + k) % filter size1,
//					wei[k,l] = in[ip.y, ip.x]*op[op.y, op.x]
//                  lcl_wei[MLO_OUT_BLK_GRP_PIX_SZ * out.grp + [op.y, op.x]] = we[k,l]
//                  accumulate the weights in the wk_item with wei_accum +=  lcl_wei[accum_id], accum_id = MLO_OUT_BLK_GRP*MLO_N_LCL_OUT_MAPS*map_id + k*(filter size 0) + l
//	write out accululate weights

// group layout
// 0 - input channel index
// 1 - output channel index
// 2 - 1

// loop over batches
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
// input/output tiles

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();

	int c_idx_base = get_group_id(1); // input map index base

// register memory
	int o_idx_base = get_group_id(2); // output map index base

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	int lcl_id = get_local_id(0);

// output global read
//	int o_blk_idx = iDiv(lcl_id, MLO_OUT_LCL_SZ);
//	int o_pix_X4 = iMod(lcl_id, o_blk_idx, MLO_OUT_LCL_SZ);
	int o_scan = iDiv(lcl_id, MLO_N_OUT_HORIZ_READS);
	int o_pix4 = iMod(lcl_id, o_scan, MLO_N_OUT_HORIZ_READS);

// input global read
	int i_blk_idx = 0; //iDiv(lcl_id, MLO_IN_LCL_SZ);
	//int c_pix_X4 = iMod(lcl_id, i_blk_idx, MLO_IN_LCL_SZ);
	int c_scan0 = iDiv(lcl_id, MLO_N_IN_HORIZ_READS);
	int c_pix4 = iMod(lcl_id,c_scan0, MLO_N_IN_HORIZ_READS);

	int w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	int w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
	int w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
	int w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);

// FILTR SPECIFIC !!!!!
	int last_tap_to_remove = (w_x0 > 0)? 0 : 1;

// mask to shut down the overshot filter tap
	__constant int filter_mask0 = ~(1 << MLO_FILTER_SIZE0);
	int working_mask = 0;
// filled with 1 evry stride bit
	for(int i = 0; i < MLO_WEI_WKITEM; ++i)
	{
		working_mask |= (1 << (i*MLO_FILTER_STRIDE0));
	}
// add strting point
	working_mask <<= w_x0;
// clamp to the filter size
	working_mask &= filter_mask0;
// move back to the relative start

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE;

	for (int i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

// zero out LDS
	for (int i = 0; i < ( MLO_LCL_SZ ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);
// zero out pvt accum
	// over all batches

#if 0
	if (c_idx == 0 && o_idx == 0 && lcl_id == 0)
	{
		printf("K:i: %d %d %d %d %d\n",
			MLO_MAX_WEI_BLK_LOOP,
			MLO_WEI_BLK_SZ,
			MLO_N_LCL_OUT_MAPS,
			MLO_WEI_BLKS_LCL_SZ,
			(MLO_LCL_SZ)
		);
	}
#endif

#if 1
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;
// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1
// LDS UPDATE !!!!
// input scan
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off = 0;
		int gbl_out_scan_off = 0;

		for(int c_scan = c_scan0;  in_y + c_scan < MLO_IN_HEIGHT && c_scan < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; c_scan += MLO_IN_VERT_READS)
		{
// TODO :: multiply on right mask if needed
			*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
// TO DO:: CHECK HEIGHT
// TO DO:: CHECK PADDING
//			*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
			}

#if 0
			if (c_idx == 0 && o_idx == 0 && c_scan == 2 /*&& (c_pix4 == 0 || c_pix4 == 1)*/)
			{
				printf("K:i: %d %d %f %f\n",
					lcl_id,
					gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
					in_rd_data[0],
					in_rd_data[2]
				);
			}
#endif

		}

		in_y +=  MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1;

// TO DO: HANDLE PADDING
// over all out blocks
		for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
               // TODO:: HANDLE multiple INPUTS
			   // overshot i shadled by out put
			gbl_in_scan_off = gbl_in_off + mul24(in_y, MLO_IN_STRIDE);

			for (int c_scan = c_scan0;i_blk_idx < MLO_N_LCL_IN_MAPS && in_y + c_scan < MLO_IN_HEIGHT && c_scan <  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1); c_scan += MLO_IN_VERT_READS)
			{

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
//					*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
					}

#if 0
					if (c_idx == 0 && o_idx == 0 && in_y + c_scan == 2 && (c_pix4 == 0 || c_pix4 == 1))
					{
						printf("K:i: %d %d %f %f\n",
							lcl_id,
							gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
							in_rd_data[0],
							in_rd_data[2]
						);
					}
#endif

			}

			// ouput scan
			__private _FLOAT out_rd_data[MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT];

			gbl_out_scan_off = gbl_out_off + mul24(out_y, MLO_OUT_STRIDE);

// over all outputs groups

			int gbl_out_scan_off1 = gbl_out_scan_off;
			for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS*MLO_OUT_CHANNEL_STRIDE )
			{

// fetch output

				for(int o = 0; o < MLO_N_LCL_OUT_MAPS
						&&  ( o_idx + o < MLO_N_OUTPUTS && out_y + o_scan < MLO_OUT_HEIGHT && o_scan < MLO_N_ALIGNED_OUT_SCAN_BLK)
						; ++o)
				{
						*(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT]
							= *(MLO_READ_TYPE*)&top_df[gbl_out_scan_off1 + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT];
#if MLO_OUT_SCAN_NOT_DIVBY4
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS-1))
						{
							for(int i = MLO_READ_UNIT-1; i >= MLO_READ_UNIT - MLO_OUT_N_PIXS_OFF; --i)
							{
	 							out_rd_data[o*MLO_READ_UNIT + i] = 0;
							}
						}
#endif
// write with MLO_OUT_HORIZ_PIX_EXT_SZ stride
//						*(MLO_READ_TYPE*)&lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT];
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + i] = out_rd_data[o*MLO_READ_UNIT + i];
						}

#if 0
						if (c_idx == 0 && o_idx == 0 && og == 0 && o == 1 && out_y + o_scan == 1)
						{
							printf("K:o: %d %d %d %f %f %f\n",
								lcl_id,
								gbl_out_scan_off + o*MLO_IN_CHANNEL_STRIDE + o_scan * MLO_IN_STRIDE + o_pix4*MLO_READ_UNIT,
								o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT,
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 0],
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 1],
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 2]
								);
						}
#endif

				}

// process
// algorithm

#if 1
				for(int j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
				{

					_FLOAT i_vals[MLO_WEI_WKITEM];
					for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
					{
							int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

					}

// if we overshoot the scanline
// out data will be 0 by initial setting

					for(int i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
					{

//						for(int w = 0; w < MLO_WEI_WKITEM; ++w)

						{
							int w_x = w_x0 + (MLO_WEI_WKITEM-1)* MLO_WEI_BLK_SZ0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[(MLO_WEI_WKITEM-1)] = i_val;

						}

						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
							_FLOAT o_val
								= lcl_top[o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

							int w = 0;
							for(/*int w = 0*/; w < MLO_WEI_WKITEM - 1; ++w )
							{

								_FLOAT i_val = i_vals[w];
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;

#if 0
								int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
								if (c_idx == 0 && o_idx == 0 && og ==1 && o == 0 && w_y ==0 && w_x == 0 && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM)
								{
									printf("K:s: %d %d %d %d %d %f %f %f\n",
										lcl_id,
										j,
										i,
										w,
										o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i),
										pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w],
										i_val,
										o_val
									);
								}
#endif
							}

// set input data to zero if MLO_FILTER_SIZE0 is not multiple of MLO_WEI_WKITEM and it's over MLO_FILTER_SIZE0

							_FLOAT i_val = i_vals[w];
// not needed becuse it's handled when "virtual" tables are trasnformed into real ones
	//						int ii_val = as_uint(i_val) & last_tap_to_remove;
	//						i_val = as_float(ii_val);

							pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;

#if 0
							int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							if (c_idx == 0 && o_idx == 0 && og == 1 && o == 0 && w_y == 0 && w_x == 0 && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM)
							{
								printf("K:s: %d %d %d %d %d %f %f %f\n",
									lcl_id,
									j,
									i,
									w,
									o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i),
									pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w],
									i_val,
									o_val
								);
							}
#endif

						}

						for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
						{
							i_vals[w] = i_vals[w + 1];

						}

					}
				}
#else
						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
							for(int w = 0; w < MLO_WEI_WKITEM; ++w)
							{
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = lcl_top[(og * MLO_N_LCL_OUT_MAPS + o)] * lcl_bot[w];
							}
						}
#endif

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

			barrier(CLK_LOCAL_MEM_FENCE);

// move data up
			for(int c_scan = c_scan0; c_scan < MLO_FILTER_SIZE1 -1; c_scan += MLO_IN_VERT_READS)
			{
					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&lcl_bot[(c_scan +  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1))*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[c_scan*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

		} // for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (int b = 0;

#endif

	int oo = iDiv(lcl_id, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i = iMod(lcl_id, oo, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
	int wei_i_x = iMod(wei_i,wei_i_y, (MLO_FILTER_SIZE0));
	// send it out
	int wei_df_off = mul24(o_idx, (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);

	int wei_lcl_off = 0;
	_FLOAT final_sum = 0;

// save in lcl and orgnize in a proper order
// outputs
//	filter size1
//	  filter size0
// TO DO DEPENDING ON THE GROUP SIZE
	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0;  w_blk_idx < MLO_MAX_WEI_BLK_LOOP && o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
// save "virtual" filter table
				int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
				wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1  + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
#if 0
				// wei_lcl_off == 1020 || wei_lcl_off == 1080 || wei_lcl_off == 1140)
				if (c_idx == 0 && o_idx == 0 && og == 1 && o == 0 && w_y == 0 && w_x == 0 && lcl_id <= 45/* && (j == 0 || j == 1) */)
				{
					printf("K:u: %d %d %d %f %f\n",
						lcl_id,
						w_blk_idx,
						wei_lcl_off,
						lcl_bot[wei_lcl_off],
						pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
#if 1
// read into real filter table

		if (lcl_id < ( MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
		{
			final_sum = 0;
			for(int i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
			{
				final_sum += lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x];
#if 0
				if (c_idx == 0 && o_idx == 0  && og == 1 && oo  == 0 && wei_i_y == 0 && wei_i_x == 0)
				{
					printf("K:fs: %d %d %d  %f %f\n",
						lcl_id,
						i,
						((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x,
						final_sum,
						lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x]
					);
				}
#endif
			}

#if 1

			weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //

#if 0
			if (og == 1 && oo == 0 && wei_i == 0)
			{

				printf("K:o: %d %d %f\n",
					wei_df_off,
					wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i,
					weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i]
					);
			}
#endif
#else
			_FLOAT t_accum = 0;

			for (int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
			{
				for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
				{
					for (int w = 0; w < MLO_WEI_WKITEM; ++w)
					{
						t_accum += pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
					}
				}
			}

			weights_df[lcl_id] = t_accum;
#endif

		}
#endif
		barrier(CLK_LOCAL_MEM_FENCE);

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// number of filter taps in the processing wk_item
#define MLO_WEI_WKITEM 4
// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
#define MLO_OUT_WEI_SCAN_BLK ((MLO_OUT_WIDTH + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_WIDTH + MLO_OUT_WEI_SCAN_BLK - 1)/MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ *MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_HORIZ_PIX_SZ + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)
#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_N_WEI_BLK)
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#define MLO_N_IN_HORIZ_READS ((((MLO_OUT_HORIZ_PIX_SZ-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#if (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u,(int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm
// data layout.
// registers:
// per wk_item:
// MLO_READ_UNIT x MLO_N_LCL_OUT_MAPS of dy
// group:
// MLO_N_OUT_BLK_GRP blocks of MLO_N_OUT_BLK_GRP_PIX_SZ of dy
//
// LDS:
// data buffer
// MLO_N_LCL_IN_MAPS blocks of MLO_IN_LCL_SZ of x

// dw local buffer keeps 1 set of dweghts calculated at each iteration
	MLO_TEMP_WEI_BUFFER_SZ

//
// inputs:
// read into LDS
// MLO_N_IN_BLK_GRP groups
// output
// fetch
// MLO_N_OUT_BLK_GRP groups

// alg
// for each batch
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for k in filter size1
//				for l in filter size 0
//					at for each out/in pixel pair op,ip in wk_item
//                  calculate the contribution to a dweights by formula ip.x (input pixel.x) = (op.x * stride.x - pad .x + l) % (filter size 0), ip.y = (op.y * stride.y - pad.y + k) % filter size1,
//					wei[k,l] = in[ip.y, ip.x]*op[op.y, op.x]
//                  lcl_wei[MLO_OUT_BLK_GRP_PIX_SZ * out.grp + [op.y, op.x]] = we[k,l]
//                  accumulate the weights in the wk_item with wei_accum +=  lcl_wei[accum_id], accum_id = MLO_OUT_BLK_GRP*MLO_N_LCL_OUT_MAPS*map_id + k*(filter size 0) + l
//	write out accululate weights

// group layout
// 0 - input channel index
// 1 - output channel index
// 2 - 1

// loop over batches
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
// input/output tiles

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();

	int c_idx_base = get_group_id(1); // input map index base

// register memory
	int o_idx_base = get_group_id(2); // output map index base

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	int lcl_id = get_local_id(0);

// output global read
//	int o_blk_idx = iDiv(lcl_id, MLO_OUT_LCL_SZ);
//	int o_pix_X4 = iMod(lcl_id, o_blk_idx, MLO_OUT_LCL_SZ);
	int o_scan = iDiv(lcl_id, MLO_N_OUT_HORIZ_READS);
	int o_pix4 = iMod(lcl_id, o_scan, MLO_N_OUT_HORIZ_READS);

// input global read
	int i_blk_idx = 0; //iDiv(lcl_id, MLO_IN_LCL_SZ);
	//int c_pix_X4 = iMod(lcl_id, i_blk_idx, MLO_IN_LCL_SZ);
	int c_scan0 = iDiv(lcl_id, MLO_N_IN_HORIZ_READS);
	int c_pix4 = iMod(lcl_id,c_scan0, MLO_N_IN_HORIZ_READS);

	int w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	int w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
	int w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
	int w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);

// FILTR SPECIFIC !!!!!
	int last_tap_to_remove = (w_x0 > 0)? 0 : 1;

// mask to shut down the overshot filter tap
	__constant int filter_mask0 = ~(1 << MLO_FILTER_SIZE0);
	int working_mask = 0;
// filled with 1 evry stride bit
	for(int i = 0; i < MLO_WEI_WKITEM; ++i)
	{
		working_mask |= (1 << (i*MLO_FILTER_STRIDE0));
	}
// add strting point
	working_mask <<= w_x0;
// clamp to the filter size
	working_mask &= filter_mask0;
// move back to the relative start

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE;

	for (int i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

// zero out LDS
	for (int i = 0; i < ( MLO_LCL_SZ ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);
// zero out pvt accum
	// over all batches

#if 0
	if (c_idx == 0 && o_idx == 0 && lcl_id == 0)
	{
		printf("K:i: %d %d %d %d %d\n",
			MLO_MAX_WEI_BLK_LOOP,
			MLO_WEI_BLK_SZ,
			MLO_N_LCL_OUT_MAPS,
			MLO_WEI_BLKS_LCL_SZ,
			(MLO_LCL_SZ)
		);
	}
#endif

#if 1
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;
// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1
// LDS UPDATE !!!!
// input scan
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off = 0;
		int gbl_out_scan_off = 0;

		for(int c_scan = c_scan0;  in_y + c_scan < MLO_IN_HEIGHT && c_scan < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; c_scan += MLO_IN_VERT_READS)
		{
// TODO :: multiply on right mask if needed
			*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
// TO DO:: CHECK HEIGHT
// TO DO:: CHECK PADDING
//			*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
			}

#if 0
			if (c_idx == 0 && o_idx == 0 && c_scan == 2 /*&& (c_pix4 == 0 || c_pix4 == 1)*/)
			{
				printf("K:i: %d %d %f %f\n",
					lcl_id,
					gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
					in_rd_data[0],
					in_rd_data[2]
				);
			}
#endif

		}

		in_y +=  MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1;

// TO DO: HANDLE PADDING
// over all out blocks
		for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
               // TODO:: HANDLE multiple INPUTS
			   // overshot i shadled by out put
			gbl_in_scan_off = gbl_in_off + mul24(in_y, MLO_IN_STRIDE);

			for (int c_scan = c_scan0;i_blk_idx < MLO_N_LCL_IN_MAPS && in_y + c_scan < MLO_IN_HEIGHT && c_scan <  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1); c_scan += MLO_IN_VERT_READS)
			{

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
//					*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
					}

#if 0
					if (c_idx == 0 && o_idx == 0 && in_y + c_scan == 2 && (c_pix4 == 0 || c_pix4 == 1))
					{
						printf("K:i: %d %d %f %f\n",
							lcl_id,
							gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT,
							in_rd_data[0],
							in_rd_data[2]
						);
					}
#endif

			}

			// ouput scan
			__private _FLOAT out_rd_data[MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT];

			gbl_out_scan_off = gbl_out_off + mul24(out_y, MLO_OUT_STRIDE);

// over all outputs groups

			int gbl_out_scan_off1 = gbl_out_scan_off;
			for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS*MLO_OUT_CHANNEL_STRIDE )
			{

// fetch output

				for(int o = 0; o < MLO_N_LCL_OUT_MAPS
						&&  ( o_idx + o < MLO_N_OUTPUTS && out_y + o_scan < MLO_OUT_HEIGHT && o_scan < MLO_N_ALIGNED_OUT_SCAN_BLK)
						; ++o)
				{
						*(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT]
							= *(MLO_READ_TYPE*)&top_df[gbl_out_scan_off1 + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT];
#if MLO_OUT_SCAN_NOT_DIVBY4
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS-1))
						{
							for(int i = MLO_READ_UNIT-1; i >= MLO_READ_UNIT - MLO_OUT_N_PIXS_OFF; --i)
							{
	 							out_rd_data[o*MLO_READ_UNIT + i] = 0;
							}
						}
#endif
// write with MLO_OUT_HORIZ_PIX_EXT_SZ stride
//						*(MLO_READ_TYPE*)&lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT];
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + i] = out_rd_data[o*MLO_READ_UNIT + i];
						}

#if 0
						if (c_idx == 0 && o_idx == 0 && og == 0 && o == 1 && out_y + o_scan == 1)
						{
							printf("K:o: %d %d %d %f %f %f\n",
								lcl_id,
								gbl_out_scan_off + o*MLO_IN_CHANNEL_STRIDE + o_scan * MLO_IN_STRIDE + o_pix4*MLO_READ_UNIT,
								o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT,
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 0],
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 1],
								lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT + 2]
								);
						}
#endif

				}

// process
// algorithm

#if 1
				for(int j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
				{

					_FLOAT i_vals[MLO_WEI_WKITEM];
					for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
					{
							int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

					}

// if we overshoot the scanline
// out data will be 0 by initial setting

					for(int i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
					{

//						for(int w = 0; w < MLO_WEI_WKITEM; ++w)

						{
							int w_x = w_x0 + (MLO_WEI_WKITEM-1)* MLO_WEI_BLK_SZ0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[(MLO_WEI_WKITEM-1)] = i_val;

						}

						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
							_FLOAT o_val
								= lcl_top[o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

							int w = 0;
							for(/*int w = 0*/; w < MLO_WEI_WKITEM - 1; ++w )
							{

								_FLOAT i_val = i_vals[w];
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;

#if 1
								int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
								if (c_idx == 0 && o_idx == 0 && og ==1 && o == 0 && w_y ==0 && w_x == 0 && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM)
								{
									printf("K:s: %d %d %d %d %d %f %f %f\n",
										lcl_id,
										j,
										i,
										w,
										o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i),
										pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w],
										i_val,
										o_val
									);
								}
#endif
							}

// set input data to zero if MLO_FILTER_SIZE0 is not multiple of MLO_WEI_WKITEM and it's over MLO_FILTER_SIZE0

							_FLOAT i_val = i_vals[w];
// not needed becuse it's handled when "virtual" tables are trasnformed into real ones
	//						int ii_val = as_uint(i_val) & last_tap_to_remove;
	//						i_val = as_float(ii_val);

							pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;

#if 1
							int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							if (c_idx == 0 && o_idx == 0 && og == 1 && o == 0 && w_y == 0 && w_x == 0 && lcl_id < MLO_OUT_WEI_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ0 * MLO_WEI_WKITEM)
							{
								printf("K:s: %d %d %d %d %d %f %f %f\n",
									lcl_id,
									j,
									i,
									w,
									o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i),
									pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w],
									i_val,
									o_val
								);
							}
#endif

						}

						for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
						{
							i_vals[w] = i_vals[w + 1];

						}

					}
				}
#else
						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
							for(int w = 0; w < MLO_WEI_WKITEM; ++w)
							{
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = lcl_top[(og * MLO_N_LCL_OUT_MAPS + o)] * lcl_bot[w];
							}
						}
#endif

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

			barrier(CLK_LOCAL_MEM_FENCE);

// move data up
			for(int c_scan = c_scan0; c_scan < MLO_FILTER_SIZE1 -1; c_scan += MLO_IN_VERT_READS)
			{
					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&lcl_bot[(c_scan +  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1))*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[c_scan*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

		} // for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (int b = 0;

#endif

	int oo = iDiv(lcl_id, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i = iMod(lcl_id, oo, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
	int wei_i_x = iMod(wei_i,wei_i_y, (MLO_FILTER_SIZE0));
	// send it out
	int wei_df_off = mul24(o_idx, (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);

	int wei_lcl_off = 0;
	_FLOAT final_sum = 0;

// save in lcl and orgnize in a proper order
// outputs
//	filter size1
//	  filter size0
// TO DO DEPENDING ON THE GROUP SIZE
	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0;  w_blk_idx < MLO_MAX_WEI_BLK_LOOP && o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
// save "virtual" filter table
				int w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
				wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1  + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
#if 1
				// wei_lcl_off == 1020 || wei_lcl_off == 1080 || wei_lcl_off == 1140)
				if (c_idx == 0 && o_idx == 0 && og == 1 && o == 0 && w_y == 0 && w_x == 0 && lcl_id <= 45/* && (j == 0 || j == 1) */)
				{
					printf("K:u: %d %d %d %f %f\n",
						lcl_id,
						w_blk_idx,
						wei_lcl_off,
						lcl_bot[wei_lcl_off],
						pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
#if 1
// read into real filter table

		if (lcl_id < ( MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
		{
			final_sum = 0;
			for(int i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
			{
				final_sum += lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x];
#if 1
				if (c_idx == 0 && o_idx == 0  && og == 1 && oo  == 0 && wei_i_y == 0 && wei_i_x == 0)
				{
					printf("K:fs: %d %d %d  %f %f\n",
						lcl_id,
						i,
						((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x,
						final_sum,
						lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x]
					);
				}
#endif
			}

#if 1

			weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //

#if 0
			if (og == 1 && oo == 0 && wei_i == 0)
			{

				printf("K:o: %d %d %f\n",
					wei_df_off,
					wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i,
					weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i]
					);
			}
#endif
#else
			_FLOAT t_accum = 0;

			for (int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
			{
				for (int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
				{
					for (int w = 0; w < MLO_WEI_WKITEM; ++w)
					{
						t_accum += pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
					}
				}
			}

			weights_df[lcl_id] = t_accum;
#endif

		}
#endif
		barrier(CLK_LOCAL_MEM_FENCE);

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

}


MIOpenConv1x1 perf improvements

MIOpenConv1x1 perf improvements

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

#if MLO_N_MAPS_PERGROUP > 1
#define MLO_WEIGHTS_PER_LOOP_MAX 8
#else
#define MLO_WEIGHTS_PER_LOOP_MAX 16
#endif
#if ((MLO_N_MAPS_PERGROUP*MLO_N_LCL_IN_MAPS) < MLO_N_INPUTS)
#define MLO_LCL_IN_ROW (MLO_N_MAPS_PERGROUP*MLO_N_LCL_IN_MAPS)
#else
#define MLO_LCL_IN_ROW (MLO_N_INPUTS)
#endif

#define MLO_WEIGHTS_PER_LOOP_TMP ((MLO_N_INPUTS + MLO_LCL_IN_ROW - 1)/MLO_LCL_IN_ROW)

#if (MLO_WEIGHTS_PER_LOOP_TMP < MLO_WEIGHTS_PER_LOOP_MAX)
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_TMP)
#else
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_MAX)
#endif
#define MLO_LCL_WEIGHTS_ROW (MLO_WEIGHTS_PER_LOOP * MLO_LCL_IN_ROW)
#define MLO_WEIGHTS_ROW (MLO_LCL_WEIGHTS_ROW* MLO_WEI_CHANNEL_STRIDE)

// size of the area for weights prefetch
#define MLO_WEIGHTS_LCL_SZ (MLO_WEIGHTS_ROW * MLO_N_LCL_OUT_MAPS)

// size of area for exchanging partial sums
#define MLO_EXCHNGE_SZ4 (MLO_MAP_SZ4*MLO_EXCHANGE_STEP * MLO_N_MAPS_PERGROUP)

#if MLO_N_MAPS_PERGROUP > 1 && ((MLO_EXCHNGE_SZ4 * MLO_READ_UNIT) > MLO_WEIGHTS_LCL_SZ)
#define MLO_LCL_MEM_SZ (MLO_EXCHNGE_SZ4 * MLO_READ_UNIT)
#else
#define MLO_LCL_MEM_SZ MLO_WEIGHTS_LCL_SZ
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24(u, d);
	return(r);
}

/*
Layout:
assuming NCHW data layout.

Data:
data has been fetch by 4 floats sequentially.
MLO_MAP_SZ4 = (map_width*map_height + 3)/4.
in case of total size not a multiple of 4 the the last pixel has a special treatment.
There are 2 cases:
MLO_N_MAPS_PERGROUP == 1
and
MLO_N_MAPS_PERGROUP > 1, when MLO_MAP_SZ4 <= GPROUP_SIZE/2, in other words when more than 1 map can be held by a group.
Case MLO_N_MAPS_PERGROUP == 1:
Data, by 4 floats, may come from MLO_N_LCL_IN_MAPS sequential input maps from MLO_N_LCL_BATCHS neighboring batches.
Weigts:
on each MLO_WEIGHTS_PER_LOOP input loop set of weight are prefetched for another MLO_WEIGHTS_PER_LOOP loops.
Each input map contributes to partial sums of MLO_N_LCL_OUT_MAPS output maps.
Case MLO_N_MAPS_PERGROUP > 1:
Similar to a previous case.
The difference is that several input sequential input maps are kept by group.
Each taking part in the calculation of partial sums of the same MLO_N_LCL_OUT_MAPS output maps.
After completion of the main MLO_IN_LOOP loop partial sums have been summed up in parallel.

*/

__kernel void MIOpenConv1x1(
       const __global _FLOAT * __restrict in_ptr,
       const __global _FLOAT * __restrict wei_ptr,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
 	  __global _FLOAT * __restrict out_ptr,
	   UNUSED _FLOAT dummy_val // nothing
	   )
{
// KERNEL
// private buffers
	__private _FLOAT in_stage[MLO_N_LCL_BATCHS][MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];
	__private _FLOAT wei_stage;
	__private _FLOAT out_tiles[MLO_N_LCL_BATCHS][MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
	__local _FLOAT lcl_wei_stage[MLO_LCL_MEM_SZ];

#if MLO_N_MAPS_PERGROUP > 1
	__local _FLOAT * lcl_out_stage = lcl_wei_stage;

#endif

	uint lcl_id0 = get_local_id(0);
	int in_map_id = 0; // map
	uint pix_id = get_global_id(0);  // inside map
	in_map_id = pix_id / MLO_MAP_SZ4; // mad id inside group
	uint out_grp_block = get_group_id(1); // block of outputs for the entire group
	uint out_block = out_grp_block;
	uint batch_block = get_group_id(2); // block of batchs
// multipe maps per group

	pix_id = (pix_id - in_map_id * MLO_MAP_SZ4);  // pixel inside map

	uint in_map_off_id = (in_map_id >= MLO_N_MAPS_PERGROUP) ? MLO_N_MAPS_PERGROUP - 1 : in_map_id;

	uint in_off = batch_block * MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE
		+ in_map_off_id * MLO_IN_CHANNEL_STRIDE
				+ pix_id * MLO_READ_UNIT;

	uint wei_off = out_grp_block * MLO_N_LCL_OUT_MAPS *
#if MLO_DIR_FORWARD==1
		MLO_WEI_BSTRIDE
#else
		MLO_WEI_CHANNEL_STRIDE
#endif
		;
	for (uint j = 0; j < MLO_N_LCL_BATCHS; ++j)
	{
		for (uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
		{
			for (uint k = 0; k < MLO_READ_UNIT; ++k)
			{
				out_tiles[j][i][k] = 0;
			}
		}
	}
// over all input maps; with step == MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP; MLO_IN_LOOP
	for (uint wc = 0; wc < MLO_IN_LOOP; wc += MLO_WEIGHTS_PER_LOOP) {
		// read array of weights
		barrier(CLK_LOCAL_MEM_FENCE);

		for (uint w = lcl_id0; w < MLO_WEIGHTS_LCL_SZ ; w += MLO_GRP_SZ0)
		{

#if (MLO_WEIGHTS_ROW) & (MLO_WEIGHTS_ROW - 1)

			uint oi = iDiv(w, MLO_WEIGHTS_ROW);
			uint lwi = iMod(w, oi, MLO_WEIGHTS_ROW);
#else
			uint oi = (w / MLO_WEIGHTS_ROW);
			uint lwi = (w & (MLO_WEIGHTS_ROW - 1));
#endif

			uint wi = (wc * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP) + lwi)*
#if MLO_DIR_FORWARD==1
				MLO_WEI_CHANNEL_STRIDE;
#else
				MLO_WEI_BSTRIDE;
#endif

			// out of range check
			uint wei_off_r = wei_off + wi + oi *
#if MLO_DIR_FORWARD==1
				MLO_WEI_BSTRIDE;
#else
				MLO_WEI_CHANNEL_STRIDE;
#endif

			wei_off_r = (wei_off_r < MLO_N_OUTPUTS *MLO_N_INPUTS) ? wei_off_r : 0;
			_FLOAT wei_val = wei_ptr[wei_off_r];
			wei_val = (wei_off_r < MLO_N_OUTPUTS *MLO_N_INPUTS) ? wei_val : 0;
			lcl_wei_stage[w] = wei_val;

		}

		barrier(CLK_LOCAL_MEM_FENCE);

#if MLO_WEIGHTS_PER_LOOP > 7
#pragma unroll (MLO_WEIGHTS_PER_LOOP / 8)
#endif
		for (uint ci = 0; ci < MLO_WEIGHTS_PER_LOOP; ++ci,
			in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP
			)
		{
			uint c = wc + ci;
			uint wei_indx = ci;

			// read data
			// over all local batchs
			uint in_off1 = in_off;
			for (uint ib = 0; ib < MLO_N_LCL_BATCHS
				; ++ib, in_off1 += MLO_IN_BATCH_STRIDE)
			{
				uint in_off2 = in_off1;
				// lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP
				for (uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS; ++ilc, in_off2 += MLO_IN_CHANNEL_STRIDE * MLO_N_MAPS_PERGROUP)
				{
					bool v =
#if MLO_BATCH_ALIGNED == 0
						(batch_block*MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ) &&
#endif
						c*MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP + in_map_id + ilc* MLO_N_MAPS_PERGROUP < MLO_N_INPUTS;
					__global const _FLOAT *in_p = &in_ptr[in_off2];
#if MLO_C1x1_PIXLEFT > 0
					// if the last one
					if (pix_id == MLO_MAP_SZ4 - 1)
					{

						for (uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
						{
#ifdef __AMDGCN__
							in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
							_FLOAT val = in_p[i];
							in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE
							if (in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}
					}
					else

#endif
					{
						for (uint i = 0; i < MLO_READ_UNIT; ++i)
						{
#ifdef __AMDGCN__
							in_stage[ib][ilc][i] = v ? in_p[i] : 0.0f;
#else
							_FLOAT val = in_p[i];
							in_stage[ib][ilc][i] = v ? val : 0.0f;
#endif
#if DBG_OUT_OF_RNGE
							if (in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}
					}
				}
			}

			// convolve
			for (uint olc = 0, lcl_wei_off = wei_indx*MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP*MLO_WEI_CHANNEL_STRIDE; olc < MLO_N_LCL_OUT_MAPS; ++olc, lcl_wei_off += MLO_WEIGHTS_ROW)
			{
				// lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP, weights are mapped accordingly
				uint lcl_wei_off1 = lcl_wei_off;
				for (uint ilc = 0; ilc < MLO_N_LCL_IN_MAPS; ++ilc, lcl_wei_off1 += MLO_N_MAPS_PERGROUP*MLO_WEI_CHANNEL_STRIDE)
				{
					// read weights
					uint lcl_wei_off2 = lcl_wei_off1 + mul24(in_map_id, (int)MLO_WEI_CHANNEL_STRIDE);
					wei_stage = lcl_wei_stage[lcl_wei_off2];
					for (uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
					{
						for (uint i = 0; i < MLO_READ_UNIT; ++i)
						{
							out_tiles[ib][olc][i] += in_stage[ib][ilc][i] * wei_stage;
						}
					}
				}
			}

		}
	}

// out of range check
	if (in_map_id >= MLO_N_MAPS_PERGROUP || in_map_id*MLO_N_LCL_IN_MAPS >= MLO_N_INPUTS)
	{
		return;
	}

	out_block = out_grp_block * MLO_N_LCL_OUT_MAPS;
	uint out_off = batch_block * MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE
		+ out_block *  MLO_OUT_CHANNEL_STRIDE
		+ pix_id * MLO_READ_UNIT;

// small groups
#if MLO_N_MAPS_PERGROUP > 1

	// calculate reduction over all partial sums
	// MLO_N_LCL_OUT_MAPS is multiple of MLO_EXCHANGE_STEP
	// write data into local memory

	for (uint ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
	{
		for (uint t = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id0 < MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP)
			{
				for (uint om = 0; om < MLO_EXCHANGE_STEP; ++om)
				{
					uint lcl_off = (om * MLO_MAP_SZ4*MLO_N_MAPS_PERGROUP + in_map_id*MLO_MAP_SZ4 + pix_id) * MLO_READ_UNIT;
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_out_stage[lcl_off + i] = out_tiles[ib][t + om][i];
					}
				}

			}
			barrier(CLK_LOCAL_MEM_FENCE);

			// sum partial sum
			// MLO_N_MAPS_PERGROUP >= MLO_EXCHANGE_STEP
			// in_map_id is an index of the output map now.
			if (in_map_id < MLO_EXCHANGE_STEP)
			{
				_FLOAT sum[MLO_READ_UNIT];
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					sum[i] = 0;
				}

				for (uint s = 0; s < MLO_N_MAPS_PERGROUP; ++s)
				{
					uint imp = in_map_id + s;
					imp = (imp >= MLO_N_MAPS_PERGROUP) ? imp - MLO_N_MAPS_PERGROUP : imp;
					uint lcl_off = (in_map_id* MLO_MAP_SZ4*MLO_N_MAPS_PERGROUP + imp*MLO_MAP_SZ4 + pix_id) * MLO_READ_UNIT;
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						sum[i] += lcl_out_stage[lcl_off + i];
					}

				}

				// write it out
				uint olc = t + in_map_id;

				if (true
#if MLO_BATCH_ALIGNED == 0
					&& (batch_block*MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
					&& out_block + olc < MLO_N_OUTPUTS
#endif
					)
				{

					uint out_off2 = out_off + ib * MLO_OUT_BATCH_STRIDE + olc * MLO_OUT_CHANNEL_STRIDE;
					__global _FLOAT *out_p = &out_ptr[out_off2];

#if MLO_CONV_BIAS
					_FLOAT  bias_val = 0;
					bias_val = bias[out_block* MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

					// if the last one
					if (pix_id == MLO_MAP_SZ4 - 1)
					{
						for (uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
						{
							out_p[i] = sum[i]
#if MLO_CONV_BIAS
								+ bias_val
#endif
								;

#if DBG_OUT_OF_RNGE
							if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:out-of-range\n");
							}
#endif

						}

					}
					else
#endif
					{

						for (uint i = 0; i < MLO_READ_UNIT; ++i)
						{
							out_p[i] = sum[i]
#if MLO_CONV_BIAS
								+ bias_val
#endif
								;
#if DBG_OUT_OF_RNGE
							if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:out-of-range\n");
							}
#endif

						}

					}

				} //if (true

			} // if (in_map_id < MLO_EXCHANGE_STEP)

		} // for (int t = 0, p = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)

	} // 	for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)

#else // #if MLO_N_MAPS_PERGROUP > 1

	uint out_off1 = out_off;
	for (uint ib = 0; ib < MLO_N_LCL_BATCHS
		; ++ib, out_off1 += MLO_OUT_BATCH_STRIDE)
	{

		uint out_off2 = out_off1;
		for (uint olc = 0; olc < MLO_N_LCL_OUT_MAPS
			; ++olc, out_off2 += MLO_OUT_CHANNEL_STRIDE)
		{
			__global _FLOAT *out_p = &out_ptr[out_off2];

		    if ( true
#if MLO_BATCH_ALIGNED == 0
			&& (batch_block*MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
			&& (out_block + olc < MLO_N_OUTPUTS)
#endif
			)
			{
#if MLO_CONV_BIAS
				_FLOAT  bias_val = 0;
				bias_val = bias[out_block* MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

			// if the last one
				if (pix_id == MLO_MAP_SZ4 - 1)
				{
					for (uint i = 0; i < MLO_C1x1_PIXLEFT; ++i)
					{
						out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
							+ bias_val
#endif
						;
#if DBG_OUT_OF_RNGE
						if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-of-range\n");
						}
#endif

					}

				}
				else
#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{

						out_p[i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
						+ bias_val
#endif
						;
#if DBG_OUT_OF_RNGE
						if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-of-range\n");
						}
#endif
					}
				}
			}
		}

	}

#endif // #if MLO_N_MAPS_PERGROUP > 1

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

#if MLO_N_MAPS_PERGROUP > 1
#define MLO_WEIGHTS_PER_LOOP_MAX 8
#else
#define MLO_WEIGHTS_PER_LOOP_MAX 16
#endif
#if ((MLO_N_MAPS_PERGROUP*MLO_N_LCL_IN_MAPS) < MLO_N_INPUTS)
#define MLO_LCL_IN_ROW (MLO_N_MAPS_PERGROUP*MLO_N_LCL_IN_MAPS)
#else
#define MLO_LCL_IN_ROW (MLO_N_INPUTS)
#endif

#define MLO_WEIGHTS_PER_LOOP_TMP ((MLO_N_INPUTS + MLO_LCL_IN_ROW - 1)/MLO_LCL_IN_ROW)

#if (MLO_WEIGHTS_PER_LOOP_TMP < MLO_WEIGHTS_PER_LOOP_MAX)
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_TMP)
#else
#define MLO_WEIGHTS_PER_LOOP (MLO_WEIGHTS_PER_LOOP_MAX)
#endif
#define MLO_LCL_WEIGHTS_ROW (MLO_WEIGHTS_PER_LOOP * MLO_LCL_IN_ROW)
#define MLO_WEIGHTS_ROW (MLO_LCL_WEIGHTS_ROW* MLO_WEI_CHANNEL_STRIDE)

// size of the area for weights prefetch
#define MLO_WEIGHTS_LCL_SZ (MLO_WEIGHTS_ROW * MLO_N_LCL_OUT_MAPS)

// size of area for exchanging partial sums
#define MLO_EXCHNGE_SZ4 (MLO_MAP_SZ4*MLO_EXCHANGE_STEP * MLO_N_MAPS_PERGROUP)

#if MLO_N_MAPS_PERGROUP > 1 && ((MLO_EXCHNGE_SZ4 * MLO_READ_UNIT) > MLO_WEIGHTS_LCL_SZ)
#define MLO_LCL_MEM_SZ (MLO_EXCHNGE_SZ4 * MLO_READ_UNIT)
#else
#define MLO_LCL_MEM_SZ MLO_WEIGHTS_LCL_SZ
#endif

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

/*
Layout:
assuming NCHW data layout.

Data:
data has been fetch by 4 floats sequentially.
MLO_MAP_SZ4 = (map_width*map_height + 3)/4.
in case of total size not a multiple of 4 the the last pixel has a special treatment.
There are 2 cases:
MLO_N_MAPS_PERGROUP == 1
and
MLO_N_MAPS_PERGROUP > 1, when MLO_MAP_SZ4 <= GPROUP_SIZE/2, in other words when more than 1 map can be held by a group.
Case MLO_N_MAPS_PERGROUP == 1:
Data, by 4 floats, may come from MLO_N_LCL_IN_MAPS sequential input maps from MLO_N_LCL_BATCHS neighboring batches.
Weigts:
on each MLO_WEIGHTS_PER_LOOP input loop set of weight are prefetched for another MLO_WEIGHTS_PER_LOOP loops.
Each input map contributes to partial sums of MLO_N_LCL_OUT_MAPS output maps.
Case MLO_N_MAPS_PERGROUP > 1:
Similar to a previous case.
The difference is that several input sequential input maps are kept by group.
Each taking part in the calculation of partial sums of the same MLO_N_LCL_OUT_MAPS output maps.
After completion of the main MLO_IN_LOOP loop partial sums have been summed up in parallel.

*/

__kernel void MIOpenConv1x1(
       const __global _FLOAT * __restrict in_ptr,
       const __global _FLOAT * __restrict wei_ptr,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
 	  __global _FLOAT * __restrict out_ptr,
	   UNUSED _FLOAT dummy_val // nothing
	   )
{
// KERNEL
// private buffers
	__private _FLOAT in_stage[MLO_N_LCL_BATCHS][MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];
	__private _FLOAT wei_stage;
	__private _FLOAT out_tiles[MLO_N_LCL_BATCHS][MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
	__local _FLOAT lcl_wei_stage[MLO_LCL_MEM_SZ];

#if MLO_N_MAPS_PERGROUP > 1
	__local _FLOAT * lcl_out_stage = lcl_wei_stage;

#endif

	int lcl_id0 = get_local_id(0);
	int in_map_id = 0; // map
	int pix_id = get_global_id(0);  // inside map
	in_map_id = pix_id / MLO_MAP_SZ4; // mad id inside group
	int out_grp_block = get_group_id(1); // block of outputs for the entire group
	int out_block = out_grp_block;
	int batch_block = get_group_id(2); // block of batchs
// multipe maps per group

	pix_id = (pix_id - in_map_id * MLO_MAP_SZ4);  // pixel inside map

	int in_map_off_id = (in_map_id >= MLO_N_MAPS_PERGROUP) ? MLO_N_MAPS_PERGROUP - 1 : in_map_id;

	int in_off = batch_block * MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE
		+ in_map_off_id * MLO_IN_CHANNEL_STRIDE
				+ pix_id * MLO_READ_UNIT;

	int wei_off = out_grp_block * MLO_N_LCL_OUT_MAPS *
#if MLO_DIR_FORWARD==1
		MLO_WEI_BSTRIDE
#else
		MLO_WEI_CHANNEL_STRIDE
#endif
		;
	for (int j = 0; j < MLO_N_LCL_BATCHS; ++j)
	{
		for (int i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
		{
			for (int k = 0; k < MLO_READ_UNIT; ++k)
			{
				out_tiles[j][i][k] = 0;
			}
		}
	}
// over all input maps; with step == MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP; MLO_IN_LOOP
	for (int c = 0, wc = 0; c < MLO_IN_LOOP; ++c,
		in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP
		)
	{

		// read array of weights
		if (wc - c == 0)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int w = lcl_id0; w < MLO_WEIGHTS_LCL_SZ ; w += MLO_GRP_SZ0)
			{

#if (MLO_WEIGHTS_ROW) & (MLO_WEIGHTS_ROW - 1)

				int oi = iDiv(w, MLO_WEIGHTS_ROW);
				int lwi = iMod(w, oi, MLO_WEIGHTS_ROW);
#else
				int oi = (int)((uint)w / MLO_WEIGHTS_ROW);
				int lwi = (int)( (uint)w & (MLO_WEIGHTS_ROW - 1));

#endif

				int wi = (wc * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP) + lwi)*
#if MLO_DIR_FORWARD==1
					MLO_WEI_CHANNEL_STRIDE;
#else
					MLO_WEI_BSTRIDE;
#endif

					// out of range check
				int wei_off_r = wei_off + wi + oi *
#if MLO_DIR_FORWARD==1
					MLO_WEI_BSTRIDE;
#else
					MLO_WEI_CHANNEL_STRIDE;
#endif

				wei_off_r = (wei_off_r < MLO_N_OUTPUTS *MLO_N_INPUTS) ? wei_off_r : 0;
				_FLOAT wei_val = wei_ptr[wei_off_r];
				wei_val = (wei_off_r < MLO_N_OUTPUTS *MLO_N_INPUTS) ? wei_val : 0;
				lcl_wei_stage[mad24(oi, MLO_WEIGHTS_ROW, lwi)] = wei_val;

			}
			wc += MLO_WEIGHTS_PER_LOOP;

			barrier(CLK_LOCAL_MEM_FENCE);

		}

		int wei_indx = c - wc + MLO_WEIGHTS_PER_LOOP;
		// read data
		// over all local batchs
		int in_off1 = in_off;
		for (int ib = 0; ib < MLO_N_LCL_BATCHS
			; ++ib, in_off1 += MLO_IN_BATCH_STRIDE)
		{
			int in_off2 = in_off1;
			// lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP
			for (int ilc = 0; ilc < MLO_N_LCL_IN_MAPS; ++ilc, in_off2 += MLO_IN_CHANNEL_STRIDE * MLO_N_MAPS_PERGROUP)
			{
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_stage[ib][ilc][i] = 0;
				}

				if (c*MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP + in_map_id + ilc* MLO_N_MAPS_PERGROUP < MLO_N_INPUTS
#if MLO_BATCH_ALIGNED == 0
					&& (batch_block*MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
				)
				{
#if MLO_C1x1_PIXLEFT > 0
					// if the last one
					if (pix_id == MLO_MAP_SZ4 - 1)
					{

						for (int i = 0; i < MLO_C1x1_PIXLEFT; ++i)
						{
							in_stage[ib][ilc][i] = in_ptr[in_off2 + i];
#if DBG_OUT_OF_RNGE
							if (in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}
					}
					else

#endif
					{
						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_stage[ib][ilc][i] = in_ptr[in_off2 + i];
#if DBG_OUT_OF_RNGE
							if (in_off2 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}
					}
				}
			}
		}

		// convolve
		for (int olc = 0, lcl_wei_off = wei_indx*MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PERGROUP*MLO_WEI_CHANNEL_STRIDE; olc < MLO_N_LCL_OUT_MAPS; ++olc, lcl_wei_off += MLO_WEIGHTS_ROW)
		{
			// lcl in maps (in data tiles) is has the stride = MLO_N_MAPS_PERGROUP, weights are mapped accordingly
			int lcl_wei_off1 = lcl_wei_off;
			for (int ilc = 0; ilc < MLO_N_LCL_IN_MAPS; ++ilc, lcl_wei_off1 += MLO_N_MAPS_PERGROUP*MLO_WEI_CHANNEL_STRIDE)
			{
				// read weights
				int lcl_wei_off2 = lcl_wei_off1 + mul24(in_map_id, (int)MLO_WEI_CHANNEL_STRIDE);
				wei_stage = lcl_wei_stage[lcl_wei_off2];
				for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						out_tiles[ib][olc][i] += in_stage[ib][ilc][i] * wei_stage;
					}
				}
			}
		}

	}

// out of range check
	if (in_map_id >= MLO_N_MAPS_PERGROUP || in_map_id*MLO_N_LCL_IN_MAPS >= MLO_N_INPUTS)
	{
		return;
	}

	out_block = out_grp_block * MLO_N_LCL_OUT_MAPS;
	int out_off = batch_block * MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE
		+ out_block *  MLO_OUT_CHANNEL_STRIDE
		+ pix_id * MLO_READ_UNIT;

// small groups
#if MLO_N_MAPS_PERGROUP > 1

	// calculate reduction over all partial sums
	// MLO_N_LCL_OUT_MAPS is multiple of MLO_EXCHANGE_STEP
	// write data into local memory

	for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)
	{
		for (int t = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id0 < MLO_MAP_SZ4 * MLO_N_MAPS_PERGROUP)
			{
				for (int om = 0; om < MLO_EXCHANGE_STEP; ++om)
				{
					int lcl_off = (om * MLO_MAP_SZ4*MLO_N_MAPS_PERGROUP + in_map_id*MLO_MAP_SZ4 + pix_id) * MLO_READ_UNIT;
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_out_stage[lcl_off + i] = out_tiles[ib][t + om][i];
					}
				}

			}
			barrier(CLK_LOCAL_MEM_FENCE);

			// sum partial sum
			// MLO_N_MAPS_PERGROUP >= MLO_EXCHANGE_STEP
			// in_map_id is an index of the output map now.
			if (in_map_id < MLO_EXCHANGE_STEP)
			{
				_FLOAT sum[MLO_READ_UNIT];
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					sum[i] = 0;
				}

				for (int s = 0; s < MLO_N_MAPS_PERGROUP; ++s)
				{
					int imp = in_map_id + s;
					imp = (imp >= MLO_N_MAPS_PERGROUP) ? imp - MLO_N_MAPS_PERGROUP : imp;
					int lcl_off = (in_map_id* MLO_MAP_SZ4*MLO_N_MAPS_PERGROUP + imp*MLO_MAP_SZ4 + pix_id) * MLO_READ_UNIT;
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						sum[i] += lcl_out_stage[lcl_off + i];
					}

				}

				// write it out
				int olc = t + in_map_id;

				if (true
#if MLO_BATCH_ALIGNED == 0
					&& (batch_block*MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
					&& out_block + olc < MLO_N_OUTPUTS
#endif
					)
				{

					int out_off2 = out_off + ib * MLO_OUT_BATCH_STRIDE + olc * MLO_OUT_CHANNEL_STRIDE;

#if MLO_CONV_BIAS
					_FLOAT  bias_val = 0;
					bias_val = bias[out_block* MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

					// if the last one
					if (pix_id == MLO_MAP_SZ4 - 1)
					{
						for (int i = 0; i < MLO_C1x1_PIXLEFT; ++i)
						{
							out_ptr[out_off2 + i] = sum[i]
#if MLO_CONV_BIAS
								+ bias_val
#endif
								;

#if DBG_OUT_OF_RNGE
							if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:out-of-range\n");
							}
#endif

						}

					}
					else
#endif
					{

						for (int i = 0; i < MLO_READ_UNIT; ++i)
						{
							out_ptr[out_off2 + i] = sum[i]
#if MLO_CONV_BIAS
								+ bias_val
#endif
								;
#if DBG_OUT_OF_RNGE
							if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:out-of-range\n");
							}
#endif

						}

					}

				} //if (true

			} // if (in_map_id < MLO_EXCHANGE_STEP)

		} // for (int t = 0, p = 0; t < MLO_N_LCL_OUT_MAPS; t += MLO_EXCHANGE_STEP)

	} // 	for (int ib = 0; ib < MLO_N_LCL_BATCHS; ++ib)

#else // #if MLO_N_MAPS_PERGROUP > 1

	int out_off1 = out_off;
	for (int ib = 0; ib < MLO_N_LCL_BATCHS
		; ++ib, out_off1 += MLO_OUT_BATCH_STRIDE)
	{

		int out_off2 = out_off1;
		for (int olc = 0; olc < MLO_N_LCL_OUT_MAPS
			; ++olc, out_off2 += MLO_OUT_CHANNEL_STRIDE)
		{

		    if ( true
#if MLO_BATCH_ALIGNED == 0
			&& (batch_block*MLO_N_LCL_BATCHS + ib < MLO_BATCH_SZ)
#endif
#if MLO_OUTPUTS_ALIGNED == 0
			&& (out_block + olc < MLO_N_OUTPUTS)
#endif
			)
			{
#if MLO_CONV_BIAS
				_FLOAT  bias_val = 0;
				bias_val = bias[out_block* MLO_N_LCL_OUT_MAPS + olc];
#endif
#if MLO_C1x1_PIXLEFT > 0

			// if the last one
				if (pix_id == MLO_MAP_SZ4 - 1)
				{
					for (int i = 0; i < MLO_C1x1_PIXLEFT; ++i)
					{
						out_ptr[out_off2 + i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
							+ bias_val
#endif
						;
#if DBG_OUT_OF_RNGE
						if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-of-range\n");
						}
#endif

					}

				}
				else
#endif
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{

						out_ptr[out_off2 + i] = out_tiles[ib][olc][i]
#if MLO_CONV_BIAS
						+ bias_val
#endif
						;
#if DBG_OUT_OF_RNGE
						if (out_off2 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-of-range\n");
						}
#endif
					}
				}
			}
		}

	}

#endif // #if MLO_N_MAPS_PERGROUP > 1

}