restore LRN Bwd

restore LRN Bwd

/*
 * Copyright (c) 2015 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_GROUP_SZ0 *MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_LCL_DATA_HEIGHT  (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0,MLO_LRN_GROUP_SZ1,MLO_LRN_GROUP_SZ2)))
__kernel void MLOpenLRNWithinChannel(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
#if MLO_LRN_DO_SCALE
	   __global _FLOAT * scale,
#endif
	   _FLOAT alphaoverarea,
	   _FLOAT alpha,
	   _FLOAT beta,
	   _FLOAT K
	   )
{

// IT's taken from POOLING AVE with stride = 1'
		__local _FLOAT bot_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
		int x = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
		int lcl_id = (lcl_id1 << MLO_LRN_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int o = ob / MLO_LRN_BATCH_SZ;
		int b = ob - o * MLO_LRN_BATCH_SZ;
		int bot_x = x;
		int bot_y = y;
		int bot_off = b * MLO_LRN_BOT_BATCH_STRIDE + o * MLO_LRN_BOT_CHANNEL_STRIDE;

		_FLOAT prv_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

// load tile
		for( int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j - MLO_LRN_PAD;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_LRN_BOT_HEIGHT);

			bot_y_act = (bot_y_act < 0) ? 0 : (bot_y_act >= MLO_LRN_BOT_HEIGHT) ? MLO_LRN_BOT_HEIGHT - 1: bot_y_act;

			int bot_y_off = bot_y_act * MLO_LRN_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_LRN_PAD;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_LRN_BOT_WIDTH);

				bot_x_act = (bot_x_act < 0) ? 0 : (bot_x_act >= MLO_LRN_BOT_WIDTH) ? MLO_LRN_BOT_WIDTH - 1: bot_x_act;

				_FLOAT bot_val = bot[bot_off + bot_y_off + bot_x_act];

				bot_val = (invisibleX || invisibleY)?
							0 :
							bot_val;

				bot_val = bot_val;

				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int top_y = (y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX);

		int lcl_y = lcl_id1 * MLO_LRN_N_VERT_OUT_PIX;
		int lcl_x = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX;
		int lcl_off = lcl_y * MLO_LRN_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++)
		{

			int hstart = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX  + k - MLO_LRN_PAD;
			int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_HEIGHT + MLO_LRN_PAD);

			for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++)
			{

				int wstart = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + l - MLO_LRN_PAD;
				int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_WIDTH + MLO_LRN_PAD);

				int adj_area_size = (hend - hstart) * (wend - wstart);

				_FLOAT accum = 0;
				for( int j = 0; j < MLO_LRN_KERNEL_SZ; j++)
				{
					for(int i = 0; i < MLO_LRN_KERNEL_SZ; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k +j)*MLO_LRN_LCL_DATA_WIDTH + (l +i)];
						accum   += bot_val * bot_val;

					}
				}

				_FLOAT adj_alphaoverarea = alpha / adj_area_size;
				prv_scale[k][l]  = K + accum * adj_alphaoverarea ;

#if 0
				if ( top_x + l == 7 && top_y + k == 0 && o == 0)
				{
					printf("K:lrn: %13.11f %13.11f %13.11f\n", prv_scale[k][l], accum, adj_alphaoverarea);
				}
#endif
			}
		}

		int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE + top_y * MLO_LRN_TOP_STRIDE + top_x;
		int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE + top_y * MLO_LRN_SCALE_STRIDE + top_x;

		for( int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++)
			{
				_FLOAT s;
				_FLOAT bot_val;

				if (top_y + k < MLO_LRN_TOP_HEIGHT && top_x + l < MLO_LRN_TOP_WIDTH)
				{

					s = native_exp((_FLOAT)-beta * native_log(prv_scale[k][l]));
//					s = pow(prv_scale[k][l], -beta);
					bot_val =  bot_data[lcl_off + (k +MLO_LRN_PAD)*MLO_LRN_LCL_DATA_WIDTH + (l + MLO_LRN_PAD)];
#if MLO_LRN_DO_SCALE
					scale[scale_off + k * MLO_LRN_SCALE_STRIDE +l] = prv_scale[k][l];
#endif
					top[top_off + k * MLO_LRN_TOP_STRIDE +l] = bot_val * s;
				}
#if 0
				if ( top_x + l == 9 && top_y + k == 4 && o == 0)
				{
					printf("K:lrn: %13.11f %13.11f %13.11f %13.11f\n", top[top_off + k * MLO_LRN_TOP_STRIDE +l], bot_val, s, prv_scale[k][l]);
				}
#endif

			}
		}

}

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0,MLO_LRN_GROUP_SZ1,MLO_LRN_GROUP_SZ2)))
__kernel void MLOpenLRNWithinChannelBwd(
       const __global _FLOAT * top,
	   const __global _FLOAT *	bot,
       const __global _FLOAT * top_df,
	   const __global _FLOAT *	scale,
       __global _FLOAT * bot_df,
	   _FLOAT ratio, //2. * alpha * beta / local_area
	   _FLOAT alpha,
	   _FLOAT beta
	   )
{
		__local _FLOAT top_df_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
		__local _FLOAT ratio_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
		int x = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
		int lcl_id = (lcl_id1 << MLO_LRN_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int o = ob / MLO_LRN_BATCH_SZ;
		int b = ob - o * MLO_LRN_BATCH_SZ;
		int top_x = x;
		int top_y = y;
		int top_df_off = b * MLO_LRN_TOPDF_BATCH_STRIDE + o * MLO_LRN_TOPDF_CHANNEL_STRIDE;
		int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE;
		int bot_x = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX;
		int bot_y = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX;

		_FLOAT prv_exp_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
//		_FLOAT prv_top_df[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

		// load top_diff and scale tiles
		for( int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
		{
			int top_y_act = top_y + b_j - MLO_LRN_PAD;

			bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

//			top_y_act = (bot_y_act < 0) ? 0 : (bot_y_act >= MLO_LRN_BOT_HEIGHT) ? MLO_LRN_BOT_HEIGHT - 1: bot_y_act;

			int top_df_y_off = top_y_act * MLO_LRN_TOPDF_STRIDE;
			int scale_y_off = top_y_act * MLO_LRN_SCALE_STRIDE;

			int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
			{

				int top_x_act = top_x + b_i - MLO_LRN_PAD;

				bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

//				bot_x_act = (bot_x_act < 0) ? 0 : (bot_x_act >= MLO_LRN_BOT_WIDTH) ? MLO_LRN_BOT_WIDTH - 1: bot_x_act;

				_FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act];
				_FLOAT scale_val = scale[scale_off + scale_y_off + top_x_act];

				top_df_val = (invisibleX || invisibleY)?
							0 :
							top_df_val;
				scale_val = (invisibleX || invisibleY)?
							1.f :
							scale_val;

				top_df_data[lcl_off_v + b_i] = top_df_val;
				ratio_data[lcl_off_v + b_i] = scale_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		// actual top_diffs and scales
		for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
		{
			int lcl_off_v = (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_PAD + j) *  MLO_LRN_LCL_DATA_WIDTH;
			for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
			{
				_FLOAT scale = ratio_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i];
				prv_exp_scale[j][i]= native_exp(-beta * native_log(scale));
//				prv_exp_scale[j][i]= pow(scale, -beta);
#if 0
				if (o==0 && b==0 && (bot_x + i) ==2 && (bot_y + j) == 0)
				{
					printf("K:scl: %d %d  %11.9f %11.9f %11.9f\n",
					i, j,
					prv_exp_scale[j][i],
					beta, scale
					);
				}
#endif

//				_FLOAT top_df_val = top_df_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i];
//				prv_top_df[j][i] = top_df_val;
			}
		}

// read top and load ratio tile
		int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE;
		for( int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
		{
			int top_y_act = top_y + b_j - MLO_LRN_PAD;

			bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

//			top_y_act = (bot_y_act < 0) ? 0 : (bot_y_act >= MLO_LRN_BOT_HEIGHT) ? MLO_LRN_BOT_HEIGHT - 1: bot_y_act;

			int top_y_off = top_y_act * MLO_LRN_TOP_STRIDE;

			int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
			{

				int top_x_act = top_x + b_i - MLO_LRN_PAD;

				bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

//				bot_x_act = (bot_x_act < 0) ? 0 : (bot_x_act >= MLO_LRN_BOT_WIDTH) ? MLO_LRN_BOT_WIDTH - 1: bot_x_act;

				_FLOAT top_val = top[top_off + top_y_off + top_x_act];

				top_val = (invisibleX || invisibleY) ? 0 : top_val;

				_FLOAT top_df_val = top_df_data[lcl_off_v + b_i];

				_FLOAT scale_val = ratio_data[lcl_off_v + b_i];

	// scale val is not 0
				_FLOAT 	ratio_dta =
						(top_df_val * top_val) / scale_val;
#if 0
				if (o==0 && b==0 && (b_i >=0 && b_i < 8) && (b_j == 0))
				{
					printf("K:rdt: %d %d  %11.9f %11.9f %11.9f %11.9f\n",
					b_i, b_j,
					top_df_val, top_val,scale_val,
					ratio_dta
					);
				}
#endif
	// replacing scale with ratio
				ratio_data[lcl_off_v + b_i] = ratio_dta;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

// caculate bot diff
		_FLOAT prv_bot_diff[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

		for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
		{
			int v_off_v =  (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + j);
			int hstart = y + v_off_v - MLO_LRN_PAD;
			int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_HEIGHT + MLO_LRN_PAD);

		// accum offset, vertical
//			int lcl_a_off_v = v_off_v *  MLO_LRN_LCL_DATA_WIDTH;
		// value offset, vertical
			int lcl_v_off_v = (v_off_v + MLO_LRN_PAD) *  MLO_LRN_LCL_DATA_WIDTH;
			for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
			{
				_FLOAT prv_ratio_accum = 0;
				int v_off_h = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + i;

				int wstart = x + v_off_h - MLO_LRN_PAD;
				int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_WIDTH + MLO_LRN_PAD);

				int adj_area_size = (hend - hstart) * (wend - wstart);

		// accum offset, horiz
				int lcl_a_off_h = v_off_h;
		//	value offset, horiz
				int lcl_v_off_h = lcl_a_off_h  + MLO_LRN_PAD;

				for(int k = 0; k < MLO_LRN_KERNEL_SZ; k++)
				{
					for(int l = 0; l < MLO_LRN_KERNEL_SZ; l++)
					{
						prv_ratio_accum += ratio_data[(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l];
#if 0
				if (o==0 && b==0 && (bot_x + i) ==2 && (bot_y + j) == 0)
				{
					printf("K:prv: %d %d  %d %d  %11.9f %11.9f\n",
					l, k,
					(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l,
					MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT,
					prv_ratio_accum,
					ratio_data[(v_off_v + k)  * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l]
					);
				}
#endif

					}
				}

				_FLOAT top_df_val = top_df_data[lcl_v_off_v + lcl_v_off_h];
				_FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *o + MLO_LRN_BOT_STRIDE * (y + v_off_v) + x + v_off_h];
				_FLOAT adj_ratio = 2.f * alpha * beta / adj_area_size;
				_FLOAT prv_accum_ratio =
					adj_ratio * bot_dta * prv_ratio_accum;
				prv_bot_diff[j][i] =
					prv_exp_scale[j][i] * top_df_val - prv_accum_ratio;
#if 0
				if (o==0 && b==0 && (bot_x + i) ==2 && (bot_y + j) == 0)
				{
					printf("K:lrn: %11.9f %11.9f %11.9f\n%11.9f %11.9f %11.9f\n%11.9f\n",
					adj_ratio, bot_dta, prv_ratio_accum,
					prv_exp_scale[j][i], top_df_val, prv_accum_ratio,
					prv_bot_diff[j][i]
					);
				}
#endif
			}
		}

		for( int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; j++)
		{
			for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
			{
				if (bot_y + j < MLO_LRN_BOT_HEIGHT && bot_x + i < MLO_LRN_BOT_WIDTH)
				{

					bot_df[MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE *o + MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i] = prv_bot_diff[j][i];
				}
			}
		}

}

#if (MLO_LRN_N_INPUTS + 2* MLO_LRN_PAD - 1 < MLO_LRN_KERNEL_SZ || MLO_LRN_N_OUTPUTSS + 2* MLO_LRN_PAD - 1 < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0,MLO_LRN_GROUP_SZ1,MLO_LRN_GROUP_SZ2)))
__kernel void MLOpenLRNAcrossChannels1(
       const __global _FLOAT * bottom,
       __global _FLOAT * top,
#if MLO_LRN_DO_SCALE
	   __global _FLOAT *scale,
#endif
	   _FLOAT alphaoverarea,
	   _FLOAT alpha,
	   _FLOAT beta,
	   _FLOAT K
	   )
{

		int x = get_global_id(0); // channel x
		int y = get_global_id(1); // channel y
		int b = get_global_id(2); // batch
		_FLOAT accum = 0;
		_FLOAT bot_in[MLO_LRN_KERNEL_SZ];
		_FLOAT bot_in2[MLO_LRN_KERNEL_SZ];
		int c_i = 0, c_o = 0;

		int top_off = 0;
		int scale_off = 0;

		for( c_i = 0; c_i < MLO_LRN_PAD; c_i++)
		{
			_FLOAT bot_val = bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *c_i + MLO_LRN_BOT_STRIDE * y + x];
#if MLO_LOW_CHNL_COUNT
			bot_in[c_i] = (c_i < MLO_LRN_N_INPUTS) ? bot_val : 0;
#endif
			bot_in2[c_i] = bot_in[c_i] * bot_in[c_i];
			accum =
				accum + bot_in2[c_i];
//				fma(bot_in[c_i + MLO_LRN_PAD], bot_in[c_i + MLO_LRN_PAD], accum);

		}

		for( ; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
		{
			_FLOAT bot_val = bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *c_i + MLO_LRN_BOT_STRIDE * y + x];

#if MLO_LOW_CHNL_COUNT
			bot_in[c_i] = (c_i < MLO_LRN_N_INPUTS) ? bot_val : 0;
#endif
			bot_in2[c_i] = bot_in[c_i] * bot_in[c_i];
			accum =
				accum + bot_in2[c_i];

			top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE + y * MLO_LRN_TOP_STRIDE +x;
			scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE + y * MLO_LRN_SCALE_STRIDE +x;
			_FLOAT prv_scale =
				(K + accum * alphaoverarea);
//				fma(accum,alphaoverarea, 1.f);

			_FLOAT exp_scale =
				native_exp(-beta * native_log(prv_scale));
//				pow(prv_scale,-beta);

			_FLOAT out_val = bot_in[c_o] * exp_scale;
#if MLO_LOW_CHNL_COUNT
			if (c_o < MLO_LRN_N_OUTPUTS)
#endif
			{
#if MLO_LRN_DO_SCALE
				scale[scale_off] = prv_scale;
#endif
				top[top_off] = out_val;
			}
		}

		for( ; c_i < MLO_LRN_N_CHANNELS; c_i++, c_o++)
		{

			_FLOAT prv_bot_in = bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *c_i + MLO_LRN_BOT_STRIDE * y + x];
#if MLO_LOW_CHNL_COUNT
			prv_bot_in = (c_i < MLO_LRN_N_INPUTS) ? prv_bot_in : 0;
#endif
			_FLOAT prv_bot_in2 = prv_bot_in * prv_bot_in;
			accum =
				accum + prv_bot_in2;

			accum =
				accum - bot_in2[0];
//				fma(-bot_in[0], bot_in[0], accum);

			for( int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
			{
				bot_in[i] = bot_in[i+1];
				bot_in2[i] = bot_in2[i+1];
			}

			bot_in[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in;
			bot_in2[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in2;

			top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE + y * MLO_LRN_TOP_STRIDE +x;
			scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE + y * MLO_LRN_SCALE_STRIDE +x;
			_FLOAT prv_scale =
				(K + accum * alphaoverarea);
//				fma(accum,alphaoverarea, 1.f);

			_FLOAT exp_scale =
				native_exp(-beta * native_log(prv_scale));
//				pow(prv_scale,-beta);

			_FLOAT out_val = bot_in[MLO_LRN_PAD] * exp_scale;
#if MLO_LOW_CHNL_COUNT
			if (c_o < MLO_LRN_N_OUTPUTS)
#endif
			{
#if MLO_LRN_DO_SCALE
				scale[scale_off] = prv_scale;
#endif
				top[top_off] = out_val;
			}

		}

		for(  ; c_i < MLO_LRN_N_CHANNELS + MLO_LRN_PAD; c_i++, c_o++)
		{

			accum =
				accum - bot_in2[0];
//				fma(-bot_in[0], bot_in[0], accum);

			for( int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
			{
				bot_in[i] = bot_in[i+1];
				bot_in2[i] = bot_in2[i+1];
			}

			top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE + y * MLO_LRN_TOP_STRIDE + x;
			scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE + y * MLO_LRN_SCALE_STRIDE +x;

			_FLOAT prv_scale =
				(K + accum * alphaoverarea);
//				fma(accum,alphaoverarea, 1.f);

			_FLOAT exp_scale =
				native_exp(-beta * native_log(prv_scale));
	//			pow(prv_scale,-beta);

			_FLOAT out_val = bot_in[MLO_LRN_PAD] * exp_scale;
#if MLO_LOW_CHNL_COUNT
			if (c_o < MLO_LRN_N_OUTPUTS)
#endif
			{
#if MLO_LRN_DO_SCALE
				scale[scale_off] = prv_scale;
#endif
				top[top_off] = out_val;
			}
		}

}

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0,MLO_LRN_GROUP_SZ1,MLO_LRN_GROUP_SZ2)))
__kernel void MLOpenLRNAcrossChannelsBwd1(
       const __global _FLOAT * top,
	   const __global _FLOAT *	bot,
       const __global _FLOAT * top_df,
	   const __global _FLOAT *	scale,
       __global _FLOAT * bot_df,
	   _FLOAT ratio, //2. * alpha * beta / local_area
	   _FLOAT alpha,
	   _FLOAT beta
	   )
{

		int x = get_global_id(0); // channel x
		int y = get_global_id(1); // channel y
		int b = get_global_id(2); // batch
		_FLOAT accum_ratio = 0;
		_FLOAT top_df_in[MLO_LRN_KERNEL_SZ];
		_FLOAT scale_in[MLO_LRN_KERNEL_SZ];
		_FLOAT ratio_dta[MLO_LRN_KERNEL_SZ];
		int c_i = 0, c_o = 0;
		int bot_df_off = 0;

		for( c_i = 0; c_i < MLO_LRN_PAD; c_i++)
		{

			top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE *c_i + MLO_LRN_TOPDF_STRIDE * y + x];
			scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i + MLO_LRN_SCALE_STRIDE * y + x];
			_FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE *b  + MLO_LRN_TOP_CHANNEL_STRIDE * c_i + MLO_LRN_TOP_STRIDE * y + x];

			ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];

#if MLO_LOW_CHNL_COUNT
			ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif
#if 0
			if ( x == 5 && y == 11/* && c_o==12 */&& b==10)
			{
				printf("K:a %d %f %f\n",
					c_i,
					accum_ratio,
					ratio_dta[c_i]

				);

			}

#endif

 			accum_ratio =
				accum_ratio + ratio_dta[c_i] ;

		}

		for( ; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
		{
			top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE *c_i + MLO_LRN_TOPDF_STRIDE * y + x];
			scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i + MLO_LRN_SCALE_STRIDE * y + x];
			_FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE *b  + MLO_LRN_TOP_CHANNEL_STRIDE * c_i + MLO_LRN_TOP_STRIDE * y + x];
			ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];
#if MLO_LOW_CHNL_COUNT
			ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

#if 0
			if ( x == 5 && y == 11/* && c_o==12 */&& b==10)
			{
				printf("K:a %d %f %f\n",
					c_i,
					accum_ratio,
					ratio_dta[c_i]

				);

			}

#endif

 			accum_ratio =
				accum_ratio + ratio_dta[c_i] ;
#if MLO_LOW_CHNL_COUNT
			if (c_o < MLO_LRN_N_INPUTS)
#endif
			{
				_FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *c_o + MLO_LRN_BOT_STRIDE * y + x];

				_FLOAT prv_scale = scale_in[c_o];

				_FLOAT exp_scale =
					native_exp(-beta * native_log(prv_scale));
//					pow(prv_scale, -beta);

				_FLOAT prv_accum_ratio =
					ratio * bot_dta * accum_ratio;

				_FLOAT out_val = top_df_in[c_o] * exp_scale - prv_accum_ratio;

				bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o + MLO_LRN_BOTDF_STRIDE * y + x;

				bot_df[bot_df_off] = out_val;
			}

		}

		for( ; c_i < MLO_LRN_N_CHANNELS; c_i++, c_o++)
		{

			_FLOAT prv_top_df_in = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE *c_i + MLO_LRN_TOPDF_STRIDE * y + x];
			_FLOAT prv_scale_in = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i + MLO_LRN_SCALE_STRIDE * y + x];
			_FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE *b  + MLO_LRN_TOP_CHANNEL_STRIDE * c_i + MLO_LRN_TOP_STRIDE * y + x];
			_FLOAT prv_ratio_dta = prv_top_df_in * top_dta / prv_scale_in;
#if MLO_LOW_CHNL_COUNT
			prv_ratio_dta = (c_i < MLO_LRN_N_OUTPUTS) ? prv_ratio_dta : 0;
#endif

#if 0
			if ( x == 5 && y == 11/* && c_o==12 */&& b==10)
			{
				printf("K:a %d %f %f\n",
					c_i,
					accum_ratio,
					prv_ratio_dta

				);

			}

#endif

 			accum_ratio =
				accum_ratio + prv_ratio_dta ;

			accum_ratio =
				accum_ratio - ratio_dta[0];

			for( int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
			{
				top_df_in[i] = top_df_in[i+1];
				scale_in[i] = scale_in[i+1];
				ratio_dta[i] = ratio_dta[i+1];
			}

			top_df_in[MLO_LRN_KERNEL_SZ - 1] = prv_top_df_in;
			scale_in[MLO_LRN_KERNEL_SZ - 1] = prv_scale_in;
			ratio_dta[MLO_LRN_KERNEL_SZ - 1]= prv_ratio_dta;

#if 0
			if ( x == 3 && y == 0/* && c_o==12 */&& b==3)
			{
				printf("K: %d %16.12f %16.12f %16.12f %16.12f %16.12f\n",
					c_i,
					accum_ratio,
					prv_ratio_dta,
					prv_top_df_in,
					top_dta,
					prv_scale_in
				);

			}

#endif

#if MLO_LOW_CHNL_COUNT
			if (c_o < MLO_LRN_N_INPUTS)
#endif
			{
				_FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *c_o + MLO_LRN_BOT_STRIDE * y + x];

				_FLOAT prv_scale = scale_in[MLO_LRN_PAD];

				_FLOAT exp_scale =
					native_exp(-beta * native_log(prv_scale));
				//				pow(prv_scale,-beta);

				_FLOAT prv_accum_ratio =
					ratio * bot_dta * accum_ratio;

				_FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

				bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o + MLO_LRN_BOTDF_STRIDE * y + x;

				bot_df[bot_df_off] = out_val;
			}

		}

		for (; c_i < MLO_LRN_N_CHANNELS + MLO_LRN_PAD; c_i++, c_o++)
		{

			accum_ratio =
				accum_ratio - ratio_dta[0];

			for (int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
			{
				top_df_in[i] = top_df_in[i + 1];
				scale_in[i] = scale_in[i + 1];
				ratio_dta[i] = ratio_dta[i + 1];

			}

#if MLO_LOW_CHNL_COUNT
			if (c_o < MLO_LRN_N_INPUTS)
#endif
			{
				_FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE *c_o + MLO_LRN_BOT_STRIDE * y + x];

				_FLOAT prv_scale = scale_in[MLO_LRN_PAD];

				_FLOAT exp_scale =
					native_exp(-beta * native_log(prv_scale));
				//				pow(prv_scale,-beta);

				_FLOAT prv_accum_ratio =
					ratio * bot_dta * accum_ratio;

				_FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

				bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o + MLO_LRN_BOTDF_STRIDE * y + x;

				bot_df[bot_df_off] = out_val;

#if 0
				if ( x == 5 && y == 11/* && c_o==12 */&& b== 10)
				{
					printf("K: %d %10.8f %10.8f %10.8f %10.8f %10.8f %10.8f %10.8f\n",
						c_i,
						bot_df[bot_df_off],
						top_df_in[MLO_LRN_PAD],
						exp_scale,
						prv_scale,
						-prv_accum_ratio,
						bot_dta,
						accum_ratio
						);

				}

#endif
			}
		}

}

#if 0
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0,MLO_LRN_GROUP_SZ1,MLO_LRN_GROUP_SZ2)))
__kernel void aDNNLRNAcrossChannels4(
       const __global _FLOAT * bottom,
       __global _FLOAT * top,
	   int bot_width,
	   int bot_height,
	   int bot_stride,
	   int bot_channel_stride,
	   int bot_batch_stride,
	   int top_stride,
	   int top_channel_stride,
	   int top_batch_stride,
	   _FLOAT padding_value,
	   _FLOAT alphaoverarea,
	   _FLOAT beta
	   )
{

		int x = get_global_id(0); // channel x
		int y = get_global_id(1); // channel y
		int b = get_global_id(2); // batch
		_FLOAT4 accum = 0;
		_FLOAT4 bot_in[MLO_LRN_LOCALAREA_SZ];
		int c_i = 0, c_o = 0;
		for( ; c_i < MLO_LRN_LOCALAREA_PAD_SZ; c_i++)
		{
			bot_in[c_i] = (_FLOAT4)padding_value;
			accum += bot_in[c_i] * bot_in[c_i];
		}

		for( ; c_i < MLO_LRN_LOCALAREA_SZ; c_i++)
		{

			bot_in[c_i] = *(__global _FLOAT4*)&bottom[bot_batch_stride * b + bot_channel_stride *c_i + bot_stride * y + x*MLO_LRN_N_HORIZ_OUT_PIX];
			accum += bot_in[c_i] * bot_in[c_i];

		}

		int top_off = 0;

		for( ; c_i < MLO_LRN_N_CHANNELS + MLO_LRN_LOCALAREA_PAD_SZ; c_i++)
		{
			top_off = b * top_batch_stride + c_o++ * top_channel_stride + top_stride *y + x*MLO_LRN_N_HORIZ_OUT_PIX;
			_FLOAT4 a = ((_FLOAT4)1.f + accum * alphaoverarea);
			_FLOAT4 scale = native_exp((_FLOAT4)-beta * native_log(a));
			_FLOAT4 out_val = bot_in[MLO_LRN_LOCALAREA_PAD_SZ + 1] * scale;
			*(__global _FLOAT4*)&top[top_off] = out_val;

			accum -= bot_in[0] * bot_in[0];
			for( int i = 0; i < MLO_LRN_LOCALAREA_SZ - 1; i++)
			{
				bot_in[i] = bot_in[i+1];
			}

			bot_in[MLO_LRN_LOCALAREA_SZ - 1] = *(__global _FLOAT4*)&bottom[bot_batch_stride * b + bot_channel_stride *c_i + bot_stride * y + x*MLO_LRN_N_HORIZ_OUT_PIX];
			accum += bot_in[MLO_LRN_LOCALAREA_SZ - 1] * bot_in[MLO_LRN_LOCALAREA_SZ - 1];
		}

		for(  ; c_i < MLO_LRN_N_CHANNELS + MLO_LRN_LOCALAREA_PAD_SZ * 2; c_i++)
		{
			top_off = b * top_batch_stride + c_o++ * top_channel_stride + top_stride *y + x*MLO_LRN_N_HORIZ_OUT_PIX;

			_FLOAT4 a = ((_FLOAT4)1.f + accum * alphaoverarea);
			_FLOAT4 scale = native_exp((_FLOAT4)-beta * native_log(a));
			_FLOAT4 out_val = bot_in[MLO_LRN_LOCALAREA_PAD_SZ + 1] * scale;
			*(__global _FLOAT4*)&top[top_off] = out_val;

			accum -= bot_in[0] * bot_in[0];
			for( int i = 0; i < MLO_LRN_LOCALAREA_SZ - 1; i++)
			{
				bot_in[i] = bot_in[i+1];
			}

			bot_in[MLO_LRN_LOCALAREA_SZ - 1] = (_FLOAT4)padding_value;
			accum += bot_in[MLO_LRN_LOCALAREA_SZ - 1] * bot_in[MLO_LRN_LOCALAREA_SZ - 1];
		}

}

#define MLO_LRN_LCL_STRIDE (MLO_LRN_GROUP_SZ0)
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0,MLO_LRN_GROUP_SZ1,MLO_LRN_GROUP_SZ2)))
__kernel void aDNNLRNWithinChannel4_3x3(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
	   int bot_width,
	   int bot_height,
	   int bot_stride,
	   int bot_channel_stride,
	   int bot_batch_stride,
	   int top_stride,
	   int top_channel_stride,
	   int top_batch_stride,
	   _FLOAT padding_value,
	   _FLOAT alphaoverarea,
	   _FLOAT beta
	   )
{

	__local _FLOAT4 lcl_sum[ MLO_LRN_LCL_STRIDE *(MLO_LRN_GROUP_SZ1 + MLO_LRN_LOCALAREA_PAD_SZ * 2)];
	int x = get_global_id(0); // channel x
	int lcl_x = get_local_id(0);
	int grp_x = get_group_id(0);
	int y = get_global_id(1); // channel y
	int lcl_y = get_local_id(1);
	int grp_y = get_group_id(1);
	int cb= get_global_id(2); // channels * batch
	int b = (int)((float)cb / MLO_LRN_N_CHANNELS);
	int c = cb - b * MLO_LRN_N_CHANNELS;
	int bot_off= 0;
	_FLOAT accum[4] = {0,0,0,0};
	_FLOAT4 accum4 = 0;
	_FLOAT4 data4;

// Y padding should not affect the DATA being read.
// the DATA is going to be scaled by the final sum.
// we needd only top and borttom part of accum, since the middle has been calculated already. Again it should not be affected by Y padding.

	int x_group_off = grp_x * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX - MLO_LRN_LOCALAREA_PAD_SZ;

	int x_off = x_group_off + lcl_x;

	int y_group_off = grp_y * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX - MLO_LRN_GROUP_SZ1;

	int y_off = y_group_off + lcl_y ;
	while (y_off - y_group_off < MLO_LRN_GROUP_SZ1 + MLO_LRN_LOCALAREA_PAD_SZ * 2)
	{

		bool top_padding = (y_off < 0);
		bool bot_padding = (y_off >= bot_height);
		if ( top_padding )
		{
			lcl_sum[(y_off + MLO_LRN_LOCALAREA_PAD_SZ) *MLO_LRN_LCL_STRIDE + lcl_x] = (_FLOAT4)padding_value;
		}
		if ( bot_padding )
		{

			lcl_sum[(bot_height - y_off + MLO_LRN_LOCALAREA_SZ + MLO_LRN_LOCALAREA_PAD_SZ) *MLO_LRN_LCL_STRIDE + lcl_x] = (_FLOAT4)padding_value;
		}
		if ( top_padding || bot_padding )
		{
			y_off += MLO_LRN_GROUP_SZ1;
			continue;
		}
// top/left with padding
// padding should not exeed 4 - 9x9 local area
		for(int i = 0; i < MLO_LRN_LOCALAREA_PAD_SZ; i++)
		{
			bool left_padding = (x_off < 0);
			x_off = (left_padding) ? 0 : x_off;

			bot_off = b * bot_batch_stride + c*bot_channel_stride + y_off* bot_stride + x_off;

			_FLOAT val = bot[bot_off];

			val = (left_padding) ? padding_value : val;

			for(int j = 0; j < i; j++)
			{
				accum[j] += val * val;
			}
			x_off++;
		}
// assuming  - data is alwways group_size*4 aligned horizontally an dvertically
// read 4 per wk-item
		bot_off = b * bot_batch_stride + c*bot_channel_stride + y_off* bot_stride + x_off;
		data4 = *(__global _FLOAT4 *)&bot[bot_off];
		_FLOAT data[4] = {data4.s0, data4.s1, data4.s2, data4.s3};
	// checlk LOOP HERE
// left padding
		for (int j = 0; j < MLO_LRN_LOCALAREA_SZ - MLO_LRN_LOCALAREA_PAD_SZ; j++)
		{
			accum[0] += data[j] * data[j];
		}
// inside
		for ( int i = 1; i < 3; i++)
		{
			for (int j = i - 1; j < i - 1 + MLO_LRN_LOCALAREA_SZ; j++)
			{
				accum[i] += data[j] * data[j];
			}
		}
// right padding

		for (int j = 2; j < 2 + MLO_LRN_LOCALAREA_SZ - MLO_LRN_LOCALAREA_PAD_SZ; j++)
		{
			accum[3] += data[j] * data[j];
		}

		x_off += MLO_LRN_N_HORIZ_OUT_PIX;
		for(int i = 0; i < MLO_LRN_LOCALAREA_PAD_SZ; i++)
		{

			x_off += i;
			bool right_padding = x_off >= bot_width;
			x_off = (right_padding) ? 0 : x_off;
			bot_off = b * bot_batch_stride + c*bot_channel_stride + y_off* bot_stride + x_off;

			_FLOAT val = bot[bot_off];

			val = (right_padding ) ? padding_value : val;

			for(int j = MLO_LRN_N_HORIZ_OUT_PIX - (MLO_LRN_LOCALAREA_SZ - MLO_LRN_LOCALAREA_PAD_SZ) + i; j < MLO_LRN_N_HORIZ_OUT_PIX; j++)
			{
				accum[j] += val * val;
			}

		}

// write into local
		accum4.s0 = accum[0];
		accum4.s1 = accum[1];
		accum4.s2 = accum[2];
		accum4.s3 = accum[3];
		lcl_sum[(y_off - y_group_off) * MLO_LRN_LCL_STRIDE + lcl_x] = accum4;
// bottom padding
		y_off += MLO_LRN_GROUP_SZ1;

	}

	barrier(CLK_LOCAL_MEM_FENCE);
// final summ
	for(int j = lcl_y; j < lcl_y + MLO_LRN_LOCALAREA_PAD_SZ; j++)
	{
		accum4 += lcl_sum[j * MLO_LRN_LCL_STRIDE + lcl_x];
	}
	for(int j = lcl_y + MLO_LRN_LOCALAREA_PAD_SZ + 1; j < lcl_y + MLO_LRN_LOCALAREA_SZ; j++)
	{
		accum4 += lcl_sum[j * MLO_LRN_LCL_STRIDE + lcl_x];
	}
// done summation
	mem_fence(CLK_LOCAL_MEM_FENCE);
// calculate scale
	_FLOAT4 a = ((_FLOAT4)1.f + accum4 * alphaoverarea);
	_FLOAT4 scale = native_exp((_FLOAT4)-beta * native_log(a));

	int top_off = b * top_batch_stride + c*top_channel_stride + y* bot_stride + x;
	*(__global _FLOAT4 *)&top[top_off] = data4 * scale;

}

#endif


Updating a test case that was missed in r369957.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@369968 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only

// This test checks that various C/C++/OpenCL C constructs are not available in
// C++ for OpenCL.

// Test that typeid is not available.
namespace std {
  // Provide a dummy std::type_info so that we can use typeid.
  class type_info {
    int a;
  };
}
__constant std::type_info int_ti = typeid(int);
// expected-error@-1 {{'typeid' is not supported in C++ for OpenCL}}

// Test that dynamic_cast is not available in C++ for OpenCL.
class A {
public:
  int a;
};

class B : public A {
  int b;
};

B *test_dynamic_cast(B *p) {
  return dynamic_cast<B *>(p);
  // expected-error@-1 {{'dynamic_cast' is not supported in C++ for OpenCL}}
}

// Test storage class qualifiers.
__constant _Thread_local int a = 1;
// expected-error@-1 {{C++ for OpenCL version 1.0 does not support the '_Thread_local' storage class specifier}}
// expected-warning@-2 {{_Thread_local is a C11 extension}}

__constant __thread int b = 2;
// expected-error@-1 {{C++ for OpenCL version 1.0 does not support the '__thread' storage class specifier}}
kernel void test_storage_classes() {
  register int x;
  // expected-error@-1 {{C++ for OpenCL version 1.0 does not support the 'register' storage class specifier}}
  thread_local int y;
  // expected-error@-1 {{C++ for OpenCL version 1.0 does not support the 'thread_local' storage class specifier}}
}

// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only

// This test checks that various C/C++/OpenCL C constructs are not available in
// C++ for OpenCL.

// Test that typeid is not available.
namespace std {
  // Provide a dummy std::type_info so that we can use typeid.
  class type_info {
    int a;
  };
}
__constant std::type_info int_ti = typeid(int);
// expected-error@-1 {{'typeid' is not supported in C++ for OpenCL}}

// Test that dynamic_cast is not available in C++ for OpenCL.
class A {
public:
  int a;
};

class B : public A {
  int b;
};

B *test_dynamic_cast(B *p) {
  return dynamic_cast<B *>(p);
  // expected-error@-1 {{'dynamic_cast' is not supported in C++ for OpenCL}}
}

// Test storage class qualifiers.
__constant _Thread_local int a = 1;
// expected-error@-1 {{C++ for OpenCL version 1.0 does not support the '_Thread_local' storage class specifier}}
// expected-warning@-2 {{_Thread_local is a C11-specific feature}}

__constant __thread int b = 2;
// expected-error@-1 {{C++ for OpenCL version 1.0 does not support the '__thread' storage class specifier}}
kernel void test_storage_classes() {
  register int x;
  // expected-error@-1 {{C++ for OpenCL version 1.0 does not support the 'register' storage class specifier}}
  thread_local int y;
  // expected-error@-1 {{C++ for OpenCL version 1.0 does not support the 'thread_local' storage class specifier}}
}


cl formatted

cl formatted

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define MLO_NRN_GROUP_SZ2 1

#define MLO_NEURON_PASTHRU 0                       // x
#define MLO_NEURON_LOGISTIC MLO_NEURON_PASTHRU + 1 //	1 / (1 + e^-x)	//Sigmoid
#define MLO_NEURON_TANH MLO_NEURON_LOGISTIC + 1    //	a * tanh( b * x)
#define MLO_NEURON_RELU MLO_NEURON_TANH + 1        //	max(0, x)
#define MLO_NEURON_SOFTRELU \
    MLO_NEURON_RELU + 1                        //	log(1 + e^x)   // bonomial normal log likelihood
#define MLO_NEURON_ABS MLO_NEURON_SOFTRELU + 1 //	abs(x)
#define MLO_NEURON_POWER MLO_NEURON_ABS + 1    // (a + b * x ) ^power
//#define MLO_NEURON_BRELU		MLO_NEURON_POWER + 1		//	min(a, max(0, x))
//#define MLO_NEURON_SQUARE		BRELU + 1			//	x^2
//#define MLO_NEURON_SQR			MLO_NEURON_SQUARE + 1		//	sqr(x)
//#define MLO_NEURON_LINEAR		MLO_NEURON_SQR	+ 1			//	a + b * x
#define MLO_NEURON_TOTAL MLO_NEURON_POWER + 1

__attribute__((always_inline)) void
ActivationFunction_PassThru(uint n, _FLOAT* res, const _FLOAT* data)
{
    for(uint i = 0; i < n; i++)
    {
        res[i] = data[i];
    }
}

__attribute__((always_inline)) void
ActivationFunction_ReLU(uint n, _FLOAT* res, const _FLOAT* data, _FLOAT slope)
{
    for(uint i = 0; i < n; ++i)
    {
        res[i] = (data[i] > 0) ? data[i] : data[i] * slope;
    }
}

__attribute__((always_inline)) void
ActivationFunction_BReLU(uint n, _FLOAT* res, const _FLOAT* data, _FLOAT alpha)
{

    for(uint i = 0; i < n; ++i)
    {
        res[i] = fmin(alpha, fmax(data[i], 0));
    }
}

__attribute__((always_inline)) void
ActivationFunction_Sigmoid(uint n, _FLOAT* res, const _FLOAT* data)
{
    for(uint i = 0; i < n; i++)
    {
        // 1/(1 + exp(-x))
        res[i] = (_FLOAT)1.f / ((_FLOAT)1.f + exp(-data[i]));
    }
}

__attribute__((always_inline)) void
ActivationFunction_TanH(int n, _FLOAT* res, const _FLOAT* data, _FLOAT alpha, _FLOAT beta)
{
    for(uint i = 0; i < n; i++)
    {
        // (exp(2x) -1) / (exp(2x) + 1)
        res[i] = alpha * tanh(beta * data[i]);
    }
}
__attribute__((always_inline)) void ActivationFunction_Abs(uint n, _FLOAT* res, const _FLOAT* data)
{
    for(uint i = 0; i < n; i++)
    {
        res[i] = fabs(data[i]);
    }
}

__attribute__((always_inline)) void
ActivationFunction_Square(uint n, _FLOAT* res, const _FLOAT* data)
{
    for(uint i = 0; i < n; i++)
    {

        res[i] = data[i] * data[i];
    }
}

__attribute__((always_inline)) void ActivationFunction_Sqrt(uint n, _FLOAT* res, const _FLOAT* data)
{
    for(uint i = 0; i < n; i++)
    {

        res[i] = sqrt(data[i]);
    }
}

__attribute__((always_inline)) void
ActivationFunction_Linear(uint n, _FLOAT* res, const _FLOAT* data, _FLOAT alpha, _FLOAT beta)
{
    for(uint i = 0; i < n; i++)
    {
        // (exp(2x) -1) / (exp(2x) + 1)
        res[i] = alpha + beta * data[i];
    }
}

__attribute__((always_inline)) void ActivationFunction_Power(
    uint n, _FLOAT* res, const _FLOAT* data, _FLOAT power, _FLOAT alpha, _FLOAT beta)
{
    for(uint i = 0; i < n; i++)
    {
        // (shift + scale * x ) ^power
        _FLOAT arg     = alpha + data[i] * beta;
        _FLOAT run_arg = (arg == (_FLOAT)0) ? (_FLOAT)1 : arg;
        res[i]         = (arg == (_FLOAT)0) ? (_FLOAT)0 : pow(run_arg, power);
    }
}

__attribute__((always_inline)) void ActivationFunction_BNLL(uint n, _FLOAT* res, const _FLOAT* data)

{
    for(uint i = 0; i < n; i++)
    {
        //	log(1 + exp(x))
        res[i] = (data[i] > 0) ? data[i] + log((_FLOAT)1.f + exp(-data[i]))
                               : log((_FLOAT)(1.f) + exp(data[i]));
    }
}

void ActivationFunction(
    uint n, _FLOAT* res, const _FLOAT* data, _FLOAT power, _FLOAT alpha, _FLOAT beta)
{
    (void)power;
    (void)alpha;
    (void)beta;
#if MLO_NRN_OP_ID == MLO_NEURON_PASTHRU

    ActivationFunction_PassThru(n, res, data);

#elif MLO_NRN_OP_ID == MLO_NEURON_LOGISTIC
    // 1/(1 + exp(-x))
    ActivationFunction_Sigmoid(n, res, data);

#elif MLO_NRN_OP_ID == MLO_NEURON_TANH
    // (exp(2x) -1) / (exp(2x) + 1)
    ActivationFunction_TanH(n, res, data, alpha, beta);

#elif MLO_NRN_OP_ID == MLO_NEURON_RELU
    ActivationFunction_ReLU(n, res, data, alpha);

//#elif	MLO_NRN_OP_ID==MLO_NEURON_BRELU
//	ActivationFunction_BReLU(n, res, data, alpha);

#elif MLO_NRN_OP_ID == MLO_NEURON_SOFTRELU
    //	log(1 + exp(x))
    ActivationFunction_BNLL(n, res, data);
#elif MLO_NRN_OP_ID == MLO_NEURON_ABS
    ActivationFunction_Abs(n, res, data);

//#elif	MLO_NRN_OP_ID==MLO_NEURON_SQUARE
//	ActivationFunction_Square(res, data);

//#elif	MLO_NRN_OP_ID==MLO_NEURON_SQR
//	ActivationFunction_Sqrt(n, res, data);

#elif MLO_NRN_OP_ID == MLO_NEURON_POWER
    // (shift + scale * x ) ^power

    ActivationFunction_Power(n, res, data, power, alpha, beta);

#endif
}

/******************************************************************************/
/*                                  DIFF                                      */
/******************************************************************************/

static __constant _FLOAT kBNLL_THRESHOLD = (_FLOAT)50.;

__attribute__((always_inline)) void ActivationFunction_ReLU_Diff(uint n,
                                                                 _FLOAT* bot_diff,
                                                                 const _FLOAT* top_diff,
                                                                 const _FLOAT* bot_data,
                                                                 UNUSED _FLOAT negative_slope)
{

    for(uint i = 0; i < n; ++i)
    {
        bot_diff[i] = top_diff[i] * (bot_data[i] > 0);
    }
}

__attribute__((always_inline)) void ActivationFunction_TanH_Diff(uint n,
                                                                 _FLOAT* bot_diff,
                                                                 const _FLOAT* top_diff,
                                                                 const _FLOAT* top_data)
{
    for(uint i = 0; i < n; i++)
    {
        // (exp(2x) -1) / (exp(2x) + 1)
        _FLOAT tanh_x = top_data[i];
        bot_diff[i]   = top_diff[i] * (1 - tanh_x * tanh_x);
    }
}

__attribute__((always_inline)) void ActivationFunction_Sigmoid_Diff(uint n,
                                                                    _FLOAT* bot_diff,
                                                                    const _FLOAT* top_diff,
                                                                    const _FLOAT* top_data)
{
    for(uint i = 0; i < n; i++)
    {
        // 1/(1 + exp(-x))
        _FLOAT sigmoid_x = top_data[i];
        bot_diff[i]      = top_diff[i] * sigmoid_x * ((_FLOAT)1.f - sigmoid_x);
    }
}

__attribute__((always_inline)) void ActivationFunction_Abs_Diff(uint n,
                                                                _FLOAT* bot_diff,
                                                                const _FLOAT* top_diff,
                                                                const _FLOAT* bot_data)
{
    for(uint i = 0; i < n; i++)
    {
        bot_diff[i] = top_diff[i] * ((bot_data[i] >= 0) ? 1 : -1);
    }
}

// Compute dy/dx = scale * power * (shift + scale * x)^(power - 1)
//               = diff_scale * y / (shift + scale * x)
__attribute__((always_inline)) void ActivationFunction_Power_Diff(uint n,
                                                                  _FLOAT* bot_diff,
                                                                  UNUSED const _FLOAT* top_diff,
                                                                  const _FLOAT* top_data,
                                                                  const _FLOAT* bot_data,
                                                                  _FLOAT diff_scale,
                                                                  UNUSED _FLOAT power,
                                                                  _FLOAT scale,
                                                                  _FLOAT shift)
{

    for(uint i = 0; i < n; i++)
    {
        _FLOAT arg = shift + bot_data[i] * scale;
//		bot_diff[i] = (arg == 0) ? 0 : diff_scale * top_data[i] / arg;
#if MIOPEN_USE_FP16 == 1
        bot_diff[i] = (fabs(arg) < (_FLOAT)0.0001) ? (_FLOAT)0 : diff_scale * top_data[i] / arg;
#endif
#if MIOPEN_USE_FP32 == 1
        bot_diff[i] = (fabs(arg) < (_FLOAT)0.000001) ? (_FLOAT)0 : diff_scale * top_data[i] / arg;
#endif
    }
}

__attribute__((always_inline)) void ActivationFunction_BNLL_Diff(uint n,
                                                                 _FLOAT* bot_diff,
                                                                 const _FLOAT* top_diff,
                                                                 const _FLOAT* bot_data)
{
    for(uint i = 0; i < n; i++)
    {
        //	(log(1 + exp(x)))' = 1/ (1 + exp(-x))
        //		_FLOAT kBNLL_THRESHOLD = (_FLOAT)50.;
        _FLOAT expval = exp(fmin(bot_data[i], kBNLL_THRESHOLD));
        bot_diff[i]   = top_diff[i] * expval / (expval + (_FLOAT)1.f);
    }
}

#ifdef LITE

/**********************************************************************************************
**********************************************************************************************/

// N - batch size
// C - # of maps
// H - map height
// W - map width
// TENS_LEN = (N*C*H*W);
// RD_BLCK = (TENS_LEN%4==0) ? 4 : (TENS_LEN%3==0)? 3 : (TENS_LEN%2==0)? 2 : 1;
// READ_TYPE = (RD_BLCK==4) ? "float4" : (RD_BLCK == 3) ? "float3" : (RD_BLC==2) ? "float2" :
// "float";
// local size = (256, 1, 1)
// global size = ((TENS_LEN/RD_BLCK), 1, 1)

__kernel void MIOpenActiveFwdLite(const __global _FLOAT* bot,
                                  __global _FLOAT* top,
                                  _FLOAT power,
                                  _FLOAT scale,
                                  _FLOAT shift,
                                  const long bot_offset,
                                  const long top_offset)
{
    (void)power;
    (void)scale;
    (void)shift;

    uint gid0 = get_global_id(0);

    uint index = gid0 * MLO_READ_UNIT;

    _FLOAT data[MLO_READ_UNIT];
    _FLOAT response[MLO_READ_UNIT];

    *((MLO_READ_TYPE*)data) = *((const __global MLO_READ_TYPE*)(bot + bot_offset + index));

    ActivationFunction(MLO_READ_UNIT, response, (const _FLOAT*)data, power, scale, shift);

    *((__global MLO_READ_TYPE*)(top + top_offset + index)) = *((MLO_READ_TYPE*)response);
}

/**********************************************************************************************
**********************************************************************************************/

__kernel void MIOpenActiveFwd2DLite(const __global _FLOAT* bot,
                                    __global _FLOAT* top,
                                    _FLOAT power,
                                    _FLOAT scale,
                                    _FLOAT shift,
                                    const long bot_offset,
                                    const long top_offset,
                                    const uint bot_stride,
                                    const uint top_stride)
{

    (void)power;
    (void)scale;
    (void)shift;

    uint x_id = get_global_id(0);
    uint y    = get_global_id(1);

    uint bot_index = y * bot_stride + x_id * MLO_READ_UNIT;
    uint top_index = y * top_stride + x_id * MLO_READ_UNIT;

    _FLOAT data[MLO_READ_UNIT];
    _FLOAT response[MLO_READ_UNIT];

    *((MLO_READ_TYPE*)data) = *((const __global MLO_READ_TYPE*)(bot + bot_offset + bot_index));

    ActivationFunction(MLO_READ_UNIT, response, (const _FLOAT*)data, power, scale, shift);

    *((__global MLO_READ_TYPE*)(top + top_offset + top_index)) = *((MLO_READ_TYPE*)response);
}

/**********************************************************************************************
**********************************************************************************************/

__kernel void MIOpenActiveBwdLite(__global _FLOAT* bot_diff,
                                  __global const _FLOAT* top_diff,
                                  __global const _FLOAT* bot,
                                  __global const _FLOAT* top,
                                  _FLOAT diff_scale,
                                  _FLOAT power,
                                  _FLOAT scale,
                                  _FLOAT shift,
                                  const long bot_diff_offset,
                                  const long top_diff_offset,
                                  const long bot_offset,
                                  const long top_offset)
{
    (void)diff_scale;
    (void)power;
    (void)scale;
    (void)shift;

    int gid0 = get_global_id(0);

    int index = gid0 * MLO_READ_UNIT;

    _FLOAT bot_diff_dat[MLO_READ_UNIT];
    _FLOAT top_diff_dat[MLO_READ_UNIT];
    _FLOAT bot_dat[MLO_READ_UNIT];
    _FLOAT top_dat[MLO_READ_UNIT];

    *((MLO_READ_TYPE*)top_diff_dat) =
        *((const __global MLO_READ_TYPE*)(top_diff + top_diff_offset + index));
    *((MLO_READ_TYPE*)bot_dat) = *((const __global MLO_READ_TYPE*)(bot + bot_offset + index));
    *((MLO_READ_TYPE*)top_dat) = *((const __global MLO_READ_TYPE*)(top + top_offset + index));

#if MLO_NRN_OP_ID == MLO_NEURON_RELU
    {
        ActivationFunction_ReLU_Diff(MLO_READ_UNIT,
                                     bot_diff_dat,
                                     (const _FLOAT*)top_diff_dat,
                                     (const _FLOAT*)bot_dat,
                                     scale);
    }
#elif MLO_NRN_OP_ID == MLO_NEURON_LOGISTIC
    // 1/(1 + exp(-x))
    ActivationFunction_Sigmoid_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)top_dat);
#elif MLO_NRN_OP_ID == MLO_NEURON_TANH
    // (exp(2x) -1) / (exp(2x) + 1)

    ActivationFunction_TanH_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)top_dat);

#elif MLO_NRN_OP_ID == MLO_NEURON_ABS

    ActivationFunction_Abs_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)bot_dat);
#elif MLO_NRN_OP_ID == MLO_NEURON_POWER
    // (shift + scale * x ) ^power

    ActivationFunction_Power_Diff(MLO_READ_UNIT,
                                  bot_diff_dat,
                                  (const _FLOAT*)top_diff_dat,
                                  (const _FLOAT*)top_dat,
                                  (const _FLOAT*)bot_dat,
                                  diff_scale,
                                  power,
                                  scale,
                                  shift);

#elif MLO_NRN_OP_ID == MLO_NEURON_SOFTRELU
    //	log(1 + exp(x))
    ActivationFunction_BNLL_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)bot_dat);
#endif

    *((__global MLO_READ_TYPE*)(bot_diff + bot_diff_offset + index)) =
        *((MLO_READ_TYPE*)bot_diff_dat);
}

/**********************************************************************************************
**********************************************************************************************/

__kernel void MIOpenActiveBwd2DLite(__global _FLOAT* bot_diff,
                                    __global const _FLOAT* top_diff,
                                    __global const _FLOAT* bot,
                                    __global const _FLOAT* top,
                                    _FLOAT diff_scale,
                                    _FLOAT power,
                                    _FLOAT scale,
                                    _FLOAT shift,
                                    const long bot_diff_offset,
                                    const long top_diff_offset,
                                    const long bot_offset,
                                    const long top_offset,
                                    const uint bot_diff_stride,
                                    const uint top_diff_stride,
                                    const uint bot_stride,
                                    const uint top_stride)
{
    (void)diff_scale;
    (void)power;
    (void)scale;
    (void)shift;

    uint x_id = get_global_id(0);
    uint y    = get_global_id(1);

    uint bot_diff_index = y * bot_diff_stride + x_id * MLO_READ_UNIT;
    uint top_diff_index = y * top_diff_stride + x_id * MLO_READ_UNIT;
    uint bot_index      = y * bot_stride + x_id * MLO_READ_UNIT;
    uint top_index      = y * top_stride + x_id * MLO_READ_UNIT;

    _FLOAT bot_diff_dat[MLO_READ_UNIT];
    _FLOAT top_diff_dat[MLO_READ_UNIT];
    _FLOAT bot_dat[MLO_READ_UNIT];
    _FLOAT top_dat[MLO_READ_UNIT];

    *((MLO_READ_TYPE*)top_diff_dat) =
        *((const __global MLO_READ_TYPE*)(top_diff + top_diff_offset + top_diff_index));
    *((MLO_READ_TYPE*)bot_dat) = *((const __global MLO_READ_TYPE*)(bot + bot_offset + bot_index));
    *((MLO_READ_TYPE*)top_dat) = *((const __global MLO_READ_TYPE*)(top + top_offset + top_index));

#if MLO_NRN_OP_ID == MLO_NEURON_RELU
    {
        ActivationFunction_ReLU_Diff(MLO_READ_UNIT,
                                     bot_diff_dat,
                                     (const _FLOAT*)top_diff_dat,
                                     (const _FLOAT*)bot_dat,
                                     scale);
    }
#elif MLO_NRN_OP_ID == MLO_NEURON_LOGISTIC
    // 1/(1 + exp(-x))
    ActivationFunction_Sigmoid_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)top_dat);
#elif MLO_NRN_OP_ID == MLO_NEURON_TANH
    // (exp(2x) -1) / (exp(2x) + 1)

    ActivationFunction_TanH_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)top_dat);

#elif MLO_NRN_OP_ID == MLO_NEURON_ABS

    ActivationFunction_Abs_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)bot_dat);
#elif MLO_NRN_OP_ID == MLO_NEURON_POWER
    // (shift + scale * x ) ^power

    ActivationFunction_Power_Diff(MLO_READ_UNIT,
                                  bot_diff_dat,
                                  (const _FLOAT*)top_diff_dat,
                                  (const _FLOAT*)top_dat,
                                  (const _FLOAT*)bot_dat,
                                  diff_scale,
                                  power,
                                  scale,
                                  shift);

#elif MLO_NRN_OP_ID == MLO_NEURON_SOFTRELU
    //	log(1 + exp(x))
    ActivationFunction_BNLL_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)bot_dat);
#endif

    *((__global MLO_READ_TYPE*)(bot_diff + bot_diff_offset + bot_diff_index)) =
        *((MLO_READ_TYPE*)bot_diff_dat);
}

/**************************************************************************************************************/

#else

/***************************************************************************************************************/
__attribute__((reqd_work_group_size(MLO_NRN_GROUP_SZ0, MLO_NRN_GROUP_SZ1, MLO_NRN_GROUP_SZ2)))
__kernel void
MIOpenNeuronFwd(const __global _FLOAT* bot,
                __global _FLOAT* top,
                _FLOAT power,
                _FLOAT scale,
                _FLOAT shift,
                const long xOffset,
                const long yOffset)
{
    int x            = get_global_id(0); // channel x

#if MLO_N_OUT_STRIDE > MLO_OUT_BLOCK_SZ
    int n_out_stride = MLO_N_OUT_STRIDE;
    int c_out        = MLO_C_OUT;
    int h_out        = MLO_H_OUT;
    int w_out        = MLO_W_OUT;
#endif
#if MLO_N_IN_STRIDE > MLO_IN_BLOCK_SZ
    int n_in_stride  = MLO_N_IN_STRIDE;
    int c_in         = MLO_C_IN;
    int h_in         = MLO_H_IN;
    int w_in         = MLO_W_IN;
#endif

    _FLOAT data[MLO_READ_UNIT];
    _FLOAT response[MLO_READ_UNIT];
#if MLO_N_PIXS_OFF > 0
    if(x == MLO_MAP_SZ_ALIGNED - 1)
    {
        int i = 0;
        for(; i < MLO_N_PIXS_OFF; ++i)
        {
#if MLO_N_IN_STRIDE > MLO_IN_BLOCK_SZ
            if(n_in_stride > c_in * h_in * w_in && c_in != 0 && h_in != 0 && w_in != 0)
            {
                int loc, n_loc, c_loc, h_loc, w_loc;
                loc   = x * MLO_READ_UNIT + i;
                n_loc = loc / (MLO_C_IN * MLO_H_IN * MLO_W_IN);
                c_loc = (loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) / (MLO_H_IN * MLO_W_IN);
                h_loc =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) / MLO_W_IN;
                w_loc =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) % MLO_W_IN;

                data[i] = bot[xOffset + n_loc * MLO_N_IN_STRIDE + c_loc * MLO_C_IN_STRIDE +
                              h_loc * MLO_H_IN_STRIDE + w_loc * MLO_W_IN_STRIDE];
            }
            else
#endif
            {
                data[i] = bot[xOffset + x * MLO_READ_UNIT + i];
            }
        }
        for(; i < MLO_READ_UNIT; ++i)
        {
            data[i] = (_FLOAT)1.f;
        }
    }
    else
#endif
    {
        for(int i = 0; i < MLO_READ_UNIT; ++i)
        {
#if MLO_N_IN_STRIDE > MLO_IN_BLOCK_SZ
            if(n_in_stride > c_in * h_in * w_in && c_in != 0 && h_in != 0 && w_in != 0)
            {
                int loc, n_loc, c_loc, h_loc, w_loc;
                loc   = x * MLO_READ_UNIT + i;
                n_loc = loc / (MLO_C_IN * MLO_H_IN * MLO_W_IN);
                c_loc = (loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) / (MLO_H_IN * MLO_W_IN);
                h_loc =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) / MLO_W_IN;
                w_loc =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) % MLO_W_IN;

                data[i] = bot[xOffset + n_loc * MLO_N_IN_STRIDE + c_loc * MLO_C_IN_STRIDE +
                              h_loc * MLO_H_IN_STRIDE + w_loc * MLO_W_IN_STRIDE];
            }
            else
#endif
            {
                data[i] = bot[xOffset + x * MLO_READ_UNIT + i];
            }
        }
    }
    ActivationFunction(MLO_READ_UNIT, response, (const _FLOAT*)data, power, scale, shift);

#if MLO_N_PIXS_OFF > 0
    if(x == MLO_MAP_SZ_ALIGNED - 1)
    {
        int i = 0;
        for(; i < MLO_N_PIXS_OFF; ++i)
        {
#if MLO_N_OUT_STRIDE > MLO_OUT_BLOCK_SZ
            if(n_out_stride > c_out * h_out * w_out && c_out != 0 && h_out != 0 && w_out != 0)
            {
                int loc, n_loc, c_loc, h_loc, w_loc;
                loc   = x * MLO_READ_UNIT + i;
                n_loc = loc / (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT);
                c_loc = (loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) / (MLO_H_OUT * MLO_W_OUT);
                h_loc = ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) /
                        MLO_W_OUT;
                w_loc = ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) %
                        MLO_W_OUT;

                top[yOffset + n_loc * MLO_N_OUT_STRIDE + c_loc * MLO_C_OUT_STRIDE +
                    h_loc * MLO_H_OUT_STRIDE + w_loc * MLO_W_OUT_STRIDE] = response[i];
            }
            else
#endif
            {
                top[yOffset + x * MLO_READ_UNIT + i] = response[i];
            }
        }
    }
    else
#endif
    {
        for(int i = 0; i < MLO_READ_UNIT; ++i)
        {
#if MLO_N_OUT_STRIDE > MLO_OUT_BLOCK_SZ
            if(n_out_stride > c_out * h_out * w_out && c_out != 0 && h_out != 0 && w_out != 0)
            {
                int loc, n_loc, c_loc, h_loc, w_loc;
                loc   = x * MLO_READ_UNIT + i;
                n_loc = loc / (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT);
                c_loc = (loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) / (MLO_H_OUT * MLO_W_OUT);
                h_loc = ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) /
                        MLO_W_OUT;
                w_loc = ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) %
                        MLO_W_OUT;

                top[yOffset + n_loc * MLO_N_OUT_STRIDE + c_loc * MLO_C_OUT_STRIDE +
                    h_loc * MLO_H_OUT_STRIDE + w_loc * MLO_W_OUT_STRIDE] = response[i];
            }
            else
#endif
            {
                top[yOffset + x * MLO_READ_UNIT + i] = response[i];
            }
        }
    }
}

__attribute__((reqd_work_group_size(MLO_NRN_GROUP_SZ0, MLO_NRN_GROUP_SZ1, MLO_NRN_GROUP_SZ2)))
__kernel void
MIOpenNeuronBwd(__global _FLOAT* bot_diff,
                __global const _FLOAT* top_diff,
                __global const _FLOAT* bot_data,
                __global const _FLOAT* top_data,
                _FLOAT diff_scale,
                _FLOAT power,
                _FLOAT scale,
                _FLOAT shift,
                const long dxOffset,
                const long dyOffset,
                const long xOffset,
                const long yOffset)
{
    (void)diff_scale;
    (void)power;
    (void)scale;
    (void)shift;
    int x             = get_global_id(0); // channel x

#if MLO_N_OUT_STRIDE > MLO_OUT_BLOCK_SZ || MLO_N_DOUT_STRIDE > MLO_DOUT_BLOCK_SZ || \
    MLO_N_IN_STRIDE > MLO_IN_BLOCK_SZ
    int n_out_stride  = MLO_N_OUT_STRIDE;
    int c_out         = MLO_C_OUT;
    int h_out         = MLO_H_OUT;
    int w_out         = MLO_W_OUT;
    int n_dout_stride = MLO_N_DOUT_STRIDE;
    int c_dout        = MLO_C_DOUT;
    int h_dout        = MLO_H_DOUT;
    int w_dout        = MLO_W_DOUT;
    int n_in_stride   = MLO_N_IN_STRIDE;
    int c_in          = MLO_C_IN;
    int h_in          = MLO_H_IN;
    int w_in          = MLO_W_IN;
#endif

#if MLO_N_DIN_STRIDE > MLO_DIN_BLOCK_SZ
    int n_din_stride  = MLO_N_DIN_STRIDE;
    int c_din         = MLO_C_DIN;
    int h_din         = MLO_H_DIN;
    int w_din         = MLO_W_DIN;
#endif

    _FLOAT bot_diff_dat[MLO_READ_UNIT];
    _FLOAT top_diff_dat[MLO_READ_UNIT];
    _FLOAT bot_dat[MLO_READ_UNIT];
    _FLOAT top_dat[MLO_READ_UNIT];
#if MLO_N_PIXS_OFF > 0
    if(x == MLO_MAP_SZ_ALIGNED - 1)
    {
        int i = 0;
        for(; i < MLO_N_PIXS_OFF; ++i)
        {
#if MLO_N_OUT_STRIDE > MLO_OUT_BLOCK_SZ || MLO_N_DOUT_STRIDE > MLO_DOUT_BLOCK_SZ || \
    MLO_N_IN_STRIDE > MLO_IN_BLOCK_SZ
            if((n_out_stride > c_out * h_out * w_out || n_dout_stride > c_dout * h_dout * w_dout ||
                n_in_stride > c_in * h_in * w_in) &&
               c_out != 0 && h_out != 0 && w_out != 0 && c_dout != 0 && h_dout != 0 &&
               w_dout != 0 && c_in != 0 && h_in != 0 && w_in != 0)
            {
                int loc, n_loc_top_diff, c_loc_top_diff, h_loc_top_diff, w_loc_top_diff, n_loc_top,
                    c_loc_top, h_loc_top, w_loc_top, n_loc_bot, c_loc_bot, h_loc_bot, w_loc_bot;
                loc = x * MLO_READ_UNIT + i;

                n_loc_top_diff = loc / (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT);
                c_loc_top_diff =
                    (loc % (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT)) / (MLO_H_DOUT * MLO_W_DOUT);
                h_loc_top_diff =
                    ((loc % (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT)) % (MLO_H_DOUT * MLO_W_DOUT)) /
                    MLO_W_DOUT;
                w_loc_top_diff =
                    ((loc % (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT)) % (MLO_H_DOUT * MLO_W_DOUT)) %
                    MLO_W_DOUT;

                n_loc_top = loc / (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT);
                c_loc_top = (loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) / (MLO_H_OUT * MLO_W_OUT);
                h_loc_top =
                    ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) /
                    MLO_W_OUT;
                w_loc_top =
                    ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) %
                    MLO_W_OUT;

                n_loc_bot = loc / (MLO_C_IN * MLO_H_IN * MLO_W_IN);
                c_loc_bot = (loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) / (MLO_H_IN * MLO_W_IN);
                h_loc_bot =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) / MLO_W_IN;
                w_loc_bot =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) % MLO_W_IN;

                top_diff_dat[i] = top_diff[dyOffset + n_loc_top_diff * MLO_N_DOUT_STRIDE +
                                           c_loc_top_diff * MLO_C_DOUT_STRIDE +
                                           h_loc_top_diff * MLO_H_DOUT_STRIDE +
                                           w_loc_top_diff * MLO_W_DOUT_STRIDE];
                bot_dat[i] =
                    bot_data[xOffset + n_loc_bot * MLO_N_IN_STRIDE + c_loc_bot * MLO_C_IN_STRIDE +
                             h_loc_bot * MLO_H_IN_STRIDE + w_loc_bot * MLO_W_IN_STRIDE];
                top_dat[i] =
                    top_data[yOffset + n_loc_top * MLO_N_OUT_STRIDE + c_loc_top * MLO_C_OUT_STRIDE +
                             h_loc_top * MLO_H_OUT_STRIDE + w_loc_top * MLO_W_OUT_STRIDE];
            }
            else
#endif
            {
                top_diff_dat[i] = top_diff[dyOffset + x * MLO_READ_UNIT + i];
                bot_dat[i]      = bot_data[xOffset + x * MLO_READ_UNIT + i];
                top_dat[i]      = top_data[yOffset + x * MLO_READ_UNIT + i];
            }
        }
        for(; i < MLO_READ_UNIT; ++i)
        {
            top_diff_dat[i] = (_FLOAT)1.f;
            bot_dat[i]      = (_FLOAT)1.f;
            top_dat[i]      = (_FLOAT)1.f;
        }
    }
    else
#endif
    {
        for(int i = 0; i < MLO_READ_UNIT; ++i)
        {
#if MLO_N_OUT_STRIDE > MLO_OUT_BLOCK_SZ || MLO_N_DOUT_STRIDE > MLO_DOUT_BLOCK_SZ || \
    MLO_N_IN_STRIDE > MLO_IN_BLOCK_SZ
            if((n_out_stride > c_out * h_out * w_out || n_dout_stride > c_dout * h_dout * w_dout ||
                n_in_stride > c_in * h_in * w_in) &&
               c_out != 0 && h_out != 0 && w_out != 0 && c_dout != 0 && h_dout != 0 &&
               w_dout != 0 && c_in != 0 && h_in != 0 && w_in != 0)
            {
                int loc, n_loc_top_diff, c_loc_top_diff, h_loc_top_diff, w_loc_top_diff, n_loc_top,
                    c_loc_top, h_loc_top, w_loc_top, n_loc_bot, c_loc_bot, h_loc_bot, w_loc_bot;
                loc = x * MLO_READ_UNIT + i;

                n_loc_top_diff = loc / (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT);
                c_loc_top_diff =
                    (loc % (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT)) / (MLO_H_DOUT * MLO_W_DOUT);
                h_loc_top_diff =
                    ((loc % (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT)) % (MLO_H_DOUT * MLO_W_DOUT)) /
                    MLO_W_DOUT;
                w_loc_top_diff =
                    ((loc % (MLO_C_DOUT * MLO_H_DOUT * MLO_W_DOUT)) % (MLO_H_DOUT * MLO_W_DOUT)) %
                    MLO_W_DOUT;

                n_loc_top = loc / (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT);
                c_loc_top = (loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) / (MLO_H_OUT * MLO_W_OUT);
                h_loc_top =
                    ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) /
                    MLO_W_OUT;
                w_loc_top =
                    ((loc % (MLO_C_OUT * MLO_H_OUT * MLO_W_OUT)) % (MLO_H_OUT * MLO_W_OUT)) %
                    MLO_W_OUT;

                n_loc_bot = loc / (MLO_C_IN * MLO_H_IN * MLO_W_IN);
                c_loc_bot = (loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) / (MLO_H_IN * MLO_W_IN);
                h_loc_bot =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) / MLO_W_IN;
                w_loc_bot =
                    ((loc % (MLO_C_IN * MLO_H_IN * MLO_W_IN)) % (MLO_H_IN * MLO_W_IN)) % MLO_W_IN;

                top_diff_dat[i] = top_diff[dyOffset + n_loc_top_diff * MLO_N_DOUT_STRIDE +
                                           c_loc_top_diff * MLO_C_DOUT_STRIDE +
                                           h_loc_top_diff * MLO_H_DOUT_STRIDE +
                                           w_loc_top_diff * MLO_W_DOUT_STRIDE];
                bot_dat[i] =
                    bot_data[xOffset + n_loc_bot * MLO_N_IN_STRIDE + c_loc_bot * MLO_C_IN_STRIDE +
                             h_loc_bot * MLO_H_IN_STRIDE + w_loc_bot * MLO_W_IN_STRIDE];
                top_dat[i] =
                    top_data[yOffset + n_loc_top * MLO_N_OUT_STRIDE + c_loc_top * MLO_C_OUT_STRIDE +
                             h_loc_top * MLO_H_OUT_STRIDE + w_loc_top * MLO_W_OUT_STRIDE];
            }
            else
#endif
            {
                top_diff_dat[i] = top_diff[dyOffset + x * MLO_READ_UNIT + i];
                bot_dat[i]      = bot_data[xOffset + x * MLO_READ_UNIT + i];
                top_dat[i]      = top_data[yOffset + x * MLO_READ_UNIT + i];
            }
        }
    }

#if MLO_NRN_OP_ID == MLO_NEURON_RELU
    {
        ActivationFunction_ReLU_Diff(MLO_READ_UNIT,
                                     bot_diff_dat,
                                     (const _FLOAT*)top_diff_dat,
                                     (const _FLOAT*)bot_dat,
                                     scale);
    }
#elif MLO_NRN_OP_ID == MLO_NEURON_LOGISTIC
    // 1/(1 + exp(-x))
    ActivationFunction_Sigmoid_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)top_dat);
#elif MLO_NRN_OP_ID == MLO_NEURON_TANH
    // (exp(2x) -1) / (exp(2x) + 1)

    ActivationFunction_TanH_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)top_dat);

#elif MLO_NRN_OP_ID == MLO_NEURON_ABS

    ActivationFunction_Abs_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)bot_dat);
#elif MLO_NRN_OP_ID == MLO_NEURON_POWER
    // (shift + scale * x ) ^power

    ActivationFunction_Power_Diff(MLO_READ_UNIT,
                                  bot_diff_dat,
                                  (const _FLOAT*)top_diff_dat,
                                  (const _FLOAT*)top_dat,
                                  (const _FLOAT*)bot_dat,
                                  diff_scale,
                                  power,
                                  scale,
                                  shift);

#elif MLO_NRN_OP_ID == MLO_NEURON_SOFTRELU
    //	log(1 + exp(x))
    ActivationFunction_BNLL_Diff(
        MLO_READ_UNIT, bot_diff_dat, (const _FLOAT*)top_diff_dat, (const _FLOAT*)bot_dat);
#endif

#if MLO_N_PIXS_OFF > 0
    if(x == MLO_MAP_SZ_ALIGNED - 1)
    {
        int i = 0;
        for(; i < MLO_N_PIXS_OFF; ++i)
        {
#if MLO_N_DIN_STRIDE > MLO_DIN_BLOCK_SZ
            if(n_din_stride > c_din * h_din * w_din && c_din != 0 && h_din != 0 && w_din != 0)
            {
                int loc, n_loc_bot_diff, c_loc_bot_diff, h_loc_bot_diff, w_loc_bot_diff;
                loc = x * MLO_READ_UNIT + i;

                n_loc_bot_diff = loc / (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN);
                c_loc_bot_diff =
                    (loc % (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN)) / (MLO_H_DIN * MLO_W_DIN);
                h_loc_bot_diff =
                    ((loc % (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN)) % (MLO_H_DIN * MLO_W_DIN)) /
                    MLO_W_DIN;
                w_loc_bot_diff =
                    ((loc % (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN)) % (MLO_H_DIN * MLO_W_DIN)) %
                    MLO_W_DIN;

                bot_diff[dxOffset + n_loc_bot_diff * MLO_N_DIN_STRIDE +
                         c_loc_bot_diff * MLO_C_DIN_STRIDE + h_loc_bot_diff * MLO_H_DIN_STRIDE +
                         w_loc_bot_diff * MLO_W_DIN_STRIDE] = bot_diff_dat[i];
            }
            else
#endif
            {
                bot_diff[dxOffset + x * MLO_READ_UNIT + i] = bot_diff_dat[i];
            }
        }
    }
    else
#endif
    {
        for(int i = 0; i < MLO_READ_UNIT; ++i)
        {
#if MLO_N_DIN_STRIDE > MLO_DIN_BLOCK_SZ
            if(n_din_stride > c_din * h_din * w_din && c_din != 0 && h_din != 0 && w_din != 0)
            {
                int loc, n_loc_bot_diff, c_loc_bot_diff, h_loc_bot_diff, w_loc_bot_diff;
                loc = x * MLO_READ_UNIT + i;

                n_loc_bot_diff = loc / (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN);
                c_loc_bot_diff =
                    (loc % (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN)) / (MLO_H_DIN * MLO_W_DIN);
                h_loc_bot_diff =
                    ((loc % (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN)) % (MLO_H_DIN * MLO_W_DIN)) /
                    MLO_W_DIN;
                w_loc_bot_diff =
                    ((loc % (MLO_C_DIN * MLO_H_DIN * MLO_W_DIN)) % (MLO_H_DIN * MLO_W_DIN)) %
                    MLO_W_DIN;

                bot_diff[dxOffset + n_loc_bot_diff * MLO_N_DIN_STRIDE +
                         c_loc_bot_diff * MLO_C_DIN_STRIDE + h_loc_bot_diff * MLO_H_DIN_STRIDE +
                         w_loc_bot_diff * MLO_W_DIN_STRIDE] = bot_diff_dat[i];
            }
            else
#endif
            {
                bot_diff[dxOffset + x * MLO_READ_UNIT + i] = bot_diff_dat[i];
            }
        }
    }
}

#endif // #ifdef LITE