Fix out-of-bounds read/write in XhadFaster

Fix an error in XhadFaster where data would be written beyond the end of zgm.
The kernel loop assumed that there was always enough work for each thread to
process WPT items, but this was not enforced. It's possible to detect the
overflow with the "canary" buffer regions, but for SHAD, kCanarySize must be
~500 (much larger than the normal 127).

This commit may improve the performance of XhadFaster, since the kernel was
performing 2x work in some cases (once over real data, once over garbage).

Courtesy of Codeplay Software Ltd.

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the Xhad kernel. It contains one fast vectorized version in case of unit
// strides (incx=incy=incz=1) and no offsets (offx=offy=offz=0). Another version is more general,
// but doesn't support vector data-types. Based on the XAXPY kernels.
//
// This kernel uses the level-1 BLAS common tuning parameters.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// A vector-vector multiply function. See also level1.opencl for a vector-scalar version
INLINE_FUNC realV MultiplyVectorVector(realV cvec, const realV aval, const realV bvec) {
  #if VW == 1
    Multiply(cvec, aval, bvec);
  #elif VW == 2
    Multiply(cvec.x, aval.x, bvec.x);
    Multiply(cvec.y, aval.y, bvec.y);
  #elif VW == 4
    Multiply(cvec.x, aval.x, bvec.x);
    Multiply(cvec.y, aval.y, bvec.y);
    Multiply(cvec.z, aval.z, bvec.z);
    Multiply(cvec.w, aval.w, bvec.w);
  #elif VW == 8
    Multiply(cvec.s0, aval.s0, bvec.s0);
    Multiply(cvec.s1, aval.s1, bvec.s1);
    Multiply(cvec.s2, aval.s2, bvec.s2);
    Multiply(cvec.s3, aval.s3, bvec.s3);
    Multiply(cvec.s4, aval.s4, bvec.s4);
    Multiply(cvec.s5, aval.s5, bvec.s5);
    Multiply(cvec.s6, aval.s6, bvec.s6);
    Multiply(cvec.s7, aval.s7, bvec.s7);
  #elif VW == 16
    Multiply(cvec.s0, aval.s0, bvec.s0);
    Multiply(cvec.s1, aval.s1, bvec.s1);
    Multiply(cvec.s2, aval.s2, bvec.s2);
    Multiply(cvec.s3, aval.s3, bvec.s3);
    Multiply(cvec.s4, aval.s4, bvec.s4);
    Multiply(cvec.s5, aval.s5, bvec.s5);
    Multiply(cvec.s6, aval.s6, bvec.s6);
    Multiply(cvec.s7, aval.s7, bvec.s7);
    Multiply(cvec.s8, aval.s8, bvec.s8);
    Multiply(cvec.s9, aval.s9, bvec.s9);
    Multiply(cvec.sA, aval.sA, bvec.sA);
    Multiply(cvec.sB, aval.sB, bvec.sB);
    Multiply(cvec.sC, aval.sC, bvec.sC);
    Multiply(cvec.sD, aval.sD, bvec.sD);
    Multiply(cvec.sE, aval.sE, bvec.sE);
    Multiply(cvec.sF, aval.sF, bvec.sF);
  #endif
  return cvec;
}

// =================================================================================================

// Full version of the kernel with offsets and strided accesses
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void Xhad(const int n, const real_arg arg_alpha, const real_arg arg_beta,
          const __global real* restrict xgm, const int x_offset, const int x_inc,
          const __global real* restrict ygm, const int y_offset, const int y_inc,
          __global real* zgm, const int z_offset, const int z_inc) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  // Loops over the work that needs to be done (allows for an arbitrary number of threads)
  for (int id = get_global_id(0); id < n; id += get_global_size(0)) {
    real xvalue = xgm[id*x_inc + x_offset];
    real yvalue = ygm[id*y_inc + y_offset];
    real zvalue = zgm[id*z_inc + z_offset];
    real result;
    real alpha_times_x;
    Multiply(alpha_times_x, alpha, xvalue);
    Multiply(result, alpha_times_x, yvalue);
    MultiplyAdd(result, beta, zvalue);
    zgm[id*z_inc + z_offset] = result;
  }
}

// Faster version of the kernel without offsets and strided accesses but with if-statement. Also
// assumes that 'n' is dividable by 'VW' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XhadFaster(const int n, const real_arg arg_alpha, const real_arg arg_beta,
                const __global realV* restrict xgm, const __global realV* restrict ygm,
                __global realV* zgm) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  const int num_desired_threads = n / (VW * WPT);

  if (get_global_id(0) < num_desired_threads) {
    #pragma unroll
    for (int _w = 0; _w < WPT; _w += 1) {
      const int id = _w * num_desired_threads + get_global_id(0);
      realV xvalue = xgm[id];
      realV yvalue = ygm[id];
      realV zvalue = zgm[id];
      realV result;
      realV alpha_times_x;
      alpha_times_x = MultiplyVector(alpha_times_x, alpha, xvalue);
      result = MultiplyVectorVector(result, alpha_times_x, yvalue);
      zgm[id] = MultiplyAddVector(result, beta, zvalue);
    }
  }
}

// Faster version of the kernel without offsets and strided accesses. Also assumes that 'n' is
// dividable by 'VW', 'WGS' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XhadFastest(const int n, const real_arg arg_alpha, const real_arg arg_beta,
                 const __global realV* restrict xgm, const __global realV* restrict ygm,
                 __global realV* zgm) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  #pragma unroll
  for (int _w = 0; _w < WPT; _w += 1) {
    const int id = _w*get_global_size(0) + get_global_id(0);
    realV xvalue = xgm[id];
    realV yvalue = ygm[id];
    realV zvalue = zgm[id];
    realV result;
    realV alpha_times_x;
    alpha_times_x = MultiplyVector(alpha_times_x, alpha, xvalue);
    result = MultiplyVectorVector(result, alpha_times_x, yvalue);
    zgm[id] = MultiplyAddVector(result, beta, zvalue);
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the Xhad kernel. It contains one fast vectorized version in case of unit
// strides (incx=incy=incz=1) and no offsets (offx=offy=offz=0). Another version is more general,
// but doesn't support vector data-types. Based on the XAXPY kernels.
//
// This kernel uses the level-1 BLAS common tuning parameters.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// A vector-vector multiply function. See also level1.opencl for a vector-scalar version
INLINE_FUNC realV MultiplyVectorVector(realV cvec, const realV aval, const realV bvec) {
  #if VW == 1
    Multiply(cvec, aval, bvec);
  #elif VW == 2
    Multiply(cvec.x, aval.x, bvec.x);
    Multiply(cvec.y, aval.y, bvec.y);
  #elif VW == 4
    Multiply(cvec.x, aval.x, bvec.x);
    Multiply(cvec.y, aval.y, bvec.y);
    Multiply(cvec.z, aval.z, bvec.z);
    Multiply(cvec.w, aval.w, bvec.w);
  #elif VW == 8
    Multiply(cvec.s0, aval.s0, bvec.s0);
    Multiply(cvec.s1, aval.s1, bvec.s1);
    Multiply(cvec.s2, aval.s2, bvec.s2);
    Multiply(cvec.s3, aval.s3, bvec.s3);
    Multiply(cvec.s4, aval.s4, bvec.s4);
    Multiply(cvec.s5, aval.s5, bvec.s5);
    Multiply(cvec.s6, aval.s6, bvec.s6);
    Multiply(cvec.s7, aval.s7, bvec.s7);
  #elif VW == 16
    Multiply(cvec.s0, aval.s0, bvec.s0);
    Multiply(cvec.s1, aval.s1, bvec.s1);
    Multiply(cvec.s2, aval.s2, bvec.s2);
    Multiply(cvec.s3, aval.s3, bvec.s3);
    Multiply(cvec.s4, aval.s4, bvec.s4);
    Multiply(cvec.s5, aval.s5, bvec.s5);
    Multiply(cvec.s6, aval.s6, bvec.s6);
    Multiply(cvec.s7, aval.s7, bvec.s7);
    Multiply(cvec.s8, aval.s8, bvec.s8);
    Multiply(cvec.s9, aval.s9, bvec.s9);
    Multiply(cvec.sA, aval.sA, bvec.sA);
    Multiply(cvec.sB, aval.sB, bvec.sB);
    Multiply(cvec.sC, aval.sC, bvec.sC);
    Multiply(cvec.sD, aval.sD, bvec.sD);
    Multiply(cvec.sE, aval.sE, bvec.sE);
    Multiply(cvec.sF, aval.sF, bvec.sF);
  #endif
  return cvec;
}

// =================================================================================================

// Full version of the kernel with offsets and strided accesses
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void Xhad(const int n, const real_arg arg_alpha, const real_arg arg_beta,
          const __global real* restrict xgm, const int x_offset, const int x_inc,
          const __global real* restrict ygm, const int y_offset, const int y_inc,
          __global real* zgm, const int z_offset, const int z_inc) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  // Loops over the work that needs to be done (allows for an arbitrary number of threads)
  for (int id = get_global_id(0); id < n; id += get_global_size(0)) {
    real xvalue = xgm[id*x_inc + x_offset];
    real yvalue = ygm[id*y_inc + y_offset];
    real zvalue = zgm[id*z_inc + z_offset];
    real result;
    real alpha_times_x;
    Multiply(alpha_times_x, alpha, xvalue);
    Multiply(result, alpha_times_x, yvalue);
    MultiplyAdd(result, beta, zvalue);
    zgm[id*z_inc + z_offset] = result;
  }
}

// Faster version of the kernel without offsets and strided accesses but with if-statement. Also
// assumes that 'n' is dividable by 'VW' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XhadFaster(const int n, const real_arg arg_alpha, const real_arg arg_beta,
                const __global realV* restrict xgm, const __global realV* restrict ygm,
                __global realV* zgm) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  if (get_global_id(0) < n / (VW)) {
    #pragma unroll
    for (int _w = 0; _w < WPT; _w += 1) {
      const int id = _w*get_global_size(0) + get_global_id(0);
      realV xvalue = xgm[id];
      realV yvalue = ygm[id];
      realV zvalue = zgm[id];
      realV result;
      realV alpha_times_x;
      alpha_times_x = MultiplyVector(alpha_times_x, alpha, xvalue);
      result = MultiplyVectorVector(result, alpha_times_x, yvalue);
      zgm[id] = MultiplyAddVector(result, beta, zvalue);
    }
  }
}

// Faster version of the kernel without offsets and strided accesses. Also assumes that 'n' is
// dividable by 'VW', 'WGS' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XhadFastest(const int n, const real_arg arg_alpha, const real_arg arg_beta,
                 const __global realV* restrict xgm, const __global realV* restrict ygm,
                 __global realV* zgm) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  #pragma unroll
  for (int _w = 0; _w < WPT; _w += 1) {
    const int id = _w*get_global_size(0) + get_global_id(0);
    realV xvalue = xgm[id];
    realV yvalue = ygm[id];
    realV zvalue = zgm[id];
    realV result;
    realV alpha_times_x;
    alpha_times_x = MultiplyVector(alpha_times_x, alpha, xvalue);
    result = MultiplyVectorVector(result, alpha_times_x, yvalue);
    zgm[id] = MultiplyAddVector(result, beta, zvalue);
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================


fixed comment issue in toSepia.cl again

fixed comment issue in toSepia.cl again

kernel void toSepia(read_only image2d_t srcImg, write_only image2d_t dstImg)
{
  const sampler_t smp =
    CLK_NORMALIZE_COORDS_FALSE | //Natural coordinates
    CLK_ADDRESS_CLAMP_TO_EDGE | //Clamp to zeros
    CLK_FILTER_LINEAR;

  int2 coord = (int2)(get_global_id(0), get_global_id(1));

  uint4 rgba = read_imageui(srcImg, smp, coord); //The byte order is RGBA

  float4 rgbafloat = convert_float4(rgba) / 255.0f; //Convert to normalized [0..1] float

  //Convert RGB to luminance (make the image sepia).
  float luminance = sqrt(0.43f * rbgafloat.z * rbgafloat.z + 0.74f * rbgafloat.y * rbgafloat.y + 1.07f * rbgafloat.x * rbgafloat.x);

  rgba.x = rgba.y = rgba.z = (uint) (rgba.z * 255.0f);
  rgba.w = 255;

  write_imageui(dstImg, coord, rgba);
}

kernel void toSepia(read_only image2d_t srcImg, write_only image2d_t dstImg)
{
  const sampler_t smp =
    CLK_NORMALIZE_COORDS_FALSE | //Natural coordinates
    CLK_ADDRESS_CLAMP_TO_EDGE | //Clamp to zeros
    CLK_FILTER_LINEAR;

  int2 coord = (int2)(get_global_id(0), get_global_id(1));

  uint4 rgba = read_imageui(srcImg, smp, coord); //The byte order is RGBA

  float4 rgbafloat = convert_float4(rgba) / 255.0f; //Convert to normalized [0..1] float

  //Convert RGB to luminance (make the image sepia).
  // float luminance = sqrt(0.43f * rbgafloat.z * rbgafloat.z + 0.74f * rbgafloat.y * rbgafloat.y + 1.07f * rbgafloat.x * rbgafloat.x);

  rgba.x = rgba.y = rgba.z = (uint) (rgba.z * 255.0f);
  rgba.w = 255;

  write_imageui(dstImg, coord, rgba);
}


improved recursion on 9/7 transform

improved recursion on 9/7 transform

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns
Right (odd) boundary column

 **/

#define BOUNDARY_X 4

#define VERTICAL_STRIDE 64  // WIN_SIZE_X/2

// two vertical neighbours: pointer diff:
#define BUFFER_SIZE            512	// VERTICAL_STRIDE * WIN_SIZE_Y

#define CHANNEL_BUFFER_SIZE     1024            // BUFFER_SIZE + BUFFER_SIZE

#define CHANNEL_BUFFER_SIZE_X2  2048
#define CHANNEL_BUFFER_SIZE_X3  3072

#define PIXEL_BUFFER_SIZE   4096

#define HORIZONTAL_EVEN_TO_PREVIOUS_ODD  511
#define HORIZONTAL_EVEN_TO_NEXT_ODD      512

#define HORIZONTAL_ODD_TO_PREVIOUS_EVEN -512
#define HORIZONTAL_ODD_TO_NEXT_EVEN     -511

CONSTANT float P1 = -1.586134342;   ///< forward 9/7 predict 1
CONSTANT float U1 = -0.05298011854;  ///< forward 9/7 update 1
CONSTANT float P2 = 0.8829110762;   ///< forward 9/7 predict 2
CONSTANT float U2 = 0.4435068522;    ///< forward 9/7 update 2
CONSTANT float U1P1 = 0.08403358545952490068;

CONSTANT float scale97Mul = 1.23017410491400f;
CONSTANT float scale97Div = 1.0 / 1.23017410491400f;

/*

Lifting scheme consists of four steps: Predict1 Update1 Predict2 Update2
followed by scaling.

If the odd predict2 pixels are calculated first, then the even update2 pixels
can be easily calculated.

Predict Calculation:

For S odd, we have:

plusOne_P1 = plusOne + P1*(current + plusTwo)

plusOne_P2 = plusOne_P1 +
             P2*(current_U1 + plusTwo_U1)
           =   plusOne + P1*(current + plusTwo) +
		         P2*(current + U1*(minusOne + plusOne) + U1P1*(minusTwo + 2*current + plusTwo) +
				     plusTwo + U1*(plusOne + plusThree) + U1P1*(current + 2*plusTwo + plusFour)  )

Update Calculation

For S even, we have

current_U1 = current + U1*(minusOne_P1 + plusOne_P1)
           = current + U1*(minusOne + P1*(minusTwo + current) + plusOne + P1*(current + plusTwo))
		   = current + U1*(minusOne + plusOne) + U1P1*(minusTwo + 2*current + plusTwo)

current_U2 = current_U1 + U2*(minusOne_P2 + plusOne_P2)

*/

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

inline int getCorrectedGlobalIdX() {
      return getGlobalId(0) - 2 * BOUNDARY_X * getGroupId(0);
}

// read pixel from local buffer
float4 readPixel( LOCAL float*  restrict  src) {
	return (float4)(*src, *(src+CHANNEL_BUFFER_SIZE),  *(src+CHANNEL_BUFFER_SIZE_X2),  *(src+CHANNEL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(float4 pix, LOCAL float*  restrict  dest) {
	*dest = pix.x;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.y;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.z;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.w;
}

// write column to destination
void writeColumnToOutput(LOCAL float* restrict currentScratch, __write_only image2d_t odata, int firstY, int inputX, int height, int halfHeight){

	int2 posOut = {inputX, firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even row

		//only need to check evens, since even point will be the first out of bound point
	    if (posOut.y >= halfHeight)
			break;

		write_imagef(odata, posOut,scale97Div * readPixel(currentScratch));

		// odd row
		currentScratch += VERTICAL_STRIDE ;
		posOut.y+= halfHeight;

		write_imagef(odata, posOut,scale97Mul * readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE;
		posOut.y -= (halfHeight - 1);
	}
}

// initial scratch offset when transforming vertically
inline int getScratchOffset(){
   return (getLocalId(0)>> 1) + (getLocalId(0)&1) * BUFFER_SIZE;
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	int inputX = getCorrectedGlobalIdX();

	int outputX = inputX;
	outputX = (outputX >> 1) + (outputX & 1)*( width >> 1);

	const int halfWinSizeX = WIN_SIZE_X >> 1;
    const unsigned int halfHeight = height >> 1;
	LOCAL float scratch[PIXEL_BUFFER_SIZE];
	const float yDelta = 1.0/(height-1);
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -4 point
	float2 posIn = (float2)(inputX, firstY - 4) /  (float2)(width-1, height-1);
	float4 minusFour = read_imagef(idata, sampler, posIn);

	posIn.y += yDelta;
	float4 minusThree = read_imagef(idata, sampler, posIn);

	// read -2 point
	posIn.y += yDelta;
	float4 minusTwo = read_imagef(idata, sampler, posIn);

	// read -1 point
	posIn.y += yDelta;
	float4 minusOne = read_imagef(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	float4 current = read_imagef(idata, sampler, posIn);

	// +1 point
	posIn.y += yDelta;
	float4 plusOne = read_imagef(idata, sampler, posIn);

	// +2 point
	posIn.y += yDelta;
	float4 plusTwo = read_imagef(idata, sampler, posIn);

	float4 minusThree_P1 = minusThree + P1*(minusFour + minusTwo);
	float4 minusOne_P1   = minusOne   + P1*(minusTwo + current);
	float4 plusOne_P1    = plusOne    + P1*(current + plusTwo);

	float4 minusTwo_U1 = minusTwo + U1*(minusThree_P1 + minusOne_P1);
	float4 current_U1  = current + U1*(minusOne_P1 + plusOne_P1);
	float4 minusOne_P2 = minusOne_P1 + P2*(minusTwo_U1 + current_U1);

	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL float* currentScratch = scratch + getScratchOffset();
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	        //read next two points

			// +3 point
			posIn.y += yDelta;
			float4 plusThree = read_imagef(idata, sampler, posIn);

	   		// +4 point
			posIn.y += yDelta;
	   		if (posIn.y > 1 + 3*yDelta)
				break;
			float4 plusFour = read_imagef(idata, sampler, posIn);

			float4 plusThree_P1    = plusThree  + P1*(plusTwo + plusFour);
			float4 plusTwo_U1      = plusTwo + U1*(plusOne_P1 + plusThree_P1);
			float4 plusOne_P2      = plusOne_P1 + P2*(current_U1 + plusTwo_U1);

			//write current U2 (even)
			writePixel(scale97Div * (current_U1 +  U2 * (minusOne_P2 + plusOne_P2)), currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE;

			//write current P2 (odd)
			writePixel(scale97Mul* plusOne_P2 , currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE;

			// shift registers up by two
			minusFour = minusTwo;
			minusThree = minusOne;
			minusTwo = current;
			minusOne = plusOne;
			current = plusTwo;
			plusOne = plusThree;
			plusTwo = plusFour;
			//update P1s
			minusThree_P1 = minusOne_P1;
			minusOne_P1 = plusOne_P1;
			plusOne_P1 = plusThree_P1;

			//update U1s
			minusTwo_U1 = current_U1;
			current_U1 = plusTwo_U1;

			//update P2s
			minusOne_P2 = plusOne_P2;
		}

		//4. transform horizontally
		currentScratch = scratch + getScratchOffset();

		localMemoryFence();
		// P2 - odd columns (skip left three boundary columns and all right boundary columns)
		if ( (getLocalId(0)&1) && (getLocalId(0) >= BOUNDARY_X-1) && (getLocalId(0) < WIN_SIZE_X-BOUNDARY_X) ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 current = readPixel(currentScratch);
				float4 minusOne = readPixel(currentScratch + HORIZONTAL_ODD_TO_PREVIOUS_EVEN);
				float4 minusTwo = readPixel(currentScratch -1);
				float4 minusThree = readPixel(currentScratch + HORIZONTAL_ODD_TO_PREVIOUS_EVEN -1);
				float4 plusOne = readPixel(currentScratch + HORIZONTAL_ODD_TO_NEXT_EVEN);
				float4 plusTwo = readPixel(currentScratch + 1);
				float4 plusThree = readPixel(currentScratch + HORIZONTAL_ODD_TO_NEXT_EVEN+1);

				float4 current_P2 =  current + P1*(minusOne + plusOne) +
		         P2*( minusOne + U1*(minusTwo + current) + U1P1*(minusThree + 2*minusOne + plusOne) +
				      plusOne + U1*(current + plusTwo) + U1P1*(minusOne + 2*plusOne + plusThree)  );
				writePixel(current_P2, currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}

		currentScratch = scratch + getScratchOffset();
		localMemoryFence();
		//U2 - even columns (skip left and right boundary columns)
		if ( !(getLocalId(0)&1) && (getLocalId(0) >= BOUNDARY_X) && (getLocalId(0) < WIN_SIZE_X-BOUNDARY_X)  ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 currentEven = readPixel(currentScratch);
				float4 prevOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_PREVIOUS_ODD);
				float4 nextOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_NEXT_ODD);
				writePixel(currentEven + U2*(prevOdd + nextOdd), currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}
		localMemoryFence();

		//5. write local buffer column to destination image
		// (only write non-boundary columns that are within the image bounds)
		if ((getLocalId(0) >= BOUNDARY_X) && ( getLocalId(0) < WIN_SIZE_X - BOUNDARY_X) && (inputX < width) && inputX >= 0) {
			writeColumnToOutput(scratch + getScratchOffset(), odata, firstY, outputX, height, halfHeight);

		}
		// move to next step
		firstY += WIN_SIZE_Y;
	}
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns
Right (odd) boundary column

 **/

#define BOUNDARY_X 4

#define VERTICAL_STRIDE 64  // WIN_SIZE_X/2

// two vertical neighbours: pointer diff:
#define BUFFER_SIZE            512	// VERTICAL_STRIDE * WIN_SIZE_Y

#define CHANNEL_BUFFER_SIZE     1024            // BUFFER_SIZE + BUFFER_SIZE

#define CHANNEL_BUFFER_SIZE_X2  2048
#define CHANNEL_BUFFER_SIZE_X3  3072

#define PIXEL_BUFFER_SIZE   4096

#define HORIZONTAL_EVEN_TO_PREVIOUS_ODD  511
#define HORIZONTAL_EVEN_TO_NEXT_ODD      512

#define HORIZONTAL_ODD_TO_PREVIOUS_EVEN -512
#define HORIZONTAL_ODD_TO_NEXT_EVEN     -511

CONSTANT float P1 = -1.586134342;   ///< forward 9/7 predict 1
CONSTANT float U1 = -0.05298011854;  ///< forward 9/7 update 1
CONSTANT float P2 = 0.8829110762;   ///< forward 9/7 predict 2
CONSTANT float U2 = 0.4435068522;    ///< forward 9/7 update 2
CONSTANT float U1P1 = 0.08403358545952490068;

CONSTANT float scale97Mul = 1.23017410491400f;
CONSTANT float scale97Div = 1.0 / 1.23017410491400f;

/*

Lifting scheme consists of four steps: Predict1 Update1 Predict2 Update2
followed by scaling.

If the odd predict2 pixels are calculated first, then the even update2 pixels
can be easily calculated.

Predict Calculation:

For S odd, we have:

plusOne_P1 = plusOne + P1*(current + plusTwo)

plusOne_P2 = plusOne_P1 +
             P2*(current_U1 + plusTwo_U1)
           =   plusOne + P1*(current + plusTwo) +
		         P2*(current + U1*(minusOne + plusOne) + U1P1*(minusTwo + 2*current + plusTwo) +
				     plusTwo + U1*(plusOne + plusThree) + U1P1*(current + 2*plusTwo + plusFour)  )

Update Calculation

For S even, we have

current_U1 = current + U1*(minusOne_P1 + plusOne_P1)
           = current + U1*(minusOne + P1*(minusTwo + current) + plusOne + P1*(current + plusTwo))
		   = current + U1*(minusOne + plusOne) + U1P1*(minusTwo + 2*current + plusTwo)

*/

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

inline int getCorrectedGlobalIdX() {
      return getGlobalId(0) - 2 * BOUNDARY_X * getGroupId(0);
}

// read pixel from local buffer
float4 readPixel( LOCAL float*  restrict  src) {
	return (float4)(*src, *(src+CHANNEL_BUFFER_SIZE),  *(src+CHANNEL_BUFFER_SIZE_X2),  *(src+CHANNEL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(float4 pix, LOCAL float*  restrict  dest) {
	*dest = pix.x;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.y;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.z;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.w;
}

// write column to destination
void writeColumnToOutput(LOCAL float* restrict currentScratch, __write_only image2d_t odata, int firstY, int inputX, int height, int halfHeight){

	int2 posOut = {inputX, firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even row

		//only need to check evens, since even point will be the first out of bound point
	    if (posOut.y >= halfHeight)
			break;

		write_imagef(odata, posOut,scale97Div * readPixel(currentScratch));

		// odd row
		currentScratch += VERTICAL_STRIDE ;
		posOut.y+= halfHeight;

		write_imagef(odata, posOut,scale97Mul * readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE;
		posOut.y -= (halfHeight - 1);
	}
}

// initial scratch offset when transforming vertically
inline int getScratchOffset(){
   return (getLocalId(0)>> 1) + (getLocalId(0)&1) * BUFFER_SIZE;
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	int inputX = getCorrectedGlobalIdX();

	int outputX = inputX;
	outputX = (outputX >> 1) + (outputX & 1)*( width >> 1);

	const int halfWinSizeX = WIN_SIZE_X >> 1;
    const unsigned int halfHeight = height >> 1;
	LOCAL float scratch[PIXEL_BUFFER_SIZE];
	const float yDelta = 1.0/(height-1);
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -4 point
	float2 posIn = (float2)(inputX, firstY - 4) /  (float2)(width-1, height-1);
	float4 minusFour = read_imagef(idata, sampler, posIn);

	posIn.y += yDelta;
	float4 minusThree = read_imagef(idata, sampler, posIn);

	// read -2 point
	posIn.y += yDelta;
	float4 minusTwo = read_imagef(idata, sampler, posIn);

	// read -1 point
	posIn.y += yDelta;
	float4 minusOne = read_imagef(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	float4 current = read_imagef(idata, sampler, posIn);

	// +1 point
	posIn.y += yDelta;
	float4 plusOne = read_imagef(idata, sampler, posIn);

	// +2 point
	posIn.y += yDelta;
	float4 plusTwo = read_imagef(idata, sampler, posIn);

	float4 minusOne_P2 =  minusOne + P1*(minusTwo + current) +
		         P2*(minusTwo + U1*(minusThree + plusOne) + U1P1*(minusTwo + 2*minusTwo + current) +
				     current + U1*(minusOne + plusOne) + U1P1*(minusTwo + 2*current + plusTwo)  );

	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL float* currentScratch = scratch + getScratchOffset();
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	        //read next two points

			// +3 point
			posIn.y += yDelta;
			float4 plusThree = read_imagef(idata, sampler, posIn);

	   		// +4 point
			posIn.y += yDelta;
	   		if (posIn.y > 1 + 3*yDelta)
				break;
			float4 plusFour = read_imagef(idata, sampler, posIn);

			float4 plusOne_P2 =   plusOne + P1*(current + plusTwo) +
		         P2*(current + U1*(minusOne + plusOne) + U1P1*(minusTwo + 2*current + plusTwo) +
				     plusTwo + U1*(plusOne + plusThree) + U1P1*(current + 2*plusTwo + plusFour)  );

			float4 current_U2 =  current +  U2 * (minusOne_P2 + plusOne_P2);

			//write current U2 (even)
			writePixel(scale97Div * current_U2, currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE;

			//write current P2 (odd)
			writePixel(scale97Mul* plusOne_P2 , currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE;

			// shift registers up by two
			minusFour = minusTwo;
			minusThree = minusOne;
			minusTwo = current;
			minusOne = plusOne;
			current = plusTwo;
			plusOne = plusThree;
			plusTwo = plusFour;
			minusOne_P2 = plusOne_P2;
		}

		//4. transform horizontally
		currentScratch = scratch + getScratchOffset();

		localMemoryFence();
		// P2 - odd columns (skip left three boundary columns and all right boundary columns)
		if ( (getLocalId(0)&1) && (getLocalId(0) >= BOUNDARY_X-1) && (getLocalId(0) < WIN_SIZE_X-BOUNDARY_X) ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 current = readPixel(currentScratch);
				float4 minusOne = readPixel(currentScratch + HORIZONTAL_ODD_TO_PREVIOUS_EVEN);
				float4 minusTwo = readPixel(currentScratch -1);
				float4 minusThree = readPixel(currentScratch + HORIZONTAL_ODD_TO_PREVIOUS_EVEN -1);
				float4 plusOne = readPixel(currentScratch + HORIZONTAL_ODD_TO_NEXT_EVEN);
				float4 plusTwo = readPixel(currentScratch + 1);
				float4 plusThree = readPixel(currentScratch + HORIZONTAL_ODD_TO_NEXT_EVEN+1);

				float4 current_P2 =  current + P1*(minusOne + plusOne) +
		         P2*( minusOne + U1*(minusTwo + current) + U1P1*(minusThree + 2*minusOne + plusOne) +
				      plusOne + U1*(current + plusTwo) + U1P1*(minusOne + 2*plusOne + plusThree)  );
				writePixel(current_P2, currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}

		currentScratch = scratch + getScratchOffset();
		localMemoryFence();
		//U2 - even columns (skip left and right boundary columns)
		if ( !(getLocalId(0)&1) && (getLocalId(0) >= BOUNDARY_X) && (getLocalId(0) < WIN_SIZE_X-BOUNDARY_X)  ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 currentEven = readPixel(currentScratch);
				float4 prevOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_PREVIOUS_ODD);
				float4 nextOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_NEXT_ODD);
				writePixel(currentEven + U2*(prevOdd + nextOdd), currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}
		localMemoryFence();

		//5. write local buffer column to destination image
		// (only write non-boundary columns that are within the image bounds)
		if ((getLocalId(0) >= BOUNDARY_X) && ( getLocalId(0) < WIN_SIZE_X - BOUNDARY_X) && (inputX < width) && inputX >= 0) {
			writeColumnToOutput(scratch + getScratchOffset(), odata, firstY, outputX, height, halfHeight);

		}
		// move to next step
		firstY += WIN_SIZE_Y;
	}
}


time to measure

time to measure

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS (MLO_IN_WIDTH)
#define MLO_PER_WAVE_READ ((MLO_IN_WIDTH + MLO_N_WAVES - 1) / MLO_N_WAVES)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_PER_WAVE_READ * MLO_N_WAVES)
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)

#define MLO_IN_VERT_READS (1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_READS*MLO_READ_UNIT - MLO_N_IN_HORIZ_PIX_READS)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ ((MLO_N_WAVES - 1) * MLO_HW_WAVE_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void ReduceKernel64(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
	// add second half
	// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

inline void readInput(int lcl_id, int c_idx, int sc, int gbl_in_scan_off, __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int c_scan = sc;

		int c = iDiv(p4, (MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS));
		int c_pix4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS*MLO_IN_VERT_READS));

		if (c_idx + c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_READ_UNIT - 1; i >= MLO_READ_UNIT - MLO_IN_N_PIXS_OFF; --i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_pix4*MLO_READ_UNIT];
			}

// stack of inputs, each has 1 line
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
#if 0
				if (c == 0 && c_scan ==1 && c_pix4 == 0)
				{
					printf("K:g: %d %d %d %d %f\n",
						lcl_id,
						c_scan,
						i,
						lcl_in_off,
						lcl_bot[lcl_in_off]
					);
				}
#endif
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output line line on number of spans by number of waves
// read MLO_FILTER_SIZE1 number of such spans into SGPS (for example 5 *7 = 35)
// read 1 input line for 64 maps into LDS
//
// alg
//
// accumulate 1 scan of data per wave per each input map per wk-item

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private int top_wave_base[MLO_N_WAVES];

	for (int i = 0; i < MLO_N_WAVES; ++i)
	{
		top_wave_base[i] = i * MLO_PER_WAVE_READ;
	}

	__private _FLOAT top_dat[MLO_PER_WAVE_READ*MLO_FILTER_SIZE1];

	for (int i = 0; i < MLO_PER_WAVE_READ*MLO_FILTER_SIZE1; ++i)
	{
		top_dat[i] = 0;
	}

	__private _FLOAT pvt_accum[(MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)];

	for (int i = 0; i < (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

//	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	if (wei_tl_idx == 4 && o_idx == 0 && c_idx == 0 && (dat_tl_idx == 0 || dat_tl_idx == 1))
	{
		printf("G:p: %d %d %d %d %d\n",
			lcl_id,
			wei_tl_idx1,
			wei_tl_idx0,
			wei_tl1,
			wei_tl0
		);
	}

#endif

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
	//	int out_y = 0;

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;
		// over all out blocks
		// processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

//		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch output
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j, gbl_out_scan_off += MLO_OUT_STRIDE)
		{
			int top_df_off = (j < MLO_OUT_HEIGHT) ? gbl_out_scan_off + top_wave_base[wave_id] : 0;
			_FLOAT mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				_FLOAT top_val = top_df[top_df_off + i] * mask;
				top_dat[j*MLO_PER_WAVE_READ + i] = top_val;
			}
		}

		// prolog
		// handling padding

		int sc = 0;
// pad0
		for (; sc < 1; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);
// processing
			for (int l = MLO_FILTER_PAD1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:ap0: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
									* top_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								top_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
								);
						}
#endif

					}

				}

			}

		}

// pad1
		for (; sc < 2 && sc < MLO_OUT_HEIGHT; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_PAD1 + 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:ap1: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								top_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			}

		}

// generic

		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			int top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? gbl_out_scan_off + top_wave_base[wave_id] : 0;
			_FLOAT mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
			// move in the last output scan
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				top_dat[(MLO_FILTER_SIZE1 - 1) *MLO_PER_WAVE_READ + i] = top_df[top_df_off + i] * mask;
			}

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:a: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}
// move up output
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
				{
					top_dat[j*MLO_PER_WAVE_READ + i] = top_dat[(j+1)*MLO_PER_WAVE_READ + i];
				}
			}

			barrier(CLK_LOCAL_MEM_FENCE);
		}

// epilog
// handling padding
// pad1
		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:a1p: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			} // for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)

			  // move up output
			for (int j = 0; j < MLO_FILTER_SIZE1 - 2; ++j)
			{
				for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
				{
					top_dat[j*MLO_PER_WAVE_READ + i] = top_dat[(j + 1)*MLO_PER_WAVE_READ + i];
				}
			}

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

// pad0
		for (; sc < MLO_OUT_HEIGHT; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			// processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:a0p: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								top_dat[(MLO_FILTER_SIZE1 - 1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

// final summation
	for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
	{
		for (int n = 0; n < MLO_FILTER_SIZE0 && wave_id > 0; ++n)
		{
			lcl[((wave_id - 1) * MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n] =
				pvt_accum[l*MLO_FILTER_SIZE0 + n];
#if 0
			if (lcl_wv_id == 0 && l == 2 && n == 3)
			{
				printf("G:s1: %f\n",
					pvt_accum[l*MLO_FILTER_SIZE0 + n]
				);
			}
#endif

		}

		barrier(CLK_LOCAL_MEM_FENCE);

		for(int w = 0; w < MLO_N_WAVES-1 && wave_id == 0; ++w)
		{

			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				pvt_accum[l*MLO_FILTER_SIZE0 + n]
					+= lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n];
#if 0
				if (lcl_wv_id == 0 && l == 2 && n == 3)
				{
					printf("G:s2: %f %f\n",
						pvt_accum[l*MLO_FILTER_SIZE0 + n],
						lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
	}

// output
// inputs are outputs
	int c = lcl_wv_id;

	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24((c_idx + c), (int)MLO_WEI_CHANNEL_STRIDE);
	if (wave_id == 0)
	{
		for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
		{
			weights_df[wei_df_off + i] = pvt_accum[i];
		}

	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS (MLO_IN_WIDTH)
#define MLO_PER_WAVE_READ ((MLO_IN_WIDTH + MLO_N_WAVES - 1) / MLO_N_WAVES)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_PER_WAVE_READ * MLO_N_WAVES)
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)

#define MLO_IN_VERT_READS (1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_READS*MLO_READ_UNIT - MLO_N_IN_HORIZ_PIX_READS)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ ((MLO_N_WAVES - 1) * MLO_HW_WAVE_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void ReduceKernel64(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
	// add second half
	// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

inline void readInput(int lcl_id, int c_idx, int sc, int gbl_in_scan_off, __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int c_scan = sc;

		int c = iDiv(p4, (MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS));
		int c_pix4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS*MLO_IN_VERT_READS));

		if (c_idx + c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_READ_UNIT - 1; i >= MLO_READ_UNIT - MLO_IN_N_PIXS_OFF; --i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_pix4*MLO_READ_UNIT];
			}

// stack of inputs, each has 1 line
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
#if 0
				if (c == 0 && c_scan ==1 && c_pix4 == 0)
				{
					printf("K:g: %d %d %d %d %f\n",
						lcl_id,
						c_scan,
						i,
						lcl_in_off,
						lcl_bot[lcl_in_off]
					);
				}
#endif
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}
/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output line line on number of spans by number of waves
// read MLO_FILTER_SIZE1 number of such spans into SGPS (for example 5 *7 = 35)
// read 1 input line for 64 maps into LDS
//
// alg
//
// accumulate 1 scan of data per wave per each input map per wk-item

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(1); // input map index base

	int o_idx_base = iDiv(get_group_id(2), (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS))); // output map index base
	int ib_base = iMod(get_group_id(2), o_idx_base, (MLO_BATCH_SZ / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS)));

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private int top_wave_base[MLO_N_WAVES];

	for (int i = 0; i < MLO_N_WAVES; ++i)
	{
		top_wave_base[i] = i * MLO_PER_WAVE_READ;
	}

	__private _FLOAT bot_dat[MLO_PER_WAVE_READ*MLO_FILTER_SIZE1];

	for (int i = 0; i < MLO_PER_WAVE_READ*MLO_FILTER_SIZE1; ++i)
	{
		bot_dat[i] = 0;
	}

	__private _FLOAT pvt_accum[(MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)];

	for (int i = 0; i < (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

//	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	if (wei_tl_idx == 4 && o_idx == 0 && c_idx == 0 && (dat_tl_idx == 0 || dat_tl_idx == 1))
	{
		printf("G:p: %d %d %d %d %d\n",
			lcl_id,
			wei_tl_idx1,
			wei_tl_idx0,
			wei_tl1,
			wei_tl0
		);
	}

#endif

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;
		// over all out blocks
		// processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

//		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch output
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j, ++out_y)
		{
			int top_df_off = (out_y < MLO_OUT_HEIGHT) ? gbl_out_scan_off + j * MLO_IN_STRIDE + top_wave_base[wave_id] : 0;
			_FLOAT mask = (out_y < MLO_OUT_HEIGHT) ? 1 : 0;
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				_FLOAT top_val = top_df[top_df_off + i] * mask;
				bot_dat[j*MLO_PER_WAVE_READ + i] = top_val;
			}
		}

		// prolog
		// handling padding

		int sc = 0;
// pad0
		for (; sc < 1; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);
// processing
			for (int l = MLO_FILTER_PAD1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:ap0: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
									* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
								);
						}
#endif

					}

				}

			}

		}

// pad1
		for (; sc < 2 && sc < MLO_OUT_HEIGHT; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_PAD1 + 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 2 && n == 3)
						{
							printf("G:ap1: %d %d  %f %f %f %f\n",
								wave_id,
								sc,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l + 1) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			}

		}

// generic

		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, gbl_in_scan_off += MLO_IN_STRIDE, ++out_y)
		{

			int top_df_off = (out_y < MLO_OUT_HEIGHT) ? gbl_out_scan_off + (sc + MLO_FILTER_PAD1) * MLO_IN_STRIDE + top_wave_base[wave_id] : 0;
			_FLOAT mask = (out_y < MLO_OUT_HEIGHT) ? 1 : 0;
			// move in the last output scan
			for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
			{
				bot_dat[(MLO_FILTER_SIZE1 - 1) *MLO_PER_WAVE_READ + i] = top_df[top_df_off + i] * mask;
			}

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= 0; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}
// move up output
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				for (int i = 0; i < MLO_PER_WAVE_READ; ++i)
				{
					bot_dat[j*MLO_PER_WAVE_READ + i] = bot_dat[(j+1)*MLO_PER_WAVE_READ + i];
				}
			}

			barrier(CLK_LOCAL_MEM_FENCE);
		}

// epilog
// handling padding
// pad1
		for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			  // processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a0p: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif

					}

				}

			} // for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1 - 1; --l)

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 1; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

// pad0
		for (; sc < MLO_OUT_HEIGHT; ++sc, gbl_in_scan_off += MLO_IN_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			// read input line
			readInput(lcl_id, c_idx, sc, gbl_in_scan_off, bot, lcl_bot);

			// processing
			for (int l = MLO_FILTER_SIZE1 - 1; l >= MLO_FILTER_PAD1; --l)
			{

				for (int m = 0; m < MLO_PER_WAVE_READ; ++m)
				{
					for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
					{
						pvt_accum[l*MLO_FILTER_SIZE0 + n]
							// each wk-item process an input
							+= lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m];
#if 0
						if (lcl_wv_id == 0 && l == 1 && n == 1)
						{
							printf("G:a1p: %d %d %d  %f %f %f %f\n",
								wave_id,
								l,
								n,
								pvt_accum[l*MLO_FILTER_SIZE0 + n],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m]
								* bot_dat[l * MLO_PER_WAVE_READ + m],
								lcl_bot[lcl_wv_id * MLO_IN_LCL_WIDTH + wave_id * MLO_PER_WAVE_READ + n + m],
								bot_dat[(MLO_FILTER_PAD1 - l) * MLO_PER_WAVE_READ + m]
							);
						}
#endif
					}

				}

			}

			barrier(CLK_LOCAL_MEM_FENCE);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

// final summation
	for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
	{
		for (int n = 0; n < MLO_FILTER_SIZE0 && wave_id > 0; ++n)
		{
			lcl[((wave_id - 1) * MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n] =
				pvt_accum[l*MLO_FILTER_SIZE0 + n];
#if 0
			if (lcl_wv_id == 0 && l == 2 && n == 3)
			{
				printf("G:s1: %f\n",
					pvt_accum[l*MLO_FILTER_SIZE0 + n]
				);
			}
#endif

		}

		barrier(CLK_LOCAL_MEM_FENCE);

		for(int w = 0; w < MLO_N_WAVES-1 && wave_id == 0; ++w)
		{

			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				pvt_accum[l*MLO_FILTER_SIZE0 + n]
					+= lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n];
#if 0
				if (lcl_wv_id == 0 && l == 2 && n == 3)
				{
					printf("G:s2: %f %f\n",
						pvt_accum[l*MLO_FILTER_SIZE0 + n],
						lcl[(w* MLO_HW_WAVE_SZ + lcl_wv_id) * MLO_FILTER_SIZE0 + n]
					);
				}
#endif

			}

		}

		barrier(CLK_LOCAL_MEM_FENCE);
	}

// output
// inputs are outputs
	int c = lcl_wv_id;

	int wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24((c_idx + c), (int)MLO_WEI_CHANNEL_STRIDE);
	if (wave_id == 0)
	{
		for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
		{
			weights_df[wei_df_off + i] = pvt_accum[i];
		}

	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}


Don't create output file in test

Don't create output file in test

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@229407 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -x cl -emit-llvm -cl-strict-aliasing < %s

// RUN: %clang_cc1 -x cl -emit-llvm -cl-strict-aliasing %s