math: Fix ilogb(double) return type

math: Fix ilogb(double) return type

Signed-off-by: Aaron Watry <awatry@gmail.com>
Reviewed-by: Jan Vesely <jan.vesely@rutgers.edu>

git-svn-id: e7f1ab7c2a259259587475a3e7520e757882f167@261714 91177308-0d34-0410-b5e6-96231b3b80d8

/*
 * Copyright (c) 2015 Advanced Micro Devices, Inc.
 * Copyright (c) 2016 Aaron Watry
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <clc/clc.h>
#include "../clcmacro.h"
#include "math.h"

_CLC_OVERLOAD _CLC_DEF int ilogb(float x) {
    uint ux = as_uint(x);
    uint ax = ux & EXSIGNBIT_SP32;
    int rs = -118 - (int) clz(ux & MANTBITS_SP32);
    int r = (int) (ax >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32;
    r = ax < 0x00800000U ? rs : r;
    r = ax > EXPBITS_SP32 | ax == 0 ? 0x80000000 : r;
    r = ax == EXPBITS_SP32 ? 0x7fffffff : r;
    return r;
}

_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, ilogb, float);

#ifdef cl_khr_fp64
#pragma OPENCL EXTENSION cl_khr_fp64 : enable

_CLC_OVERLOAD _CLC_DEF int ilogb(double x) {
    ulong ux = as_ulong(x);
    ulong ax = ux & ~SIGNBIT_DP64;
    int r = (int) (ax >> EXPSHIFTBITS_DP64) - EXPBIAS_DP64;
    int rs = -1011 - (int) clz(ax & MANTBITS_DP64);
    r = ax < 0x0010000000000000UL ? rs : r;
    r = ax > 0x7ff0000000000000UL | ax == 0UL ? 0x80000000 : r;
    r = ax == 0x7ff0000000000000UL ? 0x7fffffff : r;
    return r;
}

_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, ilogb, double);

#endif // cl_khr_fp64

/*
 * Copyright (c) 2015 Advanced Micro Devices, Inc.
 * Copyright (c) 2016 Aaron Watry
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <clc/clc.h>
#include "../clcmacro.h"
#include "math.h"

_CLC_OVERLOAD _CLC_DEF int ilogb(float x) {
    uint ux = as_uint(x);
    uint ax = ux & EXSIGNBIT_SP32;
    int rs = -118 - (int) clz(ux & MANTBITS_SP32);
    int r = (int) (ax >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32;
    r = ax < 0x00800000U ? rs : r;
    r = ax > EXPBITS_SP32 | ax == 0 ? 0x80000000 : r;
    r = ax == EXPBITS_SP32 ? 0x7fffffff : r;
    return r;
}

_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, ilogb, float);

#ifdef cl_khr_fp64
#pragma OPENCL EXTENSION cl_khr_fp64 : enable

_CLC_OVERLOAD _CLC_DEF ilogb(double x) {
    ulong ux = as_ulong(x);
    ulong ax = ux & ~SIGNBIT_DP64;
    int r = (int) (ax >> EXPSHIFTBITS_DP64) - EXPBIAS_DP64;
    int rs = -1011 - (int) clz(ax & MANTBITS_DP64);
    r = ax < 0x0010000000000000UL ? rs : r;
    r = ax > 0x7ff0000000000000UL | ax == 0UL ? 0x80000000 : r;
    r = ax == 0x7ff0000000000000UL ? 0x7fffffff : r;
    return r;
}

_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, ilogb, double);

#endif // cl_khr_fp64


Fix test expectations (#820)

Fix test expectations (#820)

// RUN: clspv %s --emit-ir=%t.ll
// RUN: FileCheck %s < %t.ll

void kernel foo(global double *out, int in)
{
  *out = in / 2.304;
}

// CHECK: target triple = "spir-unknown-unknown"
// CHECK: define
// CHECK-SAME: spir_kernel
// CHECK-SAME: void @foo
// CHECK-SAME: double addrspace(1)*{{[^%]+}}%out
// CHECK-SAME: i32{{[^%]+}}%in

// RUN: clspv %s --emit-ir=%t.ll
// RUN: FileCheck %s < %t.ll

void kernel foo(global double *out, int in)
{
  *out = in / 2.304;
}

// CHECK: target triple = "spir-unknown-unknown"
// CHECK: define
// CHECK-SAME: spir_kernel
// CHECK-SAME: void @foo
// CHECK-SAME: double addrspace(1)* {{[^%]+}}%out
// CHECK-SAME: i32 {{[^%]+}}%in


load boundary columns

load boundary columns

#include "ocl_platform.cl"

// number of banks in local memory
// LDS_BANKS

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns

 **/

 #define BOUNDARY_X_LEFT 1

#define VERTICAL_STRIDE_EVEN 66  // WIN_SIZE_X/2 + 2 boundary columns
#define VERTICAL_STRIDE_ODD 65   // WIN_SIZE_X/2 + 1 boundary column

// two vertical neighbours: pointer diff:
#define BUFFER_SIZE_EVEN            528	// VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD             520	// VERTICAL_STRIDE_ODD * WIN_SIZE_Y

#define EVEN_ODD_PADDING                     16    // LDS_BANKS - (BUFFER_SIZE_EVEN % LDS_BANKS)

#define CHANNEL_BUFFER_SIZE_UNPADDED    1064    // BUFFER_SIZE_EVEN + EVEN_ODD_PADDING +  BUFFER_SIZE_ODD
#define CHANNEL_BUFFER_PADDING  24              // LDS_BANKS - (CHANNEL_BUFFER_SIZE_UNPADDED % LDS_BANKS)
#define CHANNEL_BUFFER_SIZE     1088            // BUFFER_SIZE_EVEN + EVEN_ODD_PADDING +  BUFFER_SIZE_ODD

#define CHANNEL_BUFFER_SIZE_X2  2176
#define CHANNEL_BUFFER_SIZE_X3  3264

#define PIXEL_BUFFER_SIZE   4328

#define VERTICAL_STRIDE_LOW_TO_HIGH   544   // BUFFER_SIZE_EVEN + EVEN_ODD_PADDING
#define VERTICAL_STRIDE_HIGH_TO_LOW  -478   // VERTICAL_STRIDE_EVEN - STRIDE_HIGH_TO_LOW

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

// read pixel from local buffer
int4 readPixel( LOCAL int*  restrict  src) {
	return (int4)(*src, *(src+CHANNEL_BUFFER_SIZE),  *(src+CHANNEL_BUFFER_SIZE_X2),  *(src+CHANNEL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(int4 pix, LOCAL int*  restrict  dest) {
	*dest = pix.x;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.y;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.z;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.w;
}

void writeColumnToOutput(LOCAL int* restrict currentScratch, __write_only image2d_t odata, int firstY, int height, int halfHeight){
	// write points to destination
	int2 posOut = {getGlobalId(0), firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even
	    if (posOut.y >= halfHeight)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		// odd
		currentScratch += VERTICAL_STRIDE_LOW_TO_HIGH ;
		posOut.y+= halfHeight + 1;
		if (posOut.y >= height)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE_HIGH_TO_LOW;
		posOut.y -= halfHeight;
	}
}

// offset when transforming columns
inline int getScratchColumnOffset(){
   return BOUNDARY_X_LEFT + (getLocalId(0) >> 1);
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	if (getGlobalId(0) >= width)
	    return;

   // column 0 loads column -2 (even) with x offset -2
   // column 1 loads column -1 (odd)  with x offset -2
   // column WIN_SIZE_X -2 loads column WIN_SIZE_X (even) with x offset +2
   int srcBoundaryShiftX = 0;
   int scratchBoundaryShiftX = 0;
   if (getLocalId(0) == 0 || getLocalId(0) == 1) {
       srcBoundaryShiftX = -2;
	   scratchBoundaryShiftX = -1;
   }
   else if (getLocalId(0) == WIN_SIZE_X -2) {
       srcBoundaryShiftX = -2;
	   scratchBoundaryShiftX = 1;
   }

	const int halfWinSizeX = WIN_SIZE_X >> 1;

    const unsigned int halfHeight = height >> 1;
	LOCAL int scratch[PIXEL_BUFFER_SIZE];
	float yDelta = 1.0/height;
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)
	int4 minusOneBdy;
	int4 currentBdy;

	// read -1 point
	const float2 posIn = (float2)(getGlobalId(0), firstY - 1) /  (float2)(width, height);
	float2 posInBdy;

	int4 minusOne = read_imagei(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	int4 current = read_imagei(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagei(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) >> 1;

	if (srcBoundaryShiftX) {
	   posInBdy = posIn;
	   posInBdy.y -=  2*yDelta;
	   minusOneBdy = read_imagei(idata, sampler, posInBdy);

	  // read 0 point
  	  posInBdy.y += yDelta;
	  currentBdy = read_imagei(idata, sampler, posInBdy);

	  // transform -1 point (no need to write to local memory)
	  minusOneBdy -= ( read_imagei(idata, sampler, (float2)(posInBdy.x, (firstY - 2)*yDelta)) + current) >> 1;
	}

	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL int* currentScratch = scratch + getScratchColumnOffset();
		for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, then transform and write current (odd) and last (even)

			// read current plus one (odd) point
			posIn.y += yDelta;
			if (posIn.y >= 1)
				break;
			int4 currentPlusOne = read_imagei(idata, sampler, posIn);

			// read current plus two (even) point
			posIn.y += yDelta;
			int4 currentPlusTwo = read_imagei(idata, sampler, posIn);

			// transform current plus one (odd) point
			currentPlusOne -= (current + currentPlusTwo) >> 1;

			// transform current (even) point
			current += (minusOne + currentPlusOne + (int4)(2,2,2,2)) >> 2;

			//write current (even)
			writePixel(current, currentScratch);

			//write odd
			currentScratch += VERTICAL_STRIDE_LOW_TO_HIGH;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE_HIGH_TO_LOW;

			//update registers
			minusOne = currentPlusOne;
			current = currentPlusTwo;
		}
		//read boundary column if necessary
		if (scratchBoundaryShiftX) {
			currentScratch = scratch + getScratchColumnOffset() + scratchBoundaryShiftX;
			for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

				///////////////////////////////////////////////////////////////////////////////////////////
				// fetch next two pixels, then transform and write current (odd) and last (even)

				// read current plus one (odd) point
				posInBdy.y += yDelta;
				if (posInBdy.y >= 1)
					break;
				int4 currentPlusOneBdy = read_imagei(idata, sampler, posInBdy);

				// read current plus two (even) point
				posInBdy.y += yDelta;
				int4 currentPlusTwoBdy = read_imagei(idata, sampler, posInBdy);

				// transform current plus one (odd) point
				currentPlusOneBdy -= (currentBdy + currentPlusTwoBdy) >> 1;

				// transform current (even) point
				currentBdy += (minusOneBdy + currentPlusOneBdy + (int4)(2,2,2,2)) >> 2;

				//write current (even)
				writePixel(currentBdy, currentScratch);

				//write odd
				currentScratch += VERTICAL_STRIDE_LOW_TO_HIGH;
				writePixel(currentPlusOneBdy, currentScratch);

				//advance scratch pointer
				currentScratch += VERTICAL_STRIDE_HIGH_TO_LOW;

				//update registers
				minusOneBdy = currentPlusOneBdy;
				currentBdy = currentPlusTwoBdy;
			}
		}

		//4. transform rows

		//5. write local buffer to destination image
		writeColumnToOutput(scratch + getScratchColumnOffset(), odata, firstY, height, halfHeight);

		// move to next step
		firstY += WIN_SIZE_Y;
	}
}

#include "ocl_platform.cl"

// number of banks in local memory
// LDS_BANKS

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns

 **/

 #define BOUNDARY_X_LEFT 1

#define VERTICAL_STRIDE_EVEN 66  // WIN_SIZE_X/2 + 2 boundary columns
#define VERTICAL_STRIDE_ODD 65   // WIN_SIZE_X/2 + 1 boundary column

// two vertical neighbours: pointer diff:
#define BUFFER_SIZE_EVEN            528	// VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD             520	// VERTICAL_STRIDE_ODD * WIN_SIZE_Y

#define EVEN_ODD_PADDING                     16    // LDS_BANKS - (BUFFER_SIZE_EVEN % LDS_BANKS)

#define CHANNEL_BUFFER_SIZE_UNPADDED    1064    // BUFFER_SIZE_EVEN + EVEN_ODD_PADDING +  BUFFER_SIZE_ODD
#define CHANNEL_BUFFER_PADDING  24              // LDS_BANKS - (CHANNEL_BUFFER_SIZE_UNPADDED % LDS_BANKS)
#define CHANNEL_BUFFER_SIZE     1088            // BUFFER_SIZE_EVEN + EVEN_ODD_PADDING +  BUFFER_SIZE_ODD

#define CHANNEL_BUFFER_SIZE_X2  2176
#define CHANNEL_BUFFER_SIZE_X3  3264

#define PIXEL_BUFFER_SIZE   4328

#define VERTICAL_STRIDE_LOW_TO_HIGH   544   // BUFFER_SIZE_EVEN + EVEN_ODD_PADDING
#define VERTICAL_STRIDE_HIGH_TO_LOW  -478   // VERTICAL_STRIDE_EVEN - STRIDE_HIGH_TO_LOW

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

// read pixel from local buffer
int4 readPixel( LOCAL int*  restrict  src) {
	return (int4)(*src, *(src+CHANNEL_BUFFER_SIZE),  *(src+CHANNEL_BUFFER_SIZE_X2),  *(src+CHANNEL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(int4 pix, LOCAL int*  restrict  dest) {
	*dest = pix.x;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.y;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.z;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.w;
}

void writeColumnToOutput(LOCAL int* restrict currentScratch, __write_only image2d_t odata, int firstY, int height, int halfHeight){
	// write points to destination
	int2 posOut = {getGlobalId(0), firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even
	    if (posOut.y >= halfHeight)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		// odd
		currentScratch += VERTICAL_STRIDE_LOW_TO_HIGH ;
		posOut.y+= halfHeight + 1;
		if (posOut.y >= height)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE_HIGH_TO_LOW;
		posOut.y -= halfHeight;
	}
}

// offset when transforming columns
inline int getScratchColumnOffset(){
   return BOUNDARY_X_LEFT + (getLocalId(0) >> 1);
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	if (getGlobalId(0) >= width)
	    return;

	const int halfWinSizeX = WIN_SIZE_X >> 1;

    const unsigned int halfHeight = height >> 1;
	LOCAL int scratch[PIXEL_BUFFER_SIZE];
	float yDelta = 1.0/height;
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -1 point
	const float2 posIn = (float2)(getGlobalId(0), firstY - 1) /  (float2)(width, height);
	int4 minusOne = read_imagei(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	int4 current = read_imagei(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagei(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) >> 1;

	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL int* currentScratch = scratch + getScratchColumnOffset();
		for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, then transform and write current (odd) and last (even)

			// read current plus one (odd) point
			posIn.y += yDelta;
			if (posIn.y >= 1)
				break;
			int4 currentPlusOne = read_imagei(idata, sampler, posIn);

			// read current plus two (even) point
			posIn.y += yDelta;
			int4 currentPlusTwo = read_imagei(idata, sampler, posIn);

			// transform current plus one (odd) point
			currentPlusOne -= (current + currentPlusTwo) >> 1;

			// transform current (even) point
			current += (minusOne + currentPlusOne + (int4)(2,2,2,2)) >> 2;

			//write current (even)
			writePixel(current, currentScratch);

			//write odd
			currentScratch += VERTICAL_STRIDE_LOW_TO_HIGH;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE_HIGH_TO_LOW;

			//update registers
			minusOne = currentPlusOne;
			current = currentPlusTwo;
		}

		//4. transform rows

		//5. write local buffer to destination image
		writeColumnToOutput(scratch + getScratchColumnOffset(), odata, firstY, height, halfHeight);

		// move to next step
		firstY += WIN_SIZE_Y;
	}
}

////////////////////////////////////////////
/*
/// Returns index ranging from 0 to num work items, such that first half
/// of the work items get even indices and others get odd indices. Each work item
/// gets a different index.
/// Example: (for WIN_SiZE_X == 8)   getLocalId(0):   0  1  2  3  4  5  6  7
///									 parityIdx:		  0  2  4  6  1  3  5  7

/// @return parity-separated index of work item
inline int parityIdx() {
	return (getLocalId(0) << 1) - (WIN_SIZE_X - 1) * (getLocalId(0) / (WIN_SIZE_X >> 1));
}

// two vertical neighbours: pointer diff:
#define VERTICAL_STRIDE_EVEN 66			// 2*BOUNDARY_X + (WIN_SIZE_X / 2)

#define VERTICAL_STRIDE_ODD  65          //  BOUNDARY_X  + (WIN_SIZE_X / 2)

// size of one of two buffers (odd or even)
#define BUFFER_SIZE_EVEN     528          // VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD     528          // VERTICAL_STRIDE_ODD * WIN_SIZE_Y

/// padding between even and odd buffers: used to reduce bank conflicts
#define PADDING			32           // LDS_BANKS - ((BUFFER_SIZE + LDS_BANKS / 2) % LDS_BANKS);

// offset of the odd columns buffer from the beginning of data buffer
#define ODD_OFFSET 560              // BUFFER_SIZE + PADDING

// size of buffer for both even and odd columns
#define CHANNEL_BUFFER_SIZE  1088     // 2 * BUFFER_SIZE + PADDING;
#define CHANNEL_BUFFER_SIZE_X2  2176
#define CHANNEL_BUFFER_SIZE_X3  3264
*/
////////////////////////////////////////////


remove dead code

remove dead code

// this assumes that all matrices are column major

kernel void {{kernelname}} (
        int GlobalRows, int GlobalMids, int GlobalCols,
        int BlockRows, int BlockMids, int BlockCols,
        int blockRows, int blockMids, int blockCols,
        global float4 *C_float4, global float4 *A_float4, global float4 *B_float4,
        local float4 *B_block_float4, local float *A_block_unused) {
    global float *C = (global float *)C_float4;
    global float *B = (global float *)B_float4;
    global float *A = (global float *)A_float4;
    local float *B_block = (local float *)B_block_float4;
    int BlockRow = get_group_id(0);
    int BlockCol = get_group_id(1);
    int tid = get_local_id(0);
    int globalRow = BlockRow * blockRows + tid;

    float C_row[{{blockCols}}];
    //float *C_row = (float *)C_row_float4;
    for(int blockCol=0; blockCol < {{blockCols}}; blockCol++) {
         C_row[blockCol] = 0.0f;
    }
    {
        for(int BlockMid = 0; BlockMid < BlockMids; BlockMid++) {
            // first copy down the data from B
            // each thread will handle one column of B data, ie
            // iterate over blockMid
            // sync point (can remove if num threads == warpsize)
            //barrier(CLK_LOCAL_MEM_FENCE);
            {
                int blockCol = tid;
                int globalCol = BlockCol * blockCols + blockCol;
                for(int blockMid=0; blockMid < blockMids; blockMid++) {
                    int globalMid = BlockMid * blockMids + blockMid;
                    B_block[blockCol * blockMids + blockMid] = B[globalCol * GlobalMids + globalMid];
                }
            }

            float A_row[{{blockMids}}];
            //float *A_row = (float*)A_row_float4;
            {
                for(int blockMid=0; blockMid < {{blockMids}}; blockMid++) {
                    int globalMid = BlockMid * blockMids + blockMid;
                    A_row[blockMid] = A[globalMid * GlobalRows + globalRow];
                }
            }

            // sync point (can remove if num threads == warpsize)
            //barrier(CLK_LOCAL_MEM_FENCE);

            // calc some C :-)
            // each thread handles a row of c, so needs to iterate over columns
            // but for each column, needs to iterate over middle too
            for(int blockMid=0; blockMid < {{blockMids}}; blockMid++) {
                for(int blockCol=0; blockCol < {{blockCols}}; blockCol++) {
                    C_row[blockCol] += A_row[blockMid] * B_block[blockCol * blockMids + blockMid];
                }
            }
        }
    }
    // write C out
    #pragma unroll
    for(int blockCol=0; blockCol < {{blockCols}}; blockCol++) {
        int globalCol = BlockCol * blockCols + blockCol;
        C[globalCol * GlobalRows + globalRow] = C_row[blockCol];
        //C_float4[globalRow_globalCols_plus_BlockCol_blockCols + blockCol] = C_row_float4[blockCol];
    }
}

// this assumes that all matrices are column major

kernel void {{kernelname}} (
        int GlobalRows, int GlobalMids, int GlobalCols,
        int BlockRows, int BlockMids, int BlockCols,
        int blockRows, int blockMids, int blockCols,
        global float4 *C_float4, global float4 *A_float4, global float4 *B_float4,
        local float4 *B_block_float4, local float *A_block_unused) {
    global float *C = (global float *)C_float4;
    global float *B = (global float *)B_float4;
    global float *A = (global float *)A_float4;
    local float *B_block = (local float *)B_block_float4;
    int BlockRow = get_group_id(0);
    int BlockCol = get_group_id(1);
    int tid = get_local_id(0);
    int globalRow = BlockRow * blockRows + tid;

    float C_row[{{blockCols}}];
    //float *C_row = (float *)C_row_float4;
    for(int blockCol=0; blockCol < {{blockCols}}; blockCol++) {
         C_row[blockCol] = 0.0f;
    }
    {
        for(int BlockMid = 0; BlockMid < BlockMids; BlockMid++) {
            // first copy down the data from B
            // each thread will handle one column of B data, ie
            // iterate over blockMid
            // sync point (can remove if num threads == warpsize)
            //barrier(CLK_LOCAL_MEM_FENCE);
            {
                int blockCol = tid;
                int globalCol = BlockCol * blockCols + blockCol;
                for(int blockMid=0; blockMid < blockMids; blockMid++) {
                    int globalMid = BlockMid * blockMids + blockMid;
                    B_block[blockCol * blockMids + blockMid] = B[globalCol * GlobalMids + globalMid];
                    //B_block[blockCol * blockMids + blockMid] = 4.0f;
                }
            }

            float A_row[{{blockMids}}];
            //float *A_row = (float*)A_row_float4;
            {
                for(int blockMid=0; blockMid < {{blockMids}}; blockMid++) {
                    int globalMid = BlockMid * blockMids + blockMid;
                    A_row[blockMid] = A[globalMid * GlobalRows + globalRow];
                    //A_row[blockMid] = 10.0f;
                }
            }

            // sync point (can remove if num threads == warpsize)
            //barrier(CLK_LOCAL_MEM_FENCE);

            // calc some C :-)
            // each thread handles a row of c, so needs to iterate over columns
            // but for each column, needs to iterate over middle too
            for(int blockMid=0; blockMid < {{blockMids}}; blockMid++) {
                for(int blockCol=0; blockCol < {{blockCols}}; blockCol++) {
                    C_row[blockCol] += A_row[blockMid] * B_block[blockCol * blockMids + blockMid];
                }
            }
        }
    }
    // write C out
    #pragma unroll
    for(int blockCol=0; blockCol < {{blockCols}}; blockCol++) {
        int globalCol = BlockCol * blockCols + blockCol;
        C[globalCol * GlobalRows + globalRow] = C_row[blockCol];
        //C_float4[globalRow_globalCols_plus_BlockCol_blockCols + blockCol] = C_row_float4[blockCol];
    }
}


tests/sixtracklib/opencl: Update testlib OpenCL beam element copy kernel to use convenience functions from sixtracklib/common/beam_elements.h

#ifndef SIXTRL_TESTS_SIXTRACKLIB_OPENCL_TEST_TEST_BEAM_ELEMENTS_KERNEL_CL__
#define SIXTRL_TESTS_SIXTRACKLIB_OPENCL_TEST_TEST_BEAM_ELEMENTS_KERNEL_CL__

#if !defined( SIXTRL_NO_INCLUDES )
    #include "sixtracklib/opencl/internal/default_compile_options.h"

    #include "sixtracklib/common/definitions.h"
    #include "sixtracklib/common/buffer/managed_buffer_minimal.h"
    #include "sixtracklib/common/buffer/managed_buffer_remap.h"
    #include "sixtracklib/common/beam_elements.h"
#endif /* !defined( SIXTRL_NO_INCLUDES ) */

#pragma OPENCL_EXTENSION cl_khr_int32_extended_atomics

__kernel void NS(BeamElements_copy_beam_elements_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char const* SIXTRL_RESTRICT in_buffer_begin,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT out_buffer_begin,
    SIXTRL_DATAPTR_DEC SIXTRL_INT32_T* SIXTRL_RESTRICT ptr_success_flag )
{
    typedef NS(buffer_size_t)                               buf_size_t;
    typedef NS(object_type_id_t)                            type_id_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC NS(Object)*        obj_iter_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC NS(Object) const*  obj_const_iter_t;

    buf_size_t const  slot_size = ( buf_size_t )8u;
    SIXTRL_INT32_T success_flag = ( SIXTRL_INT32_T )-1;

    if( ( !NS(ManagedBuffer_needs_remapping)(  in_buffer_begin, slot_size ) ) &&
        ( !NS(ManagedBuffer_needs_remapping)( out_buffer_begin, slot_size ) ) &&
        (  NS(ManagedBuffer_get_num_objects)( out_buffer_begin, slot_size ) ==
           NS(ManagedBuffer_get_num_objects)(  in_buffer_begin, slot_size ) ) )
    {
        size_t beam_element_id = get_global_id( 0 );
        size_t const stride    = get_global_size( 0 );

        size_t const num_beam_elements =
            NS(ManagedBuffer_get_num_objects)( in_buffer_begin, slot_size );

        success_flag = ( SIXTRL_INT32_T )0u;

        for( ; beam_element_id < num_beam_elements ; beam_element_id += stride )
        {
            obj_const_iter_t in_obj_it  =
                NS(ManagedBuffer_get_const_objects_index_begin)(
                    in_buffer_begin, slot_size );

            obj_iter_t out_obj_it = NS(ManagedBuffer_get_objects_index_begin)(
                out_buffer_begin, slot_size );

            type_id_t const type_id = NS(Object_get_type_id)( in_obj_it );

            in_obj_it  = in_obj_it  + beam_element_id;
            out_obj_it = out_obj_it + beam_element_id;

            success_flag |= NS(BeamElements_copy_object)( out_obj_it, in_obj_it );
        }
    }
    else
    {
        if( NS(ManagedBuffer_needs_remapping)(  in_buffer_begin, slot_size ) )
        {
            success_flag |= -4;
        }

        if( NS(ManagedBuffer_needs_remapping)( out_buffer_begin, slot_size ) )
        {
            success_flag |= -8;
        }

        if( NS(ManagedBuffer_get_num_objects)( out_buffer_begin, slot_size ) !=
            NS(ManagedBuffer_get_num_objects)( in_buffer_begin,  slot_size ) )
        {
            success_flag |= -16;
        }
    }

    if( ( success_flag != ( SIXTRL_INT32_T )0u ) &&
        ( ptr_success_flag != SIXTRL_NULLPTR ) )
    {
        atomic_or( ptr_success_flag, success_flag );
    }

    return;
}

#endif /* SIXTRL_TESTS_SIXTRACKLIB_OPENCL_TEST_TEST_BEAM_ELEMENTS_KERNEL_CL__ */

/* end: tests/sixtracklib/opencl/test_beam_elements_opencl_kernel.cl */

#ifndef SIXTRL_TESTS_SIXTRACKLIB_OPENCL_TEST_TEST_BEAM_ELEMENTS_KERNEL_CL__
#define SIXTRL_TESTS_SIXTRACKLIB_OPENCL_TEST_TEST_BEAM_ELEMENTS_KERNEL_CL__

#if !defined( SIXTRL_NO_INCLUDES )
    #include "sixtracklib/opencl/internal/default_compile_options.h"

    #include "sixtracklib/common/definitions.h"
    #include "sixtracklib/common/buffer/managed_buffer_minimal.h"
    #include "sixtracklib/common/buffer/managed_buffer_remap.h"
    #include "sixtracklib/common/beam_elements.h"
#endif /* !defined( SIXTRL_NO_INCLUDES ) */

#pragma OPENCL_EXTENSION cl_khr_int32_extended_atomics

__kernel void NS(BeamElements_copy_beam_elements_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char const* SIXTRL_RESTRICT in_buffer_begin,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT out_buffer_begin,
    SIXTRL_DATAPTR_DEC int* SIXTRL_RESTRICT ptr_success_flag )
{
    typedef NS(buffer_size_t)                               buf_size_t;
    typedef NS(object_type_id_t)                            type_id_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC NS(Object)*        obj_iter_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC NS(Object) const*  obj_const_iter_t;

    buf_size_t const slot_size = ( buf_size_t )8u;
    int success_flag = ( int )0u;

    if( ( !NS(ManagedBuffer_needs_remapping)(  in_buffer_begin, slot_size ) ) &&
        ( !NS(ManagedBuffer_needs_remapping)( out_buffer_begin, slot_size ) ) &&
        (  NS(ManagedBuffer_get_num_objects)( out_buffer_begin, slot_size ) ==
           NS(ManagedBuffer_get_num_objects)(  in_buffer_begin, slot_size ) ) )
    {
        size_t const global_id = get_global_id( 0 );

        if( global_id <
            NS(ManagedBuffer_get_num_objects)( in_buffer_begin, slot_size ) )
        {
            obj_const_iter_t in_obj_it  =
                NS(ManagedBuffer_get_const_objects_index_begin)(
                    in_buffer_begin, slot_size );

            obj_iter_t out_obj_it = NS(ManagedBuffer_get_objects_index_begin)(
                out_buffer_begin, slot_size );

            type_id_t const type_id = NS(Object_get_type_id)( in_obj_it );

            in_obj_it  = in_obj_it  + global_id;
            out_obj_it = out_obj_it + global_id;

            if( type_id == NS(Object_get_type_id)( out_obj_it ) )
            {
                switch( type_id )
                {
                    case NS(OBJECT_TYPE_DRIFT):
                    {
                        typedef NS(Drift) belem_t;
                        typedef SIXTRL_BE_ARGPTR_DEC belem_t*       ptr_belem_t;
                        typedef SIXTRL_BE_ARGPTR_DEC belem_t const* ptr_const_belem_t;

                        ptr_const_belem_t in_belem = ( ptr_const_belem_t )(
                            uintptr_t )NS(Object_get_begin_addr)( in_obj_it );

                        ptr_belem_t out_belem = ( ptr_belem_t )(
                            uintptr_t )NS(Object_get_begin_addr)( out_obj_it );

                        if( ( in_belem  != SIXTRL_NULLPTR ) &&
                            ( out_belem != SIXTRL_NULLPTR ) )
                        {
                            NS(Drift_copy)( out_belem, in_belem );
                        }
                        else
                        {
                            success_flag |= -1;
                        }

                        break;
                    }

                    case NS(OBJECT_TYPE_DRIFT_EXACT):
                    {
                        typedef NS(DriftExact) belem_t;
                        typedef SIXTRL_BE_ARGPTR_DEC belem_t*       ptr_belem_t;
                        typedef SIXTRL_BE_ARGPTR_DEC belem_t const* ptr_const_belem_t;

                        ptr_const_belem_t in_belem = ( ptr_const_belem_t )(
                            uintptr_t )NS(Object_get_begin_addr)( in_obj_it );

                        ptr_belem_t out_belem = ( ptr_belem_t )(
                            uintptr_t )NS(Object_get_begin_addr)( out_obj_it );

                        if( ( in_belem  != SIXTRL_NULLPTR ) &&
                            ( out_belem != SIXTRL_NULLPTR ) )
                        {
                            NS(DriftExact_copy)( out_belem, in_belem );
                        }
                        else
                        {
                            success_flag |= -1;
                        }

                        break;
                    }

                    default:
                    {
                        success_flag |= -1;
                    }
                };
            }
            else
            {
                success_flag |= -2;
            }
        }
    }
    else
    {
        if( NS(ManagedBuffer_needs_remapping)(  in_buffer_begin, slot_size ) )
        {
            success_flag |= -4;
        }

        if( NS(ManagedBuffer_needs_remapping)( out_buffer_begin, slot_size ) )
        {
            success_flag |= -8;
        }

        if( NS(ManagedBuffer_get_num_objects)( out_buffer_begin, slot_size ) !=
            NS(ManagedBuffer_get_num_objects)( in_buffer_begin,  slot_size ) )
        {
            success_flag |= -16;
        }
    }

    if( ( success_flag     != ( int )0u ) &&
        ( ptr_success_flag != SIXTRL_NULLPTR ) )
    {
        atomic_or( ptr_success_flag, success_flag );
    }

    return;
}

#endif /* SIXTRL_TESTS_SIXTRACKLIB_OPENCL_TEST_TEST_BEAM_ELEMENTS_KERNEL_CL__ */

/* end: tests/sixtracklib/opencl/test_beam_elements_opencl_kernel.cl */