apply3.cl so far...

apply3.cl so far...

// OpenCL kernels....

// expected templated values:
// dims
// operation
// adim
// bdim
// cdim
// MAX_CUTORCH_DIMS
//
// maybe should add:
// IndexType (hardcoded to int for now)

// (Ported from cutorch's THCApply.cuh)

// Maximum number of dimensions allowed for cutorch
#define MAX_CUTORCH_DIMS 25

// Enum that indicates whether tensor arguments are read/write or
// read-only
//enum TensorArgType { ReadWrite, ReadOnly };

// kernel argument that defines tensor layout
struct TensorInfo {
  // Extracts size/stride information for the kernel.
  // Successive dimensions can be collapsed if the size/strides match
  // up and thus there are no holes between the dimensions. This is used
  // to reduce the complexity of the problem.
  // The optional `reduceDim` indicates a reduction dimension for the
  // given tensor, so that the output size for this dimension will be 1.

  float* data;
  int sizes[{{MAX_CUTORCH_DIMS}}];
  int strides[{{MAX_CUTORCH_DIMS}}];
  int dims;
};
// Contiguous tensors of more than one dimension are collapsed down
// to one tensor
bool TensorInfo_isContiguous( TensorInfo tensorInfo ) {
    return (tensorInfo.dims == 1 && tensorInfo.strides[0] == 1);
}

float op2( float val1, float val2 ) {
    return {{operation}};
}

// Translate a linear index for the apply to a float* offset;
// specialized on `Dims` to reduce nvcc compilation time
{% for dim in dims %}
int IndexToOffset_{{dim}}_get( int linearId, const TensorInfo info) {
  int offset = 0;

  // Use static dims
  for (int i = {{dim}} - 1; i >= 0; --i) {
    int curDimIndex = linearId % info.sizes[i];
    int curDimOffset = curDimIndex * info.strides[i];
    offset += curDimOffset;

    if (i > 0) {
      linearId /= info.sizes[i];
    }
  }

  return offset;
}
{% endfor %}

kernel void
THClTensor_pointwiseApply3(TensorInfo a,
                             TensorInfo b,
                             TensorInfo c,
                             int totalElements {
  for (int linearIndex = blockIdx.x * blockDim.x + threadIdx.x;
       linearIndex < totalElements;
       linearIndex += gridDim.x * blockDim.x) {
    // Convert `linearIndex` into an offset of `a`
    const int aOffset =
      IndexToOffset_{{adim}}_get(linearIndex, a);

    // Convert `linearIndex` into an offset of `b`
    const int bOffset =
      IndexToOffset_{{bdim}}_get(linearIndex, b);

    // Convert `linearIndex` into an offset of `c`
    const int cOffset =
      IndexToOffset_{{cdim}}_get(linearIndex, c);

    a.data[aOffset] = op2(b.data[bOffset], c.data[cOffset]);
  }
}

// OpenCL kernels....

// (Ported from cutorch's THCApply.cuh)

// Maximum number of dimensions allowed for cutorch
#define MAX_CUTORCH_DIMS 25

// Enum that indicates whether tensor arguments are read/write or
// read-only
enum TensorArgType { ReadWrite, ReadOnly };

// kernel argument that defines tensor layout
struct TensorInfo {
  // Extracts size/stride information for the kernel.
  // Successive dimensions can be collapsed if the size/strides match
  // up and thus there are no holes between the dimensions. This is used
  // to reduce the complexity of the problem.
  // The optional `reduceDim` indicates a reduction dimension for the
  // given tensor, so that the output size for this dimension will be 1.

  float* data;
  int sizes[MAX_CUTORCH_DIMS];
  int strides[MAX_CUTORCH_DIMS];
  int dims;
};
// Contiguous tensors of more than one dimension are collapsed down
// to one tensor
bool TensorInfo_isContiguous( TensorInfo *tensorInfo ) {
    return (tensorInfo->dims == 1 && tensorInfo->strides[0] == 1);
}

float op2( float val1, float val2 ) {
    return {{operation}};
}

// Translate a linear index for the apply to a float* offset;
// specialized on `Dims` to reduce nvcc compilation time
int IndexToOffset_A_get( int linearId, const TensorInfo *info) {
  int offset = 0;

  // Use static dims
  for (int i = {{ADims}} - 1; i >= 0; --i) {
    int curDimIndex = linearId % info.sizes[i];
    int curDimOffset = curDimIndex * info.strides[i];
    offset += curDimOffset;

    if (i > 0) {
      linearId /= info.sizes[i];
    }
  }

  return offset;
}

int IndexToOffset_B_get( int linearId, const TensorInfo *info) {
  int offset = 0;

  // Use static dims
  for (int i = {{BDims}} - 1; i >= 0; --i) {
    int curDimIndex = linearId % info.sizes[i];
    int curDimOffset = curDimIndex * info.strides[i];
    offset += curDimOffset;

    if (i > 0) {
      linearId /= info.sizes[i];
    }
  }

  return offset;
}

int IndexToOffset_C_get( int linearId, const TensorInfo *info) {
  int offset = 0;

  // Use static dims
  for (int i = {{CDims}} - 1; i >= 0; --i) {
    int curDimIndex = linearId % info.sizes[i];
    int curDimOffset = curDimIndex * info.strides[i];
    offset += curDimOffset;

    if (i > 0) {
      linearId /= info.sizes[i];
    }
  }

  return offset;
}

template <typename Op, int ADims, int BDims, int CDims>
kernel void
THClTensor_pointwiseApply3(TensorInfo a,
                             TensorInfo b,
                             TensorInfo c,
                             int totalElements,
                             Op op) {
  for (int linearIndex = blockIdx.x * blockDim.x + threadIdx.x;
       linearIndex < totalElements;
       linearIndex += gridDim.x * blockDim.x) {
    // Convert `linearIndex` into an offset of `a`
    const int aOffset =
      IndexToOffset<int, ADims>::get(linearIndex, a);

    // Convert `linearIndex` into an offset of `b`
    const int bOffset =
      IndexToOffset<int, BDims>::get(linearIndex, b);

    // Convert `linearIndex` into an offset of `c`
    const int cOffset =
      IndexToOffset<int, CDims>::get(linearIndex, c);

    a.data[aOffset] = op(b.data[bOffset], c.data[cOffset]);
  }
}


trying stuff

trying stuff

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [imageid][outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
void kernel propagate_4_by_n_outplane_smallercache( const int batchSize,
    const int pixelsPerThread,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube, local float *_pixelSums ) {
    const int globalId = get_global_id(0);

    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int n = workgroupId / gNumFilters;
    const int outPlane = workgroupId % gNumFilters;

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int numFilterPixelsPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;

    local float *_myPixelSums = _pixelSums + pixelsPerThread * localId;
    //local float *_myPixelSums = _pixelSums;

    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        _myPixelSums[pixel] = 0.0f;
    }

    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    const int minu = gPadZeros ? max( -gHalfFilterSize, -outputRow ) : -gHalfFilterSize;
    const int maxu = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputRow  - evenPadding) : gHalfFilterSize - evenPadding;
    const int minv = gPadZeros ? max( -gHalfFilterSize, -outputCol ) : - gHalfFilterSize;
    const int maxv = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputCol - evenPadding) : gHalfFilterSize - evenPadding;

    for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
        int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int i = 0; i < numUpstreamsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gInputBoardSizeSquared ) {
                _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
            }
        }
        const int filterGlobalOffset = ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared;
        for( int i = 0; i < numFilterPixelsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gFilterSizeSquared ) {
                _filterCube[thisOffset] = filters[filterGlobalOffset + thisOffset];
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

  //      for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
            const int virtualLocalId = localId + pixel * workgroupSize;

            float thissum = 0;
            if( virtualLocalId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        thissum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
            _myPixelSums[pixel] += thissum;
//            if( globalId == 0 ) results[0] = _upstreamBoard[pixel];
        }
    }
//    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
        const int virtualLocalId = localId + pixel * workgroupSize;
        if( virtualLocalId < gOutputBoardSizeSquared ) {
            float sum = _myPixelSums[pixel];
            #ifdef BIASED
                sum += biases[outPlane];
            #endif
            // results are organized like [imageid][filterid][row][col]
            int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + virtualLocalId;
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
            // results[resultIndex ] = 123;
            //if( globalId == 0 ) results[0] += 0.000001f + _perPixelSums[0];
        }
//    }
}
#endif
#endif

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [imageid][outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
void kernel propagate_4_by_n_outplane_smallercache( const int batchSize,
    const int pixelsPerThread,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube, local float *_pixelSums ) {
    const int globalId = get_global_id(0);

    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int n = workgroupId / gNumFilters;
    const int outPlane = workgroupId % gNumFilters;

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int numFilterPixelsPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;

    //local float *_myPixelSums = _pixelSums + pixelsPerThread * localId;
    local float *_myPixelSums = _pixelSums;

    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        _myPixelSums[pixel] = 0.0f;
    }

    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    const int minu = gPadZeros ? max( -gHalfFilterSize, -outputRow ) : -gHalfFilterSize;
    const int maxu = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputRow  - evenPadding) : gHalfFilterSize - evenPadding;
    const int minv = gPadZeros ? max( -gHalfFilterSize, -outputCol ) : - gHalfFilterSize;
    const int maxv = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputCol - evenPadding) : gHalfFilterSize - evenPadding;

    for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
        int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int i = 0; i < numUpstreamsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gInputBoardSizeSquared ) {
                _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
            }
        }
        const int filterGlobalOffset = ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared;
        for( int i = 0; i < numFilterPixelsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gFilterSizeSquared ) {
                _filterCube[thisOffset] = filters[filterGlobalOffset + thisOffset];
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

  //      for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
        while( pixel < pixelsPerThread ) {
            const int virtualLocalId = localId + pixel * workgroupSize;

            float thissum = 0;
            if( virtualLocalId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        thissum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
            _myPixelSums[pixel] += thissum;
            pixel++;
//            if( globalId == 0 ) results[0] = _upstreamBoard[pixel];
        }
    }
//    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
        const int virtualLocalId = localId + pixel * workgroupSize;
        if( virtualLocalId < gOutputBoardSizeSquared ) {
            float sum = _myPixelSums[pixel];
            #ifdef BIASED
                sum += biases[outPlane];
            #endif
            // results are organized like [imageid][filterid][row][col]
            int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + virtualLocalId;
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
            // results[resultIndex ] = 123;
            //if( globalId == 0 ) results[0] += 0.000001f + _perPixelSums[0];
        }
//    }
}
#endif
#endif


use chinese to say hello taiwan

use chinese to say hello taiwan

__kernel void hello_world(__global int* values) {
    int global_id = get_global_id(0);
    printf("你好台灣!!! from kernel #%d, got value: %d\n", global_id, values[global_id]);
}

__kernel void hello_world(__global int* values) {
    int global_id = get_global_id(0);
    printf("hello host from kernel #%d, got value: %d\n", global_id, values[global_id]);
}


Update imgproc_canny.cl

Update imgproc_canny.cl

Reordering condition checking.

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Peng Xiao, pengxiao@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other oclMaterials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable
#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable

#ifdef L2GRAD
inline float calc(int x, int y)
{
    return sqrt((float)(x * x + y * y));
}
#else
inline float calc(int x, int y)
{
    return (float)abs(x) + abs(y);
}
#endif //

// Smoothing perpendicular to the derivative direction with a triangle filter
// only support 3x3 Sobel kernel
// h (-1) =  1, h (0) =  2, h (1) =  1
// h'(-1) = -1, h'(0) =  0, h'(1) =  1
// thus sobel 2D operator can be calculated as:
// h'(x, y) = h'(x)h(y) for x direction
//
// src		input 8bit single channel image data
// dx_buf	output dx buffer
// dy_buf	output dy buffer
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
calcSobelRowPass
(
    __global const uchar * src,
    __global int * dx_buf,
    __global int * dy_buf,
    int rows,
    int cols,
    int src_step,
    int src_offset,
    int dx_buf_step,
    int dx_buf_offset,
    int dy_buf_step,
    int dy_buf_offset
)
{
    dx_buf_step   /= sizeof(*dx_buf);
    dx_buf_offset /= sizeof(*dx_buf);
    dy_buf_step   /= sizeof(*dy_buf);
    dy_buf_offset /= sizeof(*dy_buf);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    __local int smem[16][18];

    smem[lidy][lidx + 1] =
        src[gidx + min(gidy, rows - 1) * src_step + src_offset];
    if(lidx == 0)
    {
        smem[lidy][0]  =
            src[max(gidx - 1,  0)        + min(gidy, rows - 1) * src_step + src_offset];
        smem[lidy][17] =
            src[min(gidx + 16, cols - 1) + min(gidy, rows - 1) * src_step + src_offset];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidy < rows && gidx < cols)
    {
        dx_buf[gidx + gidy * dx_buf_step + dx_buf_offset] =
            -smem[lidy][lidx] + smem[lidy][lidx + 2];
        dy_buf[gidx + gidy * dy_buf_step + dy_buf_offset] =
            smem[lidy][lidx] + 2 * smem[lidy][lidx + 1] + smem[lidy][lidx + 2];
    }
}

// calculate the magnitude of the filter pass combining both x and y directions
// This is the buffered version(3x3 sobel)
//
// dx_buf		dx buffer, calculated from calcSobelRowPass
// dy_buf		dy buffer, calculated from calcSobelRowPass
// dx			direvitive in x direction output
// dy			direvitive in y direction output
// mag			magnitude direvitive of xy output
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
calcMagnitude_buf
(
    __global const int * dx_buf,
    __global const int * dy_buf,
    __global int * dx,
    __global int * dy,
    __global float * mag,
    int rows,
    int cols,
    int dx_buf_step,
    int dx_buf_offset,
    int dy_buf_step,
    int dy_buf_offset,
    int dx_step,
    int dx_offset,
    int dy_step,
    int dy_offset,
    int mag_step,
    int mag_offset
)
{
    dx_buf_step    /= sizeof(*dx_buf);
    dx_buf_offset  /= sizeof(*dx_buf);
    dy_buf_step    /= sizeof(*dy_buf);
    dy_buf_offset  /= sizeof(*dy_buf);
    dx_step    /= sizeof(*dx);
    dx_offset  /= sizeof(*dx);
    dy_step    /= sizeof(*dy);
    dy_offset  /= sizeof(*dy);
    mag_step   /= sizeof(*mag);
    mag_offset /= sizeof(*mag);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    __local int sdx[18][16];
    __local int sdy[18][16];

    sdx[lidy + 1][lidx] =
        dx_buf[gidx + min(gidy, rows - 1) * dx_buf_step + dx_buf_offset];
    sdy[lidy + 1][lidx] =
        dy_buf[gidx + min(gidy, rows - 1) * dy_buf_step + dy_buf_offset];
    if(lidy == 0)
    {
        sdx[0][lidx]  =
            dx_buf[gidx + min(max(gidy-1,0),rows-1) * dx_buf_step + dx_buf_offset];
        sdx[17][lidx] =
            dx_buf[gidx + min(gidy + 16, rows - 1)  * dx_buf_step + dx_buf_offset];

        sdy[0][lidx]  =
            dy_buf[gidx + min(max(gidy-1,0),rows-1) * dy_buf_step + dy_buf_offset];
        sdy[17][lidx] =
            dy_buf[gidx + min(gidy + 16, rows - 1)  * dy_buf_step + dy_buf_offset];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidx < cols && gidy < rows)
    {
        int x =  sdx[lidy][lidx] + 2 * sdx[lidy + 1][lidx] + sdx[lidy + 2][lidx];
        int y = -sdy[lidy][lidx] + sdy[lidy + 2][lidx];

        dx[gidx + gidy * dx_step + dx_offset] = x;
        dy[gidx + gidy * dy_step + dy_offset] = y;

        mag[(gidx + 1) + (gidy + 1) * mag_step + mag_offset] = calc(x, y);
    }
}

// calculate the magnitude of the filter pass combining both x and y directions
// This is the non-buffered version(non-3x3 sobel)
//
// dx_buf		dx buffer, calculated from calcSobelRowPass
// dy_buf		dy buffer, calculated from calcSobelRowPass
// dx			direvitive in x direction output
// dy			direvitive in y direction output
// mag			magnitude direvitive of xy output
__kernel
void calcMagnitude
(
    __global const int * dx,
    __global const int * dy,
    __global float * mag,
    int rows,
    int cols,
    int dx_step,
    int dx_offset,
    int dy_step,
    int dy_offset,
    int mag_step,
    int mag_offset
)
{
    dx_step    /= sizeof(*dx);
    dx_offset  /= sizeof(*dx);
    dy_step    /= sizeof(*dy);
    dy_offset  /= sizeof(*dy);
    mag_step   /= sizeof(*mag);
    mag_offset /= sizeof(*mag);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if(gidy < rows && gidx < cols)
    {
        mag[(gidx + 1) + (gidy + 1) * mag_step + mag_offset] =
            calc(
                dx[gidx + gidy * dx_step + dx_offset],
                dy[gidx + gidy * dy_step + dy_offset]
            );
    }
}

//////////////////////////////////////////////////////////////////////////////////////////
// 0.4142135623730950488016887242097 is tan(22.5)
#define CANNY_SHIFT 15
#define TG22        (int)(0.4142135623730950488016887242097*(1<<CANNY_SHIFT) + 0.5)

//First pass of edge detection and non-maximum suppression
// edgetype is set to for each pixel:
// 0 - below low thres, not an edge
// 1 - maybe an edge
// 2 - is an edge, either magnitude is greater than high thres, or
//     Given estimates of the image gradients, a search is then carried out
//     to determine if the gradient magnitude assumes a local maximum in the gradient direction.
//     if the rounded gradient angle is zero degrees (i.e. the edge is in the north-south direction) the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the west and east directions,
//     if the rounded gradient angle is 90 degrees (i.e. the edge is in the east-west direction) the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the north and south directions,
//     if the rounded gradient angle is 135 degrees (i.e. the edge is in the north east-south west direction) the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the north west and south east directions,
//     if the rounded gradient angle is 45 degrees (i.e. the edge is in the north west-south east direction)the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the north east and south west directions.
//
// dx, dy		direvitives of x and y direction
// mag			magnitudes calculated from calcMagnitude function
// map			output containing raw edge types
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
calcMap
(
    __global const int * dx,
    __global const int * dy,
    __global const float * mag,
    __global int * map,
    int rows,
    int cols,
    float low_thresh,
    float high_thresh,
    int dx_step,
    int dx_offset,
    int dy_step,
    int dy_offset,
    int mag_step,
    int mag_offset,
    int map_step,
    int map_offset
)
{
    dx_step    /= sizeof(*dx);
    dx_offset  /= sizeof(*dx);
    dy_step    /= sizeof(*dy);
    dy_offset  /= sizeof(*dy);
    mag_step   /= sizeof(*mag);
    mag_offset /= sizeof(*mag);
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    mag += mag_offset;
    map += map_offset;

    __local float smem[18][18];

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int grp_idx = get_global_id(0) & 0xFFFFF0;
    int grp_idy = get_global_id(1) & 0xFFFFF0;

    int tid = lidx + lidy * 16;
    int lx = tid % 18;
    int ly = tid / 18;
    if(ly < 14)
    {
        smem[ly][lx] =
            mag[grp_idx + lx + min(grp_idy + ly, rows - 1) * mag_step];
    }
    if(ly < 4 && grp_idy + ly + 14 <= rows && grp_idx + lx <= cols)
    {
        smem[ly + 14][lx] =
            mag[grp_idx + lx + min(grp_idy + ly + 14, rows -1) * mag_step];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidy < rows && gidx < cols)
    {
        int x = dx[gidx + gidy * dx_step];
        int y = dy[gidx + gidy * dy_step];
        const int s = (x ^ y) < 0 ? -1 : 1;
        const float m = smem[lidy + 1][lidx + 1];
        x = abs(x);
        y = abs(y);

        // 0 - the pixel can not belong to an edge
        // 1 - the pixel might belong to an edge
        // 2 - the pixel does belong to an edge
        int edge_type = 0;
        if(m > low_thresh)
        {
            const int tg22x = x * TG22;
            const int tg67x = tg22x + (x << (1 + CANNY_SHIFT));
            y <<= CANNY_SHIFT;
            if(y < tg22x)
            {
                if(m > smem[lidy + 1][lidx] && m >= smem[lidy + 1][lidx + 2])
                {
                    edge_type = 1 + (int)(m > high_thresh);
                }
            }
            else if (y > tg67x)
            {
                if(m > smem[lidy][lidx + 1]&& m >= smem[lidy + 2][lidx + 1])
                {
                    edge_type = 1 + (int)(m > high_thresh);
                }
            }
            else
            {
                if(m > smem[lidy][lidx + 1 - s]&& m > smem[lidy + 2][lidx + 1 + s])
                {
                    edge_type = 1 + (int)(m > high_thresh);
                }
            }
        }
        map[gidx + 1 + (gidy + 1) * map_step] = edge_type;
    }
}

#undef CANNY_SHIFT
#undef TG22

//////////////////////////////////////////////////////////////////////////////////////////
// do Hysteresis for pixel whose edge type is 1
//
// If candidate pixel (edge type is 1) has a neighbour pixel (in 3x3 area) with type 2, it is believed to be part of an edge and
// marked as edge. Each thread will iterate for 16 times to connect local edges.
// Candidate pixel being identified as edge will then be tested if there is nearby potiential edge points. If there is, counter will
// be incremented by 1 and the point location is stored. These potiential candidates will be processed further in next kernel.
//
// map		raw edge type results calculated from calcMap.
// st		the potiential edge points found in this kernel call
// counter	the number of potiential edge points
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
edgesHysteresisLocal
(
    __global int * map,
    __global ushort2 * st,
    __global unsigned int * counter,
    int rows,
    int cols,
    int map_step,
    int map_offset
)
{
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    map += map_offset;

    __local int smem[18][18];

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int grp_idx = get_global_id(0) & 0xFFFFF0;
    int grp_idy = get_global_id(1) & 0xFFFFF0;

    int tid = lidx + lidy * 16;
    int lx = tid % 18;
    int ly = tid / 18;
    if(ly < 14)
    {
        smem[ly][lx] =
            map[grp_idx + lx + min(grp_idy + ly, rows - 1) * map_step];
    }
    if(ly < 4 && grp_idy + ly + 14 <= rows && grp_idx + lx <= cols)
    {
        smem[ly + 14][lx] =
            map[grp_idx + lx + min(grp_idy + ly + 14, rows - 1) * map_step];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidy < rows && gidx < cols)
    {
        int n;

        #pragma unroll
        for (int k = 0; k < 16; ++k)
        {
            n = 0;

            if (smem[lidy + 1][lidx + 1] == 1)
            {
                n += smem[lidy    ][lidx    ] == 2;
                n += smem[lidy    ][lidx + 1] == 2;
                n += smem[lidy    ][lidx + 2] == 2;

                n += smem[lidy + 1][lidx    ] == 2;
                n += smem[lidy + 1][lidx + 2] == 2;

                n += smem[lidy + 2][lidx    ] == 2;
                n += smem[lidy + 2][lidx + 1] == 2;
                n += smem[lidy + 2][lidx + 2] == 2;
            }

            if (n > 0)
                smem[lidy + 1][lidx + 1] = 2;
        }

        const int e = smem[lidy + 1][lidx + 1];
        map[gidx + 1 + (gidy + 1) * map_step] = e;

        n = 0;
        if(e == 2)
        {
            n += smem[lidy    ][lidx    ] == 1;
            n += smem[lidy    ][lidx + 1] == 1;
            n += smem[lidy    ][lidx + 2] == 1;

            n += smem[lidy + 1][lidx    ] == 1;
            n += smem[lidy + 1][lidx + 2] == 1;

            n += smem[lidy + 2][lidx    ] == 1;
            n += smem[lidy + 2][lidx + 1] == 1;
            n += smem[lidy + 2][lidx + 2] == 1;
        }

        if(n > 0)
        {
            unsigned int ind = atomic_inc(counter);
            st[ind] = (ushort2)(gidx + 1, gidy + 1);
        }
    }
}

__constant int c_dx[8] = {-1,  0,  1, -1, 1, -1, 0, 1};
__constant int c_dy[8] = {-1, -1, -1,  0, 0,  1, 1, 1};

#define stack_size 512
__kernel
void
__attribute__((reqd_work_group_size(128,1,1)))
edgesHysteresisGlobal
(
    __global int * map,
    __global ushort2 * st1,
    __global ushort2 * st2,
    __global int * counter,
    int rows,
    int cols,
    int count,
    int map_step,
    int map_offset
)
{
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    map += map_offset;

    int lidx = get_local_id(0);

    int grp_idx = get_group_id(0);
    int grp_idy = get_group_id(1);

    __local unsigned int s_counter;
    __local unsigned int s_ind;

    __local ushort2 s_st[stack_size];

    if(lidx == 0)
    {
        s_counter = 0;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    int ind = mad24(grp_idy, (int)get_local_size(0), grp_idx);

    if(ind < count)
    {
        ushort2 pos = st1[ind];
        if (lidx < 8)
        {
            pos.x += c_dx[lidx];
            pos.y += c_dy[lidx];
            if (pos.x > 0 && pos.x <= cols && pos.y > 0 && pos.y <= rows && map[pos.x + pos.y * map_step] == 1)
            {
                map[pos.x + pos.y * map_step] = 2;

                ind = atomic_inc(&s_counter);

                s_st[ind] = pos;
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        while (s_counter > 0 && s_counter <= stack_size - get_local_size(0))
        {
            const int subTaskIdx = lidx >> 3;
            const int portion = min(s_counter, (uint)(get_local_size(0)>> 3));

            if (subTaskIdx < portion)
                pos = s_st[s_counter - 1 - subTaskIdx];

            if (lidx == 0)
                s_counter -= portion;
            barrier(CLK_LOCAL_MEM_FENCE);

            if (subTaskIdx < portion)
            {
                pos.x += c_dx[lidx & 7];
                pos.y += c_dy[lidx & 7];
                if (pos.x > 0 && pos.x <= cols && pos.y > 0 && pos.y <= rows && map[pos.x + pos.y * map_step] == 1)
                {
                    map[pos.x + pos.y * map_step] = 2;

                    ind = atomic_inc(&s_counter);

                    s_st[ind] = pos;
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);
        }

        if (s_counter > 0)
        {
            if (lidx == 0)
            {
                ind = atomic_add(counter, s_counter);
                s_ind = ind - s_counter;
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            ind = s_ind;

            for (int i = lidx; i < s_counter; i += get_local_size(0))
            {
                st2[ind + i] = s_st[i];
            }
        }
    }
}
#undef stack_size

//Get the edge result. egde type of value 2 will be marked as an edge point and set to 255. Otherwise 0.
// map		edge type mappings
// dst		edge output
__kernel
void getEdges
(
    __global const int * map,
    __global uchar * dst,
    int rows,
    int cols,
    int map_step,
    int map_offset,
    int dst_step,
    int dst_offset
)
{
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if(gidy < rows && gidx < cols)
    {
        dst[gidx + gidy * dst_step] = (uchar)(-(map[gidx + 1 + (gidy + 1) * map_step + map_offset] >> 1));
    }
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Peng Xiao, pengxiao@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other oclMaterials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable
#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable

#ifdef L2GRAD
inline float calc(int x, int y)
{
    return sqrt((float)(x * x + y * y));
}
#else
inline float calc(int x, int y)
{
    return (float)abs(x) + abs(y);
}
#endif //

// Smoothing perpendicular to the derivative direction with a triangle filter
// only support 3x3 Sobel kernel
// h (-1) =  1, h (0) =  2, h (1) =  1
// h'(-1) = -1, h'(0) =  0, h'(1) =  1
// thus sobel 2D operator can be calculated as:
// h'(x, y) = h'(x)h(y) for x direction
//
// src		input 8bit single channel image data
// dx_buf	output dx buffer
// dy_buf	output dy buffer
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
calcSobelRowPass
(
    __global const uchar * src,
    __global int * dx_buf,
    __global int * dy_buf,
    int rows,
    int cols,
    int src_step,
    int src_offset,
    int dx_buf_step,
    int dx_buf_offset,
    int dy_buf_step,
    int dy_buf_offset
)
{
    dx_buf_step   /= sizeof(*dx_buf);
    dx_buf_offset /= sizeof(*dx_buf);
    dy_buf_step   /= sizeof(*dy_buf);
    dy_buf_offset /= sizeof(*dy_buf);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    __local int smem[16][18];

    smem[lidy][lidx + 1] =
        src[gidx + min(gidy, rows - 1) * src_step + src_offset];
    if(lidx == 0)
    {
        smem[lidy][0]  =
            src[max(gidx - 1,  0)        + min(gidy, rows - 1) * src_step + src_offset];
        smem[lidy][17] =
            src[min(gidx + 16, cols - 1) + min(gidy, rows - 1) * src_step + src_offset];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidy < rows && gidx < cols)
    {
        dx_buf[gidx + gidy * dx_buf_step + dx_buf_offset] =
            -smem[lidy][lidx] + smem[lidy][lidx + 2];
        dy_buf[gidx + gidy * dy_buf_step + dy_buf_offset] =
            smem[lidy][lidx] + 2 * smem[lidy][lidx + 1] + smem[lidy][lidx + 2];
    }
}

// calculate the magnitude of the filter pass combining both x and y directions
// This is the buffered version(3x3 sobel)
//
// dx_buf		dx buffer, calculated from calcSobelRowPass
// dy_buf		dy buffer, calculated from calcSobelRowPass
// dx			direvitive in x direction output
// dy			direvitive in y direction output
// mag			magnitude direvitive of xy output
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
calcMagnitude_buf
(
    __global const int * dx_buf,
    __global const int * dy_buf,
    __global int * dx,
    __global int * dy,
    __global float * mag,
    int rows,
    int cols,
    int dx_buf_step,
    int dx_buf_offset,
    int dy_buf_step,
    int dy_buf_offset,
    int dx_step,
    int dx_offset,
    int dy_step,
    int dy_offset,
    int mag_step,
    int mag_offset
)
{
    dx_buf_step    /= sizeof(*dx_buf);
    dx_buf_offset  /= sizeof(*dx_buf);
    dy_buf_step    /= sizeof(*dy_buf);
    dy_buf_offset  /= sizeof(*dy_buf);
    dx_step    /= sizeof(*dx);
    dx_offset  /= sizeof(*dx);
    dy_step    /= sizeof(*dy);
    dy_offset  /= sizeof(*dy);
    mag_step   /= sizeof(*mag);
    mag_offset /= sizeof(*mag);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    __local int sdx[18][16];
    __local int sdy[18][16];

    sdx[lidy + 1][lidx] =
        dx_buf[gidx + min(gidy, rows - 1) * dx_buf_step + dx_buf_offset];
    sdy[lidy + 1][lidx] =
        dy_buf[gidx + min(gidy, rows - 1) * dy_buf_step + dy_buf_offset];
    if(lidy == 0)
    {
        sdx[0][lidx]  =
            dx_buf[gidx + min(max(gidy-1,0),rows-1) * dx_buf_step + dx_buf_offset];
        sdx[17][lidx] =
            dx_buf[gidx + min(gidy + 16, rows - 1)  * dx_buf_step + dx_buf_offset];

        sdy[0][lidx]  =
            dy_buf[gidx + min(max(gidy-1,0),rows-1) * dy_buf_step + dy_buf_offset];
        sdy[17][lidx] =
            dy_buf[gidx + min(gidy + 16, rows - 1)  * dy_buf_step + dy_buf_offset];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidx < cols && gidy < rows)
    {
        int x =  sdx[lidy][lidx] + 2 * sdx[lidy + 1][lidx] + sdx[lidy + 2][lidx];
        int y = -sdy[lidy][lidx] + sdy[lidy + 2][lidx];

        dx[gidx + gidy * dx_step + dx_offset] = x;
        dy[gidx + gidy * dy_step + dy_offset] = y;

        mag[(gidx + 1) + (gidy + 1) * mag_step + mag_offset] = calc(x, y);
    }
}

// calculate the magnitude of the filter pass combining both x and y directions
// This is the non-buffered version(non-3x3 sobel)
//
// dx_buf		dx buffer, calculated from calcSobelRowPass
// dy_buf		dy buffer, calculated from calcSobelRowPass
// dx			direvitive in x direction output
// dy			direvitive in y direction output
// mag			magnitude direvitive of xy output
__kernel
void calcMagnitude
(
    __global const int * dx,
    __global const int * dy,
    __global float * mag,
    int rows,
    int cols,
    int dx_step,
    int dx_offset,
    int dy_step,
    int dy_offset,
    int mag_step,
    int mag_offset
)
{
    dx_step    /= sizeof(*dx);
    dx_offset  /= sizeof(*dx);
    dy_step    /= sizeof(*dy);
    dy_offset  /= sizeof(*dy);
    mag_step   /= sizeof(*mag);
    mag_offset /= sizeof(*mag);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if(gidy < rows && gidx < cols)
    {
        mag[(gidx + 1) + (gidy + 1) * mag_step + mag_offset] =
            calc(
                dx[gidx + gidy * dx_step + dx_offset],
                dy[gidx + gidy * dy_step + dy_offset]
            );
    }
}

//////////////////////////////////////////////////////////////////////////////////////////
// 0.4142135623730950488016887242097 is tan(22.5)
#define CANNY_SHIFT 15
#define TG22        (int)(0.4142135623730950488016887242097*(1<<CANNY_SHIFT) + 0.5)

//First pass of edge detection and non-maximum suppression
// edgetype is set to for each pixel:
// 0 - below low thres, not an edge
// 1 - maybe an edge
// 2 - is an edge, either magnitude is greater than high thres, or
//     Given estimates of the image gradients, a search is then carried out
//     to determine if the gradient magnitude assumes a local maximum in the gradient direction.
//     if the rounded gradient angle is zero degrees (i.e. the edge is in the north-south direction) the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the west and east directions,
//     if the rounded gradient angle is 90 degrees (i.e. the edge is in the east-west direction) the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the north and south directions,
//     if the rounded gradient angle is 135 degrees (i.e. the edge is in the north east-south west direction) the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the north west and south east directions,
//     if the rounded gradient angle is 45 degrees (i.e. the edge is in the north west-south east direction)the point will be considered to be on the edge if its gradient magnitude is greater than the magnitudes in the north east and south west directions.
//
// dx, dy		direvitives of x and y direction
// mag			magnitudes calculated from calcMagnitude function
// map			output containing raw edge types
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
calcMap
(
    __global const int * dx,
    __global const int * dy,
    __global const float * mag,
    __global int * map,
    int rows,
    int cols,
    float low_thresh,
    float high_thresh,
    int dx_step,
    int dx_offset,
    int dy_step,
    int dy_offset,
    int mag_step,
    int mag_offset,
    int map_step,
    int map_offset
)
{
    dx_step    /= sizeof(*dx);
    dx_offset  /= sizeof(*dx);
    dy_step    /= sizeof(*dy);
    dy_offset  /= sizeof(*dy);
    mag_step   /= sizeof(*mag);
    mag_offset /= sizeof(*mag);
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    mag += mag_offset;
    map += map_offset;

    __local float smem[18][18];

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int grp_idx = get_global_id(0) & 0xFFFFF0;
    int grp_idy = get_global_id(1) & 0xFFFFF0;

    int tid = lidx + lidy * 16;
    int lx = tid % 18;
    int ly = tid / 18;
    if(ly < 14)
    {
        smem[ly][lx] =
            mag[grp_idx + lx + min(grp_idy + ly, rows - 1) * mag_step];
    }
    if(ly < 4 && grp_idy + ly + 14 <= rows && grp_idx + lx <= cols)
    {
        smem[ly + 14][lx] =
            mag[grp_idx + lx + min(grp_idy + ly + 14, rows -1) * mag_step];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidy < rows && gidx < cols)
    {
        int x = dx[gidx + gidy * dx_step];
        int y = dy[gidx + gidy * dy_step];
        const int s = (x ^ y) < 0 ? -1 : 1;
        const float m = smem[lidy + 1][lidx + 1];
        x = abs(x);
        y = abs(y);

        // 0 - the pixel can not belong to an edge
        // 1 - the pixel might belong to an edge
        // 2 - the pixel does belong to an edge
        int edge_type = 0;
        if(m > low_thresh)
        {
            const int tg22x = x * TG22;
            const int tg67x = tg22x + (x << (1 + CANNY_SHIFT));
            y <<= CANNY_SHIFT;
            if(y < tg22x)
            {
                if(m > smem[lidy + 1][lidx] && m >= smem[lidy + 1][lidx + 2])
                {
                    edge_type = 1 + (int)(m > high_thresh);
                }
            }
            else if (y > tg67x)
            {
                if(m > smem[lidy][lidx + 1]&& m >= smem[lidy + 2][lidx + 1])
                {
                    edge_type = 1 + (int)(m > high_thresh);
                }
            }
            else
            {
                if(m > smem[lidy][lidx + 1 - s]&& m > smem[lidy + 2][lidx + 1 + s])
                {
                    edge_type = 1 + (int)(m > high_thresh);
                }
            }
        }
        map[gidx + 1 + (gidy + 1) * map_step] = edge_type;
    }
}

#undef CANNY_SHIFT
#undef TG22

//////////////////////////////////////////////////////////////////////////////////////////
// do Hysteresis for pixel whose edge type is 1
//
// If candidate pixel (edge type is 1) has a neighbour pixel (in 3x3 area) with type 2, it is believed to be part of an edge and
// marked as edge. Each thread will iterate for 16 times to connect local edges.
// Candidate pixel being identified as edge will then be tested if there is nearby potiential edge points. If there is, counter will
// be incremented by 1 and the point location is stored. These potiential candidates will be processed further in next kernel.
//
// map		raw edge type results calculated from calcMap.
// st		the potiential edge points found in this kernel call
// counter	the number of potiential edge points
__kernel
void
__attribute__((reqd_work_group_size(16,16,1)))
edgesHysteresisLocal
(
    __global int * map,
    __global ushort2 * st,
    __global unsigned int * counter,
    int rows,
    int cols,
    int map_step,
    int map_offset
)
{
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    map += map_offset;

    __local int smem[18][18];

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int grp_idx = get_global_id(0) & 0xFFFFF0;
    int grp_idy = get_global_id(1) & 0xFFFFF0;

    int tid = lidx + lidy * 16;
    int lx = tid % 18;
    int ly = tid / 18;
    if(ly < 14)
    {
        smem[ly][lx] =
            map[grp_idx + lx + min(grp_idy + ly, rows - 1) * map_step];
    }
    if(ly < 4 && grp_idy + ly + 14 <= rows && grp_idx + lx <= cols)
    {
        smem[ly + 14][lx] =
            map[grp_idx + lx + min(grp_idy + ly + 14, rows - 1) * map_step];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if(gidy < rows && gidx < cols)
    {
        int n;

        #pragma unroll
        for (int k = 0; k < 16; ++k)
        {
            n = 0;

            if (smem[lidy + 1][lidx + 1] == 1)
            {
                n += smem[lidy    ][lidx    ] == 2;
                n += smem[lidy    ][lidx + 1] == 2;
                n += smem[lidy    ][lidx + 2] == 2;

                n += smem[lidy + 1][lidx    ] == 2;
                n += smem[lidy + 1][lidx + 2] == 2;

                n += smem[lidy + 2][lidx    ] == 2;
                n += smem[lidy + 2][lidx + 1] == 2;
                n += smem[lidy + 2][lidx + 2] == 2;
            }

            if (n > 0)
                smem[lidy + 1][lidx + 1] = 2;
        }

        const int e = smem[lidy + 1][lidx + 1];
        map[gidx + 1 + (gidy + 1) * map_step] = e;

        n = 0;
        if(e == 2)
        {
            n += smem[lidy    ][lidx    ] == 1;
            n += smem[lidy    ][lidx + 1] == 1;
            n += smem[lidy    ][lidx + 2] == 1;

            n += smem[lidy + 1][lidx    ] == 1;
            n += smem[lidy + 1][lidx + 2] == 1;

            n += smem[lidy + 2][lidx    ] == 1;
            n += smem[lidy + 2][lidx + 1] == 1;
            n += smem[lidy + 2][lidx + 2] == 1;
        }

        if(n > 0)
        {
            unsigned int ind = atomic_inc(counter);
            st[ind] = (ushort2)(gidx + 1, gidy + 1);
        }
    }
}

__constant int c_dx[8] = {-1,  0,  1, -1, 1, -1, 0, 1};
__constant int c_dy[8] = {-1, -1, -1,  0, 0,  1, 1, 1};

#define stack_size 512
__kernel
void
__attribute__((reqd_work_group_size(128,1,1)))
edgesHysteresisGlobal
(
    __global int * map,
    __global ushort2 * st1,
    __global ushort2 * st2,
    __global int * counter,
    int rows,
    int cols,
    int count,
    int map_step,
    int map_offset
)
{

    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    map += map_offset;

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    int lidx = get_local_id(0);
    int lidy = get_local_id(1);

    int grp_idx = get_group_id(0);
    int grp_idy = get_group_id(1);

    __local unsigned int s_counter;
    __local unsigned int s_ind;

    __local ushort2 s_st[stack_size];

    if(lidx == 0)
    {
        s_counter = 0;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    int ind = mad24(grp_idy, (int)get_local_size(0), grp_idx);

    if(ind < count)
    {
        ushort2 pos = st1[ind];
        if (pos.x > 0 && pos.x <= cols && pos.y > 0 && pos.y <= rows)
        {
            if (lidx < 8)
            {
                pos.x += c_dx[lidx];
                pos.y += c_dy[lidx];

                if (map[pos.x + pos.y * map_step] == 1)
                {
                    map[pos.x + pos.y * map_step] = 2;

                    ind = atomic_inc(&s_counter);

                    s_st[ind] = pos;
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            while (s_counter > 0 && s_counter <= stack_size - get_local_size(0))
            {
                const int subTaskIdx = lidx >> 3;
                const int portion = min(s_counter, (uint)(get_local_size(0)>> 3));

                pos.x = pos.y = 0;

                if (subTaskIdx < portion)
                    pos = s_st[s_counter - 1 - subTaskIdx];
                barrier(CLK_LOCAL_MEM_FENCE);

                if (lidx == 0)
                    s_counter -= portion;
                barrier(CLK_LOCAL_MEM_FENCE);

                if (pos.x > 0 && pos.x <= cols && pos.y > 0 && pos.y <= rows)
                {
                    pos.x += c_dx[lidx & 7];
                    pos.y += c_dy[lidx & 7];

                    if (map[pos.x + pos.y * map_step] == 1)
                    {
                        map[pos.x + pos.y * map_step] = 2;

                        ind = atomic_inc(&s_counter);

                        s_st[ind] = pos;
                    }
                }
                barrier(CLK_LOCAL_MEM_FENCE);
            }

            if (s_counter > 0)
            {
                if (lidx == 0)
                {
                    ind = atomic_add(counter, s_counter);
                    s_ind = ind - s_counter;
                }
                barrier(CLK_LOCAL_MEM_FENCE);

                ind = s_ind;

                for (int i = lidx; i < s_counter; i += get_local_size(0))
                {
                    st2[ind + i] = s_st[i];
                }
            }
        }
    }
}
#undef stack_size

//Get the edge result. egde type of value 2 will be marked as an edge point and set to 255. Otherwise 0.
// map		edge type mappings
// dst		edge output
__kernel
void getEdges
(
    __global const int * map,
    __global uchar * dst,
    int rows,
    int cols,
    int map_step,
    int map_offset,
    int dst_step,
    int dst_offset
)
{
    map_step   /= sizeof(*map);
    map_offset /= sizeof(*map);

    int gidx = get_global_id(0);
    int gidy = get_global_id(1);

    if(gidy < rows && gidx < cols)
    {
        dst[gidx + gidy * dst_step] = (uchar)(-(map[gidx + 1 + (gidy + 1) * map_step + map_offset] >> 1));
    }
}


Optimization in undulator.cl

Optimization in undulator.cl

//__author__ = "Konstantin Klementiev, Roman Chernikov"
//__date__ = "10 Apr 2015"

#ifdef cl_khr_fp64
#if __OPENCL_VERSION__<120
#pragma OPENCL EXTENSION cl_khr_fp64: enable
#endif
#endif

__kernel void undulator(const float alpha,
                        const float Kx,
                        const float Ky,
                        const float phase,
                        const int jend,
                        __global float* gamma,
                        __global float* wu,
                        __global float* w,
                        __global float* ww1,
                        __global float* ddphi,
                        __global float* ddpsi,
                        __global float* tg,
                        __global float* ag,
                        __global float2* Is_gl,
                        __global float2* Ip_gl)
{
    unsigned int ii = get_global_id(0);
    int j;

    float2 beta;
    float2 eucos;
    float ucos, sinucos, cosucos, sintg, costg, sintgph, costgph;
    float2 zero2 = (float2)(0,0);
    float2 Is = zero2;
    float2 Ip = zero2;
    float wgwu = w[ii] / gamma[ii] / wu[ii];
    float Kx2 = Kx * Kx;
    float Ky2 = Ky * Ky;

    for (j=0; j<jend; j++) {
        sintg = sincos(tg[j], &costg);
        sintgph = sincos(tg[j] + phase, &costgph);
        ucos = (float)(ww1[ii]) * tg[j] + wgwu *
           (-Ky * ddphi[ii] * sintg + Kx * ddpsi[ii] * sintgph +
            0.25 / gamma[ii] * (Ky2 * sintg*costg + Kx2 * sintgph*costgph));

        sinucos = sincos(ucos, &cosucos);
        eucos.x = cosucos;
        eucos.y = sinucos;

        beta.x = Ky / gamma[ii] * costg;
        beta.y = -Kx / gamma[ii] * costgph;

        Is += (ag[j] * (ddphi[ii] - beta.x)) * eucos;
        Ip += (ag[j] * (ddpsi[ii] - beta.y)) * eucos; }
    mem_fence(CLK_LOCAL_MEM_FENCE);
    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
    }

__kernel void undulator_taper(const float alpha,
                              const float Kx,
                              const float Ky,
                              const float phase,
                              const int jend,
                              __global float* gamma,
                              __global float* wu,
                              __global float* w,
                              __global float* ww1,
                              __global float* ddtheta,
                              __global float* ddpsi,
                              __global float* tg,
                              __global float* ag,
                              __global float2* Is_gl,
                              __global float2* Ip_gl)
{
    const float E2W = 1.51926751475e15;
    const float C = 2.99792458e11;
    unsigned int ii = get_global_id(0);
    int j;

    float2 eucos, beta;
    float ucos, sintg, sin2tg, costg;
    float2 zero2 = (float2)(0,0);
    float2 Is = zero2;
    float2 Ip = zero2;
    float Kx2 = Kx * Kx;
    float Ky2 = Ky * Ky;
    float alphaS = alpha * C / wu[ii] / E2W;
    float wgwu = w[ii] / gamma[ii] / wu[ii];

    for (j=0; j<jend; j++) {
        sintg = sin(tg[j]);
        costg = cos(tg[j]);
        sin2tg = 2. * sintg * costg; //sin(2 * tg[j]);
        ucos = ww1[ii] * tg[j] + wgwu *
            (-Ky * ddtheta[ii] *
                  (sintg + alphaS * (1. - costg - tg[j] * sintg)) +
             Kx * ddpsi[ii] * sin(tg[j] + phase) +
             0.125 / gamma[ii] *
                  (Ky2 * (sin2tg - 2. * alphaS *
                    (tg[j] * tg[j] + costg * costg + tg[j] * sin2tg)) +
                   Kx2 * sin(2. * (tg[j] + phase))));

        eucos.x = cos(ucos);
        eucos.y = sin(ucos);

        beta.x = Ky / gamma[ii] * costg;
        beta.y = -Kx / gamma[ii] * cos(tg[j] + phase);

        Is += ag[j] * (ddtheta[ii] - beta.x *
            (1 - alphaS * tg[j])) * eucos;
        Ip += ag[j] * (ddpsi[ii] - beta.y) * eucos; }
    mem_fence(CLK_LOCAL_MEM_FENCE);
    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

__kernel void undulator_nf(const float R0,
                            const float L0,
                            const float Kx,
                            const float Ky,
                            const float phase,
                            const int jend,
                            __global float* gamma,
                            __global float* wu,
                            __global float* w,
                            __global float* ww1,
                            __global float* ddphi,
                            __global float* ddpsi,
                            __global float* tg,
                            __global float* ag,
                            __global float2* Is_gl,
                            __global float2* Ip_gl)
{
    //const float E2W = 1.51926751475e15;
    //const float C = 2.99792458e11;
    const float PI2 = 6.283185307179586476925286766559;
    unsigned int ii = get_global_id(0);
    int j;

    float2 eucos;
    float ucos, sintg, costg, sintgph, costgph;
    float2 zero2 = (float2)(0,0);
    float2 Is = zero2;
    float2 Ip = zero2;
    float3 r, r0;
    float2 beta;
    float Kx2 = Kx * Kx;
    float Ky2 = Ky * Ky;
    float gamma2 = gamma[ii] * gamma[ii];
    float betam = 1 - (1 + 0.5 * Kx2 + 0.5 * Ky2) / 2. / gamma2;
    float wR0 = R0 * PI2 / L0;

    r0.x = wR0 * tan(ddphi[ii]);
    r0.y = wR0 * tan(ddpsi[ii]);
    r0.z = wR0 * cos(sqrt(ddphi[ii]*ddphi[ii] + ddpsi[ii]*ddpsi[ii]));

    for (j=0; j<jend; j++) {
        sintg = sincos(tg[j], &costg);
        sintgph = sincos(tg[j] + phase, &costgph);

        r.x = Ky / gamma[ii] * sintg;
        r.y = -Kx / gamma[ii] * sintgph;
        r.z = betam * tg[j] - 0.25 / gamma2 *
        (Ky2 * sintg * costg + Kx2 * sintgph * costgph);

      ucos = w[ii] / wu[ii] * (tg[j] + length(r0 - r));

      eucos.x = cos(ucos);
      eucos.y = sin(ucos);

      beta.x = Ky / gamma[ii] * costg;
      beta.y = -Kx / gamma[ii] * costgph;

      Is += ag[j] * (ddphi[ii] - beta.x) * eucos;
      Ip += ag[j] * (ddpsi[ii] - beta.y) * eucos; }

    mem_fence(CLK_LOCAL_MEM_FENCE);

    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

double2 f_beta(double Bx, double By, double Bz,
               double emcg,
               double2 beta)
{
    return emcg*(double2)(beta.y*Bz - By,
                          Bx - beta.x*Bz);
}

double3 f_traj(double revgamma, double2 beta)
{
    return (double3)(beta.x,
                     beta.y,
                     sqrt(revgamma-beta.x*beta.x-beta.y*beta.y));
}

double2 next_beta_rk(double2 beta, int iZeroStep, int iHalfStep,
                     int iFullStep, double rkStep, double emcg,
                     __global double* Bx,
                     __global double* By,
                     __global double* Bz)
{
    double2 k1Beta, k2Beta, k3Beta, k4Beta;

    k1Beta = rkStep * f_beta(Bx[iZeroStep],
                             By[iZeroStep],
                             Bz[iZeroStep],
                             emcg, beta);
    k2Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k1Beta);
    k3Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k2Beta);
    k4Beta = rkStep * f_beta(Bx[iFullStep],
                             By[iFullStep],
                             Bz[iFullStep],
                             emcg, beta + k3Beta);
    return beta + (k1Beta + 2.*k2Beta + 2.*k3Beta + k4Beta) / 6.;
}

double8 next_traj_rk(double2 beta, double3 traj, int iZeroStep, int iHalfStep,
                     int iFullStep, double rkStep, double emcg,
                     double revgamma,
                     __global double* Bx,
                     __global double* By,
                     __global double* Bz)
{
    double2 k1Beta, k2Beta, k3Beta, k4Beta;
    double3 k1Traj, k2Traj, k3Traj, k4Traj;

    k1Beta = rkStep * f_beta(Bx[iZeroStep],
                             By[iZeroStep],
                             Bz[iZeroStep],
                             emcg, beta);
    k1Traj = rkStep * f_traj(revgamma, beta);

    k2Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k1Beta);
    k2Traj = rkStep * f_traj(revgamma, beta + 0.5*k1Beta);

    k3Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k2Beta);
    k3Traj = rkStep * f_traj(revgamma, beta + 0.5*k2Beta);

    k4Beta = rkStep * f_beta(Bx[iFullStep],
                             By[iFullStep],
                             Bz[iFullStep],
                             emcg, beta + k3Beta);
    k4Traj = rkStep * f_traj(revgamma, beta + k3Beta);

    return (double8)(beta + (k1Beta + 2.*k2Beta + 2.*k3Beta + k4Beta)/6.,
                     traj + (k1Traj + 2.*k2Traj + 2.*k3Traj + k4Traj)/6.,
                     0., 0., 0.);
}

__kernel void undulator_custom(const int jend,
                                const int nwt,
                                const double lUnd,
                                __global double* gamma,
                                __global double* w,
                                __global double* ddphi,
                                __global double* ddpsi,
                                __global double* tg,
                                __global double* ag,
                                __global double* Bx,
                                __global double* By,
                                __global double* Bz,
                                __global double2* Is_gl,
                                __global double2* Ip_gl)
{
    const double E2W = 1.51926751475e15;
    const double C = 2.99792458e11;
    const double SIE0 = 1.602176565e-19;
    const double SIM0 = 9.10938291e-31;
    //const double PI = 3.1415926535897932384626433832795;
    const double PI2 = 6.283185307179586476925286766559;

    unsigned int ii = get_global_id(0);
    int j, k;
    int iBase, iZeroStep, iHalfStep, iFullStep;

    double ucos, rkStep, wu_int, betam_int;
    double emcg = lUnd * SIE0 / SIM0 / C / gamma[ii] / PI2;
    double revgamma = 1. - 1./gamma[ii]/gamma[ii];
    double8 betaTraj;
    double2 eucos;
    double2 zero2 = (double2)(0,0);
    double2 Is = zero2;
    double2 Ip = zero2;
    double2 beta, beta0;
    double3 traj, n, traj0;

    n.x = ddphi[ii];
    n.y = ddpsi[ii];
    n.z = 1. - 0.5*(n.x*n.x + n.y*n.y);

    beta = zero2;
    beta0 = zero2;
    betam_int = 0;

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            beta = next_beta_rk(beta, iZeroStep, iHalfStep, iFullStep,
                                rkStep, emcg, Bx, By, Bz);
            beta0 += beta * rkStep; } }

    mem_fence(CLK_LOCAL_MEM_FENCE);
    beta0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = (double3)(0., 0., 0.);
    traj0 = (double3)(0., 0., 0.);

    for (j=1; j<jend; j++) {
        iBase = (j-1)*2*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
                                    iZeroStep, iHalfStep, iFullStep,
                                    rkStep, emcg, revgamma, Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234;
            traj0 += traj * rkStep;
            betam_int += rkStep * sqrt(revgamma - beta.x*beta.x -
                                       beta.y*beta.y); } }

    mem_fence(CLK_LOCAL_MEM_FENCE);
    traj0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = traj0;
    betam_int /= (tg[jend-1] - tg[0]);
    wu_int = PI2 * C * betam_int / lUnd / E2W;

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
            iZeroStep, iHalfStep, iFullStep,
            rkStep, emcg, revgamma, Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234; }

        mem_fence(CLK_LOCAL_MEM_FENCE);
        ucos = w[ii] / wu_int * (tg[j]  - dot(n, traj));
        eucos.x = cos(ucos);
        eucos.y = sin(ucos);
        Is += ag[j] * (ddphi[ii] - beta.x) * eucos;
        Ip += ag[j] * (ddpsi[ii] - beta.y) * eucos; }

    mem_fence(CLK_LOCAL_MEM_FENCE);

    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

__kernel void get_trajectory(const int jend,
                             const int nwt,
                             const double emcg,
                             const double gamma,
                             __global double* tg,
                             __global double* Bx,
                             __global double* By,
                             __global double* Bz,
                             __global double* betax,
                             __global double* betay,
                             __global double* betazav,
                             __global double* trajx,
                             __global double* trajy,
                             __global double* trajz)
{
    int j, k;
    int iBase, iZeroStep, iHalfStep, iFullStep;

    double rkStep;
    double revgamma = 1. - 1./gamma/gamma;
    double betam_int = 0;
    double2 zero2 = (double2)(0,0);
    double2 beta, beta0;
    double3 traj, traj0;
    double8 betaTraj;

    beta = zero2;
    beta0 = zero2;

    for (j=1; j<jend; j++) {
        iBase = (j-1)*2*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            beta = next_beta_rk(beta, iZeroStep, iHalfStep, iFullStep,
                                rkStep, emcg, Bx, By, Bz);
            beta0 += beta * rkStep; } }
    mem_fence(CLK_LOCAL_MEM_FENCE);
    beta0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = (double3)(0., 0., 0.);
    traj0 = (double3)(0., 0., 0.);

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
                                    iZeroStep, iHalfStep, iFullStep,
                                    rkStep, emcg, revgamma, Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234;
            traj0 += traj * rkStep;
            betam_int += rkStep*sqrt(revgamma - beta.x*beta.x -
                                     beta.y*beta.y); } }
    mem_fence(CLK_LOCAL_MEM_FENCE);
    traj0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = traj0;
    betam_int /= tg[jend-1] - tg[0];

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
                                    iZeroStep, iHalfStep, iFullStep,
                                    rkStep, emcg, revgamma, Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234; }

        betax[j] = beta.x;
        betay[j] = beta.y;
        betazav[j] = betam_int;
        trajx[j] = traj.x;
        trajy[j] = traj.y;
        trajz[j] = traj.z; }

    barrier(CLK_LOCAL_MEM_FENCE);
}

__kernel void undulator_custom_filament(const int jend,
                                        const double wu,
                                        const double L0,
                                        const double R0,
                                        __global double* w,
                                        __global double* ddphi,
                                        __global double* ddpsi,
                                        __global double* tg,
                                        __global double* ag,
                                        __global double* betax,
                                        __global double* betay,
                                        __global double* trajx,
                                        __global double* trajy,
                                        __global double* trajz,
                                        __global double2* Is_gl,
                                        __global double2* Ip_gl)
{
    const double PI2 = 6.283185307179586476925286766559;

    unsigned int ii = get_global_id(0);
    int j;

    double ucos, wR0;

    double2 eucos;
    double2 Is = (double2)(0., 0.);
    double2 Ip = (double2)(0., 0.);

    double3 traj, n, r0;

    n.x = ddphi[ii];
    n.y = ddpsi[ii];
    n.z = 1. - 0.5*(n.x*n.x + n.y*n.y);

    if (R0>0) {
        wR0 = R0 * PI2 / L0;
        r0.x = wR0 * tan(ddphi[ii]);
        r0.y = wR0 * tan(ddpsi[ii]);
        r0.z = wR0 * cos(sqrt(ddphi[ii]*ddphi[ii] + ddpsi[ii]*ddpsi[ii])); }

    for (j=1; j<jend; j++) {
        traj.x = trajx[j];
        traj.y = trajy[j];
        traj.z = trajz[j];
        if (R0 > 0) {
            ucos = w[ii] / wu * (tg[j] + length(r0 - traj)); }
        else {
            ucos = w[ii] / wu * (tg[j] - dot(n, traj)); }
        eucos.x = cos(ucos);
        eucos.y = sin(ucos);
        Is += ag[j] * (n.x - betax[j]) * eucos;
        Ip += ag[j] * (n.y - betay[j]) * eucos; }

    mem_fence(CLK_LOCAL_MEM_FENCE);
    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

//__author__ = "Konstantin Klementiev, Roman Chernikov"
//__date__ = "10 Apr 2015"

#ifdef cl_khr_fp64
#if __OPENCL_VERSION__<120
#pragma OPENCL EXTENSION cl_khr_fp64: enable
#endif
#endif

__kernel void undulator(const float alpha,
                        const float Kx,
                        const float Ky,
                        const float phase,
                        const int jend,
                        __global float* gamma,
                        __global float* wu,
                        __global float* w,
                        __global float* ww1,
                        __global float* ddphi,
                        __global float* ddpsi,
                        __global float* tg,
                        __global float* ag,
                        __global float2* Is_gl,
                        __global float2* Ip_gl)
{
    unsigned int ii = get_global_id(0);
    int j;

    float2 beta;
    float2 eucos;
    float ucos, sinucos, cosucos, sintg, costg, sintgph, costgph;
    float2 zero2 = (float2)(0,0);
    float2 Is = zero2;
    float2 Ip = zero2;
    float wgwu = w[ii] / gamma[ii] / wu[ii];
    float Kx2 = Kx * Kx;
    float Ky2 = Ky * Ky;

    for (j=0; j<jend; j++) {
        sintg = sincos(tg[j], &costg);
        sintgph = sincos(tg[j] + phase, &costgph);
        ucos = (float)(ww1[ii]) * tg[j] + wgwu *
           (-Ky * ddphi[ii] * sintg + Kx * ddpsi[ii] * sintgph +
            0.25 / gamma[ii] * (Ky2 * sintg*costg + Kx2 * sintgph*costgph));

        sinucos = sincos(ucos, &cosucos);
        eucos.x = cosucos;
        eucos.y = sinucos;

        beta.x = Ky / gamma[ii] * costg;
        beta.y = -Kx / gamma[ii] * costgph;

        Is += (ag[j] * (ddphi[ii] - beta.x)) * eucos;
        Ip += (ag[j] * (ddpsi[ii] - beta.y)) * eucos; }
    mem_fence(CLK_LOCAL_MEM_FENCE);
    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
    }

__kernel void undulator_taper(const float alpha,
                              const float Kx,
                              const float Ky,
                              const float phase,
                              const int jend,
                              __global float* gamma,
                              __global float* wu,
                              __global float* w,
                              __global float* ww1,
                              __global float* ddtheta,
                              __global float* ddpsi,
                              __global float* tg,
                              __global float* ag,
                              __global float2* Is_gl,
                              __global float2* Ip_gl)
{
    const float E2W = 1.51926751475e15;
    const float C = 2.99792458e11;
    unsigned int ii = get_global_id(0);
    int j;

    float2 eucos, beta;
    float ucos, sintg, sin2tg, costg;
    float2 zero2 = (float2)(0,0);
    float2 Is = zero2;
    float2 Ip = zero2;
    float Kx2 = Kx * Kx;
    float Ky2 = Ky * Ky;
    float alphaS = alpha * C / wu[ii] / E2W;
    float wgwu = w[ii] / gamma[ii] / wu[ii];

    for (j=0; j<jend; j++) {
        sintg = sin(tg[j]);
        costg = cos(tg[j]);
        sin2tg = 2. * sintg * costg; //sin(2 * tg[j]);
        ucos = ww1[ii] * tg[j] + wgwu *
            (-Ky * ddtheta[ii] *
                  (sintg + alphaS * (1. - costg - tg[j] * sintg)) +
             Kx * ddpsi[ii] * sin(tg[j] + phase) +
             0.125 / gamma[ii] *
                  (Ky2 * (sin2tg - 2. * alphaS *
                    (tg[j] * tg[j] + costg * costg + tg[j] * sin2tg)) +
                   Kx2 * sin(2. * (tg[j] + phase))));

        eucos.x = cos(ucos);
        eucos.y = sin(ucos);

        beta.x = Ky / gamma[ii] * costg;
        beta.y = -Kx / gamma[ii] * cos(tg[j] + phase);

        Is += ag[j] * (ddtheta[ii] - beta.x *
            (1 - alphaS * tg[j])) * eucos;
        Ip += ag[j] * (ddpsi[ii] - beta.y) * eucos; }
    mem_fence(CLK_LOCAL_MEM_FENCE);
    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

__kernel void undulator_nf(const float R0,
                            const float L0,
                            const float Kx,
                            const float Ky,
                            const float phase,
                            const int jend,
                            __global float* gamma,
                            __global float* wu,
                            __global float* w,
                            __global float* ww1,
                            __global float* ddphi,
                            __global float* ddpsi,
                            __global float* tg,
                            __global float* ag,
                            __global float2* Is_gl,
                            __global float2* Ip_gl)
{
    //const float E2W = 1.51926751475e15;
    //const float C = 2.99792458e11;
    const float PI2 = 6.283185307179586476925286766559;
    unsigned int ii = get_global_id(0);
    int j;

    float2 eucos;
    float ucos, sintg, costg, sintgph, costgph;
    float2 zero2 = (float2)(0,0);
    float2 Is = zero2;
    float2 Ip = zero2;
    float3 r, r0;
    float2 beta;
    float Kx2 = Kx * Kx;
    float Ky2 = Ky * Ky;
    float gamma2 = gamma[ii] * gamma[ii];
    float betam = 1 - (1 + 0.5 * Kx2 + 0.5 * Ky2) / 2. / gamma2;
    float wR0 = R0 * PI2 / L0;

    r0.x = wR0 * tan(ddphi[ii]);
    r0.y = wR0 * tan(ddpsi[ii]);
    r0.z = wR0 * cos(sqrt(ddphi[ii]*ddphi[ii] + ddpsi[ii]*ddpsi[ii]));

    for (j=0; j<jend; j++) {
        sintg = sincos(tg[j], &costg);
        sintgph = sincos(tg[j] + phase, &costgph);

        r.x = Ky / gamma[ii] * sintg;
        r.y = -Kx / gamma[ii] * sintgph;
        r.z = betam * tg[j] - 0.25 / gamma2 *
        (Ky2 * sintg * costg + Kx2 * sintgph * costgph);

      ucos = w[ii] / wu[ii] * (tg[j] + length(r0 - r));

      eucos.x = cos(ucos);
      eucos.y = sin(ucos);

      beta.x = Ky / gamma[ii] * costg;
      beta.y = -Kx / gamma[ii] * costgph;

      Is += ag[j] * (ddphi[ii] - beta.x) * eucos;
      Ip += ag[j] * (ddpsi[ii] - beta.y) * eucos; }

    mem_fence(CLK_LOCAL_MEM_FENCE);

    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

double2 f_beta(double Bx, double By, double Bz,
               double emcg,
               double2 beta)
{
    return emcg*(double2)(beta.y*Bz - By,
                          Bx - beta.x*Bz);
}

double3 f_traj(double gamma, double2 beta)
{
    return (double3)(beta.x,
                     beta.y,
                     sqrt(1.-1./gamma/gamma-beta.x*beta.x-beta.y*beta.y));
}

double2 next_beta_rk(double2 beta, int iZeroStep, int iHalfStep,
                     int iFullStep, double rkStep, double emcg,
                     __global double* Bx,
                     __global double* By,
                     __global double* Bz)
{
    double2 k1Beta, k2Beta, k3Beta, k4Beta;

    k1Beta = rkStep * f_beta(Bx[iZeroStep],
                             By[iZeroStep],
                             Bz[iZeroStep],
                             emcg, beta);
    k2Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k1Beta);
    k3Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k2Beta);
    k4Beta = rkStep * f_beta(Bx[iFullStep],
                             By[iFullStep],
                             Bz[iFullStep],
                             emcg, beta + k3Beta);
    return beta + (k1Beta + 2.*k2Beta + 2.*k3Beta + k4Beta) / 6.;
}

double8 next_traj_rk(double2 beta, double3 traj, int iZeroStep, int iHalfStep,
                     int iFullStep, double rkStep, double emcg, double gamma,
                     __global double* Bx,
                     __global double* By,
                     __global double* Bz)
{
    double2 k1Beta, k2Beta, k3Beta, k4Beta;
    double3 k1Traj, k2Traj, k3Traj, k4Traj;

    k1Beta = rkStep * f_beta(Bx[iZeroStep],
                             By[iZeroStep],
                             Bz[iZeroStep],
                             emcg, beta);
    k1Traj = rkStep * f_traj(gamma, beta);

    k2Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k1Beta);
    k2Traj = rkStep * f_traj(gamma, beta + 0.5*k1Beta);

    k3Beta = rkStep * f_beta(Bx[iHalfStep],
                             By[iHalfStep],
                             Bz[iHalfStep],
                             emcg, beta + 0.5*k2Beta);
    k3Traj = rkStep * f_traj(gamma, beta + 0.5*k2Beta);

    k4Beta = rkStep * f_beta(Bx[iFullStep],
                             By[iFullStep],
                             Bz[iFullStep],
                             emcg, beta + k3Beta);
    k4Traj = rkStep * f_traj(gamma, beta + k3Beta);

    return (double8)(beta + (k1Beta + 2.*k2Beta + 2.*k3Beta + k4Beta)/6.,
                     traj + (k1Traj + 2.*k2Traj + 2.*k3Traj + k4Traj)/6.,
                     0., 0., 0.);
}

__kernel void undulator_custom(const int jend,
                                const int nwt,
                                const double lUnd,
                                __global double* gamma,
                                __global double* w,
                                __global double* ddphi,
                                __global double* ddpsi,
                                __global double* tg,
                                __global double* ag,
                                __global double* Bx,
                                __global double* By,
                                __global double* Bz,
                                __global double2* Is_gl,
                                __global double2* Ip_gl)
{
    const double E2W = 1.51926751475e15;
    const double C = 2.99792458e11;
    const double SIE0 = 1.602176565e-19;
    const double SIM0 = 9.10938291e-31;
    //const double PI = 3.1415926535897932384626433832795;
    const double PI2 = 6.283185307179586476925286766559;

    unsigned int ii = get_global_id(0);
    int j, k;
    int iBase, iZeroStep, iHalfStep, iFullStep;

    double ucos, rkStep, wu_int, betam_int;
    double emcg = lUnd * SIE0 / SIM0 / C / gamma[ii] / PI2;
    double revgamma = 1. - 1./gamma[ii]/gamma[ii];
    double8 betaTraj;
    double2 eucos;
    double2 zero2 = (double2)(0,0);
    double2 Is = zero2;
    double2 Ip = zero2;
    double2 beta, beta0;
    double3 traj, n, traj0;

    n.x = ddphi[ii];
    n.y = ddpsi[ii];
    n.z = 1. - 0.5*(n.x*n.x + n.y*n.y);

    beta = zero2;
    beta0 = zero2;
    betam_int = 0;

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            beta = next_beta_rk(beta, iZeroStep, iHalfStep, iFullStep,
                                rkStep, emcg, Bx, By, Bz);
            beta0 += beta * rkStep; } }

    mem_fence(CLK_LOCAL_MEM_FENCE);
    beta0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = (double3)(0., 0., 0.);
    traj0 = (double3)(0., 0., 0.);

    for (j=1; j<jend; j++) {
        iBase = (j-1)*2*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
                                    iZeroStep, iHalfStep, iFullStep,
                                    rkStep, emcg, gamma[ii], Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234;
            traj0 += traj * rkStep;
            betam_int += rkStep * sqrt(revgamma - beta.x*beta.x -
                                       beta.y*beta.y); } }

    mem_fence(CLK_LOCAL_MEM_FENCE);
    traj0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = traj0;
    betam_int /= (tg[jend-1] - tg[0]);
    wu_int = PI2 * C * betam_int / lUnd / E2W;

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
            iZeroStep, iHalfStep, iFullStep,
            rkStep, emcg, gamma[ii], Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234; }

        mem_fence(CLK_LOCAL_MEM_FENCE);
        ucos = w[ii] / wu_int * (tg[j]  - dot(n, traj));
        eucos.x = cos(ucos);
        eucos.y = sin(ucos);
        Is += ag[j] * (ddphi[ii] - beta.x) * eucos;
        Ip += ag[j] * (ddpsi[ii] - beta.y) * eucos; }

    mem_fence(CLK_LOCAL_MEM_FENCE);

    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}

__kernel void get_trajectory(const int jend,
                             const int nwt,
                             const double emcg,
                             const double gamma,
                             __global double* tg,
                             __global double* Bx,
                             __global double* By,
                             __global double* Bz,
                             __global double* betax,
                             __global double* betay,
                             __global double* betazav,
                             __global double* trajx,
                             __global double* trajy,
                             __global double* trajz)
{
    int j, k;
    int iBase, iZeroStep, iHalfStep, iFullStep;

    double rkStep;
    double revgamma = 1. - 1./gamma/gamma;
    double betam_int = 0;
    double2 zero2 = (double2)(0,0);
    double2 beta, beta0;
    double3 traj, traj0;
    double8 betaTraj;

    beta = zero2;
    beta0 = zero2;

    for (j=1; j<jend; j++) {
        iBase = (j-1)*2*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            beta = next_beta_rk(beta, iZeroStep, iHalfStep, iFullStep,
                                rkStep, emcg, Bx, By, Bz);
            beta0 += beta * rkStep; } }

    beta0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = (double3)(0., 0., 0.);
    traj0 = (double3)(0., 0., 0.);

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
                                    iZeroStep, iHalfStep, iFullStep,
                                    rkStep, emcg, gamma, Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234;
            traj0 += traj * rkStep;
            betam_int += rkStep*sqrt(revgamma - beta.x*beta.x -
                                     beta.y*beta.y); } }

    traj0 /= -(tg[jend-1] - tg[0]);
    beta = beta0;
    traj = traj0;
    betam_int /= tg[jend-1] - tg[0];

    for (j=1; j<jend; j++) {
        iBase = 2*(j-1)*nwt;
        rkStep = (tg[j] - tg[j-1]) / nwt;
        for (k=0; k<nwt; k++) {
            iZeroStep  = iBase + 2*k;
            iHalfStep = iBase + 2*k + 1;
            iFullStep = iBase + 2*(k + 1);
            betaTraj = next_traj_rk(beta, traj,
                                    iZeroStep, iHalfStep, iFullStep,
                                    rkStep, emcg, gamma, Bx, By, Bz);
            beta = betaTraj.s01;
            traj = betaTraj.s234; }

        betax[j] = beta.x;
        betay[j] = beta.y;
        betazav[j] = betam_int;
        trajx[j] = traj.x;
        trajy[j] = traj.y;
        trajz[j] = traj.z; }

    barrier(CLK_LOCAL_MEM_FENCE);
}

__kernel void undulator_custom_filament(const int jend,
                                        const double wu,
                                        const double L0,
                                        const double R0,
                                        __global double* w,
                                        __global double* ddphi,
                                        __global double* ddpsi,
                                        __global double* tg,
                                        __global double* ag,
                                        __global double* betax,
                                        __global double* betay,
                                        __global double* trajx,
                                        __global double* trajy,
                                        __global double* trajz,
                                        __global double2* Is_gl,
                                        __global double2* Ip_gl)
{
    const double PI2 = 6.283185307179586476925286766559;

    unsigned int ii = get_global_id(0);
    int j;

    double ucos, wR0;

    double2 eucos;
    double2 Is = (double2)(0., 0.);
    double2 Ip = (double2)(0., 0.);

    double3 traj, n, r0;

    n.x = ddphi[ii];
    n.y = ddpsi[ii];
    n.z = 1. - 0.5*(n.x*n.x + n.y*n.y);

    if (R0>0) {
        wR0 = R0 * PI2 / L0;
        r0.x = wR0 * tan(ddphi[ii]);
        r0.y = wR0 * tan(ddpsi[ii]);
        r0.z = wR0 * cos(sqrt(ddphi[ii]*ddphi[ii] + ddpsi[ii]*ddpsi[ii])); }

    for (j=1; j<jend; j++) {
        traj.x = trajx[j];
        traj.y = trajy[j];
        traj.z = trajz[j];
        if (R0 > 0) {
            ucos = w[ii] / wu * (tg[j] + length(r0 - traj)); }
        else {
            ucos = w[ii] / wu * (tg[j] - dot(n, traj)); }
        eucos.x = cos(ucos);
        eucos.y = sin(ucos);
        Is += ag[j] * (n.x - betax[j]) * eucos;
        Ip += ag[j] * (n.y - betay[j]) * eucos; }

    mem_fence(CLK_LOCAL_MEM_FENCE);
    Is_gl[ii] = Is;
    Ip_gl[ii] = Ip;
}