Warn that OpenCl kernel in sort-new example segfaults on CPU.

// This kernel code based on CUDPP.  Please see the notice in
// LICENSE_CUDPP.txt.

//This kernel segfaults when run on CPU. The segfault occurrs in findRadixOffsets and it looks like a serious memory corruption. Incredibly frustrating to debug, even adding printf can trigger segfault. Maybe a bug we have unknowingly adopted from SHOC?

//Works just fine when run on GPU.

#if SORT_VECTOR == 2
#define SORTVECTYPE uint2
#elif SORT_VECTOR == 4
#define SORTVECTYPE uint4
#elif SORT_VECTOR == 8
#define SORTVECTYPE uint8
#endif

#if SCAN_VECTOR == 2
#define SCANVECTYPE uint2
#elif SCAN_VECTOR == 4
#define SCANVECTYPE uint4
#elif SCAN_VECTOR == 8
#define SCANVECTYPE uint8
#endif

uint scanLSB(const uint val, __local uint* s_data)
{
    // Set first half of shared mem to 0's
    int idx = get_local_id(0);
    s_data[idx] = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    // Set 2nd half to thread local sum
    idx += get_local_size(0);

    // scan in local memory

    // Some of these __sync's are unnecessary due to warp synchronous
    // execution.  Right now these are left in to be consistent with
    // opencl version, since that has to execute on platforms where
    // thread groups are not synchronous (i.e. CPUs)
    uint t;
    s_data[idx] = val;     barrier(CLK_LOCAL_MEM_FENCE);
    for (uint i = 1; i < SORT_BLOCK_SIZE; i*=2) {
      t = s_data[idx - i];  barrier(CLK_LOCAL_MEM_FENCE);;
      s_data[idx] += t;      barrier(CLK_LOCAL_MEM_FENCE);;
    }
    return s_data[idx] - val;  // convert inclusive -> exclusive
}

SORTVECTYPE scan4(SORTVECTYPE idata, __local uint* ptr)
{
    SORTVECTYPE val4 = idata;
    SORTVECTYPE sum;

    // Scan the elements in idata within this thread
#if SORT_VECTOR == 2
    sum.x = val4.x;
    uint val = val4.y + sum.x;
#elif SORT_VECTOR == 4
    sum.x = val4.x;
    sum.y = val4.y + sum.x;
    sum.z = val4.z + sum.y;
    uint val = val4.w + sum.z;
#elif SORT_VECTOR == 8
    sum.s0 = val4.s0;
    sum.s1 = val4.s1 + sum.s0;
    sum.s2 = val4.s2 + sum.s1;
    sum.s3 = val4.s3 + sum.s2;
    sum.s4 = val4.s4 + sum.s3;
    sum.s5 = val4.s5 + sum.s4;
    sum.s6 = val4.s6 + sum.s5;
    uint val = val4.s7 + sum.s6;
#endif

    // Now scan those sums across the local work group
    val = scanLSB(val, ptr);

#if SORT_VECTOR == 2
    val4.x = val;
    val4.y = val + sum.x;
#elif SORT_VECTOR == 4
    val4.x = val;
    val4.y = val + sum.x;
    val4.z = val + sum.y;
    val4.w = val + sum.z;
#elif SORT_VECTOR == 8
    val4.s0 = val;
    val4.s1 = val + sum.s0;
    val4.s2 = val + sum.s1;
    val4.s3 = val + sum.s2;
    val4.s4 = val + sum.s3;
    val4.s5 = val + sum.s4;
    val4.s6 = val + sum.s5;
    val4.s7 = val + sum.s6;
#endif

    return val4;
}

//----------------------------------------------------------------------------
//
// radixSortBlocks sorts all blocks of data independently in shared
// memory.  Each thread block (CTA) sorts one block of SORT_VECTOR*CTA_SIZE elements
//
// The radix sort is done in two stages.  This stage calls radixSortBlock
// on each block independently, sorting on the basis of bits
// (startbit) -> (startbit + nbits)
//----------------------------------------------------------------------------

__kernel void radixSortBlocks(const uint nbits,
                              const uint startbit,
                              __global SORTVECTYPE* keysOut,
                              __global SORTVECTYPE* valuesOut,
                              __global SORTVECTYPE* keysIn,
                              __global SORTVECTYPE* valuesIn)
{
    __local uint sMem[SORT_VECTOR*SORT_BLOCK_SIZE];

    // Get Indexing information
    const uint i = get_global_id(0);
    const uint tid = get_local_id(0);
    const uint localSize = get_local_size(0);

    // Load keys and vals from global memory
    SORTVECTYPE key, value;
    key = keysIn[i];
    value = valuesIn[i];

    __local uint numtrue;

    // For each of the 4 bits
    for(uint shift = startbit; shift < (startbit + nbits); ++shift)
    {
        // Check if the LSB is 0
        SORTVECTYPE lsb;
#if SORT_VECTOR == 2
        lsb.x = !((key.x >> shift) & 0x1);
        lsb.y = !((key.y >> shift) & 0x1);
#elif SORT_VECTOR == 4
        lsb.x = !((key.x >> shift) & 0x1);
        lsb.y = !((key.y >> shift) & 0x1);
        lsb.z = !((key.z >> shift) & 0x1);
        lsb.w = !((key.w >> shift) & 0x1);
#elif SORT_VECTOR == 8
        lsb.s0 = !((key.s0 >> shift) & 0x1);
        lsb.s1 = !((key.s1 >> shift) & 0x1);
        lsb.s2 = !((key.s2 >> shift) & 0x1);
        lsb.s3 = !((key.s3 >> shift) & 0x1);
        lsb.s4 = !((key.s4 >> shift) & 0x1);
        lsb.s5 = !((key.s5 >> shift) & 0x1);
        lsb.s6 = !((key.s6 >> shift) & 0x1);
        lsb.s7 = !((key.s7 >> shift) & 0x1);
#endif

        // Do an exclusive scan of how many elems have 0's in the LSB
        // When this is finished, address.n will contain the number of
        // elems with 0 in the LSB which precede elem n
        SORTVECTYPE address = scan4(lsb, sMem);

        // Store the total number of elems with an LSB of 0
        // to shared mem
        if (tid == localSize - 1)
        {
#if SORT_VECTOR == 2
          numtrue = address.y + lsb.y;
#elif SORT_VECTOR == 4
            numtrue = address.w + lsb.w;
#elif SORT_VECTOR == 8
            numtrue = address.s7 + lsb.s7;
#endif
        }
        barrier(CLK_LOCAL_MEM_FENCE);;

        // Determine rank -- position in the block
        // If you are a 0 --> your position is the scan of 0's
        // If you are a 1 --> your position is calculated as below
        SORTVECTYPE rank;
        const int idx = tid*SORT_VECTOR;
#if SORT_VECTOR == 2
        rank.x = lsb.x ? address.x : numtrue + idx     - address.x;
        rank.y = lsb.y ? address.y : numtrue + idx + 1 - address.y;
#elif SORT_VECTOR == 4
        rank.x = lsb.x ? address.x : numtrue + idx     - address.x;
        rank.y = lsb.y ? address.y : numtrue + idx + 1 - address.y;
        rank.z = lsb.z ? address.z : numtrue + idx + 2 - address.z;
        rank.w = lsb.w ? address.w : numtrue + idx + 3 - address.w;
#elif SORT_VECTOR == 8
        rank.s0 = lsb.s0 ? address.s0 : numtrue + idx     - address.s0;
        rank.s1 = lsb.s1 ? address.s1 : numtrue + idx + 1 - address.s1;
        rank.s2 = lsb.s2 ? address.s2 : numtrue + idx + 2 - address.s2;
        rank.s3 = lsb.s3 ? address.s3 : numtrue + idx + 3 - address.s3;
        rank.s4 = lsb.s4 ? address.s4 : numtrue + idx + 4 - address.s4;
        rank.s5 = lsb.s5 ? address.s5 : numtrue + idx + 5 - address.s5;
        rank.s6 = lsb.s6 ? address.s6 : numtrue + idx + 6 - address.s6;
        rank.s7 = lsb.s7 ? address.s7 : numtrue + idx + 7 - address.s7;
#endif

        // Scatter keys into local mem
#if SORT_VECTOR == 2
        sMem[(rank.x & 1) * localSize + (rank.x >> 1)] = key.x;
        sMem[(rank.y & 1) * localSize + (rank.y >> 1)] = key.y;
#elif SORT_VECTOR == 4
        sMem[(rank.x & 3) * localSize + (rank.x >> 2)] = key.x;
        sMem[(rank.y & 3) * localSize + (rank.y >> 2)] = key.y;
        sMem[(rank.z & 3) * localSize + (rank.z >> 2)] = key.z;
        sMem[(rank.w & 3) * localSize + (rank.w >> 2)] = key.w;
#elif SORT_VECTOR == 8
        sMem[(rank.s0 & 7) * localSize + (rank.s0 >> 3)] = key.s0;
        sMem[(rank.s1 & 7) * localSize + (rank.s1 >> 3)] = key.s1;
        sMem[(rank.s2 & 7) * localSize + (rank.s2 >> 3)] = key.s2;
        sMem[(rank.s3 & 7) * localSize + (rank.s3 >> 3)] = key.s3;
        sMem[(rank.s4 & 7) * localSize + (rank.s4 >> 3)] = key.s4;
        sMem[(rank.s5 & 7) * localSize + (rank.s5 >> 3)] = key.s5;
        sMem[(rank.s6 & 7) * localSize + (rank.s6 >> 3)] = key.s6;
        sMem[(rank.s7 & 7) * localSize + (rank.s7 >> 3)] = key.s7;
#endif
        barrier(CLK_LOCAL_MEM_FENCE);

        // Read keys out of local mem into registers, in prep for
        // write out to global mem
#if SORT_VECTOR == 2
        key.x = sMem[tid];
        key.y = sMem[tid +     localSize];
#elif SORT_VECTOR == 4
        key.x = sMem[tid];
        key.y = sMem[tid +     localSize];
        key.z = sMem[tid + 2 * localSize];
        key.w = sMem[tid + 3 * localSize];
#elif SORT_VECTOR == 8
        key.s0 = sMem[tid];
        key.s1 = sMem[tid +     localSize];
        key.s2 = sMem[tid + 2 * localSize];
        key.s3 = sMem[tid + 3 * localSize];
        key.s4 = sMem[tid + 4 * localSize];
        key.s5 = sMem[tid + 5 * localSize];
        key.s6 = sMem[tid + 6 * localSize];
        key.s7 = sMem[tid + 7 * localSize];
#endif
        barrier(CLK_LOCAL_MEM_FENCE);;

        // Scatter values into local mem
#if SORT_VECTOR == 2
        sMem[(rank.x & 1) * localSize + (rank.x >> 1)] = value.x;
        sMem[(rank.y & 1) * localSize + (rank.y >> 1)] = value.y;
#elif SORT_VECTOR == 4
        sMem[(rank.x & 3) * localSize + (rank.x >> 2)] = value.x;
        sMem[(rank.y & 3) * localSize + (rank.y >> 2)] = value.y;
        sMem[(rank.z & 3) * localSize + (rank.z >> 2)] = value.z;
        sMem[(rank.w & 3) * localSize + (rank.w >> 2)] = value.w;
#elif SORT_VECTOR == 8
        sMem[(rank.s0 & 7) * localSize + (rank.s0 >> 3)] = value.s0;
        sMem[(rank.s1 & 7) * localSize + (rank.s1 >> 3)] = value.s1;
        sMem[(rank.s2 & 7) * localSize + (rank.s2 >> 3)] = value.s2;
        sMem[(rank.s3 & 7) * localSize + (rank.s3 >> 3)] = value.s3;
        sMem[(rank.s4 & 7) * localSize + (rank.s4 >> 3)] = value.s4;
        sMem[(rank.s5 & 7) * localSize + (rank.s5 >> 3)] = value.s5;
        sMem[(rank.s6 & 7) * localSize + (rank.s6 >> 3)] = value.s6;
        sMem[(rank.s7 & 7) * localSize + (rank.s7 >> 3)] = value.s7;
#endif
        barrier(CLK_LOCAL_MEM_FENCE);;

        // Read keys out of local mem into registers, in prep for
        // write out to global mem
#if SORT_VECTOR == 2
        value.x = sMem[tid];
        value.y = sMem[tid +     localSize];
#elif SORT_VECTOR == 4
        value.x = sMem[tid];
        value.y = sMem[tid +     localSize];
        value.z = sMem[tid + 2 * localSize];
        value.w = sMem[tid + 3 * localSize];
#elif SORT_VECTOR == 8
        value.s0 = sMem[tid];
        value.s1 = sMem[tid +     localSize];
        value.s2 = sMem[tid + 2 * localSize];
        value.s3 = sMem[tid + 3 * localSize];
        value.s4 = sMem[tid + 4 * localSize];
        value.s5 = sMem[tid + 5 * localSize];
        value.s6 = sMem[tid + 6 * localSize];
        value.s7 = sMem[tid + 7 * localSize];
#endif
        barrier(CLK_LOCAL_MEM_FENCE);;
    }
    keysOut[i]   = key;
    valuesOut[i] = value;
}

//----------------------------------------------------------------------------
// Given an array with blocks sorted according to a 4-bit radix group, each
// block counts the number of keys that fall into each radix in the group, and
// finds the starting offset of each radix in the block.  It then writes the
// radix counts to the counters array, and the starting offsets to the
// blockOffsets array.
//
//----------------------------------------------------------------------------
__kernel void findRadixOffsets(__global SCANVECTYPE* keys,
        __global uint* counters,
        __global uint* blockOffsets,
        uint startbit,
        uint numElements,
        uint totalBlocks)
{
    __local uint  sStartPointers[16];
    __local uint sRadix1[SCAN_VECTOR*SCAN_BLOCK_SIZE];

    uint gid = get_group_id(0);
    uint tid = get_local_id(0);
    uint localSize = get_local_size(0);
    uint i = get_global_id(0);

    SCANVECTYPE radix2;
    radix2 = keys[i];

#if SCAN_VECTOR == 2
    sRadix1[2 * tid]     = (radix2.x >> startbit) & 0xF;
    sRadix1[2 * tid + 1] = (radix2.y >> startbit) & 0xF;
#elif SCAN_VECTOR == 4
    sRadix1[4 * tid]      = (radix2.x >> startbit) & 0xF;
    sRadix1[4 * tid + 1]  = (radix2.y >> startbit) & 0xF;
    sRadix1[4 * tid + 2]  = (radix2.z >> startbit) & 0xF;
    sRadix1[4 * tid + 3]  = (radix2.w >> startbit) & 0xF;
#elif SCAN_VECTOR == 8
    sRadix1[8 * tid]      = (radix2.s0 >> startbit) & 0xF;
    sRadix1[8 * tid + 1]  = (radix2.s1 >> startbit) & 0xF;
    sRadix1[8 * tid + 2]  = (radix2.s2 >> startbit) & 0xF;
    sRadix1[8 * tid + 3]  = (radix2.s3 >> startbit) & 0xF;
    sRadix1[8 * tid + 4]  = (radix2.s4 >> startbit) & 0xF;
    sRadix1[8 * tid + 5]  = (radix2.s5 >> startbit) & 0xF;
    sRadix1[8 * tid + 6]  = (radix2.s6 >> startbit) & 0xF;
    sRadix1[8 * tid + 7]  = (radix2.s7 >> startbit) & 0xF;
#endif

    // Finds the position where the sRadix1 entries differ and stores start
    // index for each radix.
    if(tid < 16)
    {
        sStartPointers[tid] = 0;
    }
    barrier(CLK_LOCAL_MEM_FENCE);;

    if((tid > 0) && (sRadix1[tid] != sRadix1[tid - 1]) )
    {
        sStartPointers[sRadix1[tid]] = tid;
    }
    if(sRadix1[tid + localSize] != sRadix1[tid + localSize - 1])
    {
        sStartPointers[sRadix1[tid + localSize]] = tid + localSize;
    }

#if SCAN_VECTOR == 4 || SCAN_VECTOR == 8
    if (sRadix1[tid + 2*localSize] != sRadix1[tid + 2*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 2*localSize]] = tid + 2*localSize;
    }
    if (sRadix1[tid + 3*localSize] != sRadix1[tid + 3*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 3*localSize]] = tid + 3*localSize;
    }
#endif
#if SCAN_VECTOR == 8
    if (sRadix1[tid + 4*localSize] != sRadix1[tid + 4*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 4*localSize]] = tid + 4*localSize;
    }
    if (sRadix1[tid + 5*localSize] != sRadix1[tid + 5*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 5*localSize]] = tid + 5*localSize;
    }
    if (sRadix1[tid + 6*localSize] != sRadix1[tid + 6*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 6*localSize]] = tid + 6*localSize;
    }
    if (sRadix1[tid + 7*localSize] != sRadix1[tid + 7*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 7*localSize]] = tid + 7*localSize;
    }
#endif
    barrier(CLK_LOCAL_MEM_FENCE);;

    if(tid < 16)
    {
        blockOffsets[gid*16 + tid] = sStartPointers[tid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);;

    // Compute the sizes of each block.
    if((tid > 0) && (sRadix1[tid] != sRadix1[tid - 1]) )
    {
        sStartPointers[sRadix1[tid - 1]] =
            tid - sStartPointers[sRadix1[tid - 1]];
    }
    if(sRadix1[tid + localSize] != sRadix1[tid + localSize - 1] )
    {
        sStartPointers[sRadix1[tid + localSize - 1]] =
            tid + localSize - sStartPointers[sRadix1[tid +
                                                         localSize - 1]];
    }
#if SCAN_VECTOR == 4 || SCAN_VECTOR == 8
    if(sRadix1[tid + 2*localSize] != sRadix1[tid + 2*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 2*localSize - 1]] =
            tid + 2*localSize - sStartPointers[sRadix1[tid + 2*localSize - 1]];
    }
    if(sRadix1[tid + 3*localSize] != sRadix1[tid + 3*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 3*localSize - 1]] =
            tid + 3*localSize - sStartPointers[sRadix1[tid + 3*localSize - 1]];
    }
#endif
#if SCAN_VECTOR == 8
    if(sRadix1[tid + 4*localSize] != sRadix1[tid + 4*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 4*localSize - 1]] =
            tid + 4*localSize - sStartPointers[sRadix1[tid + 4*localSize - 1]];
    }
    if(sRadix1[tid + 5*localSize] != sRadix1[tid + 5*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 5*localSize - 1]] =
            tid + 5*localSize - sStartPointers[sRadix1[tid + 5*localSize - 1]];
    }
    if(sRadix1[tid + 6*localSize] != sRadix1[tid + 6*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 6*localSize - 1]] =
            tid + 6*localSize - sStartPointers[sRadix1[tid + 6*localSize - 1]];
    }
    if(sRadix1[tid + 7*localSize] != sRadix1[tid + 7*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 7*localSize - 1]] =
            tid + 7*localSize - sStartPointers[sRadix1[tid + 7*localSize - 1]];
    }
#endif

#if SCAN_VECTOR == 2
    if(tid == localSize - 1)
    {
        sStartPointers[sRadix1[2 * localSize - 1]] =
            2 * localSize - sStartPointers[sRadix1[2 * localSize - 1]];
    }
#elif SCAN_VECTOR == 4
    if (tid == localSize - 1)
    {
        sStartPointers[sRadix1[4 * localSize - 1]] =
            4 * localSize - sStartPointers[sRadix1[4 * localSize - 1]];
    }
#elif SCAN_VECTOR == 8
    if (tid == localSize - 1)
    {
        sStartPointers[sRadix1[8 * localSize - 1]] =
            8 * localSize - sStartPointers[sRadix1[8 * localSize - 1]];
    }
#endif
    barrier(CLK_LOCAL_MEM_FENCE);;

    if(tid < 16)
    {
        counters[tid * totalBlocks + gid] = sStartPointers[tid];
    }
}

//----------------------------------------------------------------------------
// reorderData shuffles data in the array globally after the radix offsets
// have been found. On compute version 1.1 and earlier GPUs, this code depends
// on SORT_BLOCK_SIZE being 16 * number of radices (i.e. 16 * 2^nbits).
//----------------------------------------------------------------------------
__kernel void reorderData(uint  startbit,
                          __global uint  *outKeys,
                          __global uint  *outValues,
                          __global SCANVECTYPE *keys,
                          __global SCANVECTYPE *values,
                          __global uint  *blockOffsets,
                          __global uint  *offsets,
                          __global uint  *sizes,
                          uint  totalBlocks)
{
    __local SCANVECTYPE sKeys2[SCAN_BLOCK_SIZE];
    __local SCANVECTYPE sValues2[SCAN_BLOCK_SIZE];
    __local uint  sOffsets[16];
    __local uint  sBlockOffsets[16];
    __local uint* sKeys1   = (__local uint*) &sKeys2;
    __local uint* sValues1 = (__local uint*) &sValues2;

    uint gid = get_group_id(0);
    uint tid = get_local_id(0);
    uint localSize = get_local_size(0);
    uint i = get_global_id(0);

    sKeys2[tid]   = keys[i];
    sValues2[tid] = values[i];

    if(tid < 16)
    {
        sOffsets[tid]      = offsets[tid * totalBlocks +
                                             gid];
        sBlockOffsets[tid] = blockOffsets[gid * 16 + tid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    uint radix = (sKeys1[tid] >> startbit) & 0xF;
    uint globalOffset = sOffsets[radix] + tid - sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid];
    outValues[globalOffset] = sValues1[tid];

    radix = (sKeys1[tid + localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + localSize];
    outValues[globalOffset] = sValues1[tid + localSize];

#if SCAN_VECTOR == 4 || SCAN_VECTOR == 8
    radix = (sKeys1[tid + 2*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 2*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 2*localSize];
    outValues[globalOffset] = sValues1[tid + 2*localSize];

    radix = (sKeys1[tid + 3*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 3*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 3*localSize];
    outValues[globalOffset] = sValues1[tid + 3*localSize];
#endif
#if SCAN_VECTOR == 8
    radix = (sKeys1[tid + 4*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 4*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 4*localSize];
    outValues[globalOffset] = sValues1[tid + 4*localSize];

    radix = (sKeys1[tid + 5*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 5*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 5*localSize];
    outValues[globalOffset] = sValues1[tid + 5*localSize];

    radix = (sKeys1[tid + 6*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 6*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 6*localSize];
    outValues[globalOffset] = sValues1[tid + 6*localSize];

    radix = (sKeys1[tid + 7*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 7*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 7*localSize];
    outValues[globalOffset] = sValues1[tid + 7*localSize];
#endif
}

uint scanLocalMem(const uint val, __local uint* s_data)
{
    // Set first half of shared mem to 0
    int idx = get_local_id(0);
    s_data[idx] = 0.0f;
    barrier(CLK_LOCAL_MEM_FENCE);

    // Set 2nd half to thread local sum
    idx += get_local_size(0);

    // Some of these __sync's are unnecessary due to warp synchronous
    // execution.  Right now these are left in to be consistent with
    // opencl version, since that has to execute on platforms where
    // thread groups are not synchronous (i.e. CPUs)
    uint t;
    s_data[idx] = val;     barrier(CLK_LOCAL_MEM_FENCE);
    for (uint i = 1; i < SCAN_BLOCK_SIZE; i*=2) {
      t = s_data[idx - i];  barrier(CLK_LOCAL_MEM_FENCE);
      s_data[idx] += t;      barrier(CLK_LOCAL_MEM_FENCE);
    }
    return s_data[idx-1];
}

__kernel void
scan(__global uint *g_odata,  __global uint* g_idata,
     __global uint* g_blockSums, const int n,
     const int fullBlock, const int storeSum)
{
    __local uint s_data[2*SCAN_BLOCK_SIZE];

    // Load data into shared mem
    SORTVECTYPE tempData;
    SORTVECTYPE threadScanT;
    uint res;
    __global SORTVECTYPE* inData  = (__global SORTVECTYPE*) g_idata;

    const int gid = get_global_id(0);
    const int tid = get_local_id(0);
    const int i = gid * SORT_VECTOR;

    // If possible, read from global mem in a SORTVECTYPE chunk
#if SORT_VECTOR == 2
    if (fullBlock || i + 1 < n)
    {
        // scan the 2 elems read in from global
        tempData       = inData[gid];
        threadScanT.x = tempData.x;
        threadScanT.y = tempData.y + threadScanT.x;
        res = threadScanT.y;
    }
    else
    {   // if not, read individual uints, scan & store in lmem
        threadScanT.x = (i < n) ? g_idata[i] : 0.0f;
        threadScanT.y = ((i+1 < n) ? g_idata[i+1] : 0.0f) + threadScanT.x;
        res = threadScanT.y;
    }
#elif SORT_VECTOR == 4
    if (fullBlock || i + 3 < n)
    {
        // scan the 4 elems read in from global
        tempData       = inData[gid];
        threadScanT.x = tempData.x;
        threadScanT.y = tempData.y + threadScanT.x;
        threadScanT.z = tempData.z + threadScanT.y;
        threadScanT.w = tempData.w + threadScanT.z;
        res = threadScanT.w;
    }
    else
    {   // if not, read individual uints, scan & store in lmem
        threadScanT.x = (i < n) ? g_idata[i] : 0.0f;
        threadScanT.y = ((i+1 < n) ? g_idata[i+1] : 0.0f) + threadScanT.x;
        threadScanT.z = ((i+2 < n) ? g_idata[i+2] : 0.0f) + threadScanT.y;
        threadScanT.w = ((i+3 < n) ? g_idata[i+3] : 0.0f) + threadScanT.z;
        res = threadScanT.w;
    }
#elif SORT_VECTOR == 8
    if (fullBlock || i + 7 < n)
    {
        // scan the 8 elems read in from global
        tempData       = inData[gid];
        threadScanT.s0 = tempData.s0;
        threadScanT.s1 = tempData.s1 + threadScanT.s0;
        threadScanT.s2 = tempData.s2 + threadScanT.s1;
        threadScanT.s3 = tempData.s3 + threadScanT.s2;
        threadScanT.s4 = tempData.s4 + threadScanT.s3;
        threadScanT.s5 = tempData.s5 + threadScanT.s4;
        threadScanT.s6 = tempData.s6 + threadScanT.s5;
        threadScanT.s7 = tempData.s7 + threadScanT.s6;
        res = threadScanT.s7;
    }
    else
    {   // if not, read individual uints, scan & store in lmem
        threadScanT.s0 = (i < n) ? g_idata[i] : 0.0f;
        threadScanT.s1 = ((i+1 < n) ? g_idata[i+1] : 0.0f) + threadScanT.s0;
        threadScanT.s2 = ((i+2 < n) ? g_idata[i+2] : 0.0f) + threadScanT.s1;
        threadScanT.s3 = ((i+3 < n) ? g_idata[i+3] : 0.0f) + threadScanT.s2;
        threadScanT.s4 = ((i+4 < n) ? g_idata[i+4] : 0.0f) + threadScanT.s3;
        threadScanT.s5 = ((i+5 < n) ? g_idata[i+5] : 0.0f) + threadScanT.s4;
        threadScanT.s6 = ((i+6 < n) ? g_idata[i+6] : 0.0f) + threadScanT.s5;
        threadScanT.s7 = ((i+7 < n) ? g_idata[i+7] : 0.0f) + threadScanT.s6;
        res = threadScanT.s7;
    }
#endif

    res = scanLocalMem(res, s_data);
    barrier(CLK_LOCAL_MEM_FENCE);

    // If we have to store the sum for the block, have the last work item
    // in the block write it out
    if (storeSum && tid == get_local_size(0)-1) {
#if SORT_VECTOR == 2
        g_blockSums[get_group_id(0)] = res + threadScanT.y;
#elif SORT_VECTOR == 4
        g_blockSums[get_group_id(0)] = res + threadScanT.w;
#elif SORT_VECTOR == 8
        g_blockSums[get_group_id(0)] = res + threadScanT.s7;
#endif
    }

    // write results to global memory
    __global SORTVECTYPE* outData = (__global SORTVECTYPE*) g_odata;

#if SORT_VECTOR == 2
    tempData.x = res;
    tempData.y = res + threadScanT.x;

    if (fullBlock || i + 1 < n)
    {
        outData[gid] = tempData;
    }
    else
    {
        if ( i    < n) { g_odata[i]   = tempData.x; }
    }
#elif SORT_VECTOR == 4
    tempData.x = res;
    tempData.y = res + threadScanT.x;
    tempData.z = res + threadScanT.y;
    tempData.w = res + threadScanT.z;

    if (fullBlock || i + 3 < n)
    {
        outData[gid] = tempData;
    }
    else
    {
        if ( i    < n) { g_odata[i]   = tempData.x;
        if ((i+1) < n) { g_odata[i+1] = tempData.y;
        if ((i+2) < n) { g_odata[i+2] = tempData.z; } } }
    }
#elif SORT_VECTOR == 8
    tempData.s0 = res;
    tempData.s1 = res + threadScanT.s0;
    tempData.s2 = res + threadScanT.s1;
    tempData.s3 = res + threadScanT.s2;
    tempData.s4 = res + threadScanT.s3;
    tempData.s5 = res + threadScanT.s4;
    tempData.s6 = res + threadScanT.s5;
    tempData.s7 = res + threadScanT.s6;

    if (fullBlock || i + 7 < n)
    {
        outData[gid] = tempData;
    }
    else
    {
        if ( i    < n) { g_odata[i]   = tempData.s0;
        if ((i+1) < n) { g_odata[i+1] = tempData.s1;
        if ((i+2) < n) { g_odata[i+2] = tempData.s2;
        if ((i+3) < n) { g_odata[i+3] = tempData.s3;
        if ((i+4) < n) { g_odata[i+4] = tempData.s4;
        if ((i+5) < n) { g_odata[i+5] = tempData.s5;
        if ((i+6) < n) { g_odata[i+6] = tempData.s6; } } } } } } }
    }
#endif
}

__kernel void
vectorAddUniform4(__global uint *d_vector, __global const uint *d_uniforms,
        const int n)
{
    __local uint uni[1];

    if (get_local_id(0) == 0)
    {
        uni[0] = d_uniforms[get_group_id(0)];
    }

    unsigned int address = get_local_id(0) + (get_group_id(0) *
            get_local_size(0) * SORT_VECTOR);

    barrier(CLK_LOCAL_MEM_FENCE);

    // SORT_VECTOR elems per thread
    for (int i = 0; i < SORT_VECTOR && address < n; i++)
    {
        d_vector[address] += uni[0];
        address += get_local_size(0);
    }
}

// This kernel code based on CUDPP.  Please see the notice in
// LICENSE_CUDPP.txt.

#if SORT_VECTOR == 2
#define SORTVECTYPE uint2
#elif SORT_VECTOR == 4
#define SORTVECTYPE uint4
#elif SORT_VECTOR == 8
#define SORTVECTYPE uint8
#endif

#if SCAN_VECTOR == 2
#define SCANVECTYPE uint2
#elif SCAN_VECTOR == 4
#define SCANVECTYPE uint4
#elif SCAN_VECTOR == 8
#define SCANVECTYPE uint8
#endif

uint scanLSB(const uint val, __local uint* s_data)
{
    // Set first half of shared mem to 0's
    int idx = get_local_id(0);
    s_data[idx] = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    // Set 2nd half to thread local sum
    idx += get_local_size(0);

    // scan in local memory

    // Some of these __sync's are unnecessary due to warp synchronous
    // execution.  Right now these are left in to be consistent with
    // opencl version, since that has to execute on platforms where
    // thread groups are not synchronous (i.e. CPUs)
    uint t;
    s_data[idx] = val;     barrier(CLK_LOCAL_MEM_FENCE);
    for (uint i = 1; i < SORT_BLOCK_SIZE; i*=2) {
      t = s_data[idx - i];  barrier(CLK_LOCAL_MEM_FENCE);;
      s_data[idx] += t;      barrier(CLK_LOCAL_MEM_FENCE);;
    }
    return s_data[idx] - val;  // convert inclusive -> exclusive
}

SORTVECTYPE scan4(SORTVECTYPE idata, __local uint* ptr)
{
    SORTVECTYPE val4 = idata;
    SORTVECTYPE sum;

    // Scan the elements in idata within this thread
#if SORT_VECTOR == 2
    sum.x = val4.x;
    uint val = val4.y + sum.x;
#elif SORT_VECTOR == 4
    sum.x = val4.x;
    sum.y = val4.y + sum.x;
    sum.z = val4.z + sum.y;
    uint val = val4.w + sum.z;
#elif SORT_VECTOR == 8
    sum.s0 = val4.s0;
    sum.s1 = val4.s1 + sum.s0;
    sum.s2 = val4.s2 + sum.s1;
    sum.s3 = val4.s3 + sum.s2;
    sum.s4 = val4.s4 + sum.s3;
    sum.s5 = val4.s5 + sum.s4;
    sum.s6 = val4.s6 + sum.s5;
    uint val = val4.s7 + sum.s6;
#endif

    // Now scan those sums across the local work group
    val = scanLSB(val, ptr);

#if SORT_VECTOR == 2
    val4.x = val;
    val4.y = val + sum.x;
#elif SORT_VECTOR == 4
    val4.x = val;
    val4.y = val + sum.x;
    val4.z = val + sum.y;
    val4.w = val + sum.z;
#elif SORT_VECTOR == 8
    val4.s0 = val;
    val4.s1 = val + sum.s0;
    val4.s2 = val + sum.s1;
    val4.s3 = val + sum.s2;
    val4.s4 = val + sum.s3;
    val4.s5 = val + sum.s4;
    val4.s6 = val + sum.s5;
    val4.s7 = val + sum.s6;
#endif

    return val4;
}

//----------------------------------------------------------------------------
//
// radixSortBlocks sorts all blocks of data independently in shared
// memory.  Each thread block (CTA) sorts one block of SORT_VECTOR*CTA_SIZE elements
//
// The radix sort is done in two stages.  This stage calls radixSortBlock
// on each block independently, sorting on the basis of bits
// (startbit) -> (startbit + nbits)
//----------------------------------------------------------------------------

__kernel void radixSortBlocks(const uint nbits,
                              const uint startbit,
                              __global SORTVECTYPE* keysOut,
                              __global SORTVECTYPE* valuesOut,
                              __global SORTVECTYPE* keysIn,
                              __global SORTVECTYPE* valuesIn)
{
    __local uint sMem[SORT_VECTOR*SORT_BLOCK_SIZE];

    // Get Indexing information
    const uint i = get_global_id(0);
    const uint tid = get_local_id(0);
    const uint localSize = get_local_size(0);

    // Load keys and vals from global memory
    SORTVECTYPE key, value;
    key = keysIn[i];
    value = valuesIn[i];

    __local uint numtrue;

    // For each of the 4 bits
    for(uint shift = startbit; shift < (startbit + nbits); ++shift)
    {
        // Check if the LSB is 0
        SORTVECTYPE lsb;
#if SORT_VECTOR == 2
        lsb.x = !((key.x >> shift) & 0x1);
        lsb.y = !((key.y >> shift) & 0x1);
#elif SORT_VECTOR == 4
        lsb.x = !((key.x >> shift) & 0x1);
        lsb.y = !((key.y >> shift) & 0x1);
        lsb.z = !((key.z >> shift) & 0x1);
        lsb.w = !((key.w >> shift) & 0x1);
#elif SORT_VECTOR == 8
        lsb.s0 = !((key.s0 >> shift) & 0x1);
        lsb.s1 = !((key.s1 >> shift) & 0x1);
        lsb.s2 = !((key.s2 >> shift) & 0x1);
        lsb.s3 = !((key.s3 >> shift) & 0x1);
        lsb.s4 = !((key.s4 >> shift) & 0x1);
        lsb.s5 = !((key.s5 >> shift) & 0x1);
        lsb.s6 = !((key.s6 >> shift) & 0x1);
        lsb.s7 = !((key.s7 >> shift) & 0x1);
#endif

        // Do an exclusive scan of how many elems have 0's in the LSB
        // When this is finished, address.n will contain the number of
        // elems with 0 in the LSB which precede elem n
        SORTVECTYPE address = scan4(lsb, sMem);

        // Store the total number of elems with an LSB of 0
        // to shared mem
        if (tid == localSize - 1)
        {
#if SORT_VECTOR == 2
          numtrue = address.y + lsb.y;
#elif SORT_VECTOR == 4
            numtrue = address.w + lsb.w;
#elif SORT_VECTOR == 8
            numtrue = address.s7 + lsb.s7;
#endif
        }
        barrier(CLK_LOCAL_MEM_FENCE);;

        // Determine rank -- position in the block
        // If you are a 0 --> your position is the scan of 0's
        // If you are a 1 --> your position is calculated as below
        SORTVECTYPE rank;
        const int idx = tid*SORT_VECTOR;
#if SORT_VECTOR == 2
        rank.x = lsb.x ? address.x : numtrue + idx     - address.x;
        rank.y = lsb.y ? address.y : numtrue + idx + 1 - address.y;
#elif SORT_VECTOR == 4
        rank.x = lsb.x ? address.x : numtrue + idx     - address.x;
        rank.y = lsb.y ? address.y : numtrue + idx + 1 - address.y;
        rank.z = lsb.z ? address.z : numtrue + idx + 2 - address.z;
        rank.w = lsb.w ? address.w : numtrue + idx + 3 - address.w;
#elif SORT_VECTOR == 8
        rank.s0 = lsb.s0 ? address.s0 : numtrue + idx     - address.s0;
        rank.s1 = lsb.s1 ? address.s1 : numtrue + idx + 1 - address.s1;
        rank.s2 = lsb.s2 ? address.s2 : numtrue + idx + 2 - address.s2;
        rank.s3 = lsb.s3 ? address.s3 : numtrue + idx + 3 - address.s3;
        rank.s4 = lsb.s4 ? address.s4 : numtrue + idx + 4 - address.s4;
        rank.s5 = lsb.s5 ? address.s5 : numtrue + idx + 5 - address.s5;
        rank.s6 = lsb.s6 ? address.s6 : numtrue + idx + 6 - address.s6;
        rank.s7 = lsb.s7 ? address.s7 : numtrue + idx + 7 - address.s7;
#endif

        // Scatter keys into local mem
#if SORT_VECTOR == 2
        sMem[(rank.x & 1) * localSize + (rank.x >> 1)] = key.x;
        sMem[(rank.y & 1) * localSize + (rank.y >> 1)] = key.y;
#elif SORT_VECTOR == 4
        sMem[(rank.x & 3) * localSize + (rank.x >> 2)] = key.x;
        sMem[(rank.y & 3) * localSize + (rank.y >> 2)] = key.y;
        sMem[(rank.z & 3) * localSize + (rank.z >> 2)] = key.z;
        sMem[(rank.w & 3) * localSize + (rank.w >> 2)] = key.w;
#elif SORT_VECTOR == 8
        sMem[(rank.s0 & 7) * localSize + (rank.s0 >> 3)] = key.s0;
        sMem[(rank.s1 & 7) * localSize + (rank.s1 >> 3)] = key.s1;
        sMem[(rank.s2 & 7) * localSize + (rank.s2 >> 3)] = key.s2;
        sMem[(rank.s3 & 7) * localSize + (rank.s3 >> 3)] = key.s3;
        sMem[(rank.s4 & 7) * localSize + (rank.s4 >> 3)] = key.s4;
        sMem[(rank.s5 & 7) * localSize + (rank.s5 >> 3)] = key.s5;
        sMem[(rank.s6 & 7) * localSize + (rank.s6 >> 3)] = key.s6;
        sMem[(rank.s7 & 7) * localSize + (rank.s7 >> 3)] = key.s7;
#endif
        barrier(CLK_LOCAL_MEM_FENCE);

        // Read keys out of local mem into registers, in prep for
        // write out to global mem
#if SORT_VECTOR == 2
        key.x = sMem[tid];
        key.y = sMem[tid +     localSize];
#elif SORT_VECTOR == 4
        key.x = sMem[tid];
        key.y = sMem[tid +     localSize];
        key.z = sMem[tid + 2 * localSize];
        key.w = sMem[tid + 3 * localSize];
#elif SORT_VECTOR == 8
        key.s0 = sMem[tid];
        key.s1 = sMem[tid +     localSize];
        key.s2 = sMem[tid + 2 * localSize];
        key.s3 = sMem[tid + 3 * localSize];
        key.s4 = sMem[tid + 4 * localSize];
        key.s5 = sMem[tid + 5 * localSize];
        key.s6 = sMem[tid + 6 * localSize];
        key.s7 = sMem[tid + 7 * localSize];
#endif
        barrier(CLK_LOCAL_MEM_FENCE);;

        // Scatter values into local mem
#if SORT_VECTOR == 2
        sMem[(rank.x & 1) * localSize + (rank.x >> 1)] = value.x;
        sMem[(rank.y & 1) * localSize + (rank.y >> 1)] = value.y;
#elif SORT_VECTOR == 4
        sMem[(rank.x & 3) * localSize + (rank.x >> 2)] = value.x;
        sMem[(rank.y & 3) * localSize + (rank.y >> 2)] = value.y;
        sMem[(rank.z & 3) * localSize + (rank.z >> 2)] = value.z;
        sMem[(rank.w & 3) * localSize + (rank.w >> 2)] = value.w;
#elif SORT_VECTOR == 8
        sMem[(rank.s0 & 7) * localSize + (rank.s0 >> 3)] = value.s0;
        sMem[(rank.s1 & 7) * localSize + (rank.s1 >> 3)] = value.s1;
        sMem[(rank.s2 & 7) * localSize + (rank.s2 >> 3)] = value.s2;
        sMem[(rank.s3 & 7) * localSize + (rank.s3 >> 3)] = value.s3;
        sMem[(rank.s4 & 7) * localSize + (rank.s4 >> 3)] = value.s4;
        sMem[(rank.s5 & 7) * localSize + (rank.s5 >> 3)] = value.s5;
        sMem[(rank.s6 & 7) * localSize + (rank.s6 >> 3)] = value.s6;
        sMem[(rank.s7 & 7) * localSize + (rank.s7 >> 3)] = value.s7;
#endif
        barrier(CLK_LOCAL_MEM_FENCE);;

        // Read keys out of local mem into registers, in prep for
        // write out to global mem
#if SORT_VECTOR == 2
        value.x = sMem[tid];
        value.y = sMem[tid +     localSize];
#elif SORT_VECTOR == 4
        value.x = sMem[tid];
        value.y = sMem[tid +     localSize];
        value.z = sMem[tid + 2 * localSize];
        value.w = sMem[tid + 3 * localSize];
#elif SORT_VECTOR == 8
        value.s0 = sMem[tid];
        value.s1 = sMem[tid +     localSize];
        value.s2 = sMem[tid + 2 * localSize];
        value.s3 = sMem[tid + 3 * localSize];
        value.s4 = sMem[tid + 4 * localSize];
        value.s5 = sMem[tid + 5 * localSize];
        value.s6 = sMem[tid + 6 * localSize];
        value.s7 = sMem[tid + 7 * localSize];
#endif
        barrier(CLK_LOCAL_MEM_FENCE);;
    }
    keysOut[i]   = key;
    valuesOut[i] = value;
}

//----------------------------------------------------------------------------
// Given an array with blocks sorted according to a 4-bit radix group, each
// block counts the number of keys that fall into each radix in the group, and
// finds the starting offset of each radix in the block.  It then writes the
// radix counts to the counters array, and the starting offsets to the
// blockOffsets array.
//
//----------------------------------------------------------------------------
__kernel void findRadixOffsets(__global SCANVECTYPE* keys,
        __global uint* counters,
        __global uint* blockOffsets,
        uint startbit,
        uint numElements,
        uint totalBlocks)
{
    __local uint  sStartPointers[16];
    __local uint sRadix1[SCAN_VECTOR*SCAN_BLOCK_SIZE];

    uint gid = get_group_id(0);
    uint tid = get_local_id(0);
    uint localSize = get_local_size(0);
    uint i = get_global_id(0);

    SCANVECTYPE radix2;
    radix2 = keys[i];

#if SCAN_VECTOR == 2
    sRadix1[2 * tid]     = (radix2.x >> startbit) & 0xF;
    sRadix1[2 * tid + 1] = (radix2.y >> startbit) & 0xF;
#elif SCAN_VECTOR == 4
    sRadix1[4 * tid]      = (radix2.x >> startbit) & 0xF;
    sRadix1[4 * tid + 1]  = (radix2.y >> startbit) & 0xF;
    sRadix1[4 * tid + 2]  = (radix2.z >> startbit) & 0xF;
    sRadix1[4 * tid + 3]  = (radix2.w >> startbit) & 0xF;
#elif SCAN_VECTOR == 8
    sRadix1[8 * tid]      = (radix2.s0 >> startbit) & 0xF;
    sRadix1[8 * tid + 1]  = (radix2.s1 >> startbit) & 0xF;
    sRadix1[8 * tid + 2]  = (radix2.s2 >> startbit) & 0xF;
    sRadix1[8 * tid + 3]  = (radix2.s3 >> startbit) & 0xF;
    sRadix1[8 * tid + 4]  = (radix2.s4 >> startbit) & 0xF;
    sRadix1[8 * tid + 5]  = (radix2.s5 >> startbit) & 0xF;
    sRadix1[8 * tid + 6]  = (radix2.s6 >> startbit) & 0xF;
    sRadix1[8 * tid + 7]  = (radix2.s7 >> startbit) & 0xF;
#endif

    // Finds the position where the sRadix1 entries differ and stores start
    // index for each radix.
    if(tid < 16)
    {
        sStartPointers[tid] = 0;
    }
    barrier(CLK_LOCAL_MEM_FENCE);;

    if((tid > 0) && (sRadix1[tid] != sRadix1[tid - 1]) )
    {
        sStartPointers[sRadix1[tid]] = tid;
    }
    if(sRadix1[tid + localSize] != sRadix1[tid + localSize - 1])
    {
        sStartPointers[sRadix1[tid + localSize]] = tid + localSize;
    }

#if SCAN_VECTOR == 4 || SCAN_VECTOR == 8
    if (sRadix1[tid + 2*localSize] != sRadix1[tid + 2*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 2*localSize]] = tid + 2*localSize;
    }
    if (sRadix1[tid + 3*localSize] != sRadix1[tid + 3*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 3*localSize]] = tid + 3*localSize;
    }
#endif
#if SCAN_VECTOR == 8
    if (sRadix1[tid + 4*localSize] != sRadix1[tid + 4*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 4*localSize]] = tid + 4*localSize;
    }
    if (sRadix1[tid + 5*localSize] != sRadix1[tid + 5*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 5*localSize]] = tid + 5*localSize;
    }
    if (sRadix1[tid + 6*localSize] != sRadix1[tid + 6*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 6*localSize]] = tid + 6*localSize;
    }
    if (sRadix1[tid + 7*localSize] != sRadix1[tid + 7*localSize - 1])
    {
      sStartPointers[sRadix1[tid + 7*localSize]] = tid + 7*localSize;
    }
#endif
    barrier(CLK_LOCAL_MEM_FENCE);;

    if(tid < 16)
    {
        blockOffsets[gid*16 + tid] = sStartPointers[tid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);;

    // Compute the sizes of each block.
    if((tid > 0) && (sRadix1[tid] != sRadix1[tid - 1]) )
    {
        sStartPointers[sRadix1[tid - 1]] =
            tid - sStartPointers[sRadix1[tid - 1]];
    }
    if(sRadix1[tid + localSize] != sRadix1[tid + localSize - 1] )
    {
        sStartPointers[sRadix1[tid + localSize - 1]] =
            tid + localSize - sStartPointers[sRadix1[tid +
                                                         localSize - 1]];
    }
#if SCAN_VECTOR == 4 || SCAN_VECTOR == 8
    if(sRadix1[tid + 2*localSize] != sRadix1[tid + 2*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 2*localSize - 1]] =
            tid + 2*localSize - sStartPointers[sRadix1[tid + 2*localSize - 1]];
    }
    if(sRadix1[tid + 3*localSize] != sRadix1[tid + 3*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 3*localSize - 1]] =
            tid + 3*localSize - sStartPointers[sRadix1[tid + 3*localSize - 1]];
    }
#endif
#if SCAN_VECTOR == 8
    if(sRadix1[tid + 4*localSize] != sRadix1[tid + 4*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 4*localSize - 1]] =
            tid + 4*localSize - sStartPointers[sRadix1[tid + 4*localSize - 1]];
    }
    if(sRadix1[tid + 5*localSize] != sRadix1[tid + 5*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 5*localSize - 1]] =
            tid + 5*localSize - sStartPointers[sRadix1[tid + 5*localSize - 1]];
    }
    if(sRadix1[tid + 6*localSize] != sRadix1[tid + 6*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 6*localSize - 1]] =
            tid + 6*localSize - sStartPointers[sRadix1[tid + 6*localSize - 1]];
    }
    if(sRadix1[tid + 7*localSize] != sRadix1[tid + 7*localSize - 1] )
    {
        sStartPointers[sRadix1[tid + 7*localSize - 1]] =
            tid + 7*localSize - sStartPointers[sRadix1[tid + 7*localSize - 1]];
    }
#endif

#if SCAN_VECTOR == 2
    if(tid == localSize - 1)
    {
        sStartPointers[sRadix1[2 * localSize - 1]] =
            2 * localSize - sStartPointers[sRadix1[2 * localSize - 1]];
    }
#elif SCAN_VECTOR == 4
    if (tid == localSize - 1)
    {
        sStartPointers[sRadix1[4 * localSize - 1]] =
            4 * localSize - sStartPointers[sRadix1[4 * localSize - 1]];
    }
#elif SCAN_VECTOR == 8
    if (tid == localSize - 1)
    {
        sStartPointers[sRadix1[8 * localSize - 1]] =
            8 * localSize - sStartPointers[sRadix1[8 * localSize - 1]];
    }
#endif
    barrier(CLK_LOCAL_MEM_FENCE);;

    if(tid < 16)
    {
        counters[tid * totalBlocks + gid] = sStartPointers[tid];
    }
}

//----------------------------------------------------------------------------
// reorderData shuffles data in the array globally after the radix offsets
// have been found. On compute version 1.1 and earlier GPUs, this code depends
// on SORT_BLOCK_SIZE being 16 * number of radices (i.e. 16 * 2^nbits).
//----------------------------------------------------------------------------
__kernel void reorderData(uint  startbit,
                          __global uint  *outKeys,
                          __global uint  *outValues,
                          __global SCANVECTYPE *keys,
                          __global SCANVECTYPE *values,
                          __global uint  *blockOffsets,
                          __global uint  *offsets,
                          __global uint  *sizes,
                          uint  totalBlocks)
{
    __local SCANVECTYPE sKeys2[SCAN_BLOCK_SIZE];
    __local SCANVECTYPE sValues2[SCAN_BLOCK_SIZE];
    __local uint  sOffsets[16];
    __local uint  sBlockOffsets[16];
    __local uint* sKeys1   = (__local uint*) &sKeys2;
    __local uint* sValues1 = (__local uint*) &sValues2;

    uint gid = get_group_id(0);
    uint tid = get_local_id(0);
    uint localSize = get_local_size(0);
    uint i = get_global_id(0);

    sKeys2[tid]   = keys[i];
    sValues2[tid] = values[i];

    if(tid < 16)
    {
        sOffsets[tid]      = offsets[tid * totalBlocks +
                                             gid];
        sBlockOffsets[tid] = blockOffsets[gid * 16 + tid];
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    uint radix = (sKeys1[tid] >> startbit) & 0xF;
    uint globalOffset = sOffsets[radix] + tid - sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid];
    outValues[globalOffset] = sValues1[tid];

    radix = (sKeys1[tid + localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + localSize];
    outValues[globalOffset] = sValues1[tid + localSize];

#if SCAN_VECTOR == 4 || SCAN_VECTOR == 8
    radix = (sKeys1[tid + 2*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 2*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 2*localSize];
    outValues[globalOffset] = sValues1[tid + 2*localSize];

    radix = (sKeys1[tid + 3*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 3*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 3*localSize];
    outValues[globalOffset] = sValues1[tid + 3*localSize];
#endif
#if SCAN_VECTOR == 8
    radix = (sKeys1[tid + 4*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 4*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 4*localSize];
    outValues[globalOffset] = sValues1[tid + 4*localSize];

    radix = (sKeys1[tid + 5*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 5*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 5*localSize];
    outValues[globalOffset] = sValues1[tid + 5*localSize];

    radix = (sKeys1[tid + 6*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 6*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 6*localSize];
    outValues[globalOffset] = sValues1[tid + 6*localSize];

    radix = (sKeys1[tid + 7*localSize] >> startbit) & 0xF;
    globalOffset = sOffsets[radix] + tid + 7*localSize -
                   sBlockOffsets[radix];

    outKeys[globalOffset]   = sKeys1[tid + 7*localSize];
    outValues[globalOffset] = sValues1[tid + 7*localSize];
#endif
}

uint scanLocalMem(const uint val, __local uint* s_data)
{
    // Set first half of shared mem to 0
    int idx = get_local_id(0);
    s_data[idx] = 0.0f;
    barrier(CLK_LOCAL_MEM_FENCE);

    // Set 2nd half to thread local sum
    idx += get_local_size(0);

    // Some of these __sync's are unnecessary due to warp synchronous
    // execution.  Right now these are left in to be consistent with
    // opencl version, since that has to execute on platforms where
    // thread groups are not synchronous (i.e. CPUs)
    uint t;
    s_data[idx] = val;     barrier(CLK_LOCAL_MEM_FENCE);
    for (uint i = 1; i < SCAN_BLOCK_SIZE; i*=2) {
      t = s_data[idx - i];  barrier(CLK_LOCAL_MEM_FENCE);
      s_data[idx] += t;      barrier(CLK_LOCAL_MEM_FENCE);
    }
    return s_data[idx-1];
}

__kernel void
scan(__global uint *g_odata,  __global uint* g_idata,
     __global uint* g_blockSums, const int n,
     const int fullBlock, const int storeSum)
{
    __local uint s_data[2*SCAN_BLOCK_SIZE];

    // Load data into shared mem
    SORTVECTYPE tempData;
    SORTVECTYPE threadScanT;
    uint res;
    __global SORTVECTYPE* inData  = (__global SORTVECTYPE*) g_idata;

    const int gid = get_global_id(0);
    const int tid = get_local_id(0);
    const int i = gid * SORT_VECTOR;

    // If possible, read from global mem in a SORTVECTYPE chunk
#if SORT_VECTOR == 2
    if (fullBlock || i + 1 < n)
    {
        // scan the 2 elems read in from global
        tempData       = inData[gid];
        threadScanT.x = tempData.x;
        threadScanT.y = tempData.y + threadScanT.x;
        res = threadScanT.y;
    }
    else
    {   // if not, read individual uints, scan & store in lmem
        threadScanT.x = (i < n) ? g_idata[i] : 0.0f;
        threadScanT.y = ((i+1 < n) ? g_idata[i+1] : 0.0f) + threadScanT.x;
        res = threadScanT.y;
    }
#elif SORT_VECTOR == 4
    if (fullBlock || i + 3 < n)
    {
        // scan the 4 elems read in from global
        tempData       = inData[gid];
        threadScanT.x = tempData.x;
        threadScanT.y = tempData.y + threadScanT.x;
        threadScanT.z = tempData.z + threadScanT.y;
        threadScanT.w = tempData.w + threadScanT.z;
        res = threadScanT.w;
    }
    else
    {   // if not, read individual uints, scan & store in lmem
        threadScanT.x = (i < n) ? g_idata[i] : 0.0f;
        threadScanT.y = ((i+1 < n) ? g_idata[i+1] : 0.0f) + threadScanT.x;
        threadScanT.z = ((i+2 < n) ? g_idata[i+2] : 0.0f) + threadScanT.y;
        threadScanT.w = ((i+3 < n) ? g_idata[i+3] : 0.0f) + threadScanT.z;
        res = threadScanT.w;
    }
#elif SORT_VECTOR == 8
    if (fullBlock || i + 7 < n)
    {
        // scan the 8 elems read in from global
        tempData       = inData[gid];
        threadScanT.s0 = tempData.s0;
        threadScanT.s1 = tempData.s1 + threadScanT.s0;
        threadScanT.s2 = tempData.s2 + threadScanT.s1;
        threadScanT.s3 = tempData.s3 + threadScanT.s2;
        threadScanT.s4 = tempData.s4 + threadScanT.s3;
        threadScanT.s5 = tempData.s5 + threadScanT.s4;
        threadScanT.s6 = tempData.s6 + threadScanT.s5;
        threadScanT.s7 = tempData.s7 + threadScanT.s6;
        res = threadScanT.s7;
    }
    else
    {   // if not, read individual uints, scan & store in lmem
        threadScanT.s0 = (i < n) ? g_idata[i] : 0.0f;
        threadScanT.s1 = ((i+1 < n) ? g_idata[i+1] : 0.0f) + threadScanT.s0;
        threadScanT.s2 = ((i+2 < n) ? g_idata[i+2] : 0.0f) + threadScanT.s1;
        threadScanT.s3 = ((i+3 < n) ? g_idata[i+3] : 0.0f) + threadScanT.s2;
        threadScanT.s4 = ((i+4 < n) ? g_idata[i+4] : 0.0f) + threadScanT.s3;
        threadScanT.s5 = ((i+5 < n) ? g_idata[i+5] : 0.0f) + threadScanT.s4;
        threadScanT.s6 = ((i+6 < n) ? g_idata[i+6] : 0.0f) + threadScanT.s5;
        threadScanT.s7 = ((i+7 < n) ? g_idata[i+7] : 0.0f) + threadScanT.s6;
        res = threadScanT.s7;
    }
#endif

    res = scanLocalMem(res, s_data);
    barrier(CLK_LOCAL_MEM_FENCE);

    // If we have to store the sum for the block, have the last work item
    // in the block write it out
    if (storeSum && tid == get_local_size(0)-1) {
#if SORT_VECTOR == 2
        g_blockSums[get_group_id(0)] = res + threadScanT.y;
#elif SORT_VECTOR == 4
        g_blockSums[get_group_id(0)] = res + threadScanT.w;
#elif SORT_VECTOR == 8
        g_blockSums[get_group_id(0)] = res + threadScanT.s7;
#endif
    }

    // write results to global memory
    __global SORTVECTYPE* outData = (__global SORTVECTYPE*) g_odata;

#if SORT_VECTOR == 2
    tempData.x = res;
    tempData.y = res + threadScanT.x;

    if (fullBlock || i + 1 < n)
    {
        outData[gid] = tempData;
    }
    else
    {
        if ( i    < n) { g_odata[i]   = tempData.x; }
    }
#elif SORT_VECTOR == 4
    tempData.x = res;
    tempData.y = res + threadScanT.x;
    tempData.z = res + threadScanT.y;
    tempData.w = res + threadScanT.z;

    if (fullBlock || i + 3 < n)
    {
        outData[gid] = tempData;
    }
    else
    {
        if ( i    < n) { g_odata[i]   = tempData.x;
        if ((i+1) < n) { g_odata[i+1] = tempData.y;
        if ((i+2) < n) { g_odata[i+2] = tempData.z; } } }
    }
#elif SORT_VECTOR == 8
    tempData.s0 = res;
    tempData.s1 = res + threadScanT.s0;
    tempData.s2 = res + threadScanT.s1;
    tempData.s3 = res + threadScanT.s2;
    tempData.s4 = res + threadScanT.s3;
    tempData.s5 = res + threadScanT.s4;
    tempData.s6 = res + threadScanT.s5;
    tempData.s7 = res + threadScanT.s6;

    if (fullBlock || i + 7 < n)
    {
        outData[gid] = tempData;
    }
    else
    {
        if ( i    < n) { g_odata[i]   = tempData.s0;
        if ((i+1) < n) { g_odata[i+1] = tempData.s1;
        if ((i+2) < n) { g_odata[i+2] = tempData.s2;
        if ((i+3) < n) { g_odata[i+3] = tempData.s3;
        if ((i+4) < n) { g_odata[i+4] = tempData.s4;
        if ((i+5) < n) { g_odata[i+5] = tempData.s5;
        if ((i+6) < n) { g_odata[i+6] = tempData.s6; } } } } } } }
    }
#endif
}

__kernel void
vectorAddUniform4(__global uint *d_vector, __global const uint *d_uniforms,
        const int n)
{
    __local uint uni[1];

    if (get_local_id(0) == 0)
    {
        uni[0] = d_uniforms[get_group_id(0)];
    }

    unsigned int address = get_local_id(0) + (get_group_id(0) *
            get_local_size(0) * SORT_VECTOR);

    barrier(CLK_LOCAL_MEM_FENCE);

    // SORT_VECTOR elems per thread
    for (int i = 0; i < SORT_VECTOR && address < n; i++)
    {
        d_vector[address] += uni[0];
        address += get_local_size(0);
    }
}


optimiiizeee!

optimiiizeee!

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

#define ITERATIONS 600000
#define timestep 0.0001
#define g 9.81
#define l1 0.285
#define k1 0.0284
#define k2 0.118
#define k3 0.448
#define k4 0.00441
#define k5 0.0254
#define FLOAT_PI 3.14159265358979323846264338327950288419716939938

// Data structure for current values during sim
typedef struct {
  float phi1;
  float phi2;
  float p1;
  float p2;
} pstate;

//////// Equations of motion and energy ////////
// Differential equations
inline float fphi1(const pstate s) {
  float phidiff = cos(s.phi1-s.phi2);
  return (k4*s.p1-k5*l1*phidiff*s.p2) / (k1*k4+l1*l1*(k3*k4-k5*k5*phidiff*phidiff));
}
inline float fphi2(const pstate s) {
  float phidiff = cos(s.phi1-s.phi2);
  return (k1*s.p2+l1*(k3*l1*s.p2-k5*s.p1*phidiff)) / (k1*k4+l1*l1*(k3*k4-k5*k5*phidiff*phidiff));
}
inline float fp1(const pstate s, float newphi1, float newphi2) {
  return -l1*newphi1*newphi2*k5*sin(s.phi1-s.phi2) - g*k2*sin(s.phi1) - g*l1*k3*sin(s.phi1);
}
inline float fp2(const pstate s, float newphi1, float newphi2) {
  return l1*newphi1*newphi2*k5*sin(s.phi1-s.phi2) - g*k5*sin(s.phi2);
}

pstate calc_s_(pstate s) {
  pstate s_;
  s_.phi1 = fphi1(s);
  s_.phi2 = fphi2(s);
  s_.p1   = fp1(s, s_.phi1, s_.phi2);
  s_.p2   = fp2(s, s_.phi1, s_.phi2);
  return s_;
}

inline pstate mystep(pstate s0, float h) {
  // see http://de.wikipedia.org/wiki/Klassisches_Runge-Kutta-Verfahren
  // sxd is the first derivation of sx

  pstate s0d = calc_s_(s0);

  pstate sa;
  sa.phi1 = s0.phi1 + h/2 * s0d.phi1;
  sa.phi2 = s0.phi2 + h/2 * s0d.phi2;
  sa.p1 =   s0.p1   + h/2 * s0d.p1;
  sa.p2 =   s0.p2   + h/2 * s0d.p2;
  pstate sad = calc_s_(sa);

  pstate sb;
  sb.phi1 = s0.phi1 + h/2 * sad.phi1;
  sb.phi2 = s0.phi2 + h/2 * sad.phi2;
  sb.p1 =   s0.p1   + h/2 * sad.p1;
  sb.p2 =   s0.p2   + h/2 * sad.p2;
  pstate sbd = calc_s_(sb);

  pstate sc;
  sc.phi1 = s0.phi1 + h   * sbd.phi1;
  sc.phi2 = s0.phi2 + h   * sbd.phi2;
  sc.p1 =   s0.p1   + h   * sbd.p1;
  sc.p2 =   s0.p2   + h   * sbd.p2;
  pstate scd = calc_s_(sc);

  pstate s1;
  s1.phi1 = s0.phi1 + h/6 * (s0d.phi1 + 2*(sad.phi1+sbd.phi1) + scd.phi1);
  s1.phi2 = s0.phi2 + h/6 * (s0d.phi2 + 2*(sad.phi2+sbd.phi2) + scd.phi2);
  s1.p1 = s0.p1 + h/6 * (s0d.p1 + 2*(sad.p1+sbd.p1) + scd.p1);
  s1.p2 = s0.p2 + h/6 * (s0d.p2 + 2*(sad.p2+sbd.p2) + scd.p2);
  return s1;
}

__kernel void simulate_pendulum(__global float *in, __global unsigned char *out_) {
  // grab our stuff from the environment
  size_t tid = get_global_id(0);
  float phi1_0 = in[2*tid+0];
  float phi2_0 = in[2*tid+1];
  __global unsigned char *out = out_+3*tid;
  unsigned char loopings_left=0, loopings_right=0;

  pstate s = {.phi1=phi1_0, .phi2 = phi2_0, .p1 = 0, .p2 = 0 };
  float phi2 = floor((fabs(s.phi2)+FLOAT_PI) / (2 * FLOAT_PI));
  float phi2new;
  int i;
  for (i=0; i<ITERATIONS; i++) {
    s = mystep(s, timestep);

    phi2new = floor((fabs(s.phi2)+FLOAT_PI) / (2 * FLOAT_PI));
    if (phi2new > phi2) loopings_left ++;
    else if (phi2new < phi2) loopings_right ++;
    phi2 = phi2new;
  }

  out[0] = 0;
  out[1] = loopings_left;
  out[2] = loopings_right;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

#define ITERATIONS 600000
#define timestep 0.0001
#define g 9.81
#define l1 0.285
#define k1 0.0284
#define k2 0.118
#define k3 0.448
#define k4 0.00441
#define k5 0.0254
#define FLOAT_PI 3.14159265358979323846264338327950288419716939938

// Data structure for current values during sim
typedef struct {
  float phi1;
  float phi2;
  float p1;
  float p2;
} pstate;

//////// Equations of motion and energy ////////
// Differential equations
inline float fphi1(const pstate s) {
  float phidiff = cos(s.phi1-s.phi2);
  return (k4*s.p1-k5*l1*phidiff*s.p2) / (k1*k4+l1*l1*(k3*k4-k5*k5*phidiff*phidiff));
}
inline float fphi2(const pstate s) {
  float phidiff = cos(s.phi1-s.phi2);
  return (k1*s.p2+l1*(k3*l1*s.p2-k5*s.p1*phidiff)) / (k1*k4+l1*l1*(k3*k4-k5*k5*phidiff*phidiff));
}
float fp1(const pstate s) {
  return -l1*fphi1(s)*fphi2(s)*k5*sin(s.phi1-s.phi2) - g*k2*sin(s.phi1) - g*l1*k3*sin(s.phi1);
}
float fp2(const pstate s) {
  return l1*fphi1(s)*fphi2(s)*k5*sin(s.phi1-s.phi2) - g*k5*sin(s.phi2);
}

pstate calc_s_(pstate s) {
  pstate s_;
  s_.phi1 = fphi1(s);
  s_.phi2 = fphi2(s);
  s_.p1   = fp1(s);
  s_.p2   = fp2(s);
  return s_;
}

inline pstate mystep(pstate s0, float h) {
  // see http://de.wikipedia.org/wiki/Klassisches_Runge-Kutta-Verfahren
  // sxd is the first derivation of sx

  pstate s0d = calc_s_(s0);

  pstate sa;
  sa.phi1 = s0.phi1 + h/2 * s0d.phi1;
  sa.phi2 = s0.phi2 + h/2 * s0d.phi2;
  sa.p1 =   s0.p1   + h/2 * s0d.p1;
  sa.p2 =   s0.p2   + h/2 * s0d.p2;
  pstate sad = calc_s_(sa);

  pstate sb;
  sb.phi1 = s0.phi1 + h/2 * sad.phi1;
  sb.phi2 = s0.phi2 + h/2 * sad.phi2;
  sb.p1 =   s0.p1   + h/2 * sad.p1;
  sb.p2 =   s0.p2   + h/2 * sad.p2;
  pstate sbd = calc_s_(sb);

  pstate sc;
  sc.phi1 = s0.phi1 + h   * sbd.phi1;
  sc.phi2 = s0.phi2 + h   * sbd.phi2;
  sc.p1 =   s0.p1   + h   * sbd.p1;
  sc.p2 =   s0.p2   + h   * sbd.p2;
  pstate scd = calc_s_(sc);

  pstate s1;
  s1.phi1 = s0.phi1 + h/6 * (s0d.phi1 + 2*(sad.phi1+sbd.phi1) + scd.phi1);
  s1.phi2 = s0.phi2 + h/6 * (s0d.phi2 + 2*(sad.phi2+sbd.phi2) + scd.phi2);
  s1.p1 = s0.p1 + h/6 * (s0d.p1 + 2*(sad.p1+sbd.p1) + scd.p1);
  s1.p2 = s0.p2 + h/6 * (s0d.p2 + 2*(sad.p2+sbd.p2) + scd.p2);
  return s1;
}

__kernel void simulate_pendulum(__global float *in, __global unsigned char *out_) {
  // grab our stuff from the environment
  size_t tid = get_global_id(0);
  float phi1_0 = in[2*tid+0];
  float phi2_0 = in[2*tid+1];
  __global unsigned char *out = out_+3*tid;
  unsigned char loopings_left=0, loopings_right=0;

  pstate s = {.phi1=phi1_0, .phi2 = phi2_0, .p1 = 0, .p2 = 0 };
  float phi2, phi2new;
  int i;
  for (i=0; i<ITERATIONS; i++) {
    phi2 = floor((fabs(s.phi2)+FLOAT_PI) / (2 * FLOAT_PI));

    s = mystep(s, timestep);

    phi2new = floor((fabs(s.phi2)+FLOAT_PI) / (2 * FLOAT_PI));
    if (phi2new > phi2) loopings_left ++;
    else if (phi2new < phi2) loopings_right ++;
  }

  out[0] = 0;
  out[1] = loopings_left;
  out[2] = loopings_right;
}


Fix numeric error

Fix numeric error

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

/*************************************************************************
DEFINES
**************************************************************************/
#define PI 3.14159265358979323846f
#define KERNEL __kernel
#define GLOBAL __global
#define INLINE __attribute__((always_inline))
#define HIT_MARKER 1
#define MISS_MARKER -1
#define INVALID_IDX -1

/*************************************************************************
EXTENSIONS
**************************************************************************/
#ifdef AMD_MEDIA_OPS
#pragma OPENCL EXTENSION cl_amd_media_ops2 : enable
#endif

/*************************************************************************
TYPES
**************************************************************************/

// Axis aligned bounding box
typedef struct
{
    float4 pmin;
    float4 pmax;
} bbox;

// Ray definition
typedef struct
{
    float4 o;
    float4 d;
    int2 extra;
    int2 padding;
} ray;

// Intersection definition
typedef struct
{
    int shape_id;
    int prim_id;
    int2 padding;

    float4 uvwt;
} Intersection;

/*************************************************************************
HELPER FUNCTIONS
**************************************************************************/
INLINE
int ray_get_mask(ray const* r)
{
    return r->extra.x;
}

INLINE
int ray_is_active(ray const* r)
{
    return r->extra.y;
}

INLINE
float ray_get_maxt(ray const* r)
{
    return r->o.w;
}

INLINE
float ray_get_time(ray const* r)
{
    return r->d.w;
}

/*************************************************************************
FUNCTIONS
**************************************************************************/
#ifndef APPLE
INLINE
float4 make_float4(float x, float y, float z, float w)
{
    float4 res;
    res.x = x;
    res.y = y;
    res.z = z;
    res.w = w;
    return res;
}
INLINE
float3 make_float3(float x, float y, float z)
{
    float3 res;
    res.x = x;
    res.y = y;
    res.z = z;
    return res;
}
INLINE
float2 make_float2(float x, float y)
{
    float2 res;
    res.x = x;
    res.y = y;
    return res;
}
INLINE
int2 make_int2(int x, int y)
{
    int2 res;
    res.x = x;
    res.y = y;
    return res;
}
INLINE
int3 make_int3(int x, int y, int z)
{
    int3 res;
    res.x = x;
    res.y = y;
    res.z = z;
    return res;
}
#endif

INLINE float min3(float a, float b, float c)
{
#ifdef AMD_MEDIA_OPS
    return amd_min3(a, b, c);
#else
    return min(min(a,b), c);
#endif
}

INLINE float max3(float a, float b, float c)
{
#ifdef AMD_MEDIA_OPS
    return amd_max3(a, b, c);
#else
    return max(max(a,b), c);
#endif
}

// Intersect ray against a triangle and return intersection interval value if it is in
// (0, t_max], return t_max otherwise.
INLINE
float fast_intersect_triangle(ray r, float3 v1, float3 v2, float3 v3, float t_max)
{
    float3 const e1 = v2 - v1;
    float3 const e2 = v3 - v1;
    float3 const s1 = cross(r.d.xyz, e2);

    float denom = dot(s1, e1);
    if (denom == 0.f)
    {
        return t_max;
    }

#ifdef USE_SAFE_MATH
    float const invd = 1.f / denom;
#else
    float const invd = native_recip(denom);
#endif

    float3 const d = r.o.xyz - v1;
    float const b1 = dot(d, s1) * invd;
    float3 const s2 = cross(d, e1);
    float const b2 = dot(r.d.xyz, s2) * invd;
    float const temp = dot(e2, s2) * invd;

    if (b1 < 0.f || b1 > 1.f || b2 < 0.f || b1 + b2 > 1.f || temp < 0.f || temp > t_max)
    {
        return t_max;
    }
    else
    {
        return temp;
    }
}

INLINE
float3 safe_invdir(ray r)
{
    float const dirx = r.d.x;
    float const diry = r.d.y;
    float const dirz = r.d.z;
    float const ooeps = 1e-8;
    float3 invdir;
    invdir.x = 1.0f / (fabs(dirx) > ooeps ? dirx : copysign(ooeps, dirx));
    invdir.y = 1.0f / (fabs(diry) > ooeps ? diry : copysign(ooeps, diry));
    invdir.z = 1.0f / (fabs(dirz) > ooeps ? dirz : copysign(ooeps, dirz));
    return invdir;
}

// Intersect rays vs bbox and return intersection span.
// Intersection criteria is ret.x <= ret.y
INLINE
float2 fast_intersect_bbox1(bbox box, float3 invdir, float3 oxinvdir, float t_max)
{
    float3 const f = mad(box.pmax.xyz, invdir, oxinvdir);
    float3 const n = mad(box.pmin.xyz, invdir, oxinvdir);
    float3 const tmax = max(f, n);
    float3 const tmin = min(f, n);
    float const t1 = min(min3(tmax.x, tmax.y, tmax.z), t_max);
    float const t0 = max(max3(tmin.x, tmin.y, tmin.z), 0.f);
    return make_float2(t0, t1);
}

// Given a point in triangle plane, calculate its barycentrics
INLINE
float2 triangle_calculate_barycentrics(float3 p, float3 v1, float3 v2, float3 v3)
{
    float3 const e1 = v2 - v1;
    float3 const e2 = v3 - v1;
    float3 const e = p - v1;
    float const d00 = dot(e1, e1);
    float const d01 = dot(e1, e2);
    float const d11 = dot(e2, e2);
    float const d20 = dot(e, e1);
    float const d21 = dot(e, e2);

#ifdef USE_SAFE_MATH
    float const invdenom = 1.f / (d00 * d11 - d01 * d01);
#else
    float const invdenom = native_recip(d00 * d11 - d01 * d01);
#endif

    float const b1 = (d11 * d20 - d01 * d21) * invdenom;
    float const b2 = (d00 * d21 - d01 * d20) * invdenom;
    return make_float2(b1, b2);
}

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

/*************************************************************************
DEFINES
**************************************************************************/
#define PI 3.14159265358979323846f
#define KERNEL __kernel
#define GLOBAL __global
#define INLINE __attribute__((always_inline))
#define HIT_MARKER 1
#define MISS_MARKER -1
#define INVALID_IDX -1

/*************************************************************************
EXTENSIONS
**************************************************************************/
#ifdef AMD_MEDIA_OPS
#pragma OPENCL EXTENSION cl_amd_media_ops2 : enable
#endif

/*************************************************************************
TYPES
**************************************************************************/

// Axis aligned bounding box
typedef struct
{
    float4 pmin;
    float4 pmax;
} bbox;

// Ray definition
typedef struct
{
    float4 o;
    float4 d;
    int2 extra;
    int2 padding;
} ray;

// Intersection definition
typedef struct
{
    int shape_id;
    int prim_id;
    int2 padding;

    float4 uvwt;
} Intersection;

/*************************************************************************
HELPER FUNCTIONS
**************************************************************************/
INLINE
int ray_get_mask(ray const* r)
{
    return r->extra.x;
}

INLINE
int ray_is_active(ray const* r)
{
    return r->extra.y;
}

INLINE
float ray_get_maxt(ray const* r)
{
    return r->o.w;
}

INLINE
float ray_get_time(ray const* r)
{
    return r->d.w;
}

/*************************************************************************
FUNCTIONS
**************************************************************************/
#ifndef APPLE
INLINE
float4 make_float4(float x, float y, float z, float w)
{
    float4 res;
    res.x = x;
    res.y = y;
    res.z = z;
    res.w = w;
    return res;
}
INLINE
float3 make_float3(float x, float y, float z)
{
    float3 res;
    res.x = x;
    res.y = y;
    res.z = z;
    return res;
}
INLINE
float2 make_float2(float x, float y)
{
    float2 res;
    res.x = x;
    res.y = y;
    return res;
}
INLINE
int2 make_int2(int x, int y)
{
    int2 res;
    res.x = x;
    res.y = y;
    return res;
}
INLINE
int3 make_int3(int x, int y, int z)
{
    int3 res;
    res.x = x;
    res.y = y;
    res.z = z;
    return res;
}
#endif

INLINE float min3(float a, float b, float c)
{
#ifdef AMD_MEDIA_OPS
    return amd_min3(a, b, c);
#else
    return min(min(a,b), c);
#endif
}

INLINE float max3(float a, float b, float c)
{
#ifdef AMD_MEDIA_OPS
    return amd_max3(a, b, c);
#else
    return max(max(a,b), c);
#endif
}

// Intersect ray against a triangle and return intersection interval value if it is in
// (0, t_max], return t_max otherwise.
INLINE
float fast_intersect_triangle(ray r, float3 v1, float3 v2, float3 v3, float t_max)
{
    float3 const e1 = v2 - v1;
    float3 const e2 = v3 - v1;
    float3 const s1 = cross(r.d.xyz, e2);

#ifdef USE_SAFE_MATH
    float const invd = 1.f / dot(s1, e1);
#else
    float const invd = native_recip(dot(s1, e1));
#endif

    float3 const d = r.o.xyz - v1;
    float const b1 = dot(d, s1) * invd;
    float3 const s2 = cross(d, e1);
    float const b2 = dot(r.d.xyz, s2) * invd;
    float const temp = dot(e2, s2) * invd;

    if (b1 < 0.f || b1 > 1.f || b2 < 0.f || b1 + b2 > 1.f || temp < 0.f || temp > t_max)
    {
        return t_max;
    }
    else
    {
        return temp;
    }
}

INLINE
float3 safe_invdir(ray r)
{
    float const dirx = r.d.x;
    float const diry = r.d.y;
    float const dirz = r.d.z;
    float const ooeps = 1e-8;
    float3 invdir;
    invdir.x = 1.0f / (fabs(dirx) > ooeps ? dirx : copysign(ooeps, dirx));
    invdir.y = 1.0f / (fabs(diry) > ooeps ? diry : copysign(ooeps, diry));
    invdir.z = 1.0f / (fabs(dirz) > ooeps ? dirz : copysign(ooeps, dirz));
    return invdir;
}

// Intersect rays vs bbox and return intersection span.
// Intersection criteria is ret.x <= ret.y
INLINE
float2 fast_intersect_bbox1(bbox box, float3 invdir, float3 oxinvdir, float t_max)
{
    float3 const f = mad(box.pmax.xyz, invdir, oxinvdir);
    float3 const n = mad(box.pmin.xyz, invdir, oxinvdir);
    float3 const tmax = max(f, n);
    float3 const tmin = min(f, n);
    float const t1 = min(min3(tmax.x, tmax.y, tmax.z), t_max);
    float const t0 = max(max3(tmin.x, tmin.y, tmin.z), 0.f);
    return make_float2(t0, t1);
}

// Given a point in triangle plane, calculate its barycentrics
INLINE
float2 triangle_calculate_barycentrics(float3 p, float3 v1, float3 v2, float3 v3)
{
    float3 const e1 = v2 - v1;
    float3 const e2 = v3 - v1;
    float3 const e = p - v1;
    float const d00 = dot(e1, e1);
    float const d01 = dot(e1, e2);
    float const d11 = dot(e2, e2);
    float const d20 = dot(e, e1);
    float const d21 = dot(e, e2);

#ifdef USE_SAFE_MATH
    float const invdenom = 1.f / (d00 * d11 - d01 * d01);
#else
    float const invdenom = native_recip(d00 * d11 - d01 * d01);
#endif

    float const b1 = (d11 * d20 - d01 * d21) * invdenom;
    float const b2 = (d00 * d21 - d01 * d20) * invdenom;
    return make_float2(b1, b2);
}


working great rand_particles_left + need more homogeneous sphere particles repartition

static float back_ease_out(float t, float b, float c, float d) {
    float s = 1.70158f;
    float inner_t = (t / d) - 1.0f;
    return (c * (inner_t * inner_t * ((s + 1.0f) * inner_t + s) + 1.0f) + b);
}

static float elastic_ease_out(float t, float b, float c, float d) {
    if (t == 0.0f) {
        return b;
    }

    float inner_t = t / d;
    if (inner_t == 1.0f) {
        return b + c;
    }

    float p = d * 0.3f;
    float a = c;
    float s = p / 4.0f;
    float temp = (inner_t * d - s) * (2.0f * M_PI_F) / p;
    return (a * pow(2.0f, -10.0f * inner_t) * sin(temp) + c + b);
}

#define EASING_ANIMATION elastic_ease_out

__kernel void update_animation(global float3 const * const restrict from_vec,
                               global float3 const * const restrict to_vec,
                               global float3 * const restrict positions,
                               float time, float duration) {
    size_t const idx = get_global_id(0);
    float3 const from = from_vec[idx];
    float3 const to = to_vec[idx];

    positions[idx] = (float3)(EASING_ANIMATION(time, from.x, to.x - from.x, duration),
                              EASING_ANIMATION(time, from.y, to.y - from.y, duration),
                              EASING_ANIMATION(time, from.z, to.z - from.z, duration));
}

static size_t  xorshift64star(size_t x) {
    x ^= x >> 12; // a
    x ^= x << 25; // b
    x ^= x >> 27; // c
    return x * size_t(2685821657736338717);
}

__kernel void init_rand_sphere_animation(global float3 const * const restrict positions,
                                         global float3 * const restrict from_vec,
                                         global float3 * const restrict to_vec,
                                         global float3 * const restrict velocities) {
    size_t const idx = get_global_id(0);

    float const scaling = 1.f / 20.f;
    size_t const radius = 10;

    size_t const scal_rad = (radius / 2) * 10000;
    float const u = radians((float)(xorshift64star(idx >> 3) % 360));
    float const v = radians((float)(xorshift64star(idx << 2) % 360));
    float const radius_rand = ((float)(xorshift64star(idx >> 2) % scal_rad)
                                + ((radius * 10000) - scal_rad)) / 10000.f;

    // http://www.wolframalpha.com/input/?i=sphere
    float const x = radius_rand * cos(u) * sin(v);
    float const y = radius_rand * sin(u) * sin(v);
    float const z = radius_rand * cos(v);

    from_vec[idx] = positions[idx];
    to_vec[idx] = (float3)(x, y, z);
    to_vec[idx] *= scaling;
    velocities[idx] = (float3)(0.0f, 0.0f, 0.0f);
}

__kernel void init_rand_cube_animation(global float3 const * const restrict positions,
                                       global float3 * const restrict from_vec,
                                       global float3 * const restrict to_vec,
                                       global float3 * const restrict velocities) {
    size_t const idx = get_global_id(0);

    float const scaling = 1.f / 20.f;
    size_t const diameter = 20;
    float3 const center = (float3)(10.f, 10.f, 10.f);
    float const x = (float)(xorshift64star(idx << 3) % (diameter * 10000)) / 10000.f;
    float const y = (float)(xorshift64star(idx >> 2) % (diameter * 10000)) / 10000.f;
    float const z = (float)(xorshift64star(idx << 2) % (diameter * 10000)) / 10000.f;

    from_vec[idx] = positions[idx];
    to_vec[idx] = (float3)(x, y, z);
    to_vec[idx] -= center;
    to_vec[idx] *= scaling;
    velocities[idx] = (float3)(0.0f, 0.0f, 0.0f);
}

__kernel void init_cube_animation(global float3 const * const restrict positions,
                                  global float3 * const restrict from_vec,
                                  global float3 * const restrict to_vec,
                                  global float3 * const restrict velocities) {
    size_t const idx = get_global_id(0);
    size_t const total_parts = get_global_size(0);

    size_t const side_parts = cbrt((float)total_parts); // FIXME compute this one time
    size_t const parts_left = total_parts - (side_parts * side_parts * side_parts);
    float const spacing = 1.0f / (float)side_parts;

    from_vec[idx] = positions[idx];
    if (idx >= total_parts - parts_left) {
        size_t const rand_idx = xorshift64star(idx << 3) % (total_parts - parts_left);
        to_vec[idx] = (float3)((rand_idx / (side_parts * side_parts)) * spacing,
                               ((rand_idx / side_parts) % side_parts) * spacing,
                               (rand_idx % side_parts) * spacing);
    }
    else {
        to_vec[idx] = (float3)((idx / (side_parts * side_parts)) * spacing,
                               ((idx / side_parts) % side_parts) * spacing,
                               (idx % side_parts) * spacing);
    }
    to_vec[idx] -= ((float)(side_parts - 1) / 2.0f) * spacing;
    velocities[idx] = (float3)(0.0f, 0.0f, 0.0f);
}

__kernel void update_gravitation(global float3 * const restrict positions,
                                 global float3 * const restrict velocities,
                                 float3 gravity_point,
                                 float t) {
    size_t const idx = get_global_id(0);
    positions[idx] += gravity_point;
}

static float back_ease_out(float t, float b, float c, float d) {
    float s = 1.70158f;
    float inner_t = (t / d) - 1.0f;
    return (c * (inner_t * inner_t * ((s + 1.0f) * inner_t + s) + 1.0f) + b);
}

static float elastic_ease_out(float t, float b, float c, float d) {
    if (t == 0.0f) {
        return b;
    }

    float inner_t = t / d;
    if (inner_t == 1.0f) {
        return b + c;
    }

    float p = d * 0.3f;
    float a = c;
    float s = p / 4.0f;
    float temp = (inner_t * d - s) * (2.0f * M_PI_F) / p;
    return (a * pow(2.0f, -10.0f * inner_t) * sin(temp) + c + b);
}

#define EASING_ANIMATION elastic_ease_out

__kernel void update_animation(global float3 const * const restrict from_vec,
                               global float3 const * const restrict to_vec,
                               global float3 * const restrict positions,
                               float time, float duration) {
    size_t const idx = get_global_id(0);
    float3 const from = from_vec[idx];
    float3 const to = to_vec[idx];

    positions[idx] = (float3)(EASING_ANIMATION(time, from.x, to.x - from.x, duration),
                              EASING_ANIMATION(time, from.y, to.y - from.y, duration),
                              EASING_ANIMATION(time, from.z, to.z - from.z, duration));
}

static size_t  xorshift64star(size_t x) {
    x ^= x >> 12; // a
    x ^= x << 25; // b
    x ^= x >> 27; // c
    return x * size_t(2685821657736338717);
}

__kernel void init_rand_sphere_animation(global float3 const * const restrict positions,
                                         global float3 * const restrict from_vec,
                                         global float3 * const restrict to_vec,
                                         global float3 * const restrict velocities) {
    size_t const idx = get_global_id(0);
    size_t const number_particles = get_global_size(0);

    float const scaling = 1.f / 20.f;
    size_t const radius = 10;
    float radius_rand;
    // FIXME understand this (wanted: half generated between radius/2 and 0)
    if (idx > number_particles) {
        radius_rand = (float)(xorshift64star(idx >> 2) % (radius * 100));
        radius_rand /= 100.f;
    }
    else {
        size_t const scal_rad = (radius / 2) * 100;
        radius_rand = (float)(xorshift64star(idx >> 2) % scal_rad) + scal_rad;
        radius_rand /= 100.f;
    }
    float const u = radians((float)(xorshift64star(idx >> 3) % 360));
    float const v = radians((float)(xorshift64star(idx << 2) % 360));
    float const x = radius_rand * cos(u) * sin(v);
    float const y = radius_rand * sin(u) * sin(v);
    float const z = radius_rand * cos(v);
    from_vec[idx] = positions[idx];
    to_vec[idx] = (float3)(x, y, z);
    to_vec[idx] *= scaling;
    velocities[idx] = (float3)(0.0f, 0.0f, 0.0f);
}

__kernel void init_rand_cube_animation(global float3 const * const restrict positions,
                                       global float3 * const restrict from_vec,
                                       global float3 * const restrict to_vec,
                                       global float3 * const restrict velocities) {
    size_t const idx = get_global_id(0);

    float const scaling = 1.f / 20.f;
    size_t const diameter = 20;
    float const x = (float)(xorshift64star(idx << 3) % (diameter * 10)) / 10.f;
    float const y = (float)(xorshift64star(idx >> 2) % (diameter * 10)) / 10.f;
    float const z = (float)(xorshift64star(idx << 2) % (diameter * 10)) / 10.f;

    from_vec[idx] = positions[idx];
    float3 center = (float3)(10.f, 10.f, 10.f);
    to_vec[idx] = (float3)(x, y, z);
    float dist = distance(to_vec[idx], center);
    if (dist > diameter / 2.f) {
        to_vec[idx] = to_vec[idx];
    }
    to_vec[idx] -= center;
    to_vec[idx] *= scaling;
    velocities[idx] = (float3)(0.0f, 0.0f, 0.0f);
}

__kernel void init_cube_animation(global float3 const * const restrict positions,
                                  global float3 * const restrict from_vec,
                                  global float3 * const restrict to_vec,
                                  global float3 * const restrict velocities) {
    size_t const idx = get_global_id(0);
    size_t const number_particles = get_global_size(0);
    size_t const side_particles = cbrt((float)number_particles); // FIXME compute this one time
    size_t const particles_left = number_particles - (side_particles * side_particles * side_particles);
    float const spacing = 1.0f / (float)side_particles;
    from_vec[idx] = positions[idx];
    if (idx >= number_particles - particles_left) {
        to_vec[idx] = (float3)(0.0f, 0.0f, 0.0f); // FIXME random position
    }
    else {
        to_vec[idx] = (float3)((idx / (side_particles * side_particles)) * spacing,
                               ((idx / side_particles) % side_particles) * spacing,
                               (idx % side_particles) * spacing);
    }
    to_vec[idx] -= ((float)(side_particles - 1) / 2.0f) * spacing;
    velocities[idx] = (float3)(0.0f, 0.0f, 0.0f);
}

__kernel void update_gravitation(global float3 * const restrict positions,
                                 global float3 * const restrict velocities,
                                 float3 gravity_point,
                                 float t) {
    size_t const idx = get_global_id(0);
    positions[idx] += gravity_point;
}