int instead of uint

int instead of uint

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8
#define _INT_MASK_GLOBAL uchar
#define _INT_MASK_LOCAL uchar

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE 0
#define MLO_POOLING_OP_MAX 1
#define MLO_POOLING_OP_STC 2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A, B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A, B) (A + B);
#endif

/*********************************************************************************

**********************************************************************************/

#define MLO_BOT_DATA_SZ0 \
    (MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_BOT_DATA_SZ1 \
    (MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,
                                    MLO_POOLING_GROUP_SZ1,
                                    MLO_POOLING_GROUP_SZ2))) __kernel void
mloPoolingG(const __global _FLOAT* bot,
            __global _FLOAT* top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
            UNUSED
#endif
                __global _INT_MASK_GLOBAL* mask)
{

    uint x       = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
    uint y       = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
    uint lcl_id0 = get_local_id(0);
    uint lcl_id1 = get_local_id(1);
    //		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
    uint ob      = get_global_id(2); // output * batch_sz
    uint b       = ob / MLO_POOLING_N_OUTPUTS;
    uint o       = ob - b * MLO_POOLING_N_OUTPUTS;
    uint bot_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX) * MLO_POOLING_STRIDE0;
    uint bot_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX) * MLO_POOLING_STRIDE1;
    uint bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

    _FLOAT bot_data[MLO_BOT_DATA_SZ1][MLO_BOT_DATA_SZ0];
    _FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
    _INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
    for(int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            res[k][l] = -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
            res[k][l]         = 0;
#endif
        }
    }

    for(uint j = 0; j < MLO_BOT_DATA_SZ1; ++j)
    {
        int run_y = (int)bot_y + j - MLO_POOLING_PAD1;

        for(uint i = 0; i < MLO_BOT_DATA_SZ0; ++i)
        {
            int run_x        = (int)bot_x + i - MLO_POOLING_PAD0;
            uint bot_gbl_off = bot_off + (uint)run_y * MLO_POOLING_BOT_STRIDE + (uint)run_x;
            bool vis         = ((run_y >= 0 && run_y < MLO_POOLING_BOT_HEIGHT) &&
                        (run_x >= 0 && run_x < MLO_POOLING_BOT_WIDTH))
                           ? true
                           : false;
            bot_data[j][i] = (vis) ? bot[bot_gbl_off] : 0;
        }
    }

    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
        uint y_dst  = y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
        int hstart = (int)y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
        int hend   = min((hstart + MLO_POOLING_KERNEL_SZ1),
                        (int)(MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1));
#endif
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
            uint x_dst  = x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
            int wstart = (int)x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
            int wend   = min((wstart + MLO_POOLING_KERNEL_SZ0),
                            (int)(MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0));
            uint pool_size = (hend - hstart) * (wend - wstart);
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            mask_private[k][l] = 0xFF;
#endif

            for(uint j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
            {
                for(uint i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
                {

                    _FLOAT bot_val =
                        bot_data[j + k * MLO_POOLING_STRIDE1][i + l * MLO_POOLING_STRIDE0];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                    if(bot_val > res[k][l])
                    {
                        res[k][l]          = bot_val;
                        mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
                    }
#else
                    res[k][l] = MLO_POOLING_OP(res[k][l], bot_val);
#endif
                }
            }

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
            res[k][l] *= (_FLOAT)1.f / (_FLOAT)pool_size;
#endif
        }
    }

    uint top_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
    uint top_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
    uint top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE +
                   top_y * MLO_POOLING_TOP_STRIDE + top_x;
    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
            if(top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
            {
                top[top_off + k * MLO_POOLING_TOP_STRIDE + l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
            }
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8
#define _INT_MASK_GLOBAL uchar
#define _INT_MASK_LOCAL uchar

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE 0
#define MLO_POOLING_OP_MAX 1
#define MLO_POOLING_OP_STC 2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A, B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A, B) (A + B);
#endif

/*********************************************************************************

**********************************************************************************/

#define MLO_BOT_DATA_SZ0 \
    (MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_BOT_DATA_SZ1 \
    (MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,
                                    MLO_POOLING_GROUP_SZ1,
                                    MLO_POOLING_GROUP_SZ2))) __kernel void
mloPoolingG(const __global _FLOAT* bot,
            __global _FLOAT* top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
            UNUSED
#endif
                __global _INT_MASK_GLOBAL* mask)
{

    uint x       = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
    uint y       = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
    uint lcl_id0 = get_local_id(0);
    uint lcl_id1 = get_local_id(1);
    //		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
    uint ob      = get_global_id(2); // output * batch_sz
    uint b       = ob / MLO_POOLING_N_OUTPUTS;
    uint o       = ob - b * MLO_POOLING_N_OUTPUTS;
    uint bot_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX) * MLO_POOLING_STRIDE0;
    uint bot_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX) * MLO_POOLING_STRIDE1;
    uint bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

    _FLOAT bot_data[MLO_BOT_DATA_SZ1][MLO_BOT_DATA_SZ0];
    _FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
    _INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
    for(int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            res[k][l] = -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
            res[k][l]         = 0;
#endif
        }
    }

    for(uint j = 0; j < MLO_BOT_DATA_SZ1; ++j)
    {
        int run_y = (int)bot_y + j - MLO_POOLING_PAD1;

        for(uint i = 0; i < MLO_BOT_DATA_SZ0; ++i)
        {
            int run_x        = (int)bot_x + i - MLO_POOLING_PAD0;
            uint bot_gbl_off = bot_off + (uint)run_y * MLO_POOLING_BOT_STRIDE + (uint)run_x;
            bool vis         = ((run_y >= 0 && run_y < MLO_POOLING_BOT_HEIGHT) &&
                        (run_x >= 0 && run_x < MLO_POOLING_BOT_WIDTH))
                           ? true
                           : false;
            bot_data[j][i] = (vis) ? bot[bot_gbl_off] : 0;
        }
    }

    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
        uint y_dst  = y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
        uint hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
        uint hend   = min((uint)(hstart + MLO_POOLING_KERNEL_SZ1),
                        (uint)(MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1));
#endif
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
            uint x_dst  = x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
            uint wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
            uint wend   = min((uint)(wstart + MLO_POOLING_KERNEL_SZ0),
                            (uint)(MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0));
            uint pool_size = (hend - hstart) * (wend - wstart);
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            mask_private[k][l] = 0xFF;
#endif

            for(uint j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
            {
                for(uint i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
                {

                    _FLOAT bot_val =
                        bot_data[j + k * MLO_POOLING_STRIDE1][i + l * MLO_POOLING_STRIDE0];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                    if(bot_val > res[k][l])
                    {
                        res[k][l]          = bot_val;
                        mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
                    }
#else
                    res[k][l] = MLO_POOLING_OP(res[k][l], bot_val);
#endif
                }
            }

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
            res[k][l] *= (_FLOAT)1.f / (_FLOAT)pool_size;
#endif
        }
    }

    uint top_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
    uint top_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
    uint top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE +
                   top_y * MLO_POOLING_TOP_STRIDE + top_x;
    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
            if(top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
            {
                top[top_off + k * MLO_POOLING_TOP_STRIDE + l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
            }
        }
    }
}


Fails about 1 out of 100 runs, looking for error

typedef struct GPUConstants
{
     int numRadices;
     int numBlocks;
     int numGroupsPerBlock;
     int R;
     int numThreadsPerGroup;
     int numElementsPerGroup;
	 int numRadicesPerBlock;
     int bitMask;
     int L;
     int numThreadsPerBlock;
     int numTotalElements;
}Constants;

kernel void zeroes(global uint* counters) {
  counters[get_global_id(0)] = 0;
}

inline void PrefixLocal(__local uint* inout, int p_length, int numThreads){
  __private uint glocalID = get_local_id(0);
  __private int inc = 2;
  while(inc <= p_length){
    for(int i = ((inc>>1) - 1) + (glocalID * inc)  ; (i + inc) < p_length ; i+= numThreads*inc){
        inout[i + (inc>>1)] = inout[i] + inout[i + (inc>>1)];
    }
    inc = inc <<1;
    barrier(CLK_GLOBAL_MEM_FENCE);
  }
  inout[p_length-1] = 0;
  barrier(CLK_GLOBAL_MEM_FENCE);
  while(inc >=2){
    for (int i = ((inc>>1) - 1) + (glocalID * inc)  ; (i + (inc>>1)) <= p_length ; i+= numThreads*inc)
    {
      uint tmp = inout[i + (inc >>1)];
      inout[i + (inc >>1)] = inout[i] + inout[i + (inc >>1 )];
      inout[i] = tmp;
    }
    inc = inc>>1;
    barrier(CLK_GLOBAL_MEM_FENCE);
  }
}

__kernel void SetupAndCount(__global  uint* cellIdIn,
    							          __global volatile uint* counters,
							              Constants dConst,
							              uint bitOffset)
{
	__private uint gLocalId = get_local_id(0);
	__private uint gBlockId = get_group_id(0);
	__private uint threadGroup = gLocalId / dConst.R;
  __private int actBlock = gBlockId * dConst.numGroupsPerBlock * dConst.numElementsPerGroup ;
  __private int actGroup = (gLocalId / dConst.R ) * dConst.numElementsPerGroup;
  __private uint idx = actBlock + actGroup + gLocalId % dConst.R;
  __private int boarder = actBlock +actGroup + dConst.numElementsPerGroup;
  boarder = (boarder > dConst.numTotalElements)? dConst.numTotalElements : boarder;
  __private uint countersPerRadix = dConst.numBlocks * dConst.numGroupsPerBlock;
  __private uint counterGroupOffset = gBlockId * dConst.numGroupsPerBlock;
	for(;idx < boarder; idx += dConst.numThreadsPerGroup){
		__private uint actRadix = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
   for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
      if(gLocalId % dConst.R == tmpIdx){
          //counters[ (actRadix * countersPerRadix)  +counterGroupOffset+ threadGroup ]++;
          atomic_inc(&counters[(actRadix * countersPerRadix)  +counterGroupOffset+ threadGroup ]);
      }
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
	}
}

__kernel void SumIt(__global uint* cellIdIn,
                    __global volatile uint* counters,
                    __global uint* radixPrefixes,
                    __local uint* groupcnt,
                    Constants dConst,
                    uint bitOffset)
{
	__private uint globalId = get_global_id(0);
	__private uint gLocalId = get_local_id(0);
	__private uint gBlockId = get_group_id(0);
  __private uint countersPerRadix = dConst.numBlocks * dConst.numGroupsPerBlock;
	__private uint actRadix = dConst.numRadicesPerBlock * gBlockId;
  for(int i = 0; i < dConst.numRadicesPerBlock; i++){
    int numIter = 0;
    uint boarder = ((actRadix+1) * countersPerRadix);
    for(int j = (actRadix * countersPerRadix) + gLocalId ; j < boarder; j+= dConst.numThreadsPerBlock){
      groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++] = counters[j];
      barrier(CLK_LOCAL_MEM_FENCE);
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    PrefixLocal(groupcnt, countersPerRadix ,dConst.numThreadsPerBlock );
    barrier(CLK_LOCAL_MEM_FENCE);
    if(gLocalId == 1){
      radixPrefixes[actRadix] = groupcnt[(countersPerRadix) -1] + counters[((actRadix+1) * countersPerRadix)-1];
    }
    barrier(CLK_GLOBAL_MEM_FENCE);
    numIter = 0;
    for(int j = (actRadix * countersPerRadix) + gLocalId ; j < ((actRadix+1) * countersPerRadix); j+= dConst.numThreadsPerBlock){
      counters[j] = groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++];
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
    actRadix++;
  }
}

__kernel void ReorderingKeysOnly(__global uint* cellIdIn,
                            __global uint* cellIdOut,
                            __global uint* counters,
                            __global uint* radixPrefixes,
                            __local uint* localCounters,
                            __local uint* localPrefix,
                             Constants dConst,
                            uint bitOffset)
{
  int globalId = get_global_id(0);
  __private uint gLocalId = get_local_id(0);
  __private uint gBlockId = get_group_id(0);
  __private uint threadGroup= gLocalId / dConst.R;
  __private uint actRadix = dConst.numRadicesPerBlock * gBlockId;
  __private uint countersPerRadix = dConst.numGroupsPerBlock * dConst.numBlocks;
  __private int radixCounterOffset = actRadix * countersPerRadix;
  __private  uint blockidx ;
  for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
    for(uint   blockidx = gLocalId;
                blockidx < countersPerRadix ;
                 blockidx+= dConst.numThreadsPerBlock){
      localCounters[ i* countersPerRadix  +  blockidx] = counters[radixCounterOffset +  i* countersPerRadix  +  blockidx];
    }
    barrier(CLK_LOCAL_MEM_FENCE);
  }
  for(int i = gLocalId ; i< dConst.numRadices ; i+= dConst.numThreadsPerBlock){
    localPrefix[i] = radixPrefixes[i];
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  PrefixLocal(localPrefix, dConst.numRadices, dConst.numThreadsPerBlock);
  barrier(CLK_LOCAL_MEM_FENCE);
  for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
    int numIter = 0;
    for(int j =  gLocalId ; j < countersPerRadix; j+= dConst.numThreadsPerBlock){
      localCounters[ i* countersPerRadix  +  j] += localPrefix[actRadix+i];
    }
    barrier(CLK_LOCAL_MEM_FENCE);
  }
  for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
    for(uint   blockidx =gLocalId;
                blockidx <  countersPerRadix;
                 blockidx+= dConst.numThreadsPerBlock){
      counters[radixCounterOffset +  i* countersPerRadix  +  blockidx] = localCounters[ i* countersPerRadix  +  blockidx];
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
    barrier(CLK_GLOBAL_MEM_FENCE);
  }
  barrier(CLK_GLOBAL_MEM_FENCE);
  __private int actBlock = gBlockId * dConst.numGroupsPerBlock * dConst.numElementsPerGroup ;
  __private int actBlockCounter = gBlockId * dConst.numGroupsPerBlock  ;

  __private int actGroup = (gLocalId / dConst.R ) * dConst.numElementsPerGroup;

  __private  uint idx = actBlock + actGroup +  gLocalId % dConst.R;
  int boundary = actBlock+ actGroup +  dConst.numElementsPerGroup;
  boundary = (actBlock+ actGroup +  dConst.numElementsPerGroup < dConst.numTotalElements)? boundary : dConst.numTotalElements;
  for(;idx <   boundary ; idx += dConst.numThreadsPerGroup){
    uint tmpRdx = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
    for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
      if(gLocalId % dConst.R == tmpIdx){
          cellIdOut[counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]] = cellIdIn[idx];
          atomic_inc(&counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]);
      }
      barrier(CLK_GLOBAL_MEM_FENCE);
    }
  }
}

__kernel void ReorderingKeyValue(__global uint* cellIdIn,
                            __global uint* cellIdOut,
                            __global uint* valueIn,
                            __global uint* valueOut,
                            __global uint* counters,
                            __global uint* radixPrefixes,
                            __local uint* localCounters,
                            __local uint* localPrefix,
                             Constants dConst,
                            uint bitOffset)
{
  int globalId = get_global_id(0);
    __private uint gLocalId = get_local_id(0);
    __private uint gBlockId = get_group_id(0);
    __private uint threadGroup= gLocalId / dConst.R;
    __private uint actRadix = dConst.numRadicesPerBlock * gBlockId;
    __private uint countersPerRadix = dConst.numGroupsPerBlock * dConst.numBlocks;
    __private int radixCounterOffset = actRadix * countersPerRadix;
    // erst abschließen der radix summierung
    __private  uint blockidx ;
    for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        for(uint   blockidx =gLocalId;
                    blockidx <  countersPerRadix ;
                     blockidx+= dConst.numThreadsPerBlock){
            // The Num_Groups counters of the radix are read from global memory to shared memory.
            // Jeder Thread liest die Counter basierend auf der groupId aus
            localCounters[ i* countersPerRadix  +  blockidx] = counters[radixCounterOffset +  i* countersPerRadix  +  blockidx];
        }
    }

    // Read radix prefixes to localMemory
    for(int i = gLocalId ; i< dConst.numRadices ; i+= dConst.numThreadsPerBlock){
        localPrefix[i] = radixPrefixes[i];
    }

    // Präfixsumme über die RadixCounter bilden.
    barrier(CLK_GLOBAL_MEM_FENCE);
    PrefixLocal(localPrefix, dConst.numRadices, dConst.numThreadsPerBlock);
    barrier(CLK_GLOBAL_MEM_FENCE);

    // Die Präfixsumme des Radixe auf alle subcounter der radixes addieren
  for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        int numIter = 0;
        //for(int j = ((actRadix+i) * dConst.numBlocks * dConst.numGroupsPerBlock) + localId ; j < ((dConst.numRadicesPerBlock * gBlockId + 1) * dConst.numBlocks * dConst.numGroupsPerBlock); j+= dConst.numThreadsPerBlock){
        for(int j =  gLocalId ; j < countersPerRadix; j+= dConst.numThreadsPerBlock){
            //groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++] = counters[j];
            //if(gLocalId == 0 && gBlockId ==0 && i==0 && j == gLocalId )
            localCounters[ i* countersPerRadix  +  j] += localPrefix[actRadix+i];

        }
       // barrier(CLK_GLOBAL_MEM_FENCE);
    }

    // Zurückschreiben der Radixe mit entsprechedem offset.
    for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        for(uint   blockidx =gLocalId;
                    blockidx <  countersPerRadix;
                     blockidx+= dConst.numThreadsPerBlock){
            // The Num_Groups counters of the radix are read from global memory to shared memory.
            // Jeder Thread liest die Counter basierend auf der groupId aus
            counters[radixCounterOffset +  i* countersPerRadix  +  blockidx] = localCounters[ i* countersPerRadix  +  blockidx];
        }
    }

    barrier(CLK_GLOBAL_MEM_FENCE);

    __private int actBlock = gBlockId * dConst.numGroupsPerBlock * dConst.numElementsPerGroup ;
    __private int actBlockCounter = gBlockId * dConst.numGroupsPerBlock  ;

    __private int actGroup = (gLocalId / dConst.R ) * dConst.numElementsPerGroup;

    __private  uint idx = actBlock + actGroup +  gLocalId % dConst.R;
    int boundary = actBlock+ actGroup +  dConst.numElementsPerGroup;
    boundary = (actBlock+ actGroup +  dConst.numElementsPerGroup < dConst.numTotalElements)? boundary : dConst.numTotalElements;
    for(;idx <   boundary ; idx += dConst.numThreadsPerGroup){
        uint tmpRdx = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
        //uint outputIdx = counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++;
        for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
            if(gLocalId % dConst.R == tmpIdx){

                cellIdOut[counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]] = cellIdIn[idx];
                valueOut[counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]++] = valueIn[idx];
                //cellIdOut[idx] = gLocalId+1;

            }
            barrier(CLK_GLOBAL_MEM_FENCE);

        }

    }

/*

    __private  uint idx = (gBlockId * dConst.numGroupsPerBlock + gLocalId / dConst.R) * dConst.numElementsPerGroup + gLocalId % dConst.R;

    for(;idx < (gBlockId * dConst.numGroupsPerBlock + threadGroup+1) * dConst.numElementsPerGroup; idx += dConst.numThreadsPerGroup){
        uint actRadix = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
        //uint outputIdx = counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++;
        for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
            if(gLocalId % dConst.R == tmpIdx){
                cellIdOut[counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]] = cellIdIn[idx];
                valueOut[counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++] = valueIn[idx];
            }
            barrier(CLK_GLOBAL_MEM_FENCE);

        }

    }
*/

}

typedef struct GPUConstants
{
     int numRadices;
     int numBlocks;
     int numGroupsPerBlock;
     int R;
     int numThreadsPerGroup;
     int numElementsPerGroup;
	 int numRadicesPerBlock;
     int bitMask;
     int L;
     int numThreadsPerBlock;
     int numTotalElements;
}Constants;

kernel void zeroes(global uint* counters) {
  counters[get_global_id(0)] = 0;
}

inline void PrefixLocal(__local uint* inout, int p_length, int numThreads){
    __private uint glocalID = get_local_id(0);
    __private int inc = 2;

    // reduce
    while(inc <= p_length){

        for(int i = ((inc>>1) - 1) + (glocalID * inc)  ; (i + inc) < p_length ; i+= numThreads*inc){
            inout[i + (inc>>1)] = inout[i] + inout[i + (inc>>1)];
        }
        inc = inc <<1;
        barrier(CLK_GLOBAL_MEM_FENCE);
    }
    // Downsweep
    inout[p_length-1] = 0;
    barrier(CLK_GLOBAL_MEM_FENCE);

    while(inc >=2){
        for (int i = ((inc>>1) - 1) + (glocalID * inc)  ; (i + (inc>>1)) <= p_length ; i+= numThreads*inc)
        {
            uint tmp = inout[i + (inc >>1)];
            inout[i + (inc >>1)] = inout[i] + inout[i + (inc >>1 )];
            inout[i] = tmp;
        }
        inc = inc>>1;
        barrier(CLK_GLOBAL_MEM_FENCE);
    }
}

inline void PrefixGlobal(__global uint* inout, int p_length, int numThreads){
    __private uint glocalID = get_local_id(0);
    __private int inc = 2;

    // reduce
    while(inc <= p_length){

        for(int i = ((inc>>1) - 1) + (glocalID * inc)  ; (i + inc) < p_length ; i+= numThreads*inc){
            inout[i + (inc>>1)] = inout[i] + inout[i + (inc>>1)];
        }
        inc = inc <<1;
        barrier(CLK_GLOBAL_MEM_FENCE);
    }
    // Downsweep
    inout[p_length-1] = 0;
    barrier(CLK_GLOBAL_MEM_FENCE);

    while(inc >=2){
        for (int i = ((inc>>1) - 1) + (glocalID * inc)  ; (i + (inc>>1)) <= p_length ; i+= numThreads*inc)
        {
            uint tmp = inout[i + (inc >>1)];
            inout[i + (inc >>1)] = inout[i] + inout[i + (inc >>1 )];
            inout[i] = tmp;
        }
        inc = inc>>1;
        barrier(CLK_GLOBAL_MEM_FENCE);
    }
}

__kernel void SetupAndCount(	__global  uint* cellIdIn,
							__global volatile uint* counters,
							Constants dConst,
							uint bitOffset)
{
	__private uint gLocalId = get_local_id(0);
	__private uint gBlockId = get_group_id(0);

    // Current threadGroup -> is based of the localId(Block internal)
	__private uint threadGroup = gLocalId / dConst.R;

    // Startindex of the datablock that corresponds to the Threadblock
    __private int actBlock = gBlockId * dConst.numGroupsPerBlock * dConst.numElementsPerGroup ;

    // Offset inside the block for the threadgroup of the current thread
    __private int actGroup = (gLocalId / dConst.R ) * dConst.numElementsPerGroup;

    // Startindex for the current thread
    __private uint idx = actBlock + actGroup + gLocalId % dConst.R;

    // Set the boarder
    __private int boarder = actBlock +actGroup + dConst.numElementsPerGroup;
    boarder = (boarder > dConst.numTotalElements)? dConst.numTotalElements : boarder;

    // Number of counters for each radix
    __private uint countersPerRadix = dConst.numBlocks * dConst.numGroupsPerBlock;
    // Each Threadgroup has its own counter for each radix -> Calculating offset based on current block
    __private uint counterGroupOffset = gBlockId * dConst.numGroupsPerBlock;

	for(;idx < boarder; idx += dConst.numThreadsPerGroup){
		__private uint actRadix = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
        // The following code ensures that the counters of each Threadgroup are sequentially incremented
         for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
            if(gLocalId % dConst.R == tmpIdx){
                counters[ (actRadix * countersPerRadix)  +counterGroupOffset+ threadGroup ]++;
            }
            barrier(CLK_GLOBAL_MEM_FENCE);
        }
	}
}

__kernel void SumIt(	__global uint* cellIdIn,
                            __global volatile uint* counters,
							__global uint* radixPrefixes,
							__local uint* groupcnt,
							 Constants dConst,
							uint bitOffset)
{
	__private uint globalId = get_global_id(0);
	__private uint gLocalId = get_local_id(0);
	__private uint gBlockId = get_group_id(0);

    __private uint countersPerRadix = dConst.numBlocks * dConst.numGroupsPerBlock;

	__private uint actRadix = dConst.numRadicesPerBlock * gBlockId;

    for(int i = 0 ; i< dConst.numRadicesPerBlock; i++){
        // The Num_Groups counters of the radix are read from global memory to shared memory.
        // Jeder Thread liest die Counter basierend auf der localid aus
        int numIter = 0;
        uint boarder = ((actRadix+1) * countersPerRadix);
       // boarder = (boarder > dConst.numBlocks * dConst.numGroupsPerBlock)? dConst.numBlocks * dConst.numGroupsPerBlock : boarder;
        for(int j = (actRadix * countersPerRadix) + gLocalId ; j < boarder; j+= dConst.numThreadsPerBlock){
            groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++] = counters[j];
            //numIter += dConst.numThreadsPerBlock;
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        // Die einzelnen RadixCounter sind nun in dem groupcnt local Memory
        //prefixSum(&counters[actRadix * dConst.numBlocks * dConst.numGroupsPerBlock], groupcnt, tmpPrefix,dConst.numBlocks * dConst.numGroupsPerBlock);
    	PrefixLocal(groupcnt, countersPerRadix ,dConst.numThreadsPerBlock );

        // PrefixSum wurde gebildet..
        barrier(CLK_GLOBAL_MEM_FENCE);

        // Gesamtprefix für den aktuellen radix berechnen
        if(gLocalId == 1 ){
            radixPrefixes[actRadix] = groupcnt[(countersPerRadix) -1] + counters[((actRadix+1) * countersPerRadix)-1];
        }

        // Errechnete Prefixsumme zurück in den global memory schreiben
        barrier(CLK_GLOBAL_MEM_FENCE);
        numIter = 0;
        for(int j = (actRadix * countersPerRadix) + gLocalId ; j < ((actRadix+1) * countersPerRadix); j+= dConst.numThreadsPerBlock){
            counters[j] = groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++];
            //numIter += dConst.numThreadsPerBlock;
        }

        actRadix++;
    }
}

__kernel void ReorderingKeysOnly(    __global uint* cellIdIn,
                            __global uint* cellIdOut,
                            __global uint* counters,
                            __global uint* radixPrefixes,
                            __local uint* localCounters,
                            __local uint* localPrefix,
                             Constants dConst,
                            uint bitOffset)
{
    int globalId = get_global_id(0);
    __private uint gLocalId = get_local_id(0);
    __private uint gBlockId = get_group_id(0);

    __private uint threadGroup= gLocalId / dConst.R;

    __private uint actRadix = dConst.numRadicesPerBlock * gBlockId;
    __private uint countersPerRadix = dConst.numGroupsPerBlock * dConst.numBlocks;

    __private int radixCounterOffset = actRadix * countersPerRadix;

    // erst abschließen der radix summierung
    __private  uint blockidx ;
    for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        for(uint   blockidx =gLocalId;
                    blockidx <  countersPerRadix ;
                     blockidx+= dConst.numThreadsPerBlock){
            // The Num_Groups counters of the radix are read from global memory to shared memory.
            // Jeder Thread liest die Counter basierend auf der groupId aus
            localCounters[ i* countersPerRadix  +  blockidx] = counters[radixCounterOffset +  i* countersPerRadix  +  blockidx];
        }
    }

    // Read radix prefixes to localMemory
    for(int i = gLocalId ; i< dConst.numRadices ; i+= dConst.numThreadsPerBlock){
        localPrefix[i] = radixPrefixes[i];
    }
    // Präfixsumme über die RadixCounter bilden.
    barrier(CLK_GLOBAL_MEM_FENCE);
    PrefixLocal(localPrefix, dConst.numRadices, dConst.numThreadsPerBlock);
    barrier(CLK_GLOBAL_MEM_FENCE);

    // Die Präfixsumme des Radixe auf alle subcounter der radixes addieren
  for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        int numIter = 0;
        //for(int j = ((actRadix+i) * dConst.numBlocks * dConst.numGroupsPerBlock) + localId ; j < ((dConst.numRadicesPerBlock * gBlockId + 1) * dConst.numBlocks * dConst.numGroupsPerBlock); j+= dConst.numThreadsPerBlock){
        for(int j =  gLocalId ; j < countersPerRadix; j+= dConst.numThreadsPerBlock){
            //groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++] = counters[j];
            //if(gLocalId == 0 && gBlockId ==0 && i==0 && j == gLocalId )
            localCounters[ i* countersPerRadix  +  j] += localPrefix[actRadix+i];

        }
       // barrier(CLK_GLOBAL_MEM_FENCE);
    }

    // Zurückschreiben der Radixe mit entsprechedem offset.
    for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        for(uint   blockidx =gLocalId;
                    blockidx <  countersPerRadix;
                     blockidx+= dConst.numThreadsPerBlock){
            // The Num_Groups counters of the radix are read from global memory to shared memory.
            // Jeder Thread liest die Counter basierend auf der groupId aus
            counters[radixCounterOffset +  i* countersPerRadix  +  blockidx] = localCounters[ i* countersPerRadix  +  blockidx];
        }
    }

    barrier(CLK_GLOBAL_MEM_FENCE);

    __private int actBlock = gBlockId * dConst.numGroupsPerBlock * dConst.numElementsPerGroup ;
    __private int actBlockCounter = gBlockId * dConst.numGroupsPerBlock  ;

    __private int actGroup = (gLocalId / dConst.R ) * dConst.numElementsPerGroup;

    __private  uint idx = actBlock + actGroup +  gLocalId % dConst.R;
    int boundary = actBlock+ actGroup +  dConst.numElementsPerGroup;
    boundary = (actBlock+ actGroup +  dConst.numElementsPerGroup < dConst.numTotalElements)? boundary : dConst.numTotalElements;
    for(;idx <   boundary ; idx += dConst.numThreadsPerGroup){
        uint tmpRdx = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
        //uint outputIdx = counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++;
        for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
            if(gLocalId % dConst.R == tmpIdx){
                cellIdOut[counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]++] = cellIdIn[idx];
                //cellIdOut[idx] = gLocalId+1;

            }
            barrier(CLK_GLOBAL_MEM_FENCE);

        }

    }

}

__kernel void ReorderingKeyValue(    __global uint* cellIdIn,
                            __global uint* cellIdOut,
                            __global uint* valueIn,
                            __global uint* valueOut,
                            __global uint* counters,
                            __global uint* radixPrefixes,
                            __local uint* localCounters,
                            __local uint* localPrefix,
                             Constants dConst,
                            uint bitOffset)
{
  int globalId = get_global_id(0);
    __private uint gLocalId = get_local_id(0);
    __private uint gBlockId = get_group_id(0);

    __private uint threadGroup= gLocalId / dConst.R;

    __private uint actRadix = dConst.numRadicesPerBlock * gBlockId;
    __private uint countersPerRadix = dConst.numGroupsPerBlock * dConst.numBlocks;

    __private int radixCounterOffset = actRadix * countersPerRadix;

    // erst abschließen der radix summierung
    __private  uint blockidx ;
    for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        for(uint   blockidx =gLocalId;
                    blockidx <  countersPerRadix ;
                     blockidx+= dConst.numThreadsPerBlock){
            // The Num_Groups counters of the radix are read from global memory to shared memory.
            // Jeder Thread liest die Counter basierend auf der groupId aus
            localCounters[ i* countersPerRadix  +  blockidx] = counters[radixCounterOffset +  i* countersPerRadix  +  blockidx];
        }
    }

    // Read radix prefixes to localMemory
    for(int i = gLocalId ; i< dConst.numRadices ; i+= dConst.numThreadsPerBlock){
        localPrefix[i] = radixPrefixes[i];
    }

    // Präfixsumme über die RadixCounter bilden.
    barrier(CLK_GLOBAL_MEM_FENCE);
    PrefixLocal(localPrefix, dConst.numRadices, dConst.numThreadsPerBlock);
    barrier(CLK_GLOBAL_MEM_FENCE);

    // Die Präfixsumme des Radixe auf alle subcounter der radixes addieren
  for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        int numIter = 0;
        //for(int j = ((actRadix+i) * dConst.numBlocks * dConst.numGroupsPerBlock) + localId ; j < ((dConst.numRadicesPerBlock * gBlockId + 1) * dConst.numBlocks * dConst.numGroupsPerBlock); j+= dConst.numThreadsPerBlock){
        for(int j =  gLocalId ; j < countersPerRadix; j+= dConst.numThreadsPerBlock){
            //groupcnt[gLocalId + dConst.numThreadsPerBlock * numIter++] = counters[j];
            //if(gLocalId == 0 && gBlockId ==0 && i==0 && j == gLocalId )
            localCounters[ i* countersPerRadix  +  j] += localPrefix[actRadix+i];

        }
       // barrier(CLK_GLOBAL_MEM_FENCE);
    }

    // Zurückschreiben der Radixe mit entsprechedem offset.
    for(int i = 0 ; i< dConst.numRadicesPerBlock ; i++){
        for(uint   blockidx =gLocalId;
                    blockidx <  countersPerRadix;
                     blockidx+= dConst.numThreadsPerBlock){
            // The Num_Groups counters of the radix are read from global memory to shared memory.
            // Jeder Thread liest die Counter basierend auf der groupId aus
            counters[radixCounterOffset +  i* countersPerRadix  +  blockidx] = localCounters[ i* countersPerRadix  +  blockidx];
        }
    }

    barrier(CLK_GLOBAL_MEM_FENCE);

    __private int actBlock = gBlockId * dConst.numGroupsPerBlock * dConst.numElementsPerGroup ;
    __private int actBlockCounter = gBlockId * dConst.numGroupsPerBlock  ;

    __private int actGroup = (gLocalId / dConst.R ) * dConst.numElementsPerGroup;

    __private  uint idx = actBlock + actGroup +  gLocalId % dConst.R;
    int boundary = actBlock+ actGroup +  dConst.numElementsPerGroup;
    boundary = (actBlock+ actGroup +  dConst.numElementsPerGroup < dConst.numTotalElements)? boundary : dConst.numTotalElements;
    for(;idx <   boundary ; idx += dConst.numThreadsPerGroup){
        uint tmpRdx = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
        //uint outputIdx = counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++;
        for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
            if(gLocalId % dConst.R == tmpIdx){

                cellIdOut[counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]] = cellIdIn[idx];
                valueOut[counters[tmpRdx * countersPerRadix + actBlockCounter+ threadGroup]++] = valueIn[idx];
                //cellIdOut[idx] = gLocalId+1;

            }
            barrier(CLK_GLOBAL_MEM_FENCE);

        }

    }

/*

    __private  uint idx = (gBlockId * dConst.numGroupsPerBlock + gLocalId / dConst.R) * dConst.numElementsPerGroup + gLocalId % dConst.R;

    for(;idx < (gBlockId * dConst.numGroupsPerBlock + threadGroup+1) * dConst.numElementsPerGroup; idx += dConst.numThreadsPerGroup){
        uint actRadix = (cellIdIn[idx] >> bitOffset) & dConst.bitMask;
        //uint outputIdx = counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++;
        for(uint tmpIdx = 0 ; tmpIdx < dConst.R; tmpIdx++){
            if(gLocalId % dConst.R == tmpIdx){
                cellIdOut[counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]] = cellIdIn[idx];
                valueOut[counters[actRadix * countersPerRadix + gBlockId * dConst.numGroupsPerBlock + threadGroup]++] = valueIn[idx];
            }
            barrier(CLK_GLOBAL_MEM_FENCE);

        }

    }
*/

}


[OpenCL] make test independent of optimizer

There shouldn't be any tests that run the entire optimizer here,
but the last test in this file is definitely going to break with
a change in LLVM IR canonicalization. Change that part to check
the unoptimized IR because that's the real intent of this file.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@332473 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -x cl -O1 -emit-llvm  %s -o - -triple x86_64-linux-gnu | FileCheck %s -check-prefix=OPT
// RUN: %clang_cc1 -x cl -O0 -emit-llvm  %s -o - -triple x86_64-linux-gnu | FileCheck %s -check-prefix=NOOPT

// OpenCL essentially reduces all shift amounts to the last word-size
// bits before evaluating. Test this both for variables and constants
// evaluated in the front-end.

// OPT: @gtest1 = local_unnamed_addr constant i64 2147483648
__constant const unsigned long gtest1 = 1UL << 31;

// NOOPT: @negativeShift32
int negativeShift32(int a,int b) {
  // NOOPT: %array0 = alloca [256 x i8]
  char array0[((int)1)<<40];
  // NOOPT: %array1 = alloca [256 x i8]
  char array1[((int)1)<<(-24)];

  // NOOPT: ret i32 65536
  return ((int)1)<<(-16);
}

//OPT: @positiveShift32
int positiveShift32(int a,int b) {
  //OPT: [[M32:%.+]] = and i32 %b, 31
  //OPT-NEXT: [[C32:%.+]] = shl i32 %a, [[M32]]
  int c = a<<b;
  int d = ((int)1)<<33;
  //OPT-NEXT: [[E32:%.+]] = add nsw i32 [[C32]], 2
  int e = c + d;
  //OPT-NEXT: ret i32 [[E32]]
  return e;
}

//OPT: @positiveShift64
long positiveShift64(long a,long b) {
  //OPT: [[M64:%.+]] = and i64 %b, 63
  //OPT-NEXT: [[C64:%.+]] = ashr i64 %a, [[M64]]
  long c = a>>b;
  long d = ((long)8)>>65;
  //OPT-NEXT: [[E64:%.+]] = add nsw i64 [[C64]], 4
  long e = c + d;
  //OPT-NEXT: ret i64 [[E64]]
  return e;
}

typedef __attribute__((ext_vector_type(4))) int int4;

//OPT: @vectorVectorTest
int4 vectorVectorTest(int4 a,int4 b) {
  //OPT: [[VM:%.+]] = and <4 x i32> %b, <i32 31, i32 31, i32 31, i32 31>
  //OPT-NEXT: [[VC:%.+]] = shl <4 x i32> %a, [[VM]]
  int4 c = a << b;
  //OPT-NEXT: [[VF:%.+]] = add <4 x i32> [[VC]], <i32 2, i32 4, i32 16, i32 8>
  int4 d = {1, 1, 1, 1};
  int4 e = {33, 34, -28, -29};
  int4 f = c + (d << e);
  //OPT-NEXT: ret <4 x i32> [[VF]]
  return f;
}

//NOOPT-LABEL: @vectorScalarTest
int4 vectorScalarTest(int4 a,int b) {
  //NOOPT: [[SP0:%.+]] = insertelement <4 x i32> undef
  //NOOPT: [[SP1:%.+]] = shufflevector <4 x i32> [[SP0]], <4 x i32> undef, <4 x i32> zeroinitializer
  //NOOPT: [[VSM:%.+]] = and <4 x i32> [[SP1]], <i32 31, i32 31, i32 31, i32 31>
  //NOOPT: [[VSC:%.+]] = shl <4 x i32> [[VSS:%.+]], [[VSM]]
  int4 c = a << b;
  //NOOPT: [[VSF:%.+]] = shl <4 x i32> [[VSC1:%.+]], <i32 2, i32 2, i32 2, i32 2>
  //NOOPT: [[VSA:%.+]] = add <4 x i32> [[VSC2:%.+]], [[VSF]]
  int4 d = {1, 1, 1, 1};
  int4 f = c + (d << 34);
  //NOOPT: ret <4 x i32>
  return f;
}

// RUN: %clang_cc1 -x cl -O1 -emit-llvm  %s -o - -triple x86_64-linux-gnu | FileCheck %s -check-prefix=OPT
// RUN: %clang_cc1 -x cl -O0 -emit-llvm  %s -o - -triple x86_64-linux-gnu | FileCheck %s -check-prefix=NOOPT

// OpenCL essentially reduces all shift amounts to the last word-size
// bits before evaluating. Test this both for variables and constants
// evaluated in the front-end.

// OPT: @gtest1 = local_unnamed_addr constant i64 2147483648
__constant const unsigned long gtest1 = 1UL << 31;

// NOOPT: @negativeShift32
int negativeShift32(int a,int b) {
  // NOOPT: %array0 = alloca [256 x i8]
  char array0[((int)1)<<40];
  // NOOPT: %array1 = alloca [256 x i8]
  char array1[((int)1)<<(-24)];

  // NOOPT: ret i32 65536
  return ((int)1)<<(-16);
}

//OPT: @positiveShift32
int positiveShift32(int a,int b) {
  //OPT: [[M32:%.+]] = and i32 %b, 31
  //OPT-NEXT: [[C32:%.+]] = shl i32 %a, [[M32]]
  int c = a<<b;
  int d = ((int)1)<<33;
  //OPT-NEXT: [[E32:%.+]] = add nsw i32 [[C32]], 2
  int e = c + d;
  //OPT-NEXT: ret i32 [[E32]]
  return e;
}

//OPT: @positiveShift64
long positiveShift64(long a,long b) {
  //OPT: [[M64:%.+]] = and i64 %b, 63
  //OPT-NEXT: [[C64:%.+]] = ashr i64 %a, [[M64]]
  long c = a>>b;
  long d = ((long)8)>>65;
  //OPT-NEXT: [[E64:%.+]] = add nsw i64 [[C64]], 4
  long e = c + d;
  //OPT-NEXT: ret i64 [[E64]]
  return e;
}

typedef __attribute__((ext_vector_type(4))) int int4;

//OPT: @vectorVectorTest
int4 vectorVectorTest(int4 a,int4 b) {
  //OPT: [[VM:%.+]] = and <4 x i32> %b, <i32 31, i32 31, i32 31, i32 31>
  //OPT-NEXT: [[VC:%.+]] = shl <4 x i32> %a, [[VM]]
  int4 c = a << b;
  //OPT-NEXT: [[VF:%.+]] = add <4 x i32> [[VC]], <i32 2, i32 4, i32 16, i32 8>
  int4 d = {1, 1, 1, 1};
  int4 e = {33, 34, -28, -29};
  int4 f = c + (d << e);
  //OPT-NEXT: ret <4 x i32> [[VF]]
  return f;
}

//OPT: @vectorScalarTest
int4 vectorScalarTest(int4 a,int b) {
  //OPT: [[SP0:%.+]] = insertelement <4 x i32> undef, i32 %b, i32 0
  //OPT: [[SP1:%.+]] = shufflevector <4 x i32> [[SP0]], <4 x i32> undef, <4 x i32> zeroinitializer
  //OPT: [[VSM:%.+]] = and <4 x i32> [[SP1]], <i32 31, i32 31, i32 31, i32 31>
  //OPT-NEXT: [[VSC:%.+]] = shl <4 x i32> %a, [[VSM]]
  int4 c = a << b;
  //OPT-NEXT: [[VSF:%.+]] = add <4 x i32> [[VSC]], <i32 4, i32 4, i32 4, i32 4>
  int4 d = {1, 1, 1, 1};
  int4 f = c + (d << 34);
  //OPT-NEXT: ret <4 x i32> [[VSF]]
  return f;
}


Correct implementation of "length"

Correct implementation of "length"

#ifdef cl_khr_fp16
__attribute__((__overloadable__))
half length(half p)
{
  return fabs(p);
}

__attribute__((__overloadable__))
half length(half2 p)
{
  p = fabs(p);
  half maxp = max(p.s0, p.s1);
  if (maxp == (half)0.0f) return (half)0.0f;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

__attribute__((__overloadable__))
half length(half3 p)
{
  p = fabs(p);
  half maxp = max(max(p.s0, p.s1), p.s2);
  if (maxp == (half)0.0f) return (half)0.0f;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

__attribute__((__overloadable__))
half length(half4 p)
{
  p = fabs(p);
  half maxp = max(max(p.s0, p.s1), max(p.s2, p.s3));
  if (maxp == (half)0.0f) return (half)0.0f;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}
#endif

__attribute__((__overloadable__))
float length(float p)
{
  return fabs(p);
}

__attribute__((__overloadable__))
float length(float2 p)
{
  p = fabs(p);
  float maxp = max(p.s0, p.s1);
  if (maxp == 0.0f) return 0.0f;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

__attribute__((__overloadable__))
float length(float3 p)
{
  p = fabs(p);
  float maxp = max(max(p.s0, p.s1), p.s2);
  if (maxp == 0.0f) return 0.0f;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

__attribute__((__overloadable__))
float length(float4 p)
{
  p = fabs(p);
  float maxp = max(max(p.s0, p.s1), max(p.s2, p.s3));
  if (maxp == 0.0f) return 0.0f;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

#ifdef cl_khr_fp64
__attribute__((__overloadable__))
double length(double p)
{
  return fabs(p);
}

__attribute__((__overloadable__))
double length(double2 p)
{
  p = fabs(p);
  double maxp = max(p.s0, p.s1);
  if (maxp == 0.0) return 0.0;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

__attribute__((__overloadable__))
double length(double3 p)
{
  p = fabs(p);
  double maxp = max(max(p.s0, p.s1), p.s2);
  if (maxp == 0.0) return 0.0;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}

__attribute__((__overloadable__))
double length(double4 p)
{
  p = fabs(p);
  double maxp = max(max(p.s0, p.s1), max(p.s2, p.s3));
  if (maxp == 0.0) return 0.0;
  p /= maxp;
  return sqrt(dot(p, p)) * maxp;
}
#endif

#ifdef cl_khr_fp16
__attribute__((__overloadable__))
half length(half p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
half length(half2 p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
half length(half3 p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
half length(half4 p)
{
  return sqrt(dot(p, p));
}
#endif

__attribute__((__overloadable__))
float length(float p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
float length(float2 p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
float length(float3 p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
float length(float4 p)
{
  return sqrt(dot(p, p));
}

#ifdef cl_khr_fp64
__attribute__((__overloadable__))
double length(double p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
double length(double2 p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
double length(double3 p)
{
  return sqrt(dot(p, p));
}

__attribute__((__overloadable__))
double length(double4 p)
{
  return sqrt(dot(p, p));
}
#endif


Added a first version of the direct version of GEMM with local memory

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This is a generic GEMM kernel that works for all sizes and configurations: it doesn't require any
// pre and and post-processing kernels.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Caches global off-chip memory into local (shared) memory on-chip. This function is specific for
// caching the A input matrix.
inline void GlobalToLocalDirectA(const __global realM* restrict agm, __local real* alm,
                                 const int a_ld, const int a_offset, const int tid, const int kwg,
                                 const int a_transpose, const int a_conjugate) {
  const int la0 = tid % MDIMA;
  const int la1 = tid / MDIMA;
  #pragma unroll
  for (int mia=0; mia<MWA/VWM; ++mia) {
    #pragma unroll
    for (int kia=0; kia<KWA; ++kia) {

      // Computes the indices for the global memory
      int mg = mia + la0*(MWA/VWM);
      int kg = kia + la1*KWA;
      int idm = (a_transpose) ? mg + kwg/VWM : mg + GetGroupID0()*(MWG/VWM);
      int idk = (a_transpose) ? kg + GetGroupID0()*MWG : kg + kwg;

      // Loads the data from global memory into the local memory
      const realM avec = agm[idk*(a_ld/VWM) + idm + a_offset];
      #if VWM == 1
         alm[kg*MWG + mg] = avec;
      #elif VWM == 2
         alm[kg*MWG + mg*VWM + 0] = avec.x;
         alm[kg*MWG + mg*VWM + 1] = avec.y;
      #elif VWM == 4
         alm[kg*MWG + mg*VWM + 0] = avec.x;
         alm[kg*MWG + mg*VWM + 1] = avec.y;
         alm[kg*MWG + mg*VWM + 2] = avec.z;
         alm[kg*MWG + mg*VWM + 3] = avec.w;
      #elif VWM == 8
         alm[kg*MWG + mg*VWM + 0] = avec.s0;
         alm[kg*MWG + mg*VWM + 1] = avec.s1;
         alm[kg*MWG + mg*VWM + 2] = avec.s2;
         alm[kg*MWG + mg*VWM + 3] = avec.s3;
         alm[kg*MWG + mg*VWM + 4] = avec.s4;
         alm[kg*MWG + mg*VWM + 5] = avec.s5;
         alm[kg*MWG + mg*VWM + 6] = avec.s6;
         alm[kg*MWG + mg*VWM + 7] = avec.s7;
      #elif VWM == 16
         alm[kg*MWG + mg*VWM + 0] = avec.s0;
         alm[kg*MWG + mg*VWM + 1] = avec.s1;
         alm[kg*MWG + mg*VWM + 2] = avec.s2;
         alm[kg*MWG + mg*VWM + 3] = avec.s3;
         alm[kg*MWG + mg*VWM + 4] = avec.s4;
         alm[kg*MWG + mg*VWM + 5] = avec.s5;
         alm[kg*MWG + mg*VWM + 6] = avec.s6;
         alm[kg*MWG + mg*VWM + 7] = avec.s7;
         alm[kg*MWG + mg*VWM + 8] = avec.s8;
         alm[kg*MWG + mg*VWM + 9] = avec.s9;
         alm[kg*MWG + mg*VWM + 10] = avec.sA;
         alm[kg*MWG + mg*VWM + 11] = avec.sB;
         alm[kg*MWG + mg*VWM + 12] = avec.sC;
         alm[kg*MWG + mg*VWM + 13] = avec.sD;
         alm[kg*MWG + mg*VWM + 14] = avec.sE;
         alm[kg*MWG + mg*VWM + 15] = avec.sF;
      #endif
      if (a_conjugate) {
        for (int vm=0; vm<VWM; ++vm) {
          COMPLEX_CONJUGATE(alm[kg*MWG + mg*VWM + vm]);
        }
      }
    }
  }
}

// Same as above, but now for the B input matrix
inline void GlobalToLocalDirectB(const __global realN* restrict bgm, __local real* blm,
                                 const int b_ld, const int b_offset, const int tid, const int kwg,
                                 const int b_transpose, const int b_conjugate) {
  const int lb0 = tid % NDIMB;
  const int lb1 = tid / NDIMB;
  #pragma unroll
  for (int kib=0; kib<KWB; ++kib) {
    #pragma unroll
    for (int nib=0; nib<NWB/VWN; ++nib) {

      // Computes the indices for the global memory
      int ng = nib + lb0*(NWB/VWN);
      int kg = kib + lb1*KWB;
      int idn = (b_transpose) ? ng + kwg/VWN : ng + GetGroupID1()*(NWG/VWN);
      int idk = (b_transpose) ? kg + GetGroupID1()*NWG : kg + kwg;

      // Loads the data from global memory into the local memory
      const realM bvec = bgm[idk*(b_ld/VWN) + idn + b_offset];
      #if VWN == 1
         blm[kg*NWG + ng] = bvec;
      #elif VWN == 2
         blm[kg*NWG + ng*VWN + 0] = bvec.x;
         blm[kg*NWG + ng*VWN + 1] = bvec.y;
      #elif VWN == 4
         blm[kg*NWG + ng*VWN + 0] = bvec.x;
         blm[kg*NWG + ng*VWN + 1] = bvec.y;
         blm[kg*NWG + ng*VWN + 2] = bvec.z;
         blm[kg*NWG + ng*VWN + 3] = bvec.w;
      #elif VWN == 8
         blm[kg*NWG + ng*VWN + 0] = bvec.s0;
         blm[kg*NWG + ng*VWN + 1] = bvec.s1;
         blm[kg*NWG + ng*VWN + 2] = bvec.s2;
         blm[kg*NWG + ng*VWN + 3] = bvec.s3;
         blm[kg*NWG + ng*VWN + 4] = bvec.s4;
         blm[kg*NWG + ng*VWN + 5] = bvec.s5;
         blm[kg*NWG + ng*VWN + 6] = bvec.s6;
         blm[kg*NWG + ng*VWN + 7] = bvec.s7;
      #elif VWN == 16
         blm[kg*NWG + ng*VWN + 0] = bvec.s0;
         blm[kg*NWG + ng*VWN + 1] = bvec.s1;
         blm[kg*NWG + ng*VWN + 2] = bvec.s2;
         blm[kg*NWG + ng*VWN + 3] = bvec.s3;
         blm[kg*NWG + ng*VWN + 4] = bvec.s4;
         blm[kg*NWG + ng*VWN + 5] = bvec.s5;
         blm[kg*NWG + ng*VWN + 6] = bvec.s6;
         blm[kg*NWG + ng*VWN + 7] = bvec.s7;
         blm[kg*NWG + ng*VWN + 8] = bvec.s8;
         blm[kg*NWG + ng*VWN + 9] = bvec.s9;
         blm[kg*NWG + ng*VWN + 10] = bvec.sA;
         blm[kg*NWG + ng*VWN + 11] = bvec.sB;
         blm[kg*NWG + ng*VWN + 12] = bvec.sC;
         blm[kg*NWG + ng*VWN + 13] = bvec.sD;
         blm[kg*NWG + ng*VWN + 14] = bvec.sE;
         blm[kg*NWG + ng*VWN + 15] = bvec.sF;
      #endif
      if (b_conjugate) {
        for (int vn=0; vn<VWN; ++vn) {
          COMPLEX_CONJUGATE(blm[kg*NWG + ng*VWN + vn]);
        }
      }
    }
  }
}

// =================================================================================================

// Caches on-chip local memory into per-thread private memory (registers). This function is specific
// for caching the A input matrix.
inline void LocalToPrivateDirectA(__local real* alm, real apm[MWI], const int kg,
                                  const int a_transpose) {
  #pragma unroll
  for (int mi=0; mi<MWI; ++mi) {
    const int mg = mi + get_local_id(0)*MWI;
    const int index = (a_transpose) ? mg*KWG + kg : kg*MWG + mg;
    apm[mi] = alm[index];
  }
}

// Same as above, but now for the B input matrix
inline void LocalToPrivateDirectB(__local real* blm, real bpm[NWI], const int kg,
                                  const int b_transpose) {
  #pragma unroll
  for (int ni=0; ni<NWI; ++ni) {
    const int ng = ni + get_local_id(1)*NWI;
    const int index = (b_transpose) ? ng*KWG + kg : kg*NWG + ng;
    bpm[ni] = blm[index];
  }
}

// =================================================================================================

// Initializes the accumulation registers to zero
inline void InitAccRegistersDirect(real cpm[NWI][MWI]) {
  #pragma unroll
  for (int mi=0; mi<MWI; ++mi) {
    #pragma unroll
    for (int ni=0; ni<NWI; ++ni) {
      SetToZero(cpm[ni][mi]);
    }
  }
}

// =================================================================================================

// Performs the actual computation: Cpm += Apm * Bpm
inline void MultiplyAccumulateDirect(real cpm[NWI][MWI], real apm[MWI], real bpm[NWI]) {
  #pragma unroll
  for (int ni=0; ni<NWI; ++ni) {
    #pragma unroll
    for (int mi=0; mi<MWI; ++mi) {
      MultiplyAdd(cpm[ni][mi], apm[mi], bpm[ni]);
    }
  }
}

// =================================================================================================

// Merges the results in Cpm with the global array in Cgm. This also performs the multiplication
// with the constants: Cgm = alpha*A*B + beta*Cgm = alpha*Cpm + beta*Cgm
inline void StoreResultsDirect(__global real* cgm, real cpm[NWI][MWI],
                               const int kSizeM, const int kSizeN,
                               const real alpha, const real beta,
                               const int c_ld, const int c_offset, const int c_transpose) {
  #pragma unroll
  for (int ni=0; ni<NWI; ++ni) {
    #pragma unroll
    for (int mi=0; mi<MWI; ++mi) {
      int mg = mi + get_local_id(0)*MWI;
      int ng = ni + get_local_id(1)*NWI;
      int idm = mg + GetGroupID0() * MWG;
      int idn = ng + GetGroupID1() * NWG;

      // Determines the destination index
      const int c_index = (c_transpose) ? idm*c_ld + idn : idn*c_ld + idm;

      // The final multiplication with alpha and the addition with beta*C
      real result;
      AXPBY(result, alpha, cpm[ni][mi], beta, cgm[c_index + c_offset]);
      cgm[c_index + c_offset] = result;
    }
  }
}

// =================================================================================================

// Main entry point of the kernel. This is the direct version without restrictions.
__attribute__((reqd_work_group_size(MDIMC, NDIMC, 1)))
__kernel void XgemmDirect(const int kSizeM, const int kSizeN, const int kSizeK,
                          const real_arg arg_alpha,
                          const real_arg arg_beta,
                          const __global realM* restrict agm, const int a_offset, const int a_ld,
                          const __global realN* restrict bgm, const int b_offset, const int b_ld,
                          __global real* cgm, const int c_offset, const int c_ld,
                          const int a_transpose, const int b_transpose, const int c_transpose,
                          const int a_conjugate, const int b_conjugate) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  // Extra pointers to scalar versions of global memory
  const __global real* restrict agms = (const __global real* restrict) agm;
  const __global real* restrict bgms = (const __global real* restrict) bgm;

  // Allocates workgroup-private memory (local memory)
  __local real alm[KWG * MWG];
  __local real blm[KWG * NWG];

  // Combined thread identifier (volatile to disable caching)
  volatile int tid = get_local_id(0) + MDIMC*get_local_id(1);

  // Allocates workitem-private memory (registers)
  real apm[MWI];
  real bpm[NWI];
  real cpm[NWI][MWI];

  // Initializes the accumulation registers
  InitAccRegistersDirect(cpm);

  // The faster version of GEMM is not allowed on the (incomplete) borders. Therefore, this section
  // processes only the main parts: output blocks of MWG by NWG.
  const int idm = get_local_id(0) * MWI + GetGroupID0() * MWG;
  const int idn = get_local_id(1) * NWI + GetGroupID1() * NWG;
  if ((idm < (kSizeM/MWG)*MWG) && (idn < (kSizeN/NWG)*NWG) &&
      (a_ld % VWM == 0) && (b_ld % VWN == 0)) {

    // Loops over all complete workgroup tiles
    int kwg = 0;
    for (; kwg < (kSizeK/KWG) * KWG; kwg+=KWG) {

      // Loads data: off-chip --> local (matrix A and B)
      GlobalToLocalDirectA(agm, alm, a_ld, a_offset, tid, kwg, a_transpose, a_conjugate);
      GlobalToLocalDirectB(bgm, blm, b_ld, b_offset, tid, kwg, b_transpose, b_conjugate);
      barrier(CLK_LOCAL_MEM_FENCE);

      // Loops over all workitem tiles, unrolled by a factor KWI
      for (int pwi=0; pwi<KWG; pwi+=KWI) {
        #pragma unroll
        for (int pit=0; pit<KWI; ++pit) {
          int kg = pwi + pit;

          // Loads data: local --> private (matrix A)
          LocalToPrivateDirectA(alm, apm, kg, a_transpose);

          // Loads data: local --> private (matrix B)
          LocalToPrivateDirectB(blm, bpm, kg, b_transpose);

          // Performs the accumulation (Cpm += Apm * Bpm)
          MultiplyAccumulateDirect(cpm, apm, bpm);
        }
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }

    // Loop over the remaining part (incomplete tile in K-dimension)
    for (; kwg < kSizeK; ++kwg) {
      const int idk = kwg;

      // Loads A into register memory
      #pragma unroll
      for (int mi=0; mi<MWI; ++mi) {
        const int a_index = (a_transpose) ? (idm + mi)*a_ld + idk : idk*a_ld + (idm + mi);
        apm[mi] = agms[a_index + a_offset];
        if (a_conjugate) { COMPLEX_CONJUGATE(apm[mi]); }
      }

      // Loads B into register memory
      #pragma unroll
      for (int ni=0; ni<NWI; ++ni) {
        const int b_index = (b_transpose) ? (idn + ni)*b_ld + idk : idk*b_ld + (idn + ni);
        bpm[ni] = bgms[b_index + b_offset];
        if (b_conjugate) { COMPLEX_CONJUGATE(bpm[ni]); }
      }

      // Performs the accumulation (Cpm += Apm * Bpm)
      MultiplyAccumulateDirect(cpm, apm, bpm);
    }

    #if GLOBAL_MEM_FENCE == 1
      barrier(CLK_GLOBAL_MEM_FENCE);
    #endif

    // Stores a tile of results and performs the multiplication with alpha and beta
    StoreResultsDirect(cgm, cpm, kSizeM, kSizeN, alpha, beta, c_ld, c_offset, c_transpose);
  }

  // Simple but slow version for the parts on the edge (incomplete tiles in M and N-dimensions)
  else {

    // Loop over the K-dimension
    for (int idk = 0; idk < kSizeK; ++idk) {

      // Loads A into register memory
      #pragma unroll
      for (int mi=0; mi<MWI; ++mi) {
        if (idm + mi < kSizeM) {
          const int a_index = (a_transpose) ? (idm + mi)*a_ld + idk : idk*a_ld + (idm + mi);
          apm[mi] = agms[a_index + a_offset];
          if (a_conjugate) { COMPLEX_CONJUGATE(apm[mi]); }
        }
        else {
          SetToZero(apm[mi]);
        }
      }

      // Loads B into register memory
      #pragma unroll
      for (int ni=0; ni<NWI; ++ni) {
        if (idn + ni < kSizeN) {
          const int b_index = (b_transpose) ? (idn + ni)*b_ld + idk : idk*b_ld + (idn + ni);
          bpm[ni] = bgms[b_index + b_offset];
          if (b_conjugate) { COMPLEX_CONJUGATE(bpm[ni]); }
        }
        else {
          SetToZero(bpm[ni]);
        }
      }

      // Performs the accumulation (Cpm += Apm * Bpm)
      MultiplyAccumulateDirect(cpm, apm, bpm);
    }

    // Stores the results
    #pragma unroll
    for (int ni=0; ni<NWI; ++ni) {
      #pragma unroll
      for (int mi=0; mi<MWI; ++mi) {
        if ((idm + mi) < kSizeM && (idn + ni) < kSizeN) {

          // Determines the destination index
          const int c_index = (c_transpose) ? (idm + mi)*c_ld + (idn + ni) : (idn + ni)*c_ld + (idm + mi);

          // Computes and stores the result
          real result;
          AXPBY(result, alpha, cpm[ni][mi], beta, cgm[c_index + c_offset]);
          cgm[c_index + c_offset] = result;
        }
      }
    }
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This is a generic GEMM kernel that works for all sizes and configurations: it doesn't require any
// pre and and post-processing kernels.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Initializes the accumulation registers to zero
inline void InitAccRegistersDirect(real cpm[NWI][MWI]) {
  #pragma unroll
  for (int mi=0; mi<MWI; ++mi) {
    #pragma unroll
    for (int ni=0; ni<NWI; ++ni) {
      SetToZero(cpm[ni][mi]);
    }
  }
}
// =================================================================================================

// Performs the actual computation: Cpm += Apm * Bpm
inline void MultiplyAccumulateDirect(real cpm[NWI][MWI], real apm[MWI], real bpm[NWI]) {
  #pragma unroll
  for (int ni=0; ni<NWI; ++ni) {
    #pragma unroll
    for (int mi=0; mi<MWI; ++mi) {
      MultiplyAdd(cpm[ni][mi], apm[mi], bpm[ni]);
    }
  }
}

// =================================================================================================

// Merges the results in Cpm with the global array in Cgm. This also performs the multiplication
// with the constants: Cgm = alpha*A*B + beta*Cgm = alpha*Cpm + beta*Cgm
inline void StoreResultsDirect(__global real* cgm, real cpm[NWI][MWI],
                               const int kSizeM, const int kSizeN,
                               const real alpha, const real beta,
                               const int c_ld, const int c_offset, const int c_transpose) {
  #pragma unroll
  for (int ni=0; ni<NWI; ++ni) {
    #pragma unroll
    for (int mi=0; mi<MWI; ++mi) {
      int mg = mi + get_local_id(0)*MWI;
      int ng = ni + get_local_id(1)*NWI;
      int idm = mg + GetGroupID0() * MWG;
      int idn = ng + GetGroupID1() * NWG;

      // Determines the destination index
      const int c_index = (c_transpose) ? idm*c_ld + idn : idn*c_ld + idm;

      // The final multiplication with alpha and the addition with beta*C
      real result;
      AXPBY(result, alpha, cpm[ni][mi], beta, cgm[c_index + c_offset]);
      cgm[c_index + c_offset] = result;
    }
  }
}

// =================================================================================================

// Main entry point of the kernel. This is the direct version without restrictions.
__attribute__((reqd_work_group_size(MDIMC, NDIMC, 1)))
__kernel void XgemmDirect(const int kSizeM, const int kSizeN, const int kSizeK,
                          const real_arg arg_alpha,
                          const real_arg arg_beta,
                          const __global realM* restrict agm, const int a_offset, const int a_ld,
                          const __global realN* restrict bgm, const int b_offset, const int b_ld,
                          __global real* cgm, const int c_offset, const int c_ld,
                          const int a_transpose, const int b_transpose, const int c_transpose,
                          const int a_conjugate, const int b_conjugate) {
  const real alpha = GetRealArg(arg_alpha);
  const real beta = GetRealArg(arg_beta);

  // Extra pointers to scalar versions of global memory
  const __global real* restrict agms = (const __global real* restrict) agm;
  const __global real* restrict bgms = (const __global real* restrict) bgm;

  // Allocates workitem-private memory (registers)
  real apm[MWI];
  real bpm[NWI];
  real cpm[NWI][MWI];

  // Initializes the accumulation registers
  InitAccRegistersDirect(cpm);

  // The faster version of GEMM is not allowed on the (incomplete) borders. Therefore, this section
  // processes only the main parts: output blocks of NWI by MWI.
  const int idm = get_local_id(0) * MWI + GetGroupID0() * MWG;
  const int idn = get_local_id(1) * NWI + GetGroupID1() * NWG;
  if ((idm < kSizeM - MWI) && (idn < kSizeN - NWI)) {

    // Loops over all complete workgroup tiles
    int kwg = 0;
    // TODO: Implement a faster version with local memory and vector loads
    // for (; kwg < kSizeK - KWG; kwg+=KWG) { }

    // Loop over the remaining part (incomplete tile in K-dimension)
    for (; kwg < kSizeK; ++kwg) {
      const int idk = kwg;

      // Loads A into register memory
      #pragma unroll
      for (int mi=0; mi<MWI; ++mi) {
        const int a_index = (a_transpose) ? (idm + mi)*a_ld + idk : idk*a_ld + (idm + mi);
        apm[mi] = agms[a_index + a_offset];
        if (a_conjugate) { COMPLEX_CONJUGATE(apm[mi]); }
      }

      // Loads B into register memory
      #pragma unroll
      for (int ni=0; ni<NWI; ++ni) {
        const int b_index = (b_transpose) ? (idn + ni)*b_ld + idk : idk*b_ld + (idn + ni);
        bpm[ni] = bgms[b_index + b_offset];
        if (b_conjugate) { COMPLEX_CONJUGATE(bpm[ni]); }
      }

      // Performs the accumulation (Cpm += Apm * Bpm)
      MultiplyAccumulateDirect(cpm, apm, bpm);
    }

    #if GLOBAL_MEM_FENCE == 1
      barrier(CLK_GLOBAL_MEM_FENCE);
    #endif

    // Stores a tile of results and performs the multiplication with alpha and beta
    StoreResultsDirect(cgm, cpm, kSizeM, kSizeN, alpha, beta, c_ld, c_offset, c_transpose);
  }

  // Simple but slow version for the parts on the edge (incomplete tiles in M and N-dimensions)
  else {

    // Loop over the K-dimension
    for (int idk = 0; idk < kSizeK; ++idk) {

      // Loads A into register memory
      #pragma unroll
      for (int mi=0; mi<MWI; ++mi) {
        if (idm + mi < kSizeM) {
          const int a_index = (a_transpose) ? (idm + mi)*a_ld + idk : idk*a_ld + (idm + mi);
          apm[mi] = agms[a_index + a_offset];
          if (a_conjugate) { COMPLEX_CONJUGATE(apm[mi]); }
        }
        else {
          SetToZero(apm[mi]);
        }
      }

      // Loads B into register memory
      #pragma unroll
      for (int ni=0; ni<NWI; ++ni) {
        if (idn + ni < kSizeN) {
          const int b_index = (b_transpose) ? (idn + ni)*b_ld + idk : idk*b_ld + (idn + ni);
          bpm[ni] = bgms[b_index + b_offset];
          if (b_conjugate) { COMPLEX_CONJUGATE(bpm[ni]); }
        }
        else {
          SetToZero(bpm[ni]);
        }
      }

      // Performs the accumulation (Cpm += Apm * Bpm)
      MultiplyAccumulateDirect(cpm, apm, bpm);
    }

    // Stores the results
    #pragma unroll
    for (int ni=0; ni<NWI; ++ni) {
      #pragma unroll
      for (int mi=0; mi<MWI; ++mi) {
        if ((idm + mi) < kSizeM && (idn + ni) < kSizeN) {

          // Determines the destination index
          const int c_index = (c_transpose) ? (idm + mi)*c_ld + (idn + ni) : (idn + ni)*c_ld + (idm + mi);

          // Computes and stores the result
          real result;
          AXPBY(result, alpha, cpm[ni][mi], beta, cgm[c_index + c_offset]);
          cgm[c_index + c_offset] = result;
        }
      }
    }
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================


Remove unused variables

Remove unused variables

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

void g(__local uint *v_lo, __local uint *v_hi, uint4 i)
{
    ulong a, b, c, d;
    a = upsample(v_hi[i.s0], v_lo[i.s0]);
    b = upsample(v_hi[i.s1], v_lo[i.s1]);
    c = upsample(v_hi[i.s2], v_lo[i.s2]);
    d = upsample(v_hi[i.s3], v_lo[i.s3]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    v_lo[i.s0] = (uint)a;
    v_lo[i.s1] = (uint)b;
    v_lo[i.s2] = (uint)c;
    v_lo[i.s3] = (uint)d;

    v_hi[i.s0] = (uint)(a >> 32);
    v_hi[i.s1] = (uint)(b >> 32);
    v_hi[i.s2] = (uint)(c >> 32);
    v_hi[i.s3] = (uint)(d >> 32);
}

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7U;
    uint hash_lane = (thread >> 0) & 0x3U;
    uint4 u0123 = (uint4)(0U, 1U, 2U, 3U);
    uint4 index0 = 4 * u0123 + hash_lane;
    uint4 index1 = 4 * u0123 + (u0123 + hash_lane) % 4;

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index1);

    barrier(CLK_LOCAL_MEM_FENCE);

    index0 = 16 * (index0 / 2) + index0 % 2;
    index1 = 16 * (index1 / 2) + index1 % 2;

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] = prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] = prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] ^= prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] ^= prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = mem_prev->data[pos];
        prev->lo[pos] = (uint)in;
        prev->hi[pos] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos = i * THREADS_PER_LANE + thread;
                    ulong in = mem_curr->data[pos];
                    curr->lo[pos] = (uint)in;
                    curr->hi[pos] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos = i * THREADS_PER_LANE + thread;
                mem_curr->data[pos] = upsample(curr->hi[pos], curr->lo[pos]);
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
            }

            ++mem_curr;
        }
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

void g(__local uint *v_lo, __local uint *v_hi, uint4 i)
{
    ulong a, b, c, d;
    a = upsample(v_hi[i.s0], v_lo[i.s0]);
    b = upsample(v_hi[i.s1], v_lo[i.s1]);
    c = upsample(v_hi[i.s2], v_lo[i.s2]);
    d = upsample(v_hi[i.s3], v_lo[i.s3]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    v_lo[i.s0] = (uint)a;
    v_lo[i.s1] = (uint)b;
    v_lo[i.s2] = (uint)c;
    v_lo[i.s3] = (uint)d;

    v_hi[i.s0] = (uint)(a >> 32);
    v_hi[i.s1] = (uint)(b >> 32);
    v_hi[i.s2] = (uint)(c >> 32);
    v_hi[i.s3] = (uint)(d >> 32);
}

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7U;
    uint hash_lane = (thread >> 0) & 0x3U;
    uint4 u0123 = (uint4)(0U, 1U, 2U, 3U);
    uint4 index0 = 4 * u0123 + hash_lane;
    uint4 index1 = 4 * u0123 + (u0123 + hash_lane) % 4;

    __local ulong *v_lo, *v_hi;

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index1);

    barrier(CLK_LOCAL_MEM_FENCE);

    index0 = 16 * (index0 / 2) + index0 % 2;
    index1 = 16 * (index1 / 2) + index1 % 2;

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] = prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] = prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] ^= prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] ^= prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = mem_prev->data[pos];
        prev->lo[pos] = (uint)in;
        prev->hi[pos] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos = i * THREADS_PER_LANE + thread;
                    ulong in = mem_curr->data[pos];
                    curr->lo[pos] = (uint)in;
                    curr->hi[pos] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos = i * THREADS_PER_LANE + thread;
                mem_curr->data[pos] = upsample(curr->hi[pos], curr->lo[pos]);
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
            }

            ++mem_curr;
        }
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
        mem_curr = mem_lane;
    }
}