Conditional compilation of half-precision code.

/*
 * Copyright (c) 2016 ARM Limited.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifdef cl_khr_fp16
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#endif

__kernel void mm_fp32( __global float *const mtx_a,
                       __global float *const mtx_b,
                       __global float       *mtx_out
                     )
{
    /* src_addr.s0 = address of matrix A */
    /* src_addr.s1 = address of matrix B */

    /* Compute address for matrix A and B */
    uint2 src_addr = (uint2)(get_global_id (1), get_global_id (0)) * (uint2)(MATRIX_A_INTERLEAVED_STRIDE,
                                                                             MATRIX_B_TRANSPOSED_STRIDE);

    /* Compute end row address for matrix B */
    uint end_row_mtx_b = src_addr.s1 + MATRIX_B_TRANSPOSED_STRIDE;

    /* Reset accumulators */
    float4 c00 = 0.0f;
    float4 c10 = 0.0f;
    float4 c20 = 0.0f;
    float4 c30 = 0.0f;

    for(; src_addr.s1 < end_row_mtx_b; src_addr += (uint2)(8, 8))
    {
        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        float4 a0 = vload4(0, mtx_a + src_addr.s0);
        float4 b0 = vload4(0, mtx_b + src_addr.s1);

        c00 += (float4)a0.s0 * b0;
        c10 += (float4)a0.s1 * b0;
        c20 += (float4)a0.s2 * b0;
        c30 += (float4)a0.s3 * b0;

        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        a0 = vload4(0, mtx_a + src_addr.s0 + 4);
        b0 = vload4(0, mtx_b + src_addr.s1 + 4);

        c00 += (float4)a0.s0 * b0;
        c10 += (float4)a0.s1 * b0;
        c20 += (float4)a0.s2 * b0;
        c30 += (float4)a0.s3 * b0;
    }

    /* Wait for all work-items to finish */
    barrier(CLK_GLOBAL_MEM_FENCE);

    /* Compute destination address */
    uint dst_addr = (get_global_id (0) + get_global_id (1) * COL_MTX_C) * 4;

    /* Store 4x4 block */
    vstore4(c00, 0, mtx_out + dst_addr + 0 * COL_MTX_C);
    vstore4(c10, 0, mtx_out + dst_addr + 1 * COL_MTX_C);
    vstore4(c20, 0, mtx_out + dst_addr + 2 * COL_MTX_C);
    vstore4(c30, 0, mtx_out + dst_addr + 3 * COL_MTX_C);
}

#ifdef cl_khr_fp16
__kernel void mm_fp16( __global half *const mtx_a,
                       __global half *const mtx_b,
                       __global half       *mtx_out
                     )
{
    /* src_addr.s0 = address of matrix A */
    /* src_addr.s1 = address of matrix B */

    /* Compute address for matrix A and B */
    uint2 src_addr = (uint2)(get_global_id (1), get_global_id (0)) * (uint2)(MATRIX_A_INTERLEAVED_STRIDE,
                                                                             MATRIX_B_TRANSPOSED_STRIDE);

    /* Compute end row address for matrix B */
    uint end_row_mtx_b = src_addr.s1 + MATRIX_B_TRANSPOSED_STRIDE;

    /* Reset accumulators */
    half8 c00 = 0.0f;
    half8 c10 = 0.0f;
    half8 c20 = 0.0f;
    half8 c30 = 0.0f;

    for(; src_addr.s1 < end_row_mtx_b; src_addr += (uint2)(8, 16))
    {
        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        half4 a0 = vload4(0, mtx_a + src_addr.s0);
        half8 b0 = vload8(0, mtx_b + src_addr.s1);

        c00 += (half8)a0.s0 * b0;
        c10 += (half8)a0.s1 * b0;
        c20 += (half8)a0.s2 * b0;
        c30 += (half8)a0.s3 * b0;

        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        a0 = vload4(0, mtx_a + src_addr.s0 + 4);
        b0 = vload8(0, mtx_b + src_addr.s1 + 8);

        c00 += (half8)a0.s0 * b0;
        c10 += (half8)a0.s1 * b0;
        c20 += (half8)a0.s2 * b0;
        c30 += (half8)a0.s3 * b0;
    }

    /* Wait for all work-items to finish */
    barrier(CLK_GLOBAL_MEM_FENCE);

    /* Compute destination address */
    uint dst_addr = (get_global_id (0) * 2 + get_global_id (1) * COL_MTX_C) * 4;

    /* Store 8x4 block */
    vstore8(c00, 0, mtx_out + dst_addr + 0 * COL_MTX_C);
    vstore8(c10, 0, mtx_out + dst_addr + 1 * COL_MTX_C);
    vstore8(c20, 0, mtx_out + dst_addr + 2 * COL_MTX_C);
    vstore8(c30, 0, mtx_out + dst_addr + 3 * COL_MTX_C);
}
#endif

#ifdef cl_khr_fp16
__kernel void mm_hybrid( __global half *const mtx_a,
                         __global half *const mtx_b,
                         __global half       *mtx_out
                       )
{
    /* src_addr.s0 = address of matrix A */
    /* src_addr.s1 = address of matrix B */

    /* Compute address for matrix A and B */
    uint2 src_addr = (uint2)(get_global_id (1), get_global_id (0)) * (uint2)(MATRIX_A_INTERLEAVED_STRIDE,
                                                                             MATRIX_B_TRANSPOSED_STRIDE);

    /* Compute end row address for matrix B */
    uint end_row_mtx_b = src_addr.s1 + MATRIX_B_TRANSPOSED_STRIDE;

    /* Reset accumulators */
    float4 c00 = 0.0f;
    float4 c10 = 0.0f;
    float4 c20 = 0.0f;
    float4 c30 = 0.0f;

    for(; src_addr.s1 < end_row_mtx_b; src_addr += (uint2)(8, 8))
    {
        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        half8 a0_a1 = vload8(0, mtx_a + src_addr.s0);
        half8 b0_b1 = vload8(0, mtx_b + src_addr.s1);

        c00 += convert_float4((half4)a0_a1.s0 * b0_b1.s0123);
        c00 += convert_float4((half4)a0_a1.s4 * b0_b1.s4567);
        c10 += convert_float4((half4)a0_a1.s1 * b0_b1.s0123);
        c10 += convert_float4((half4)a0_a1.s5 * b0_b1.s4567);

        c20 += convert_float4((half4)a0_a1.s2 * b0_b1.s0123);
        c20 += convert_float4((half4)a0_a1.s6 * b0_b1.s4567);
        c30 += convert_float4((half4)a0_a1.s3 * b0_b1.s0123);
        c30 += convert_float4((half4)a0_a1.s7 * b0_b1.s4567);
    }

    /* Wait for all work-items to finish */
    barrier(CLK_GLOBAL_MEM_FENCE);

    /* Compute destination address */
    uint dst_addr = (get_global_id (0) + get_global_id (1) * COL_MTX_C) * 4;

    /* Store 4x4 block */
    vstore_half4(c00, 0, mtx_out + dst_addr + 0 * COL_MTX_C);
    vstore_half4(c10, 0, mtx_out + dst_addr + 1 * COL_MTX_C);
    vstore_half4(c20, 0, mtx_out + dst_addr + 2 * COL_MTX_C);
    vstore_half4(c30, 0, mtx_out + dst_addr + 3 * COL_MTX_C);

}
#endif

__kernel void transpose_fp32( __global float const *mtx_b,
                              __global float       *mtx_out
                            )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix B - source */
    uint src_addr = x * 4 + y * COL_MTX_B;

    /* Compute address for Matrix B transposed - destination */
    uint dst_addr = y * 4 + x * MATRIX_B_TRANSPOSED_STRIDE;

    float4 b0 = vload4(0, mtx_b + src_addr);

    vstore4(b0, 0, mtx_out + dst_addr);
}

#ifdef cl_khr_fp16
__kernel void transpose_fp16( __global half const *mtx_b,
                              __global half       *mtx_out
                            )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix B - source */
    uint src_addr = x * 8 + y * COL_MTX_B;

    /* Compute address for Matrix B transposed - destination */
    uint dst_addr = y * 8 + x * MATRIX_B_TRANSPOSED_STRIDE;

    half8 b0 = vload8(0, mtx_b + src_addr);

    vstore8(b0, 0, mtx_out + dst_addr);
}
#endif

#ifdef cl_khr_fp16
__kernel void transpose_hybrid( __global half const *mtx_b,
                                __global half       *mtx_out
                              )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix B - source */
    uint src_addr = x * 4 + y * COL_MTX_B;

    /* Compute address for Matrix B transposed - destination */
    uint dst_addr = y * 4 + x * MATRIX_B_TRANSPOSED_STRIDE;

    half4 b0 = vload4(0, mtx_b + src_addr);

    vstore4(b0, 0, mtx_out + dst_addr);
}
#endif

__kernel void interleave_fp32( __global float const *mtx_a,
                               __global float       *mtx_out
                             )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix A - source */
    uint src_addr = x * 4 + y * 4 * COL_MTX_A;

    /* Compute address for Matrix A interleaved - destination */
    uint dst_addr = x * 16 + y * MATRIX_A_INTERLEAVED_STRIDE;

    /* Load values from Matrix A */
    float4 a0 = vload4(0, mtx_a + src_addr + 0 * COL_MTX_A);
    float4 a1 = vload4(0, mtx_a + src_addr + 1 * COL_MTX_A);
    float4 a2 = vload4(0, mtx_a + src_addr + 2 * COL_MTX_A);
    float4 a3 = vload4(0, mtx_a + src_addr + 3 * COL_MTX_A);

    float4 val0 = (float4)(a0.s0, a1.s0, a2.s0, a3.s0);
    vstore4(val0, 0, mtx_out + dst_addr);

    val0 = (float4)(a0.s1, a1.s1, a2.s1, a3.s1);
    vstore4(val0, 0, mtx_out + dst_addr + 4);

    val0 = (float4)(a0.s2, a1.s2, a2.s2, a3.s2);
    vstore4(val0, 0, mtx_out + dst_addr + 8);

    val0 = (float4)(a0.s3, a1.s3, a2.s3, a3.s3);
    vstore4(val0, 0, mtx_out + dst_addr + 12);
}

#ifdef cl_khr_fp16
__kernel void interleave_fp16( __global half const *mtx_a,
                               __global half       *mtx_out
                             )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix A - source */
    uint src_addr = x * 4 + y * 4 * COL_MTX_A;

    /* Compute address for Matrix A interleaved - destination */
    uint dst_addr = x * 16 + y * MATRIX_A_INTERLEAVED_STRIDE;

    half4 a0 = vload4(0, mtx_a + src_addr + 0 * COL_MTX_A);
    half4 a1 = vload4(0, mtx_a + src_addr + 1 * COL_MTX_A);
    half4 a2 = vload4(0, mtx_a + src_addr + 2 * COL_MTX_A);
    half4 a3 = vload4(0, mtx_a + src_addr + 3 * COL_MTX_A);

    half4 val0 = (half4)(a0.s0, a1.s0, a2.s0, a3.s0);
    vstore4(val0, 0, mtx_out + dst_addr);

    val0 = (half4)(a0.s1, a1.s1, a2.s1, a3.s1);
    vstore4(val0, 0, mtx_out + dst_addr + 4);

    val0 = (half4)(a0.s2, a1.s2, a2.s2, a3.s2);
    vstore4(val0, 0, mtx_out + dst_addr + 8);

    val0 = (half4)(a0.s3, a1.s3, a2.s3, a3.s3);
    vstore4(val0, 0, mtx_out + dst_addr + 12);
}
#endif

__kernel void finalize_fp32( __global float const *mtx_c,
                             __global float       *mtx_axb,
                             float                alpha,
                             float                beta
                           )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address */
    uint addr = x * 4 + y * COL_MTX_C;

    /* Load values from A x B */
    float4 axb = vload4(0, mtx_axb + addr);

    /* Load values from Matrix C */
    float4 c = vload4(0, mtx_c + addr);

    /* Computes alpha * axb + beta * c */
    float4 out = (float4)alpha * axb + (float4)beta * c;

    /* Store final result in axb matrix */
    vstore4(out, 0, mtx_axb + addr);
}

#ifdef cl_khr_fp16
__kernel void finalize_fp16( __global half const *mtx_c,
                             __global half        *mtx_axb,
                             float               alpha,
                             float               beta
                           )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address */
    uint addr = x * 8 + y * COL_MTX_C;

    half8 axb = vload8(0, mtx_axb + addr);
    half8 c = vload8(0, mtx_c + addr);

    /* Computes alpha * acc + beta * c. The sum is in fp32 */
    float8 out = convert_float8((half8)alpha * axb) + convert_float8((half8)beta * c);

    vstore_half8(out, 0, mtx_axb + addr);
}
#endif

#ifdef cl_khr_fp16
#pragma OPENCL EXTENSION cl_khr_fp16 : disable
#endif

/*
 * Copyright (c) 2016 ARM Limited.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma OPENCL EXTENSION cl_khr_fp16 : enable

__kernel void mm_fp32( __global float *const mtx_a,
                       __global float *const mtx_b,
                       __global float       *mtx_out
                     )
{
    /* src_addr.s0 = address of matrix A */
    /* src_addr.s1 = address of matrix B */

    /* Compute address for matrix A and B */
    uint2 src_addr = (uint2)(get_global_id (1), get_global_id (0)) * (uint2)(MATRIX_A_INTERLEAVED_STRIDE,
                                                                             MATRIX_B_TRANSPOSED_STRIDE);

    /* Compute end row address for matrix B */
    uint end_row_mtx_b = src_addr.s1 + MATRIX_B_TRANSPOSED_STRIDE;

    /* Reset accumulators */
    float4 c00 = 0.0f;
    float4 c10 = 0.0f;
    float4 c20 = 0.0f;
    float4 c30 = 0.0f;

    for(; src_addr.s1 < end_row_mtx_b; src_addr += (uint2)(8, 8))
    {
        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        float4 a0 = vload4(0, mtx_a + src_addr.s0);
        float4 b0 = vload4(0, mtx_b + src_addr.s1);

        c00 += (float4)a0.s0 * b0;
        c10 += (float4)a0.s1 * b0;
        c20 += (float4)a0.s2 * b0;
        c30 += (float4)a0.s3 * b0;

        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        a0 = vload4(0, mtx_a + src_addr.s0 + 4);
        b0 = vload4(0, mtx_b + src_addr.s1 + 4);

        c00 += (float4)a0.s0 * b0;
        c10 += (float4)a0.s1 * b0;
        c20 += (float4)a0.s2 * b0;
        c30 += (float4)a0.s3 * b0;
    }

    /* Wait for all work-items to finish */
    barrier(CLK_GLOBAL_MEM_FENCE);

    /* Compute destination address */
    uint dst_addr = (get_global_id (0) + get_global_id (1) * COL_MTX_C) * 4;

    /* Store 4x4 block */
    vstore4(c00, 0, mtx_out + dst_addr + 0 * COL_MTX_C);
    vstore4(c10, 0, mtx_out + dst_addr + 1 * COL_MTX_C);
    vstore4(c20, 0, mtx_out + dst_addr + 2 * COL_MTX_C);
    vstore4(c30, 0, mtx_out + dst_addr + 3 * COL_MTX_C);
}

__kernel void mm_fp16( __global half *const mtx_a,
                       __global half *const mtx_b,
                       __global half       *mtx_out
                     )
{
    /* src_addr.s0 = address of matrix A */
    /* src_addr.s1 = address of matrix B */

    /* Compute address for matrix A and B */
    uint2 src_addr = (uint2)(get_global_id (1), get_global_id (0)) * (uint2)(MATRIX_A_INTERLEAVED_STRIDE,
                                                                             MATRIX_B_TRANSPOSED_STRIDE);

    /* Compute end row address for matrix B */
    uint end_row_mtx_b = src_addr.s1 + MATRIX_B_TRANSPOSED_STRIDE;

    /* Reset accumulators */
    half8 c00 = 0.0f;
    half8 c10 = 0.0f;
    half8 c20 = 0.0f;
    half8 c30 = 0.0f;

    for(; src_addr.s1 < end_row_mtx_b; src_addr += (uint2)(8, 16))
    {
        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        half4 a0 = vload4(0, mtx_a + src_addr.s0);
        half8 b0 = vload8(0, mtx_b + src_addr.s1);

        c00 += (half8)a0.s0 * b0;
        c10 += (half8)a0.s1 * b0;
        c20 += (half8)a0.s2 * b0;
        c30 += (half8)a0.s3 * b0;

        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        a0 = vload4(0, mtx_a + src_addr.s0 + 4);
        b0 = vload8(0, mtx_b + src_addr.s1 + 8);

        c00 += (half8)a0.s0 * b0;
        c10 += (half8)a0.s1 * b0;
        c20 += (half8)a0.s2 * b0;
        c30 += (half8)a0.s3 * b0;
    }

    /* Wait for all work-items to finish */
    barrier(CLK_GLOBAL_MEM_FENCE);

    /* Compute destination address */
    uint dst_addr = (get_global_id (0) * 2 + get_global_id (1) * COL_MTX_C) * 4;

    /* Store 8x4 block */
    vstore8(c00, 0, mtx_out + dst_addr + 0 * COL_MTX_C);
    vstore8(c10, 0, mtx_out + dst_addr + 1 * COL_MTX_C);
    vstore8(c20, 0, mtx_out + dst_addr + 2 * COL_MTX_C);
    vstore8(c30, 0, mtx_out + dst_addr + 3 * COL_MTX_C);
}

__kernel void mm_hybrid( __global half *const mtx_a,
                         __global half *const mtx_b,
                         __global half       *mtx_out
                       )
{
    /* src_addr.s0 = address of matrix A */
    /* src_addr.s1 = address of matrix B */

    /* Compute address for matrix A and B */
    uint2 src_addr = (uint2)(get_global_id (1), get_global_id (0)) * (uint2)(MATRIX_A_INTERLEAVED_STRIDE,
                                                                             MATRIX_B_TRANSPOSED_STRIDE);

    /* Compute end row address for matrix B */
    uint end_row_mtx_b = src_addr.s1 + MATRIX_B_TRANSPOSED_STRIDE;

    /* Reset accumulators */
    float4 c00 = 0.0f;
    float4 c10 = 0.0f;
    float4 c20 = 0.0f;
    float4 c30 = 0.0f;

    for(; src_addr.s1 < end_row_mtx_b; src_addr += (uint2)(8, 8))
    {
        /* Load values from matrix A (interleaved) and matrix B (transposed) */
        half8 a0_a1 = vload8(0, mtx_a + src_addr.s0);
        half8 b0_b1 = vload8(0, mtx_b + src_addr.s1);

        c00 += convert_float4((half4)a0_a1.s0 * b0_b1.s0123);
        c00 += convert_float4((half4)a0_a1.s4 * b0_b1.s4567);
        c10 += convert_float4((half4)a0_a1.s1 * b0_b1.s0123);
        c10 += convert_float4((half4)a0_a1.s5 * b0_b1.s4567);

        c20 += convert_float4((half4)a0_a1.s2 * b0_b1.s0123);
        c20 += convert_float4((half4)a0_a1.s6 * b0_b1.s4567);
        c30 += convert_float4((half4)a0_a1.s3 * b0_b1.s0123);
        c30 += convert_float4((half4)a0_a1.s7 * b0_b1.s4567);
    }

    /* Wait for all work-items to finish */
    barrier(CLK_GLOBAL_MEM_FENCE);

    /* Compute destination address */
    uint dst_addr = (get_global_id (0) + get_global_id (1) * COL_MTX_C) * 4;

    /* Store 4x4 block */
    vstore_half4(c00, 0, mtx_out + dst_addr + 0 * COL_MTX_C);
    vstore_half4(c10, 0, mtx_out + dst_addr + 1 * COL_MTX_C);
    vstore_half4(c20, 0, mtx_out + dst_addr + 2 * COL_MTX_C);
    vstore_half4(c30, 0, mtx_out + dst_addr + 3 * COL_MTX_C);

}

__kernel void transpose_fp32( __global float const *mtx_b,
                              __global float       *mtx_out
                            )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix B - source */
    uint src_addr = x * 4 + y * COL_MTX_B;

    /* Compute address for Matrix B transposed - destination */
    uint dst_addr = y * 4 + x * MATRIX_B_TRANSPOSED_STRIDE;

    float4 b0 = vload4(0, mtx_b + src_addr);

    vstore4(b0, 0, mtx_out + dst_addr);
}

__kernel void transpose_fp16( __global half const *mtx_b,
                              __global half       *mtx_out
                            )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix B - source */
    uint src_addr = x * 8 + y * COL_MTX_B;

    /* Compute address for Matrix B transposed - destination */
    uint dst_addr = y * 8 + x * MATRIX_B_TRANSPOSED_STRIDE;

    half8 b0 = vload8(0, mtx_b + src_addr);

    vstore8(b0, 0, mtx_out + dst_addr);
}

__kernel void transpose_hybrid( __global half const *mtx_b,
                                __global half       *mtx_out
                              )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix B - source */
    uint src_addr = x * 4 + y * COL_MTX_B;

    /* Compute address for Matrix B transposed - destination */
    uint dst_addr = y * 4 + x * MATRIX_B_TRANSPOSED_STRIDE;

    half4 b0 = vload4(0, mtx_b + src_addr);

    vstore4(b0, 0, mtx_out + dst_addr);
}

__kernel void interleave_fp32( __global float const *mtx_a,
                               __global float       *mtx_out
                             )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix A - source */
    uint src_addr = x * 4 + y * 4 * COL_MTX_A;

    /* Compute address for Matrix A interleaved - destination */
    uint dst_addr = x * 16 + y * MATRIX_A_INTERLEAVED_STRIDE;

    /* Load values from Matrix A */
    float4 a0 = vload4(0, mtx_a + src_addr + 0 * COL_MTX_A);
    float4 a1 = vload4(0, mtx_a + src_addr + 1 * COL_MTX_A);
    float4 a2 = vload4(0, mtx_a + src_addr + 2 * COL_MTX_A);
    float4 a3 = vload4(0, mtx_a + src_addr + 3 * COL_MTX_A);

    float4 val0 = (float4)(a0.s0, a1.s0, a2.s0, a3.s0);
    vstore4(val0, 0, mtx_out + dst_addr);

    val0 = (float4)(a0.s1, a1.s1, a2.s1, a3.s1);
    vstore4(val0, 0, mtx_out + dst_addr + 4);

    val0 = (float4)(a0.s2, a1.s2, a2.s2, a3.s2);
    vstore4(val0, 0, mtx_out + dst_addr + 8);

    val0 = (float4)(a0.s3, a1.s3, a2.s3, a3.s3);
    vstore4(val0, 0, mtx_out + dst_addr + 12);
}

__kernel void interleave_fp16( __global half const *mtx_a,
                               __global half       *mtx_out
                             )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address for Matrix A - source */
    uint src_addr = x * 4 + y * 4 * COL_MTX_A;

    /* Compute address for Matrix A interleaved - destination */
    uint dst_addr = x * 16 + y * MATRIX_A_INTERLEAVED_STRIDE;

    half4 a0 = vload4(0, mtx_a + src_addr + 0 * COL_MTX_A);
    half4 a1 = vload4(0, mtx_a + src_addr + 1 * COL_MTX_A);
    half4 a2 = vload4(0, mtx_a + src_addr + 2 * COL_MTX_A);
    half4 a3 = vload4(0, mtx_a + src_addr + 3 * COL_MTX_A);

    half4 val0 = (half4)(a0.s0, a1.s0, a2.s0, a3.s0);
    vstore4(val0, 0, mtx_out + dst_addr);

    val0 = (half4)(a0.s1, a1.s1, a2.s1, a3.s1);
    vstore4(val0, 0, mtx_out + dst_addr + 4);

    val0 = (half4)(a0.s2, a1.s2, a2.s2, a3.s2);
    vstore4(val0, 0, mtx_out + dst_addr + 8);

    val0 = (half4)(a0.s3, a1.s3, a2.s3, a3.s3);
    vstore4(val0, 0, mtx_out + dst_addr + 12);
}

__kernel void finalize_fp32( __global float const *mtx_c,
                             __global float       *mtx_axb,
                             float                alpha,
                             float                beta
                           )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address */
    uint addr = x * 4 + y * COL_MTX_C;

    /* Load values from A x B */
    float4 axb = vload4(0, mtx_axb + addr);

    /* Load values from Matrix C */
    float4 c = vload4(0, mtx_c + addr);

    /* Computes alpha * axb + beta * c */
    float4 out = (float4)alpha * axb + (float4)beta * c;

    /* Store final result in axb matrix */
    vstore4(out, 0, mtx_axb + addr);
}

__kernel void finalize_fp16( __global half const *mtx_c,
                             __global half        *mtx_axb,
                             float               alpha,
                             float               beta
                           )
{
    uint x = get_global_id(0);
    uint y = get_global_id(1);

    /* Compute address */
    uint addr = x * 8 + y * COL_MTX_C;

    half8 axb = vload8(0, mtx_axb + addr);
    half8 c = vload8(0, mtx_c + addr);

    /* Computes alpha * acc + beta * c. The sum is in fp32 */
    float8 out = convert_float8((half8)alpha * axb) + convert_float8((half8)beta * c);

    vstore_half8(out, 0, mtx_axb + addr);
}


Update SpGEMM_ESC_bitonic_kernels.cl

Update SpGEMM_ESC_bitonic_kernels.cl

/* ************************************************************************
* The MIT License (MIT)
* Copyright 2014-2015 weifengliu

*  Permission is hereby granted, free of charge, to any person obtaining a copy
*  of this software and associated documentation files (the "Software"), to deal
*  in the Software without restriction, including without limitation the rights
*  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
*  copies of the Software, and to permit persons to whom the Software is
*  furnished to do so, subject to the following conditions:

*  The above copyright notice and this permission notice shall be included in
*  all copies or substantial portions of the Software.

*  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
*  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
*  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
*  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
*  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
*  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
*  THE SOFTWARE.
* ************************************************************************ */
//////////////////////////////////////////////////////////////////////////
// < A CUDA/OpenCL General Sparse Matrix-Matrix Multiplication Program >
//
// < See paper:
// Weifeng Liu and Brian Vinter, "An Efficient GPU General Sparse
// Matrix-Matrix Multiplication for Irregular Data," Parallel and
// Distributed Processing Symposium, 2014 IEEE 28th International
// (IPDPS '14), pp.370-381, 19-23 May 2014
// for details. >
//////////////////////////////////////////////////////////////////////////

#pragma OPENCL EXTENSION cl_khr_fp64 : enable
#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

#define TUPLE_QUEUE 6
typedef double   vT;

inline
void coex(__local  int      *keyA,
          __local  vT    *valA,
          __local  int      *keyB,
          __local  vT    *valB,
          const int          dir)
{
    int t;
    vT v;

    if ((*keyA > *keyB) == dir)
    {
        t = *keyA;
        *keyA = *keyB;
        *keyB = t;
        v = *valA;
        *valA = *valB;
        *valB = v;
    }
}

inline
void oddeven(__local int   *s_key,
                     __local vT *s_val,
                     int    arrayLength)
{
    int dir = 1;
    int local_id = get_local_id(0);

    for (int size = 2; size <= arrayLength; size <<= 1)
    {
        int stride = size >> 1;
        int offset = local_id & (stride - 1);

        {
            barrier(CLK_LOCAL_MEM_FENCE);
            int pos = 2 * local_id - (local_id & (stride - 1));
            coex(&s_key[pos], &s_val[pos], &s_key[pos + stride], &s_val[pos + stride], dir);

            stride >>= 1;
        }

        for (; stride > 0; stride >>= 1)
        {
            barrier(CLK_LOCAL_MEM_FENCE);
            int pos = 2 * local_id - (local_id & (stride - 1));
            if (offset >= stride)
                coex(&s_key[pos - stride], &s_val[pos - stride], &s_key[pos], &s_val[pos], dir);
        }
    }
}

inline
void scan_32(__local volatile short *s_scan)
{
    int local_id = get_local_id(0);
    int ai, bi;
    int baseai = 1 + 2 * local_id;
    int basebi = baseai + 1;
    short temp;

    ai = baseai - 1;     bi = basebi - 1;     s_scan[bi] += s_scan[ai];
    if (local_id < 8)  { ai = 2 * baseai - 1;  bi = 2 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 4)  { ai = 4 * baseai - 1;  bi = 4 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 2)  { ai = 8 * baseai - 1;  bi = 8 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id == 0) { s_scan[31] += s_scan[15]; s_scan[32] = s_scan[31]; s_scan[31] = 0; temp = s_scan[15]; s_scan[15] = 0; s_scan[31] += temp; }
    if (local_id < 2)  { ai = 8 * baseai - 1;  bi = 8 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 4)  { ai = 4 * baseai - 1;  bi = 4 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 8)  { ai = 2 * baseai - 1;  bi = 2 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    ai = baseai - 1;   bi = basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;
}

inline
void scan_64(__local volatile short *s_scan)
{
    int local_id = get_local_id(0);
    int ai, bi;
    int baseai = 1 + 2 * local_id;
    int basebi = baseai + 1;
    short temp;

    ai = baseai - 1;     bi = basebi - 1;     s_scan[bi] += s_scan[ai];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 16) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 8)  { ai = 4 * baseai - 1;  bi = 4 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 4)  { ai = 8 * baseai - 1;  bi = 8 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 2)  { ai = 16 * baseai - 1;  bi = 16 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id == 0) { s_scan[63] += s_scan[31]; s_scan[64] = s_scan[63]; s_scan[63] = 0; temp = s_scan[31]; s_scan[31] = 0; s_scan[63] += temp; }
    if (local_id < 2)  { ai = 16 * baseai - 1;  bi = 16 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 4)  { ai = 8 * baseai - 1;  bi = 8 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 8)  { ai = 4 * baseai - 1;  bi = 4 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 16) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    ai = baseai - 1;   bi = basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;
}

inline
void scan_128(__local volatile short *s_scan)
{
    int local_id = get_local_id(0);
    int ai, bi;
    int baseai = 1 + 2 * local_id;
    int basebi = baseai + 1;
    short temp;

    ai = baseai - 1;     bi = basebi - 1;     s_scan[bi] += s_scan[ai];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 32) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 16) { ai =  4 * baseai - 1;  bi =  4 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 8)  { ai = 8 * baseai - 1;  bi = 8 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 4)  { ai = 16 * baseai - 1;  bi = 16 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 2)  { ai = 32 * baseai - 1;  bi = 32 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id == 0) { s_scan[127] += s_scan[63]; s_scan[128] = s_scan[127]; s_scan[127] = 0; temp = s_scan[63]; s_scan[63] = 0; s_scan[127] += temp; }
    if (local_id < 2)  { ai = 32 * baseai - 1;  bi = 32 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 4)  { ai = 16 * baseai - 1;  bi = 16 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 8)  { ai = 8 * baseai - 1;  bi = 8 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 16) { ai =  4 * baseai - 1;  bi =  4 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 32) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    barrier(CLK_LOCAL_MEM_FENCE);
    ai = baseai - 1;   bi = basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;
}

inline
void scan_256(__local volatile short *s_scan)
{
    int local_id = get_local_id(0);
    int ai, bi;
    int baseai = 1 + 2 * local_id;
    int basebi = baseai + 1;
    short temp;

    ai = baseai - 1;     bi = basebi - 1;     s_scan[bi] += s_scan[ai];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 64) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 32) { ai =  4 * baseai - 1;  bi =  4 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 16) { ai =  8 * baseai - 1;  bi =  8 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 8)  { ai = 16 * baseai - 1;  bi = 16 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 4)  { ai = 32 * baseai - 1;  bi = 32 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 2)  { ai = 64 * baseai - 1;  bi = 64 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id == 0) { s_scan[255] += s_scan[127]; s_scan[256] = s_scan[255]; s_scan[255] = 0; temp = s_scan[127]; s_scan[127] = 0; s_scan[255] += temp; }
    if (local_id < 2)  { ai = 64 * baseai - 1;  bi = 64 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 4)  { ai = 32 * baseai - 1;  bi = 32 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 8)  { ai = 16 * baseai - 1;  bi = 16 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 16) { ai =  8 * baseai - 1;  bi =  8 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 32) { ai =  4 * baseai - 1;  bi =  4 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 64) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    barrier(CLK_LOCAL_MEM_FENCE);
    ai = baseai - 1;   bi = basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;
}

inline
void scan_512(__local volatile short *s_scan)
{
    int local_id = get_local_id(0);
    int ai, bi;
    int baseai = 1 + 2 * local_id;
    int basebi = baseai + 1;
    short temp;

    ai = baseai - 1;     bi = basebi - 1;     s_scan[bi] += s_scan[ai];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 128) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 64)  { ai =  4 * baseai - 1;  bi =  4 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 32) { ai =  8 * baseai - 1;  bi =  8 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 16) { ai =  16 * baseai - 1;  bi =  16 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 8)  { ai = 32 * baseai - 1;  bi = 32 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 4)  { ai = 64 * baseai - 1;  bi = 64 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id < 2)  { ai = 128 * baseai - 1;  bi = 128 * basebi - 1;   s_scan[bi] += s_scan[ai]; }
    if (local_id == 0) { s_scan[511] += s_scan[255]; s_scan[512] = s_scan[511]; s_scan[511] = 0; temp = s_scan[255]; s_scan[255] = 0; s_scan[511] += temp; }
    if (local_id < 2)  { ai = 128 * baseai - 1;  bi = 128 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 4)  { ai = 64 * baseai - 1;  bi = 64 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 8)  { ai = 32 * baseai - 1;  bi = 32 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 16) { ai =  16 * baseai - 1;  bi =  16 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    if (local_id < 32) { ai =  8 * baseai - 1;  bi =  8 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 64) { ai =  4 * baseai - 1;  bi =  4 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    barrier(CLK_LOCAL_MEM_FENCE);
    if (local_id < 128) { ai =  2 * baseai - 1;  bi =  2 * basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;}
    barrier(CLK_LOCAL_MEM_FENCE);
    ai = baseai - 1;   bi = basebi - 1;   temp = s_scan[ai]; s_scan[ai] = s_scan[bi]; s_scan[bi] += temp;
}

inline
void compression_scan(__local volatile short *s_scan,
                      __local int            *s_key,
                      __local vT          *s_val,
                      const int               local_counter,
                      const int               local_size,
                      const int               local_id,
                      const int               local_id_halfwidth)
{
    // compression - prefix sum
    bool duplicate = 1;
    bool duplicate_halfwidth = 1;

    // generate bool value in registers
    if (local_id < local_counter && local_id > 0)
        duplicate = (s_key[local_id] != s_key[local_id - 1]);
    if (local_id_halfwidth < local_counter)
        duplicate_halfwidth = (s_key[local_id_halfwidth] != s_key[local_id_halfwidth - 1]);

    // copy bool values from register to local memory (s_scan)
    s_scan[local_id]                    = duplicate;
    s_scan[local_id_halfwidth]          = duplicate_halfwidth;
    barrier(CLK_LOCAL_MEM_FENCE);

    // in-place exclusive prefix-sum scan on s_scan
    switch (local_size)
    {
    case 16:
        scan_32(s_scan);
        break;
    case 32:
        scan_64(s_scan);
        break;
    case 64:
        scan_128(s_scan);
        break;
    case 128:
        scan_256(s_scan);
        break;
    case 256:
        scan_512(s_scan);
        break;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // compute final position and final value in registers
    int   move_pointer;
    short final_position, final_position_halfwidth;
    int   final_key,      final_key_halfwidth;
    vT final_value,    final_value_halfwidth;

    if (local_id < local_counter && duplicate == 1)
    {
        final_position = s_scan[local_id];
        final_key = s_key[local_id];
        final_value = s_val[local_id];
        move_pointer = local_id + 1;

        while (s_scan[move_pointer] == s_scan[move_pointer + 1])
        {
            final_value += s_val[move_pointer];
            move_pointer++;
        }
    }

    if (local_id_halfwidth < local_counter && duplicate_halfwidth == 1)
    {
        final_position_halfwidth = s_scan[local_id_halfwidth];
        final_key_halfwidth = s_key[local_id_halfwidth];
        final_value_halfwidth = s_val[local_id_halfwidth];
        move_pointer = local_id_halfwidth + 1;

        while (s_scan[move_pointer] == s_scan[move_pointer + 1] && move_pointer < 2 * local_size)
        {
            final_value_halfwidth += s_val[move_pointer];
            move_pointer++;
        }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // write final_positions and final_values to s_key and s_val
    if (local_id < local_counter && duplicate == 1)
    {
        s_key[final_position] = final_key;
        s_val[final_position] = final_value;
    }
    if (local_id_halfwidth < local_counter && duplicate_halfwidth == 1)
    {
        s_key[final_position_halfwidth] = final_key_halfwidth;
        s_val[final_position_halfwidth] = final_value_halfwidth;
    }
}

__kernel
void ESC_bitonic_scan(__global const int      *d_queue,
                      __global const int      *d_csrRowPtrA,
                      __global const int      *d_csrColIndA,
                      __global const vT    *d_csrValA,
                      __global const int      *d_csrRowPtrB,
                      __global const int      *d_csrColIndB,
                      __global const vT    *d_csrValB,
                      __global int            *d_csrRowPtrC,
                      __global const int      *d_csrRowPtrCt,
                      __global int            *d_csrColIndCt,
                      __global vT          *d_csrValCt,
                      __local  int            *s_key,
                      __local  vT          *s_val,
                      __local  volatile short *s_scan,
                      const int                queue_offset,
                      const int                n)
{
    int local_id = get_local_id(0); //local_id;
    int group_id = get_group_id(0); //blockIdx.x;
    int local_size = get_local_size(0);
    int width = local_size * 2;

    int i, local_counter = 0;
    int strideB, local_offset, global_offset;
    int invalid_width;
    int local_id_halfwidth = local_id + local_size;

    int row_id_B; // index_type

    int row_id;// index_type
    row_id = d_queue[TUPLE_QUEUE * (queue_offset + group_id)];

    int start_col_index_A, stop_col_index_A;  // index_type
    int start_col_index_B, stop_col_index_B;  // index_type
    vT value_A;                            // value_type

    start_col_index_A = d_csrRowPtrA[row_id];
    stop_col_index_A  = d_csrRowPtrA[row_id + 1];

    // i is both col index of A and row index of B
    for (i = start_col_index_A; i < stop_col_index_A; i++)
    {
        row_id_B = d_csrColIndA[i];
        value_A  = d_csrValA[i];

        start_col_index_B = d_csrRowPtrB[row_id_B];
        stop_col_index_B  = d_csrRowPtrB[row_id_B + 1];

        strideB = stop_col_index_B - start_col_index_B;

        if (local_id < strideB)
        {
            local_offset = local_counter + local_id;
            global_offset = start_col_index_B + local_id;

            s_key[local_offset] = d_csrColIndB[global_offset];
            s_val[local_offset] = d_csrValB[global_offset] * value_A;
        }

        if (local_id_halfwidth < strideB)
        {
            local_offset = local_counter + local_id_halfwidth;
            global_offset = start_col_index_B + local_id_halfwidth;

            s_key[local_offset] = d_csrColIndB[global_offset];
            s_val[local_offset] = d_csrValB[global_offset] * value_A;
        }

        local_counter += strideB;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    invalid_width = width - local_counter;

    // to meet 2^N, set the rest elements to n (number of columns of C)
    if (local_id < invalid_width)
        s_key[local_counter + local_id] = n;
    //if (local_id_halfwidth < invalid_width)
    //    s_key[local_counter + local_id_halfwidth] = n;
    barrier(CLK_LOCAL_MEM_FENCE);

    // bitonic/oddeven sort
    oddeven(s_key, s_val, width);
    barrier(CLK_LOCAL_MEM_FENCE);

    // compression - scan
    compression_scan(s_scan, s_key, s_val, local_counter,
                     local_size, local_id, local_id_halfwidth);
    barrier(CLK_LOCAL_MEM_FENCE);

    local_counter = s_scan[width] - invalid_width;
    if (local_id == 0)
        d_csrRowPtrC[row_id] = local_counter;

    // write compressed lists to global mem
    int row_offset = d_queue[TUPLE_QUEUE * (queue_offset + group_id) + 1]; //d_csrRowPtrCt[row_id];

    if (local_id < local_counter)
    {
        global_offset = row_offset + local_id;

        d_csrColIndCt[global_offset] = s_key[local_id];
        d_csrValCt[global_offset] = s_val[local_id];
    }
    if (local_id_halfwidth < local_counter)
    {
        global_offset = row_offset + local_id_halfwidth;

        d_csrColIndCt[global_offset] = s_key[local_id_halfwidth];
        d_csrValCt[global_offset] = s_val[local_id_halfwidth];
    }
}


fixed openCL extension

fixed openCL extension

#define INDEX_SIZE ${size}
#define BLOCK_SIZE ${block}
#define STEP_SIZE ${stepSize}

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable

/* self.d_compAll, self.d_comp,
   self.d_distances,
   self.d_validComps,
   self.d_seqs,
   self.d_index_increment,
   numpy.float32(self.compositionScale),
   numpy.int32(len(seqs)),
   numpy.int32(len(keys)),
   numpy.float32(self.sliceDistance)
   */
__kernel void calculateDistance(
		__global int *index,
		__global int *query,
		__global float *distances,
		__global unsigned int *validComps,
		__global unsigned int *seqs,
		volatile __global unsigned int *indexIncrement,
		float scale,
		unsigned int numSeqs,
		unsigned int length,
		float sliceDistance){

	//unsigned int block = (get_group_id(0) * BLOCK_SIZE + get_group_id(1))*(INDEX_SIZE+1);
	//unsigned int threadPlus1 = get_local_id(0)+1;
	//unsigned int thread = get_local_id(0);

	unsigned int seq = get_group_id(1) / BLOCK_SIZE;
	unsigned int blockY = get_group_id(1) % BLOCK_SIZE;
	unsigned int comp = get_group_id(0) * BLOCK_SIZE + blockY;
	unsigned int block = (comp)*(INDEX_SIZE+1);
	unsigned int threadPlus1 = get_local_id(0)+1;
	unsigned int thread = get_local_id(0);

	if (comp < length) {
		__local float s_distances[INDEX_SIZE];

		s_distances[thread] = (float) index[block+threadPlus1] - (float)query[threadPlus1+(seq*(INDEX_SIZE+1))];
		s_distances[thread] *= s_distances[thread];

		unsigned int offset = 1;

		for (int i=INDEX_SIZE>>1; i > 0 ; i >>=1 ) {
			barrier(CLK_LOCAL_MEM_FENCE);
			if (thread < i) {
			    int ai = offset*(2*thread+1)-1;
			    int bi = offset*(2*thread+2)-1;
			    s_distances[bi] += s_distances[ai];
			}
			offset *= 2;
		}
		barrier(CLK_LOCAL_MEM_FENCE);
		if (thread == 0){
			//distances[get_group_id(0) * BLOCK_SIZE + get_group_id(1)] =  sqrt(s_distances[INDEX_SIZE-1])/scale;
			s_distances[INDEX_SIZE-1] = sqrt(s_distances[INDEX_SIZE-1])/scale;

			if (s_distances[INDEX_SIZE-1] < sliceDistance){
				unsigned int indices = atom_inc(&indexIncrement[0]);
				distances[indices] =  s_distances[INDEX_SIZE-1];
				validComps[indices] = comp;
				seqs[indices] = seq;
			}
		}
	}
}

__kernel void setToZero(__global int *comps){
	unsigned int index = 1+ get_local_id(0) + (INDEX_SIZE+1) * (get_group_id(0)*BLOCK_SIZE + get_group_id(1));
	comps[index] = 0;
}

__kernel void scaleComp(__global float *comps, __global int *comps_int, float fraction){
	unsigned int index = 1+ get_local_id(0) + (INDEX_SIZE+1) * (get_group_id(0)*BLOCK_SIZE + get_group_id(1));
	comps[index] = (float)comps_int[index] / fraction;
	if (!get_local_id(0)) { // copy windows
		index--;
		comps[index] = (float)comps_int[index];
	}
}

__kernel void calculateQgrams(__global char *sequence, unsigned int q, unsigned int length, volatile __global int *comps, float windowLength, float step, int fraction) {
	unsigned int seqLocation = get_local_id(0) + INDEX_SIZE * get_group_id(0);

	if (seqLocation < length - q) {
		int localQgram = 0;
		int bit = 1;
		//for (int i=q-1; i >= 0; i--) {
		for (int i=0; i < q; i++) {
			if (localQgram >= 0) {
				char character = sequence[seqLocation+i];
				switch (character) {
					case 'A' : break;
					case 'T' : localQgram+=1*bit; break;
					case 'C' : localQgram+=2*bit; break;
					case 'G' : localQgram+=3*bit; break;
					default : localQgram = -1;
				}
				bit *= q;
			}
		}
		if (localQgram >= 0) {
			localQgram++;
			unsigned int startWindow = (seqLocation-windowLength) < 0 ? 0 : (unsigned int)ceil((seqLocation - windowLength) / step);
			unsigned int endWindow = (unsigned int)floor(seqLocation/step) < STEP_SIZE? (unsigned int)floor(seqLocation/step) : STEP_SIZE-1;
			comps[(startWindow * (INDEX_SIZE+1))]= (int)windowLength;

			for (unsigned int i =startWindow; i < endWindow; i++){
				atom_add(&comps[i * (INDEX_SIZE+1)+localQgram], fraction);
			}
		}

	}

}

#define INDEX_SIZE ${size}
#define BLOCK_SIZE ${block}
#define STEP_SIZE ${stepSize}

#pragma OPENCL EXTENSION cl_khr_int32_base_atomics : enable

/* self.d_compAll, self.d_comp,
   self.d_distances,
   self.d_validComps,
   self.d_seqs,
   self.d_index_increment,
   numpy.float32(self.compositionScale),
   numpy.int32(len(seqs)),
   numpy.int32(len(keys)),
   numpy.float32(self.sliceDistance)
   */
__kernel void calculateDistance(
		__global int *index,
		__global int *query,
		__global float *distances,
		__global unsigned int *validComps,
		__global unsigned int *seqs,
		volatile __global unsigned int *indexIncrement,
		float scale,
		unsigned int numSeqs,
		unsigned int length,
		float sliceDistance){

	//unsigned int block = (get_group_id(0) * BLOCK_SIZE + get_group_id(1))*(INDEX_SIZE+1);
	//unsigned int threadPlus1 = get_local_id(0)+1;
	//unsigned int thread = get_local_id(0);

	unsigned int seq = get_group_id(1) / BLOCK_SIZE;
	unsigned int blockY = get_group_id(1) % BLOCK_SIZE;
	unsigned int comp = get_group_id(0) * BLOCK_SIZE + blockY;
	unsigned int block = (comp)*(INDEX_SIZE+1);
	unsigned int threadPlus1 = get_local_id(0)+1;
	unsigned int thread = get_local_id(0);

	if (comp < length) {
		__local float s_distances[INDEX_SIZE];

		s_distances[thread] = (float) index[block+threadPlus1] - (float)query[threadPlus1+(seq*(INDEX_SIZE+1))];
		s_distances[thread] *= s_distances[thread];

		unsigned int offset = 1;

		for (int i=INDEX_SIZE>>1; i > 0 ; i >>=1 ) {
			barrier(CLK_LOCAL_MEM_FENCE);
			if (thread < i) {
			    int ai = offset*(2*thread+1)-1;
			    int bi = offset*(2*thread+2)-1;
			    s_distances[bi] += s_distances[ai];
			}
			offset *= 2;
		}
		barrier(CLK_LOCAL_MEM_FENCE);
		if (thread == 0){
			//distances[get_group_id(0) * BLOCK_SIZE + get_group_id(1)] =  sqrt(s_distances[INDEX_SIZE-1])/scale;
			s_distances[INDEX_SIZE-1] = sqrt(s_distances[INDEX_SIZE-1])/scale;

			if (s_distances[INDEX_SIZE-1] < sliceDistance){
				unsigned int indices = atom_inc(&indexIncrement[0]);
				distances[indices] =  s_distances[INDEX_SIZE-1];
				validComps[indices] = comp;
				seqs[indices] = seq;
			}
		}
	}
}

__kernel void setToZero(__global int *comps){
	unsigned int index = 1+ get_local_id(0) + (INDEX_SIZE+1) * (get_group_id(0)*BLOCK_SIZE + get_group_id(1));
	comps[index] = 0;
}

__kernel void scaleComp(__global float *comps, __global int *comps_int, float fraction){
	unsigned int index = 1+ get_local_id(0) + (INDEX_SIZE+1) * (get_group_id(0)*BLOCK_SIZE + get_group_id(1));
	comps[index] = (float)comps_int[index] / fraction;
	if (!get_local_id(0)) { // copy windows
		index--;
		comps[index] = (float)comps_int[index];
	}
}

__kernel void calculateQgrams(__global char *sequence, unsigned int q, unsigned int length, volatile __global int *comps, float windowLength, float step, int fraction) {
	unsigned int seqLocation = get_local_id(0) + INDEX_SIZE * get_group_id(0);

	if (seqLocation < length - q) {
		int localQgram = 0;
		int bit = 1;
		//for (int i=q-1; i >= 0; i--) {
		for (int i=0; i < q; i++) {
			if (localQgram >= 0) {
				char character = sequence[seqLocation+i];
				switch (character) {
					case 'A' : break;
					case 'T' : localQgram+=1*bit; break;
					case 'C' : localQgram+=2*bit; break;
					case 'G' : localQgram+=3*bit; break;
					default : localQgram = -1;
				}
				bit *= q;
			}
		}
		if (localQgram >= 0) {
			localQgram++;
			unsigned int startWindow = (seqLocation-windowLength) < 0 ? 0 : (unsigned int)ceil((seqLocation - windowLength) / step);
			unsigned int endWindow = (unsigned int)floor(seqLocation/step) < STEP_SIZE? (unsigned int)floor(seqLocation/step) : STEP_SIZE-1;
			comps[(startWindow * (INDEX_SIZE+1))]= (int)windowLength;

			for (unsigned int i =startWindow; i < endWindow; i++){
				atom_add(&comps[i * (INDEX_SIZE+1)+localQgram], fraction);
			}
		}

	}

}


Moved work_group_reduction_sum_horizontal from Neanderthal to reducitons.cl

#ifndef ACCUMULATOR
    #define ACCUMULATOR double
#endif

#ifndef WGS
    #define WGS 256
#endif

#ifndef WGSm
#define WGSm 16
#endif

#ifndef WGSn
#define WGSn 16
#endif

// ================= Sum reduction =============================================

inline ACCUMULATOR work_group_reduction_sum (const ACCUMULATOR value) {

    uint local_id = get_local_id(0);

    __local ACCUMULATOR lacc[WGS];
    lacc[local_id] = value;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    ACCUMULATOR pacc = value;
    uint i = get_local_size(0);
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_id == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            pacc += lacc[local_id + i + 1];
        }
        if (local_id < i) {
            pacc += lacc[local_id + i];
            lacc[local_id] = pacc;
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    return lacc[0];
}

inline void work_group_reduction_sum_horizontal
(__global REAL* acc, const REAL value) {

    uint local_row = get_local_id(0);
    uint local_col = get_local_id(1);
    uint local_m = get_local_size(0);

    __local REAL lacc[WGSm * WGSn];
    lacc[local_row + local_col * local_m] = value;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    REAL pacc = value;
    uint i = get_local_size(1);
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_col == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            pacc += lacc[local_row + (local_col + i + 1) * local_m];
        }
        if (local_col < i) {
            pacc += lacc[local_row + (local_col + i) * local_m];
            lacc[local_row + local_col * local_m] = pacc;
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    if(local_col == 0) {
        acc[get_global_size(0) * get_group_id(1) + get_global_id(0)] = pacc;
    }
}

#ifndef ACCUMULATOR
    #define ACCUMULATOR double
#endif

#ifndef WGS
    #define WGS 256
#endif

// ================= Sum reduction =============================================

inline ACCUMULATOR work_group_reduction_sum (const ACCUMULATOR value) {

    uint local_id = get_local_id(0);

    __local ACCUMULATOR lacc[WGS];
    lacc[local_id] = value;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    ACCUMULATOR pacc = value;
    uint i = get_local_size(0);
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_id == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            pacc += lacc[local_id + i + 1];
        }
        if (local_id < i) {
            pacc += lacc[local_id + i];
            lacc[local_id] = pacc;
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    return lacc[0];
}


fixed a bug in NLM cl code

fixed a bug in NLM cl code

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

__kernel void noneLocalMeans(
		__read_only image2d_t input,
		__write_only image2d_t output,
		//__private int group,
		//__private int window,
		__private float strength2,
		__private float sigma2

		){

    const int dataType = get_image_channel_data_type(input);
    const float pi = 3.14159265359;
    const int2 iD = get_image_dim(input);
    const int2 pos = {get_global_id(0), get_global_id(1)};

    //variables
    //const int hG = (group-1)/2;
    const int g2 = ((GROUP+1)*2)*((GROUP+1)*2)-1;
    //const int hW = (window-1)/2;
    float normSum = 0.0f;
    float totSum = 0.0f;
    float value = 0.0f;
    float indi = 0.0f;
    float groupTot = 0.0f;
    //const float strength2 = strength * strength;
    float K = 0.0f;
    //const float K = native_divide(1.0f,g2);
    //float STRENGTH2 = strength;
    if(KVERSION == 0){
        K = native_divide(1.0f,g2);
    }else if(KVERSION ==  1){
        K = native_divide(1.0f,sigma2);
    }else if(KVERSION == 2){
        K = 1;
    }else{
        K = native_divide(1.0f,g2);
    }
    //int mX = 0;
    //int mY = 0;

    for(int i = pos.x - WINDOW; i < pos.x + WINDOW; i++){
        for(int j = pos.y - WINDOW; j < pos.y + WINDOW; j++){
            int mX = pos.x - GROUP;
            int mY = pos.y - GROUP;

            if(i != pos.x && j != pos.y){
                for(int k = i - GROUP; k < i + GROUP; k++,mX++){
                    for(int l = j - GROUP; l < j + GROUP; l++,mY++){
                        int2 mPos = {mX,mY};
                        int2 sPos = {k,l};

                        if(dataType == CLK_FLOAT){
                            indi = read_imagef(input, sampler,mPos).x - read_imagef(input, sampler, sPos).x;
                        }else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16){
                            indi = read_imageui(input, sampler,mPos).x - read_imageui(input, sampler, sPos).x;
                        }else{
                            indi = read_imagei(input, sampler,mPos).x - read_imagei(input, sampler, sPos).x;
                        }
                        indi = fabs(indi*indi);

                        if(EUCLID ==  1){
                            indi = native_exp( - (native_divide(indi,strength2)));
                            groupTot += indi;
                        }else if(EUCLID == 2){
                            indi = native_exp( - (native_divide(indi,strength2)));
                            groupTot += indi;
                        }else{
                            groupTot += K * indi;
                        }
                        //groupTot += K * indi;

                        //indi = indi *  indi;
                        //groupTot += K * indi;

                    }

                }
                int2 coord = {i,j};
                if(dataType == CLK_FLOAT){
                    value = read_imagef(input,sampler,coord).x;
                }else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16){
                    value = read_imageui(input,sampler,coord).x;
                }else{
                    value = read_imagei(input,sampler,coord).x;
                }
                if(EUCLID ==  1){
                    float dist = ((pos.x - coord.x) * (pos.x - coord.x)) + ((pos.y - coord.y) * (pos.y - coord.y));
                    dist = native_sqrt(fabs(dist));
                    dist = native_exp( - native_divide( dist, (2.0f * sigma2) ) );
                    groupTot += (1.0f / (2.0f * pi * sigma2) * dist);
                }else if(EUCLID == 2){
                    float2 dist = convert_float2(coord - pos);
                    float gaussWeight = native_exp( - native_divide( dot(dist, dist), (2.0f * sigma2) ) );
                    //gaussWeight = native_divide(gaussWeight, 2.0f * pi * sigma2);
                    gaussWeight = native_divide(gaussWeight, 2.0f * sigma2);
                    groupTot *= gaussWeight;
                }else{
                    groupTot = native_exp(-groupTot/strength2);
                }
                //groupTot = K * groupTot;
                //groupTot = native_exp(-groupTot/(float)STRENGTH2);

                normSum += groupTot;
                totSum += groupTot * value;
                groupTot = 0.0f;
            }
        }
    }
    value = native_divide(totSum,normSum);
    if(value < 0){
        value = 0;
    }
    if(value > 1.0){
        value = 1.0f;
    }

    int outputDataType = get_image_channel_data_type(output);
    if(outputDataType == CLK_FLOAT) {
        write_imagef(output, pos, value);
    } else if(outputDataType == CLK_UNSIGNED_INT8 || outputDataType == CLK_UNSIGNED_INT16) {
        write_imageui(output, pos, round(value));
    } else {
        write_imagei(output, pos, round(value));
    }
    /*
    if(outputDataType == CLK_FLOAT) {
        float holder = native_divide(totSum,normSum);
        if(holder > 0){
            value = holder;
        }else if(holder > 1){
            value = 1.0f;
        }else{
            value = 0.0f;
        }
    write_imagef(output,pos,value);
    }else if(outputDataType == CLK_UNSIGNED_INT8 || outputDataType == CLK_UNSIGNED_INT16) {
        int holder = 0;
        if(value > -1 && value < 2){
            holder = value * 255;
        }else if(value > 1){
            holder = 255;
        }
    holder = min(max(holder,0),255);
    write_imageui(output, pos, holder);
    }else {
        int holder = 0;
        if(value > -1 && value < 2){
            holder = value * 255;
        }else if(value > 1){
            holder = 255;
        }
    holder = min(max(holder,0),255);
    write_imagei(output, pos, holder);
    }
    */
}

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

__kernel void noneLocalMeans(
		__read_only image2d_t input,
		__write_only image2d_t output,
		//__private int group,
		//__private int window,
		__private float strength2,
		__private float sigma2

		){

    const int dataType = get_image_channel_data_type(input);
    const float pi = 3.14159265359;
    const int2 iD = get_image_dim(input);
    const int2 pos = {get_global_id(0), get_global_id(1)};

    //variables
    //const int hG = (group-1)/2;
    const int g2 = ((GROUP+1)*2)*((GROUP+1)*2)-1;
    //const int hW = (window-1)/2;
    float normSum = 0.0f;
    float totSum = 0.0f;
    float value = 0.0f;
    float indi = 0.0f;
    float groupTot = 0.0f;
    //const float strength2 = strength * strength;
    float K = 0.0f;
    //const float K = native_divide(1.0f,g2);
    //float STRENGTH2 = strength;
    if(KVERSION == 0){
        K = native_divide(1.0f,g2);
    }else if(KVERSION ==  1){
        K = native_divide(1.0f,sigma2);
    }else if(KVERSION == 2){
        K = 1;
    }else{
        K = native_divide(1.0f,g2);
    }
    //int mX = 0;
    //int mY = 0;

    for(int i = pos.x - WINDOW; i < pos.x + WINDOW; i++){
        for(int j = pos.y - WINDOW; j < pos.y + WINDOW; j++){
            int mX = pos.x - GROUP;
            int mY = pos.y - GROUP;

            if(i != pos.x && j != pos.y){
                for(int k = i - GROUP; k < i + GROUP; k++,mX++){
                    for(int l = j - GROUP; l < j + GROUP; l++,mY++){
                        int2 mPos = {mX,mY};
                        int2 sPos = {k,l};

                        if(dataType == CLK_FLOAT){
                            indi = read_imagef(input, sampler,mPos).x - read_imagef(input, sampler, sPos).x;
                        }else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16){
                            indi = read_imageui(input, sampler,mPos).x - read_imageui(input, sampler, sPos).x;
                        }else{
                            indi = read_imagei(input, sampler,mPos).x - read_imagei(input, sampler, sPos).x;
                        }
                        indi = fabs(indi*indi);

                        if(EUCLID ==  1){
                            indi = native_exp( - (native_divide(indi,strength2)));
                            groupTot += indi;
                        }else if(EUCLID == 2){
                            indi = native_exp( - (native_divide(indi,strength2)));
                            groupTot += indi;
                        }else{
                            groupTot += K * indi;
                        }
                        //groupTot += K * indi;

                        //indi = indi *  indi;
                        //groupTot += K * indi;

                    }

                }
                int2 coord = {i,j};
                if(dataType == CLK_FLOAT){
                    value = read_imagef(input,sampler,coord).x;
                }else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16){
                    value = read_imageui(input,sampler,coord).x;
                }else{
                    value = read_imagei(input,sampler,coord).x;
                }
                if(EUCLID ==  1){
                    float dist = ((pos.x - coord.x) * (pos.x - coord.x)) + ((pos.y - coord.y) * (pos.y - coord.y));
                    dist = native_sqrt(fabs(dist));
                    dist = native_exp( - native_divide( dist, (2.0f * sigma2) ) );
                    groupTot += (1.0f / (2.0f * pi * sigma2) * dist);
                }else if(EUCLID == 2){
                    float2 dist = convert_float(coord - pos);
                    float gaussWeight = native_exp( - native_divide( dot(dist, dist), (2.0f * sigma2) ) );
                    //gaussWeight = native_divide(gaussWeight, 2.0f * pi * sigma2);
                    gaussWeight = native_divide(gaussWeight, 2.0f * sigma2);
                    groupTot *= gaussWeight;
                }else{
                    groupTot = native_exp(-groupTot/strength2);
                }
                //groupTot = K * groupTot;
                //groupTot = native_exp(-groupTot/(float)STRENGTH2);

                normSum += groupTot;
                totSum += groupTot * value;
                groupTot = 0.0f;
            }
        }
    }
    value = native_divide(totSum,normSum);
    if(value < 0){
        value = 0;
    }
    if(value > 1.0){
        value = 1.0f;
    }

    int outputDataType = get_image_channel_data_type(output);
    if(outputDataType == CLK_FLOAT) {
        write_imagef(output, pos, value);
    } else if(outputDataType == CLK_UNSIGNED_INT8 || outputDataType == CLK_UNSIGNED_INT16) {
        write_imageui(output, pos, round(value));
    } else {
        write_imagei(output, pos, round(value));
    }
    /*
    if(outputDataType == CLK_FLOAT) {
        float holder = native_divide(totSum,normSum);
        if(holder > 0){
            value = holder;
        }else if(holder > 1){
            value = 1.0f;
        }else{
            value = 0.0f;
        }
    write_imagef(output,pos,value);
    }else if(outputDataType == CLK_UNSIGNED_INT8 || outputDataType == CLK_UNSIGNED_INT16) {
        int holder = 0;
        if(value > -1 && value < 2){
            holder = value * 255;
        }else if(value > 1){
            holder = 255;
        }
    holder = min(max(holder,0),255);
    write_imageui(output, pos, holder);
    }else {
        int holder = 0;
        if(value > -1 && value < 2){
            holder = value * 255;
        }else if(value > 1){
            holder = 255;
        }
    holder = min(max(holder,0),255);
    write_imagei(output, pos, holder);
    }
    */
}


more test case for vector load/store function

/* test OpenCL 1.1 Vector Data Load/Store Functions (section 6.11.7) */
kernel void compiler_vector_load_store() {
  float p[16], f;
  float4 f4;
  f4 = vload4(0, p);
  vstore4(f4, 0, p);

  long x[16], l;
  long16 l16;
  l = vload16(0, x);
  vstore16(l16, 0, x);

  half h[16];
  half4 h4;
  f = vload_half(0, h);
  f4 = vload_half4(0, h);
  vstore_half(f, 0, h);
}

/* test OpenCL 1.1 Vector Data Load/Store Functions (section 6.11.7) */
kernel void compiler_vector_load_store() {
  float p[16], f;
  float4 f4;
  f4 = vload4(0, p);
  vstore4(f4, 0, p);

  half h[16];
  half4 h4;
  f = vload_half(0, h);
  f4 = vload_half4(0, h);
  vstore_half(f, 0, h);
}