fixed issues in tensor_ops kernel

fixed issues in tensor_ops kernel

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#ifndef MIOPEN_TENSOR_DIMS
#define MIOPEN_TENSOR_DIMS 4
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              const int a_nstride,
                              const int a_cstride,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              const int b_nstride,
                              const int b_cstride,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const float alpha0,
                              const float alpha1,
                              const float beta,
                              const int work_per_wg,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if INCR_WG == 1
    int o_n = gid / b_c;
    int o_c = gid % b_c;
    // MIOPEN_TYPE operand = b[o_c + Boffset];
    int a_sub_index = o_n * a_nstride + o_c * a_cstride;
    int b_sub_index = o_n * b_nstride + o_c * b_cstride;
    int c_sub_index = o_n * c_nstride + o_c * c_cstride;

    while(lid < work_per_wg)
    {
        int aindex = a_sub_index + lid;
        int bindex = b_sub_index + lid;
        int cindex = c_sub_index + lid;
        c_off[cindex] =
            MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, b_off[bindex] * alpha1) + beta * c_off[cindex];
        lid += get_local_size(0);
    }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
    // MIOPEN_TYPE operand = b[gid + Boffset];
    int work_off = work_per_wg / c_n;

    while(lid < work_per_wg)
    {
        int o_hw   = lid % work_off;
        int o_n    = lid / work_off;
        int aindex = o_n * a_nstride + gid * a_cstride + o_hw;
        int bindex = o_n * b_nstride + gid * b_cstride + o_hw;
        int cindex = o_n * c_nstride + gid * c_cstride + o_hw;
        // int index = o_n * c_nstride + gid * c_cstride + o_hw;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        // c[index + Coffset] =
        // alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];
        c_off[cindex] =
            MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, b_off[bindex] * alpha1) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#endif // INCR_WG
}

// DLOWELL : cutting out this section
//#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)
#if(0)

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  global MIOPEN_TYPE* b,
                                  global MIOPEN_TYPE* c,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_c,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_h,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_cstride,

                                  const float alpha0,
                                  const float alpha1,
                                  const float beta,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset)
{

/* Special case for leading ones where the total no. of threads is the
 * inner_product of the tensor dims.  Each thread just updates one value
 */
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid             = get_global_id(0);
    MIOPEN_TYPE operand = b[tid + Boffset];

    int o_w = tid % c_w;
    int o_h = (tid / c_w) % c_h;
    int o_c = (tid / (c_w * c_h)) % c_c;
    int o_n = tid / (c_w * c_h * c_c);

    int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
    int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
    int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
    // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
    c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, b_off[bindex]) + beta * c_off[cindex];

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b[gid + Boffset];

    int o_h = gid % c_h;
    int o_c = (gid / c_h) % c_c;
    int o_n = gid / (c_c * c_h);

    while(lid < work_per_wg)
    {
        int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + lid;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c[index + Coffset] =
            alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];

        lid += get_local_size(0);
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b[gid + Boffset];

    int o_c = gid % c_c;
    int o_n = gid / c_c;

    while(lid < work_per_wg)
    {
        int index = o_n * c_nstride + o_c * c_cstride + lid;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c[index + Coffset] =
            alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];
        lid += get_local_size(0);
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b[gid + Boffset];

    while(lid < work_per_wg)
    {
        int index = gid * c_nstride + lid;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c[index + Coffset] =
            alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];

        lid += get_local_size(0);
    }
#endif
}

#endif

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_c_gid_off = (gid / b_w / b_h) % b_c;
    int o_n_gid_off = (gid / b_w / b_h) / b_c;

    int bindex =
        o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // if(gid>=b_nstride) return;
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
    int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);
    // printf("lid: %d, o_[h,d,c,n]_div: %d, %d, %d, %d\n",lid, o_h_div, o_d_div, o_c_div, o_n_div);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_d_gid_off = (gid / b_w / b_h) % b_d;
    int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
    int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

    int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                 o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
        int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;
        // printf("lid: %d, o_[w,h,d,c,n]: %d, %d, %d, %d, %d\n",lid, o_w, o_h, o_d, o_c, o_n);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;
        // printf("lid: %d, index: %d\n",lid, index);
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
    int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

    int o_h_gid_off = gid % b_h;
    int o_c_gid_off = (gid / b_h) % b_c;
    // int o_n_gid_off = gid / b_nstride;
    int o_n_gid_off = (gid / b_h) / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
        int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : (lid / o_c_div) / o_n_div;

        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    int o_c_gid_off = gid % b_c;
    int o_n_gid_off = gid / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_c       = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
        int o_n       = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
        int aindex    = o_n * a_nstride + o_c;
        int cindex    = o_n * c_nstride + o_c;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }
}

// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
    // MIOPEN_TYPE operand = b[gid + Boffset];

    int o_n_gid_off     = gid % b_n;
    int bindex          = o_n_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
        c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
        lid += get_local_size(0);
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#ifndef MIOPEN_TENSOR_DIMS
#define MIOPEN_TENSOR_DIMS 4
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              const int a_nstride,
                              const int a_cstride,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              const int b_nstride,
                              const int b_cstride,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const float alpha0,
                              const float alpha1,
                              const float beta,
                              const int work_per_wg,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if INCR_WG == 1
    int o_n = gid / b_c;
    int o_c = gid % b_c;
    // MIOPEN_TYPE operand = b[o_c + Boffset];
    int a_sub_index = o_n* a_nstride + o_c* a_cstride int b_sub_index =
                          o_n* b_nstride + o_c* b_cstride int c_sub_index =
                              o_n * c_nstride +
                              o_c * c_cstride

                                  while(lid < work_per_wg)
    {
        int aindex = a_sub_index + lid;
        int bindex = b_sub_index + lid;
        int cindex = c_sub_index + lid;
        c_off[cindex] =
            MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, b_off[bindex] * alpha1) + beta * c_off[cindex];
        lid += get_local_size(0);
    }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
    // MIOPEN_TYPE operand = b[gid + Boffset];
    int work_off = work_per_wg / c_n;

    while(lid < work_per_wg)
    {
        int o_hw   = lid % work_off;
        int o_n    = lid / work_off;
        int aindex = o_n * a_nstride + gid * a_cstride + o_hw;
        int bindex = o_n * b_nstride + gid * b_cstride + o_hw;
        int cindex = o_n * c_nstride + gid * c_cstride + o_hw;
        // int index = o_n * c_nstride + gid * c_cstride + o_hw;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        // c[index + Coffset] =
        // alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];
        c_off[cindex] =
            MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, b_off[bindex] * alpha1) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#endif // INCR_WG
}

// DLOWELL : cutting out this section
//#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)
#if(0)

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  global MIOPEN_TYPE* b,
                                  global MIOPEN_TYPE* c,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_c,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_h,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_cstride,

                                  const float alpha0,
                                  const float alpha1,
                                  const float beta,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset)
{

/* Special case for leading ones where the total no. of threads is the
 * inner_product of the tensor dims.  Each thread just updates one value
 */
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid             = get_global_id(0);
    MIOPEN_TYPE operand = b[tid + Boffset];

    int o_w = tid % c_w;
    int o_h = (tid / c_w) % c_h;
    int o_c = (tid / (c_w * c_h)) % c_c;
    int o_n = tid / (c_w * c_h * c_c);

    int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
    int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
    int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
    // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
    c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, b_off[bindex]) + beta * c_off[cindex];

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b[gid + Boffset];

    int o_h = gid % c_h;
    int o_c = (gid / c_h) % c_c;
    int o_n = gid / (c_c * c_h);

    while(lid < work_per_wg)
    {
        int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + lid;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c[index + Coffset] =
            alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];

        lid += get_local_size(0);
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b[gid + Boffset];

    int o_c = gid % c_c;
    int o_n = gid / c_c;

    while(lid < work_per_wg)
    {
        int index = o_n * c_nstride + o_c * c_cstride + lid;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c[index + Coffset] =
            alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];
        lid += get_local_size(0);
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b[gid + Boffset];

    while(lid < work_per_wg)
    {
        int index = gid * c_nstride + lid;
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c[index + Coffset] =
            alpha * MIOPEN_TENSOR_OP(a[index + Aoffset], operand) + beta * c[index + Coffset];

        lid += get_local_size(0);
    }
#endif
}

#endif

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_c_gid_off = (gid / b_w / b_h) % b_c;
    int o_n_gid_off = (gid / b_w / b_h) / b_c;

    int bindex =
        o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // if(gid>=b_nstride) return;
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
    int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);
    // printf("lid: %d, o_[h,d,c,n]_div: %d, %d, %d, %d\n",lid, o_h_div, o_d_div, o_c_div, o_n_div);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_d_gid_off = (gid / b_w / b_h) % b_d;
    int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
    int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

    int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                 o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
        int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;
        // printf("lid: %d, o_[w,h,d,c,n]: %d, %d, %d, %d, %d\n",lid, o_w, o_h, o_d, o_c, o_n);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;
        // printf("lid: %d, index: %d\n",lid, index);
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
    int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

    int o_h_gid_off = gid % b_h;
    int o_c_gid_off = (gid / b_h) % b_c;
    // int o_n_gid_off = gid / b_nstride;
    int o_n_gid_off = (gid / b_h) / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
        int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : (lid / o_c_div) / o_n_div;

        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    int o_c_gid_off = gid % b_c;
    int o_n_gid_off = gid / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_c       = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
        int o_n       = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
        int aindex    = o_n * a_nstride + o_c;
        int cindex    = o_n * c_nstride + o_c;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }
}

// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
    // MIOPEN_TYPE operand = b[gid + Boffset];

    int o_n_gid_off     = gid % b_n;
    int bindex          = o_n_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
        c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
        lid += get_local_size(0);
    }
}


Fixing build failure by adding triple option to new test condition.

Adding -triple option to ensure target supports double for fpmath test.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@289552 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -emit-llvm -o - -triple spir-unknown-unknown | FileCheck --check-prefix=CHECK --check-prefix=NODIVOPT %s
// RUN: %clang_cc1 %s -emit-llvm -o - -triple spir-unknown-unknown -cl-fp32-correctly-rounded-divide-sqrt | FileCheck --check-prefix=CHECK --check-prefix=DIVOPT %s
// RUN: %clang_cc1 %s -emit-llvm -o - -DNOFP64 -cl-std=CL1.2 -triple r600-unknown-unknown -target-cpu r600 -pedantic | FileCheck --check-prefix=CHECK-FLT %s
// RUN: %clang_cc1 %s -emit-llvm -o - -DFP64 -cl-std=CL1.2 -triple spir-unknown-unknown -pedantic | FileCheck --check-prefix=CHECK-DBL %s

typedef __attribute__(( ext_vector_type(4) )) float float4;

float spscalardiv(float a, float b) {
  // CHECK: @spscalardiv
  // CHECK: #[[ATTR:[0-9]+]]
  // CHECK: fdiv{{.*}},
  // NODIVOPT: !fpmath ![[MD:[0-9]+]]
  // DIVOPT-NOT: !fpmath ![[MD:[0-9]+]]
  return a / b;
}

float4 spvectordiv(float4 a, float4 b) {
  // CHECK: @spvectordiv
  // CHECK: #[[ATTR]]
  // CHECK: fdiv{{.*}},
  // NODIVOPT: !fpmath ![[MD]]
  // DIVOPT-NOT: !fpmath ![[MD]]
  return a / b;
}

#if __OPENCL_C_VERSION__ >=120
void printf(constant char* fmt, ...);

void testdbllit(long *val) {
  // CHECK-FLT: float 2.000000e+01
  // CHECK-DBL: double 2.000000e+01
  printf("%f", 20.0);
}

#endif

#ifndef NOFP64
#pragma OPENCL EXTENSION cl_khr_fp64 : enable
double dpscalardiv(double a, double b) {
  // CHECK: @dpscalardiv
  // CHECK: #[[ATTR]]
  // CHECK-NOT: !fpmath
  return a / b;
}
#endif

// CHECK: attributes #[[ATTR]] = {
// NODIVOPT: "correctly-rounded-divide-sqrt-fp-math"="false"
// DIVOPT: "correctly-rounded-divide-sqrt-fp-math"="true"
// CHECK: }
// NODIVOPT: ![[MD]] = !{float 2.500000e+00}

// RUN: %clang_cc1 %s -emit-llvm -o - -triple spir-unknown-unknown | FileCheck --check-prefix=CHECK --check-prefix=NODIVOPT %s
// RUN: %clang_cc1 %s -emit-llvm -o - -triple spir-unknown-unknown -cl-fp32-correctly-rounded-divide-sqrt | FileCheck --check-prefix=CHECK --check-prefix=DIVOPT %s
// RUN: %clang_cc1 %s -emit-llvm -o - -DNOFP64 -cl-std=CL1.2 -triple r600-unknown-unknown -target-cpu r600 -pedantic | FileCheck --check-prefix=CHECK-FLT %s
// RUN: %clang_cc1 %s -emit-llvm -o - -DFP64 -cl-std=CL1.2 -pedantic | FileCheck --check-prefix=CHECK-DBL %s

typedef __attribute__(( ext_vector_type(4) )) float float4;

float spscalardiv(float a, float b) {
  // CHECK: @spscalardiv
  // CHECK: #[[ATTR:[0-9]+]]
  // CHECK: fdiv{{.*}},
  // NODIVOPT: !fpmath ![[MD:[0-9]+]]
  // DIVOPT-NOT: !fpmath ![[MD:[0-9]+]]
  return a / b;
}

float4 spvectordiv(float4 a, float4 b) {
  // CHECK: @spvectordiv
  // CHECK: #[[ATTR]]
  // CHECK: fdiv{{.*}},
  // NODIVOPT: !fpmath ![[MD]]
  // DIVOPT-NOT: !fpmath ![[MD]]
  return a / b;
}

#if __OPENCL_C_VERSION__ >=120
void printf(constant char* fmt, ...);

void testdbllit(long *val) {
  // CHECK-FLT: float 2.000000e+01
  // CHECK-DBL: double 2.000000e+01
  printf("%f", 20.0);
}

#endif

#ifndef NOFP64
#pragma OPENCL EXTENSION cl_khr_fp64 : enable
double dpscalardiv(double a, double b) {
  // CHECK: @dpscalardiv
  // CHECK: #[[ATTR]]
  // CHECK-NOT: !fpmath
  return a / b;
}
#endif

// CHECK: attributes #[[ATTR]] = {
// NODIVOPT: "correctly-rounded-divide-sqrt-fp-math"="false"
// DIVOPT: "correctly-rounded-divide-sqrt-fp-math"="true"
// CHECK: }
// NODIVOPT: ![[MD]] = !{float 2.500000e+00}


Minor corrections to scan by key first kernel

/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

static char calculate_head_flags(const __global Tk *kptr, int id, int previd)
{
    char flag;
    if (id == 0) {
        flag = 1;
    } else {
        flag = (kptr[id] != kptr[previd]);
    }
    return flag;
}

void scan_first_by_key_core(const bool invalid,
        To *val, char *flag,
        const __global Ti *in, const To init_val,
        __local To *l_val0, __local To *l_val1,
        __local char *l_flg0, __local char *l_flg1,
        __local To *last_val, __local char *last_flag,
        const int lid, const int lidx, const int lidy)
{
    bool flip = 0;
    __local To *l_val = l_val0;
    __local char *l_flg = l_flg0;
    *val = invalid? init_val : transform(*in);

    if ((lidx == 0) && (flag == 0)) {
        *val = binOp(*val, last_val[lidy]);
        *flag = *flag | last_flag[lidy];
    }

    l_val0[lid] = *val;
    l_flg0[lid] = *flag;
    barrier(CLK_LOCAL_MEM_FENCE);

    for (int off = 1; off < DIMX; off *= 2) {

        if (lidx >= off) {
            *val = l_flg[lid] ? *val : binOp(*val, l_val[lid - off]);
            *flag = l_flg[lid] | l_flg[lid - off];
        }
        flip = 1 - flip;
        l_val = flip ? l_val1 : l_val0;
        l_flg = flip ? l_flg1 : l_flg0;
        l_val[lid] = *val;
        l_flg[lid] = *flag;
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

__kernel
void scan_first_by_key_nonfinal_kernel(__global To *oData, KParam oInfo,
                       __global To *tData, KParam tInfo,
                       __global char *tfData, KParam tfInfo,
                       __global int *tiData, KParam tiInfo,
                       const __global Ti *iData, KParam iInfo,
                       const __global Tk *kData, KParam kInfo,
                       uint groups_x, uint groups_y,
                       uint lim)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int lid  = lidy * get_local_size(0) + lidx;

    const int zid = get_group_id(0) / groups_x;
    const int wid = get_group_id(1) / groups_y;
    const int groupId_x = get_group_id(0) - (groups_x) * zid;
    const int groupId_y = get_group_id(1) - (groups_y) * wid;
    const int xid = groupId_x * get_local_size(0) * lim + lidx;
    const int yid = groupId_y * get_local_size(1) + lidy;

    bool cond_yzw = (yid < oInfo.dims[1]) && (zid < oInfo.dims[2]) && (wid < oInfo.dims[3]);

    iData += wid * iInfo.strides[3] + zid * iInfo.strides[2] +
        yid * iInfo.strides[1] + iInfo.offset;

    kData += wid * kInfo.strides[3] + zid * kInfo.strides[2] +
        yid * kInfo.strides[1] + kInfo.offset;

    tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] +
        yid * tInfo.strides[1] + tInfo.offset;

    tfData += wid * tfInfo.strides[3] + zid * tfInfo.strides[2] +
        yid * tfInfo.strides[1] + tfInfo.offset;

    tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] +
        yid * tiInfo.strides[1] + tiInfo.offset;

    oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
        yid * oInfo.strides[1] + oInfo.offset;

    __local To l_val0[SHARED_MEM_SIZE];
    __local To l_val1[SHARED_MEM_SIZE];
    __local char l_flg0[SHARED_MEM_SIZE];
    __local char l_flg1[SHARED_MEM_SIZE];
    __local To last_val[DIMY];
    __local char last_flag[DIMY];
    __local int boundaryid;

    const To init_val = init;
    int id = xid;
    To val = init_val;
    const bool isLast = (lidx == (DIMX - 1));

    char flag = 0;
    if (!inclusive_scan) {
        iData -= 1;
    }

    if (isLast) {
        last_val[lidy] = val;
        last_flag[lidy] = 0;
        boundaryid = -1;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    __local char *prev;
    if (lidx == 0) {
        prev = &last_flag[lidy];
    } else {
        prev = &l_flg0[lidx-1];
    }
    __local char *curr = &l_flg0[lidx];

    for (int k = 0; k < lim; k++) {

        bool cond = ((id < oInfo.dims[0]) && cond_yzw);

        if (cond) {
            flag = calculate_head_flags(kData, id, id - 1);
        } else {
            flag = 0;
        }

        bool invalid = !cond;
        if (!inclusive_scan) invalid = invalid || (id == 0) || flag;

        scan_first_by_key_core(invalid, &val, &flag, iData, init_val,
                l_val0, l_val1, l_flg0, l_flg1, last_val, last_flag,
                lid, lidx, lidy);

        if ((*prev == 0) && (*curr == 1)) {
            boundaryid = id;
        }

        if (cond) oData[id] = val;
        if (isLast) {
            last_val[lidy] = val;
            last_flag[lidy] = flag;
        }
        id += DIMX;
        barrier(CLK_LOCAL_MEM_FENCE); //FIXME: May be needed only for non nvidia gpus
    }

    if (isLast && cond_yzw) {
        tData[groupId_x] = val;
        tfData[groupId_x] = flag;
        tiData[groupId_x] = boundaryid;
    }
}

__kernel
void scan_first_by_key_final_kernel(__global To *oData, KParam oInfo,
                       const __global Ti *iData, KParam iInfo,
                       const __global Tk *kData, KParam kInfo,
                       uint groups_x, uint groups_y,
                       uint lim)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int lid  = lidy * get_local_size(0) + lidx;

    const int zid = get_group_id(0) / groups_x;
    const int wid = get_group_id(1) / groups_y;
    const int groupId_x = get_group_id(0) - (groups_x) * zid;
    const int groupId_y = get_group_id(1) - (groups_y) * wid;
    const int xid = groupId_x * get_local_size(0) * lim + lidx;
    const int yid = groupId_y * get_local_size(1) + lidy;

    bool cond_yzw = (yid < oInfo.dims[1]) && (zid < oInfo.dims[2]) && (wid < oInfo.dims[3]);

    iData += wid * iInfo.strides[3] + zid * iInfo.strides[2] +
        yid * iInfo.strides[1] + iInfo.offset;

    kData += wid * kInfo.strides[3] + zid * kInfo.strides[2] +
        yid * kInfo.strides[1] + kInfo.offset;

    oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
        yid * oInfo.strides[1] + oInfo.offset;

    __local To l_val0[SHARED_MEM_SIZE];
    __local To l_val1[SHARED_MEM_SIZE];
    __local char l_flg0[SHARED_MEM_SIZE];
    __local char l_flg1[SHARED_MEM_SIZE];
    __local To last_val[DIMY];
    __local char last_flag[DIMY];

    const To init_val = init;
    int id = xid;
    To val = init_val;
    const bool isLast = (lidx == (DIMX - 1));

    if (!inclusive_scan) {
        iData -= 1;
    }

    if (isLast) {
        last_val[lidy] = val;
        last_flag[lidy] = 0;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    for (int k = 0; k < lim; k++) {
        char flag = 0;

        bool cond = ((id < oInfo.dims[0]) && cond_yzw);

        if (calculateFlags) {
            if (cond) {
                flag = calculate_head_flags(kData, id, id - 1);
            } else {
                flag = 0;
            }
        } else {
            flag = kData[id];
        }

        bool invalid = !cond;
        if (!inclusive_scan) invalid = invalid || (id == 0) || flag;

        scan_first_by_key_core(invalid, &val, &flag, iData, init_val,
                l_val0, l_val1, l_flg0, l_flg1, last_val, last_flag,
                lid, lidx, lidy);

        if (cond) oData[id] = val;
        if (isLast) {
            last_val[lidy] = val;
            last_flag[lidy] = flag;
        }
        id += DIMX;
        barrier(CLK_LOCAL_MEM_FENCE); //FIXME: May be needed only for non nvidia gpus
    }
}

__kernel
void bcast_first_kernel(__global To *oData, KParam oInfo,
                        const __global To *tData, KParam tInfo,
                        const __global int *tiData, KParam tiInfo,
                        uint groups_x, uint groups_y, uint lim)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int lid  = lidy * get_local_size(0) + lidx;

    const int zid = get_group_id(0) / groups_x;
    const int wid = get_group_id(1) / groups_y;
    const int groupId_x = get_group_id(0) - (groups_x) * zid;
    const int groupId_y = get_group_id(1) - (groups_y) * wid;
    const int xid = groupId_x * get_local_size(0) * lim + lidx;
    const int yid = groupId_y * get_local_size(1) + lidy;

    if (groupId_x != 0) {
        bool cond = (yid < oInfo.dims[1]) && (zid < oInfo.dims[2]) && (wid < oInfo.dims[3]);

        if (cond) {

            tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] +
                yid * tiInfo.strides[1] + tiInfo.offset;

            tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] +
                yid * tInfo.strides[1] + tInfo.offset;

            oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
                yid * oInfo.strides[1] + oInfo.offset;

            int boundary = tiData[groupId_x];
            To accum = tData[groupId_x - 1];

            for (int k = 0, id = xid;
                 k < lim && id < boundary;
                 k++, id += DIMX) {

                oData[id] = binOp(accum, oData[id]);
            }
        }
    }
}

/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

static char calculate_head_flags(const __global Tk *kptr, int id, int previd)
{
    char flag;
    if (id == 0) {
        flag = 1;
    } else {
        flag = (kptr[id] != kptr[previd]);
    }
    return flag;
}

void scan_first_by_key_core(const bool invalid,
        To *val, char *flag,
        const __global Ti *in, const To init_val,
        __local To *l_val0, __local To *l_val1,
        __local char *l_flg0, __local char *l_flg1,
        __local To *last_val, __local char *last_flag,
        const int lid, const int lidx, const int lidy)
{
    bool flip = 0;
    __local To *l_val = l_val0;
    __local char *l_flg = l_flg0;
    *val = invalid? init_val : transform(*in);

    if ((lidx == 0) && (flag == 0)) {
        *val = binOp(*val, last_val[lidy]);
        *flag = *flag | last_flag[lidy];
    }

    l_val0[lid] = *val;
    l_flg0[lid] = *flag;
    barrier(CLK_LOCAL_MEM_FENCE);

    for (int off = 1; off < DIMY; off *= 2) {

        if (lidy >= off) {
            *val = l_flg[lid] ? *val : binOp(*val, l_val[lid - off]);
            *flag = l_flg[lid] | l_flg[lid - off];
        }
        flip = 1 - flip;
        l_val = flip ? l_val1 : l_val0;
        l_flg = flip ? l_flg1 : l_flg0;
        l_val[lid] = *val;
        l_flg[lid] = *flag;
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

__kernel
void scan_first_by_key_nonfinal_kernel(__global To *oData, KParam oInfo,
                       __global To *tData, KParam tInfo,
                       __global char *tfData, KParam tfInfo,
                       __global int *tiData, KParam tiInfo,
                       const __global Ti *iData, KParam iInfo,
                       const __global Tk *kData, KParam kInfo,
                       uint groups_x, uint groups_y,
                       uint lim)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int lid  = lidy * get_local_size(0) + lidx;

    const int zid = get_group_id(0) / groups_x;
    const int wid = get_group_id(1) / groups_y;
    const int groupId_x = get_group_id(0) - (groups_x) * zid;
    const int groupId_y = get_group_id(1) - (groups_y) * wid;
    const int xid = groupId_x * get_local_size(0) * lim + lidx;
    const int yid = groupId_y * get_local_size(1) + lidy;

    bool cond_yzw = (yid < oInfo.dims[1]) && (zid < oInfo.dims[2]) && (wid < oInfo.dims[3]);

    iData += wid * iInfo.strides[3] + zid * iInfo.strides[2] +
        yid * iInfo.strides[1] + iInfo.offset;

    kData += wid * kInfo.strides[3] + zid * kInfo.strides[2] +
        yid * kInfo.strides[1] + kInfo.offset;

    tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] +
        yid * tInfo.strides[1] + tInfo.offset;

    tfData += wid * tfInfo.strides[3] + zid * tfInfo.strides[2] +
        yid * tfInfo.strides[1] + tfInfo.offset;

    tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] +
        yid * tiInfo.strides[1] + tiInfo.offset;

    oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
        yid * oInfo.strides[1] + oInfo.offset;

    __local To l_val0[SHARED_MEM_SIZE];
    __local To l_val1[SHARED_MEM_SIZE];
    __local char l_flg0[SHARED_MEM_SIZE];
    __local char l_flg1[SHARED_MEM_SIZE];
    __local To last_val[DIMY];
    __local char last_flag[DIMY];
    __local int boundaryid;

    const To init_val = init;
    int id = xid;
    To val = init_val;
    const bool isLast = (lidx == (DIMX - 1));

    char flag = 0;
    if (!inclusive_scan) {
        iData -= 1;
    }

    if (isLast) {
        last_val[lidy] = val;
        last_flag[lidy] = 0;
        boundaryid = -1;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    __local char *prev;
    if (lidx == 0) {
        prev = &last_flag[lidy];
    } else {
        prev = &l_flg0[lidx-1];
    }
    __local char *curr = &l_flg0[lidx];

    for (int k = 0; k < lim; k++) {

        bool cond = ((id < oInfo.dims[0]) && cond_yzw);

        if (cond) {
            flag = calculate_head_flags(kData, id, id - 1);
        } else {
            flag = 0;
        }

        bool invalid = !cond;
        if (!inclusive_scan) invalid = invalid || (id == 0) || flag;

        scan_first_by_key_core(invalid, &val, &flag, iData, init_val,
                l_val0, l_val1, l_flg0, l_flg1, last_val, last_flag,
                lid, lidx, lidy);

        if ((*prev == 0) && (*curr == 1)) {
            boundaryid = id;
        }

        if (cond) oData[id] = val;
        if (isLast) {
            last_val[lidy] = val;
            last_flag[lidy] = flag;
        }
        id += DIMX;
        barrier(CLK_LOCAL_MEM_FENCE); //FIXME: May be needed only for non nvidia gpus
    }

    if (isLast && cond_yzw) {
        tData[groupId_x] = val;
        tfData[groupId_x] = flag;
        tiData[groupId_x] = boundaryid;
    }
}

__kernel
void scan_first_by_key_final_kernel(__global To *oData, KParam oInfo,
                       const __global Ti *iData, KParam iInfo,
                       const __global Tk *kData, KParam kInfo,
                       uint groups_x, uint groups_y,
                       uint lim)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int lid  = lidy * get_local_size(0) + lidx;

    const int zid = get_group_id(0) / groups_x;
    const int wid = get_group_id(1) / groups_y;
    const int groupId_x = get_group_id(0) - (groups_x) * zid;
    const int groupId_y = get_group_id(1) - (groups_y) * wid;
    const int xid = groupId_x * get_local_size(0) * lim + lidx;
    const int yid = groupId_y * get_local_size(1) + lidy;

    bool cond_yzw = (yid < oInfo.dims[1]) && (zid < oInfo.dims[2]) && (wid < oInfo.dims[3]);

    iData += wid * iInfo.strides[3] + zid * iInfo.strides[2] +
        yid * iInfo.strides[1] + iInfo.offset;

    kData += wid * kInfo.strides[3] + zid * kInfo.strides[2] +
        yid * kInfo.strides[1] + kInfo.offset;

    oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
        yid * oInfo.strides[1] + oInfo.offset;

    __local To l_val0[SHARED_MEM_SIZE];
    __local To l_val1[SHARED_MEM_SIZE];
    __local char l_flg0[SHARED_MEM_SIZE];
    __local char l_flg1[SHARED_MEM_SIZE];
    __local To last_val[DIMY];
    __local char last_flag[DIMY];

    const To init_val = init;
    int id = xid;
    To val = init_val;
    const bool isLast = (lidx == (DIMX - 1));

    if (!inclusive_scan) {
        iData -= 1;
    }

    if (isLast) {
        last_val[lidy] = val;
        last_flag[lidy] = 0;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    for (int k = 0; k < lim; k++) {
        char flag = 0;

        bool cond = ((id < oInfo.dims[0]) && cond_yzw);

        if (calculateFlags) {
            if (cond) {
                flag = calculate_head_flags(kData, id, id - 1);
            } else {
                flag = 0;
            }
        } else {
            flag = kData[id];
        }

        bool invalid = !cond;
        if (!inclusive_scan) invalid = invalid || (id == 0) || flag;

        scan_first_by_key_core(invalid, &val, &flag, iData, init_val,
                l_val0, l_val1, l_flg0, l_flg1, last_val, last_flag,
                lid, lidx, lidy);

        if (cond) oData[id] = val;
        if (isLast) {
            last_val[lidy] = val;
            last_flag[lidy] = flag;
        }
        id += DIMX;
        barrier(CLK_LOCAL_MEM_FENCE); //FIXME: May be needed only for non nvidia gpus
    }
}

__kernel
void bcast_first_kernel(__global To *oData, KParam oInfo,
                        const __global To *tData, KParam tInfo,
                        const __global int *tiData, KParam tiInfo,
                        uint groups_x, uint groups_y, uint lim)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int lid  = lidy * get_local_size(0) + lidx;

    const int zid = get_group_id(0) / groups_x;
    const int wid = get_group_id(1) / groups_y;
    const int groupId_x = get_group_id(0) - (groups_x) * zid;
    const int groupId_y = get_group_id(1) - (groups_y) * wid;
    const int xid = groupId_x * get_local_size(0) * lim + lidx;
    const int yid = groupId_y * get_local_size(1) + lidy;

    if (groupId_x != 0) {
        bool cond = (yid < oInfo.dims[1]) && (zid < oInfo.dims[2]) && (wid < oInfo.dims[3]);

        if (cond) {

            tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] +
                yid * tiInfo.strides[1] + tiInfo.offset;

            tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] +
                yid * tInfo.strides[1] + tInfo.offset;

            oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
                yid * oInfo.strides[1] + oInfo.offset;

            int boundary = tiData[groupId_x];
            To accum = tData[groupId_x - 1];

            for (int k = 0, id = xid;
                 k < lim && id < boundary;
                 k++, id += DIMX) {

                oData[id] = binOp(accum, oData[id]);
            }
        }
    }
}


Adding a boatload of comments to the CG kernel

since it's important to fully understand it

/*
* Solve a symmetric sparse matrix system using the Conjugate Gradient method
* Each kernel works on a single row of values in the matrix @ row = id
* and updates the values x[id], r[id] and p[id]
* @param dim Dimension of the matrix we're trying to solve (square only)
* @param nVals Number of non-zero values in the sparse matrix
* @param r The residual vector. Each kernel only updates r[id] but makes use
* 	of everyone's results to compute r dot r needed for finding p[id]
* @param x The result vector, each kernel works on x[id] and sums in the
*	results of the iterations, each loop: x[id] += alpha * p[id]
* @param Atimsp The result of Ap, each kernel only computes the value at its
*	id, but uses the result of everyones work to find p dot Ap
* @param p The current search direction, again each kernel updates the value @ id
*	but uses the result of all kernels updates each step for the direction
* @param rows The row indices of the non-zero matrix elements
* @param cols The col indices of the non-zero matrix elements
* @param A The sparse matrix values. The matrix is read like so:
*	non-zero entry 2: row[2] = 1, col[2] = 5, val[2] = 15.4
* @param b The right side of the system Ax = b
* @param result The final iteration count, residual lenght and x vector are
*	written to this buffer so that the host can read back information about the solution
*	along with the solution itself
*/
__kernel void conjGrad(int dim, int nVals, __local float *r, __local float *x,
	__local float *Atimesp, __local float *p, __global int *rows, __global int *cols,
	__global float *A, __global float *b, __global float *result)
{
	//Todo: the book uses local id, and runs everything as one work group
	//which is ok, but what if the matrix is bigger dimensions than max work group size?
	//what if I want to use the preferred work group size?
	int id = get_local_id(0);
	//We use these to select the range of i to read row/col from
	int startIdx = -1;
	int endIdx = -1;
	//Determine the indices containing the values for the row this kernel will be working on
	for (int i = id; i < nVals; ++i){
		if (rows[i] == id && startIdx == -1)
			startIdx = i;
		else if (rows[i] == id + 1 && endIdx == -1){
			endIdx = i - 1;
			break;
		}
		else if (i == nVals - 1 && endIdx == -1)
			endIdx = i;
	}

	//Set initial guess, residual and dir
	x[id] = 0.0f;
	r[id] = b[id];
	p[id] = b[id];
	//Wait for everyone to get setup
	barrier(CLK_LOCAL_MEM_FENCE);

	//What if instead of only having the lead kernel do this
	//we used a seperate kernel that would more evenly divide the work?
	//Compute the initial dot product and length of r in the lead kernel
	local float oldRdotR, rLen;
	if (id == 0){
		oldRdotR = 0.0f;
		for (int i = 0; i < dim; ++i)
			oldRdotR += r[i] * r[i];

		rLen = sqrt(oldRdotR);
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	//Do the CG iterations
	local float alpha, newRdotR, ApDotp;
	local int iteration;
	iteration = 0;
	while (iteration < 1000 && rLen >= 0.01f){
		Atimesp[id] = 0.0f;
		for (int i = startIdx; i <= endIdx; ++i)
			Atimesp[id] += A[i] * p[cols[i]];

		barrier(CLK_LOCAL_MEM_FENCE);

		//Lead kernel computes alpha
		if (id == 0){
			ApDotp = 0.0f;
			for (int i = 0; i < dim; ++i)
				ApDotp += Atimesp[i] * p[i];

			alpha = oldRdotR / ApDotp;
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		//Determine next guess & residual this work item will investigate
		x[id] += alpha * p[id];
		r[id] -= alpha * Atimesp[id];
		barrier(CLK_LOCAL_MEM_FENCE);

		//Lead kernel computes r dot r, again could this benefit
		//from being split into its own kernel for even work distribution?
		//Or would the overhead of running another kernel outweight benefit?
		if (id == 0){
			newRdotR = 0.0f;
			for (int i = 0; i < dim; ++i)
				newRdotR += r[i] * r[i];

			rLen = sqrt(newRdotR);
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		//Update dir
		p[id] = r[id] + (newRdotR / oldRdotR) * p[id];
		barrier(CLK_LOCAL_MEM_FENCE);

		oldRdotR = newRdotR;

		//Lead kernel incremenents the iteration count and we wait for everyone to finish
		//their iteration
		if (id == 0)
			++iteration;

		barrier(CLK_LOCAL_MEM_FENCE);
	}
	result[0] = iteration * 1.0f;
	result[1] = rLen;
	//We also want access to the result so write that out as well
	//for the final kernel I want to use in the sim I'll probably skip
	//outputting the iteration count and rLen
	result[id + 2] = x[id];
}

//Solve a symmetric sparse matrix system using conjugate gradient method
__kernel void conjGrad(int dim, int nVals, __local float *r, __local float *x,
	__local float *Atimesp, __local float *p, __global int *rows, __global int *cols,
	__global float *A, __global float *b, __global float *result)
{
	//Todo: the book uses local id, and runs everything as one work group
	//which is ok, but what if the matrix is bigger dimensions than max work group size?
	//what if I want to use the preferred work group size?
	int id = get_local_id(0);
	//We use these to select the range of i to read row/col from
	int startIdx = -1;
	int endIdx = -1;
	//Determine the indices
	for (int i = id; i < nVals; ++i){
		if (rows[i] == id && startIdx == -1)
			startIdx = i;
		else if (rows[i] == id + 1 && endIdx == -1){
			endIdx = i - 1;
			break;
		}
		else if (i == nVals - 1 && endIdx == -1)
			endIdx = i;
	}

	//Set initial guess, residual and dir
	x[id] = 0.0f;
	r[id] = b[id];
	p[id] = b[id];
	//Wait for everyone to get setup
	barrier(CLK_LOCAL_MEM_FENCE);

	//What if instead of only having the lead kernel do this
	//we used a seperate kernel that would more evenly divide the work?
	//Compute the initial dot product and length of r in the lead kernel
	local float oldRdotR, rLen;
	if (id == 0){
		oldRdotR = 0.0f;
		for (int i = 0; i < dim; ++i)
			oldRdotR += r[i] * r[i];

		rLen = sqrt(oldRdotR);
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	//Do the computation iteration
	local float alpha, newRdotR, ApDotp;
	local int iteration;
	iteration = 0;
	while (iteration < 1000 && rLen >= 0.01f){
		Atimesp[id] = 0.0f;
		for (int i = startIdx; i <= endIdx; ++i)
			Atimesp[id] += A[i] * p[cols[i]];

		barrier(CLK_LOCAL_MEM_FENCE);

		//Lead kernel computes alpha
		if (id == 0){
			ApDotp = 0.0f;
			for (int i = 0; i < dim; ++i)
				ApDotp += Atimesp[i] * p[i];

			//Compute alpha
			alpha = oldRdotR / ApDotp;
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		//Determine next guess & residual this work item will investigate
		x[id] += alpha * p[id];
		r[id] -= alpha * Atimesp[id];
		barrier(CLK_LOCAL_MEM_FENCE);

		//Lead kernel computes r dot r, again could this benefit
		//from being split into its own kernel for even work distribution?
		//Or would the overhead of running another kernel outweight benefit?
		if (id == 0){
			newRdotR = 0.0f;
			for (int i = 0; i < dim; ++i)
				newRdotR += r[i] * r[i];

			rLen = sqrt(newRdotR);
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		//Update dir
		p[id] = r[id] + (newRdotR / oldRdotR) * p[id];
		barrier(CLK_LOCAL_MEM_FENCE);

		oldRdotR = newRdotR;

		//Lead id incremenents the iteration count and we wait for everyone to finish
		//their iteration
		if (id == 0)
			++iteration;

		barrier(CLK_LOCAL_MEM_FENCE);
	}
	result[0] = iteration * 1.0f;
	result[1] = rLen;
	//Is this reasonable to get x out? x is a local buffer so we need to
	//make its values accessible from some other buffer
	result[id + 2] = x[id];
}


OpenCL: Added writeImageFloat16f and  writeImageFloat32f to image3d_write.cl

/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2013-2015 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#ifndef IVW_IMAGE3D_WRITE_CL
#define IVW_IMAGE3D_WRITE_CL

// https://www.khronos.org/registry/cl/sdk/1.2/docs/man/xhtml/write_image3d.html

// cl_khr_3d_image_writes is defined but not supported
// on some NVIDIA drivers (355.82 for example)
// This means that we cannot rely on cl_khr_3d_image_writes
// being correctly defined and must use our own define.
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    #pragma OPENCL_EXTENSION cl_khr_3d_image_writes : enable
    #define image_3d_write_uint8_t __write_only image3d_t
    #define image_3d_write_uint16_t __write_only image3d_t
    #define image_3d_write_uint32_t __write_only image3d_t
    #define image_3d_write_float16_t __write_only image3d_t
    #define image_3d_write_float32_t __write_only image3d_t

    // Vec2 types
    #define image_3d_write_vec2_uint16_t __write_only image3d_t

    // Vec4 types
    #define image_3d_write_vec4_float16_t __write_only image3d_t
    #define image_3d_write_vec4_float32_t __write_only image3d_t

#else
    #define image_3d_write_uint8_t __global uchar*
    #define image_3d_write_uint16_t __global ushort*
    #define image_3d_write_uint32_t __global uint*
    #define image_3d_write_float16_t __global half*
    #define image_3d_write_float32_t __global float*

    // vec2 types
    #define image_3d_write_vec2_uint16_t __global ushort2*

    // vec4 types
    #define image_3d_write_vec4_float16_t __global half*
    #define image_3d_write_vec4_float32_t __global float4*
#endif

// 8.3.1.2 Converting floating-point values
// to normalized integer channel data types
// https://www.khronos.org/registry/cl/specs/opencl-1.0.pdf

// Write value, val in [0 1], at location coord.
// 0.f will convert to 0
// 1.f will convert to 255
// @note write_imagef can only be used with image objects created with
// image_channel_data_type set to one of the pre-defined packed formats or set to
// CL_SNORM_INT8, CL_UNORM_INT8, CL_SNORM_INT16, CL_UNORM_INT16, CL_HALF_FLOAT or CL_FLOAT.
void writeImageUInt8f(image_3d_write_uint8_t image, int4 coord, int4 dimension, float val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imagef(image, coord, (float4)(val));
#else
    // The preferred method for conversions from floating-point values to normalized integer
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = convert_uchar_sat_rte(val*255.f);
#endif
}

//
// write_imageui can only be used with image objects created with
// image_channel_data_type set to one of the following values:
// CL_UNSIGNED_INT8, CL_UNSIGNED_INT16, or CL_UNSIGNED_INT32.
void write_imageUInt8ui(image_3d_write_uint8_t image, int4 coord, int4 dimension, uchar val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imageui(image, coord, (uint4)(convert_uint(val)));
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = val;
#endif
}

// Write value at location coord.
void writeImageFloat16f(image_3d_write_float16_t image, int4 coord, int4 dimension, float val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imagef(image, coord, (float4)(val));
#else
    vstore_half(val, coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y, image);
#endif
}

// Write value at location coord.
void writeImageFloat32f(image_3d_write_float32_t image, int4 coord, int4 dimension, float val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imagef(image, coord, (float4)(val));
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = val;
#endif
}

// ------------ vec2 ---------------- //

// Write value, val in [0 1], at location coord.
// 0.f will convert to 0
// 1.f will convert to 65535
void writeImageVec2UInt16f(image_3d_write_vec2_uint16_t image, int4 coord, int4 dimension, float2 val) {
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    write_imagef(image, coord, (float4)(val, val));
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = convert_ushort2_sat_rte(val * 65535.f);
#endif
}

// ------------ vec4 ---------------- //

// Write value at location coord.
void writeImageVec4Float16f(image_3d_write_vec4_float16_t image, int4 coord, int4 dimension, float4 val) {
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    write_imagef(image, coord, val);
#else
    vstore_half4(val, coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y, image);
#endif
}

// Write value at location coord.
void writeImageVec4Float32f(image_3d_write_vec4_float32_t image, int4 coord, int4 dimension, float4 val) {
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    write_imagef(image, coord, val);
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = val;
#endif
}

#endif // IVW_IMAGE3D_WRITE_CL

/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2013-2015 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#ifndef IVW_IMAGE3D_WRITE_CL
#define IVW_IMAGE3D_WRITE_CL

// https://www.khronos.org/registry/cl/sdk/1.2/docs/man/xhtml/write_image3d.html

// cl_khr_3d_image_writes is defined but not supported
// on some NVIDIA drivers (355.82 for example)
// This means that we cannot rely on cl_khr_3d_image_writes
// being correctly defined and must use our own define.
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    #pragma OPENCL_EXTENSION cl_khr_3d_image_writes : enable
    #define image_3d_write_uint8_t __write_only image3d_t
    #define image_3d_write_uint16_t __write_only image3d_t
    #define image_3d_write_uint32_t __write_only image3d_t
    #define image_3d_write_float16_t __write_only image3d_t
    #define image_3d_write_float32_t __write_only image3d_t

    // Vec2 types
    #define image_3d_write_vec2_uint16_t __write_only image3d_t

    // Vec4 types
    #define image_3d_write_vec4_float16_t __write_only image3d_t
    #define image_3d_write_vec4_float32_t __write_only image3d_t

#else
    #define image_3d_write_uint8_t __global uchar*
    #define image_3d_write_uint16_t __global ushort*
    #define image_3d_write_uint32_t __global uint*
    #define image_3d_write_float16_t __global half*
    #define image_3d_write_float32_t __global float*

    // vec2 types
    #define image_3d_write_vec2_uint16_t __global ushort2*

    // vec4 types
    #define image_3d_write_vec4_float16_t __global half*
    #define image_3d_write_vec4_float32_t __global float4*
#endif

// 8.3.1.2 Converting floating-point values
// to normalized integer channel data types
// https://www.khronos.org/registry/cl/specs/opencl-1.0.pdf

// Write value, val in [0 1], at location coord.
// 0.f will convert to 0
// 1.f will convert to 255
// @note write_imagef can only be used with image objects created with
// image_channel_data_type set to one of the pre-defined packed formats or set to
// CL_SNORM_INT8, CL_UNORM_INT8, CL_SNORM_INT16, CL_UNORM_INT16, CL_HALF_FLOAT or CL_FLOAT.
void writeImageUInt8f(image_3d_write_uint8_t image, int4 coord, int4 dimension, float val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imagef(image, coord, (float4)(val));
#else
    // The preferred method for conversions from floating-point values to normalized integer
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = convert_uchar_sat_rte(val*255.f);
#endif
}

//
// write_imageui can only be used with image objects created with
// image_channel_data_type set to one of the following values:
// CL_UNSIGNED_INT8, CL_UNSIGNED_INT16, or CL_UNSIGNED_INT32.
void write_imageUInt8ui(image_3d_write_uint8_t image, int4 coord, int4 dimension, uchar val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imageui(image, coord, (uint4)(convert_uint(val)));
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = val;
#endif
}

// Write value, val in [0 1], at location coord.
// 0.f will convert to 0
// 1.f will convert to 65535
// @note write_imagef can only be used with image objects created with
// image_channel_data_type set to one of the pre-defined packed formats or set to
// CL_SNORM_INT8, CL_UNORM_INT8, CL_SNORM_INT16, CL_UNORM_INT16, CL_HALF_FLOAT or CL_FLOAT.
void writeImageUInt16f(image_3d_write_uint16_t image, int4 coord, int4 dimension, float val) {
#ifdef SUPPORTS_VOLUME_WRITE
    write_imagef(image, coord, (float4)(val));
#else
    // The preferred method for conversions from floating-point values to normalized integer
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = convert_ushort_sat_rte(val * 65535.f);
#endif
}

// ------------ vec2 ---------------- //

// Write value, val in [0 1], at location coord.
// 0.f will convert to 0
// 1.f will convert to 65535
void writeImageVec2UInt16f(image_3d_write_vec2_uint16_t image, int4 coord, int4 dimension, float2 val) {
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    write_imagef(image, coord, (float4)(val, val));
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = convert_ushort2_sat_rte(val * 65535.f);
#endif
}

// ------------ vec4 ---------------- //

// Write value at location coord.
void writeImageVec4Float16f(image_3d_write_vec4_float16_t image, int4 coord, int4 dimension, float4 val) {
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    write_imagef(image, coord, val);
#else
    vstore_half4(val, coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y, image);
#endif
}

// Write value at location coord.
void writeImageVec4Float32f(image_3d_write_vec4_float32_t image, int4 coord, int4 dimension, float4 val) {
#ifdef SUPPORTS_VOLUME_WRITE //cl_khr_3d_image_writes
    write_imagef(image, coord, val);
#else
    image[coord.x + coord.y*dimension.x + coord.z*dimension.x*dimension.y] = val;
#endif
}

#endif // IVW_IMAGE3D_WRITE_CL