added comments

added comments

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#ifndef MIOPEN_TENSOR_DIMS
#define MIOPEN_TENSOR_DIMS 4
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const int work_per_wg,
                              const float alpha0,
                              const float alpha1,
                              const float beta,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset,
                              const int num_wg)
{
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int gid = get_group_id(0);
    int lid = get_local_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

#if INCR_WG == 1
        int o_n             = gid / b_c;
        int o_c             = gid % b_c;
        MIOPEN_TYPE operand = b_off[o_c] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid] * alpha1;
        int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_hw     = lid % work_off;
            int o_n      = lid / work_off;
            int index    = o_n * c_nstride + gid * c_cstride + o_hw;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

__kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a,
                                     const int a_nstride,
                                     const int a_cstride,
                                     const int a_hstride,
                                     global MIOPEN_TYPE* b,
#if INCR_WG == 0
                                     UNUSED
#endif
                                     const int b_c,
                                     const int b_cstride,
                                     global MIOPEN_TYPE* c,
#if INCR_WG == 1
                                     UNUSED
#endif
                                     const int c_n,
                                     const int c_w,
                                     const int c_nstride,
                                     const int c_cstride,
                                     const int c_hstride,
                                     const float alpha0,
                                     const float alpha1,
                                     const float beta,
                                     const int work_per_wg,
                                     const long Aoffset,
                                     const long Boffset,
                                     const long Coffset,
                                     const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

#if INCR_WG == 1

        int o_c = gid % b_c;
        int o_n = gid / b_c;

        int bindex          = o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid * b_cstride] * alpha1;
        // int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_n    = lid % c_n;
            int o_h    = (lid / c_n) / c_w;
            int o_w    = (lid / c_n) % c_w;
            int aindex = o_n * a_nstride + gid * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + gid * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

// DLOWELL : cutting out this section
#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  global MIOPEN_TYPE* b,
                                  global MIOPEN_TYPE* c,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_c,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_h,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_cstride,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const float alpha0,
                                  const float alpha1,
                                  const float beta,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset,
                                  const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1

    int tid = get_global_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        MIOPEN_TYPE operand = b_off[tid] * alpha1;

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w;
        c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
    }
#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_c = gid % c_c;
        int o_n = gid / c_c;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = gid * c_nstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#endif
}

#endif

// DLOWELL : cutting out this section
#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)

__kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a,
                                         const int a_nstride,
                                         const int a_cstride,
                                         const int a_hstride,
                                         global MIOPEN_TYPE* b,
                                         const int b_nstride,
#if FIRST_NOT_ONE == 0
                                         UNUSED
#endif
                                         const int b_cstride,
#if FIRST_NOT_ONE <= 1
                                         UNUSED
#endif
                                         const int b_hstride,
                                         global MIOPEN_TYPE* c,
                                         const int c_c,
#if FIRST_NOT_ONE == 1
                                         UNUSED
#endif
                                         const int c_h,
#if FIRST_NOT_ONE == 0 || FIRST_NOT_ONE == 2
                                         UNUSED
#endif
                                         const int c_w,
                                         const int c_nstride,
                                         const int c_cstride,
                                         const int c_hstride,
                                         const float alpha0,
                                         const float alpha1,
                                         const float beta,
#if FIRST_NOT_ONE == 3
                                         UNUSED
#endif
                                         const int work_per_wg,
                                         const long Aoffset,
                                         const long Boffset,
                                         const long Coffset,
                                         const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid = get_global_id(0);
    // MIOPEN_TYPE operand = b[tid];

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] =
            MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], alpha1 * b_off[bindex]) + beta * c_off[cindex];
    }

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        int bindex          = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + lid;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + lid;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_c = gid % c_c;
        int o_n = gid / c_c;

        int bindex          = o_n * b_nstride + o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid * b_nstride] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = lid % c_c;
            int o_h    = (lid / c_c) % c_h;
            int o_w    = (lid / c_c) / c_h;
            int aindex = gid * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = gid * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#endif
}

#endif

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_c_gid_off = (gid / b_w / b_h) % b_c;
        int o_n_gid_off = (gid / b_w / b_h) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride +
                     o_w_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
        int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_d_gid_off = (gid / b_w / b_h) % b_d;
        int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
        int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                     o_h_gid_off * b_hstride + o_w_gid_off;

        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
            int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;

            int aindex =
                o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
            int cindex =
                o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid = get_local_id(0);
        // MIOPEN_TYPE operand = b[gid + Boffset];
        int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
        int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

        int o_h_gid_off = gid % b_h;
        int o_c_gid_off = (gid / b_h) % b_c;
        int o_n_gid_off = (gid / b_h) / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
            int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid         = get_local_id(0);
        int o_c_gid_off = gid % b_c;
        int o_n_gid_off = gid / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
            int o_n    = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
            int aindex = o_n * a_nstride + o_c;
            int cindex = o_n * c_nstride + o_c;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }
}

// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        int o_n_gid_off     = gid % b_n;
        int bindex          = o_n_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;
        while(lid < work_per_wg)
        {
            int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
            c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
            lid += get_local_size(0);
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#ifndef MIOPEN_TENSOR_DIMS
#define MIOPEN_TENSOR_DIMS 4
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const int work_per_wg,
                              const float alpha0,
                              const float alpha1,
                              const float beta,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset,
                              const int num_wg)
{
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int gid = get_group_id(0);
    int lid = get_local_id(0);

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

#if INCR_WG == 1
        int o_n             = gid / b_c;
        int o_c             = gid % b_c;
        MIOPEN_TYPE operand = b_off[o_c] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid] * alpha1;
        int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_hw     = lid % work_off;
            int o_n      = lid / work_off;
            int index    = o_n * c_nstride + gid * c_cstride + o_hw;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

__kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a,
                                     const int a_nstride,
                                     const int a_cstride,
                                     const int a_hstride,
                                     global MIOPEN_TYPE* b,
#if INCR_WG == 0
                                     UNUSED
#endif
                                     const int b_c,
                                     const int b_cstride,
                                     global MIOPEN_TYPE* c,
#if INCR_WG == 1
                                     UNUSED
#endif
                                     const int c_n,
                                     const int c_w,
                                     const int c_nstride,
                                     const int c_cstride,
                                     const int c_hstride,
                                     const float alpha0,
                                     const float alpha1,
                                     const float beta,
                                     const int work_per_wg,
                                     const long Aoffset,
                                     const long Boffset,
                                     const long Coffset,
                                     const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

#if INCR_WG == 1

        int o_c = gid % b_c;
        int o_n = gid / b_c;

        int bindex          = o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid * b_cstride] * alpha1;
        // int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_n    = lid % c_n;
            int o_h    = (lid / c_n) / c_w;
            int o_w    = (lid / c_n) % c_w;
            int aindex = o_n * a_nstride + gid * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + gid * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

// DLOWELL : cutting out this section
#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  global MIOPEN_TYPE* b,
                                  global MIOPEN_TYPE* c,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_c,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_h,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_cstride,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const float alpha0,
                                  const float alpha1,
                                  const float beta,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset,
                                  const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1

    int tid = get_global_id(0);
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        MIOPEN_TYPE operand = b_off[tid] * alpha1;

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w;
        c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
    }
#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_c = gid % c_c;
        int o_n = gid / c_c;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = gid * c_nstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#endif
}

#endif

// DLOWELL : cutting out this section
#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)

__kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a,
                                         const int a_nstride,
                                         const int a_cstride,
                                         const int a_hstride,
                                         global MIOPEN_TYPE* b,
                                         const int b_nstride,
#if FIRST_NOT_ONE == 0
                                         UNUSED
#endif
                                         const int b_cstride,
#if FIRST_NOT_ONE <= 1
                                         UNUSED
#endif
                                         const int b_hstride,
                                         global MIOPEN_TYPE* c,
                                         const int c_c,
#if FIRST_NOT_ONE == 1
                                         UNUSED
#endif
                                         const int c_h,
#if FIRST_NOT_ONE == 0 || FIRST_NOT_ONE == 2
                                         UNUSED
#endif
                                         const int c_w,
                                         const int c_nstride,
                                         const int c_cstride,
                                         const int c_hstride,
                                         const float alpha0,
                                         const float alpha1,
                                         const float beta,
#if FIRST_NOT_ONE == 3
                                         UNUSED
#endif
                                         const int work_per_wg,
                                         const long Aoffset,
                                         const long Boffset,
                                         const long Coffset,
                                         const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid = get_global_id(0);
    // MIOPEN_TYPE operand = b[tid];

    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] =
            MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], alpha1 * b_off[bindex]) + beta * c_off[cindex];
    }

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        int bindex          = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + lid;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + lid;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_c = gid % c_c;
        int o_n = gid / c_c;

        int bindex          = o_n * b_nstride + o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid * b_nstride] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = lid % c_c;
            int o_h    = (lid / c_c) % c_h;
            int o_w    = (lid / c_c) / c_h;
            int aindex = gid * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = gid * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#endif
}

#endif

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_c_gid_off = (gid / b_w / b_h) % b_c;
        int o_n_gid_off = (gid / b_w / b_h) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride +
                     o_w_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
        int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_d_gid_off = (gid / b_w / b_h) % b_d;
        int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
        int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                     o_h_gid_off * b_hstride + o_w_gid_off;

        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
            int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;

            int aindex =
                o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
            int cindex =
                o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid = get_local_id(0);
        // MIOPEN_TYPE operand = b[gid + Boffset];
        int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
        int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

        int o_h_gid_off = gid % b_h;
        int o_c_gid_off = (gid / b_h) % b_c;
        int o_n_gid_off = (gid / b_h) / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
            int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid         = get_local_id(0);
        int o_c_gid_off = gid % b_c;
        int o_n_gid_off = gid / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
            int o_n    = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
            int aindex = o_n * a_nstride + o_c;
            int cindex = o_n * c_nstride + o_c;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }
}

// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        int o_n_gid_off     = gid % b_n;
        int bindex          = o_n_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;
        while(lid < work_per_wg)
        {
            int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
            c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
            lid += get_local_size(0);
        }
    }
}


Updates to simple add to support vectors of dimensions > workgroup size

// Need to define a simplified iteration id:
inline int get_index(int nelems, int index ) {
  if (index == -1)  {
    index = get_global_id(0);
  } else {
    index += get_global_size(0);
  }

  if (index >= nelems) index = -1;

  return index;
}

__kernel void add(__global const float *x, __global const float *y, __global float *z, int N) {

  int id = get_index(N, -1);
  while(id >= 0) {
     z[id] = x[id] + y[id];
     id = get_index(N, id);
  }

}

__kernel void add(__global const float *x, __global const float *y, __global float *z, int N) {
  int id = get_global_id(0);
  if (id >= N) return;
  z[id] = x[id] + y[id];
}


Updated vector functions in kmeans.cl to match gmm.cl

typedef union
{
   float s[2];
   float2 v2;
} Vector2;

#define vectorInitZero(a) \
   (a)->s[0] = 0; \
   (a)->s[1] = 0;

#define vectorAdd(a, b) \
   (a)->s[0] += (b)->s[0]; \
   (a)->s[1] += (b)->s[1];

#define vectorScale(a, c) \
   (a)->s[0] *= (c); \
   (a)->s[1] *= (c);

#define SQR(x) ((x)*(x))
#define vectorDiffNorm(a, b) \
   sqrt(SQR((a)->s[0] - (b)->s[0]) + SQR((a)->s[1] - (b)->s[1]))

/**
 * Implementation of the MWC64X random number generator.
 *
 * @param state
 */
uint rand(uint2 *state)
{
   enum { A = 4294883355U };
   uint x = (*state).x, c = (*state).y;  // Unpack the state
   uint res = x ^ c;                     // Calculate the result
   uint hi = mul_hi(x, A);               // Step the RNG
   x = x * A + c;
   c = hi + (x < c);
   *state = (uint2)(x, c);               // Pack the state back up

   return res;                           // Return the next result
}

/**
 * Fetch and build data matrix X for a pair of genes, skipping any expressions
 * that are missing for either gene.
 *
 * @param expressions
 * @param size
 * @param indexA
 * @param indexB
 * @param X
 * @param labels
 * @return number of rows in X
 */
int fetchData(
   __global const float *expressions, int size,
   int indexA, int indexB,
   __global Vector2 *X,
   __global int *labels)
{
   int numSamples = 0;

   indexA *= size;
   indexB *= size;

   // build data matrix from expressions and indices
   for ( int i = 0; i < size; ++i )
   {
      if ( !isnan(expressions[indexA + i]) && !isnan(expressions[indexB + i]) )
      {
         // if both expressions exist add expressions to new lists and increment
         X[numSamples].v2 = (float2) (
            expressions[indexA + i],
            expressions[indexB + i]
         );
         numSamples++;

         labels[i] = 0;
      }
      else
      {
         labels[i] = -1;
      }
   }

   // return size of X
   return numSamples;
}

/**
 * Compute the log-likelihood of a K-means model given data X.
 *
 * @param X
 * @param N
 * @param y
 * @param Mu
 * @param K
 */
float computeLogLikelihood(
   __global const Vector2 *X, int N,
   __global const int *y,
   __global const Vector2 *Mu, int K)
{
   // compute within-class scatter
   float S = 0;

   for ( int k = 0; k < K; ++k )
   {
      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != k )
         {
            continue;
         }

         float dist = vectorDiffNorm(&X[i], &Mu[k]);

         S += dist * dist;
      }
   }

   return -S;
}

/**
 * Compute a K-means clustering model from a dataset.
 */
void computeKmeans(
   __global const Vector2 *X, int N, int K,
   __global int *y,
   float *logL,
   __global Vector2 *Mu,
   __global int *y_next)
{
   uint2 state = (get_global_id(0), get_global_id(1));

   // initialize means randomly from X
   for ( int k = 0; k < K; ++k )
   {
      int i = rand(&state) % N;
      Mu[k] = X[i];
   }

   // iterate K means until convergence
   while ( true )
   {
      // compute new labels
      for ( int i = 0; i < N; ++i )
      {
         // find k that minimizes norm(x_i - mu_k)
         int min_k = -1;
         float min_dist;

         for ( int k = 0; k < K; ++k )
         {
            float dist = vectorDiffNorm(&X[i], &Mu[k]);

            if ( min_k == -1 || dist < min_dist )
            {
               min_k = k;
               min_dist = dist;
            }
         }

         y_next[i] = min_k;
      }

      // check for convergence
      bool converged = true;

      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != y_next[i] )
         {
            converged = false;
            break;
         }
      }

      if ( converged )
      {
         break;
      }

      // update labels
      for ( int i = 0; i < N; ++i )
      {
         y[i] = y_next[i];
      }

      // update means
      for ( int k = 0; k < K; ++k )
      {
         // compute mu_k = mean of all x_i in cluster k
         int n_k = 0;

         vectorInitZero(&Mu[k]);

         for ( int i = 0; i < N; ++i )
         {
            if ( y[i] == k )
            {
               vectorAdd(&Mu[k], &X[i]);
               n_k++;
            }
         }

         vectorScale(&Mu[k], 1.0f / n_k);
      }
   }

   *logL = computeLogLikelihood(X, N, y, Mu, K);
}

/**
 * Compute the Bayes information criterion of a K-means model.
 *
 * @param K
 * @param logL
 * @param N
 * @param D
 */
float computeBIC(int K, float logL, int N, int D)
{
   int p = K * D;

   return log((float) N) * p - 2 * logL;
}

/**
 * Compute a block of K-means models given a block of gene pairs.
 *
 * For each gene pair, several models are computed and the best model
 * is selected according to a criterion (BIC). The selected K and the
 * resulting sample mask for each pair is returned.
 */
__kernel void computeKmeansBlock(
   __global const float *expressions, int size,
   __global const int2 *pairs,
   int minSamples, int minClusters, int maxClusters,
   __global Vector2 *work_X,
   __global int *work_y,
   __global Vector2 *work_Mu,
   __global int *work_ynext,
   __global int *result_K,
   __global int *result_labels)
{
   // initialize workspace variables
   int i = get_global_id(0);
   __global Vector2 *X = &work_X[i * size];
   __global int *y = &work_y[i * size];
   __global Vector2 *Mu = &work_Mu[i * maxClusters];
   __global int *y_next = &work_ynext[i * size];
   __global int *bestK = &result_K[i];
   __global int *bestLabels = &result_labels[i * size];

   // fetch data matrix X from expression matrix
   int numSamples = fetchData(expressions, size, pairs[i].x, pairs[i].y, X, bestLabels);

   // initialize output
   *bestK = 0;

   // make sure minimum number of related samples is reached
   if ( numSamples >= minSamples )
   {
      float bestValue = INFINITY;

      for ( int K = minClusters; K <= maxClusters; ++K )
      {
         // run each clustering model
         float logL;
         computeKmeans(X, numSamples, K, y, &logL, Mu, y_next);

         // evaluate model
         float value = computeBIC(K, logL, numSamples, 2);

         // save the best model
         if ( value < bestValue )
         {
            *bestK = K;
            bestValue = value;

            for ( int i = 0, j = 0; i < numSamples; ++i )
            {
               if ( bestLabels[i] != -1 )
               {
                  bestLabels[i] = y[j];
                  ++j;
               }
            }
         }
      }
   }
}

typedef float2 Vector2;

void vectorInitZero(__global Vector2 *a)
{
   a->x = 0;
   a->y = 0;
}

void vectorAdd(__global Vector2 *a, __global const Vector2 *b)
{
   a->x += b->x;
   a->y += b->y;
}

void vectorScale(__global Vector2 *a, float c)
{
   a->x *= c;
   a->y *= c;
}

float vectorDiffNorm(__global const Vector2 *a, __global const Vector2 *b)
{
   float dist = 0;
   dist += (a->x - b->x) * (a->x - b->x);
   dist += (a->y - b->y) * (a->y - b->y);

   return sqrt(dist);
}

/**
 * Implementation of the MWC64X random number generator.
 *
 * @param state
 */
uint rand(uint2 *state)
{
   enum { A = 4294883355U };
   uint x = (*state).x, c = (*state).y;  // Unpack the state
   uint res = x ^ c;                     // Calculate the result
   uint hi = mul_hi(x, A);               // Step the RNG
   x = x * A + c;
   c = hi + (x < c);
   *state = (uint2)(x, c);               // Pack the state back up

   return res;                           // Return the next result
}

/**
 * Fetch and build data matrix X for a pair of genes, skipping any expressions
 * that are missing for either gene.
 *
 * @param expressions
 * @param size
 * @param indexA
 * @param indexB
 * @param X
 * @param labels
 * @return number of rows in X
 */
int fetchData(
   __global const float *expressions, int size,
   int indexA, int indexB,
   __global Vector2 *X,
   __global int *labels)
{
   int numSamples = 0;

   indexA *= size;
   indexB *= size;

   // build data matrix from expressions and indices
   for ( int i = 0; i < size; ++i )
   {
      if ( !isnan(expressions[indexA + i]) && !isnan(expressions[indexB + i]) )
      {
         // if both expressions exist add expressions to new lists and increment
         X[numSamples] = (Vector2) (
            expressions[indexA + i],
            expressions[indexB + i]
         );
         numSamples++;

         labels[i] = 0;
      }
      else
      {
         labels[i] = -1;
      }
   }

   // return size of X
   return numSamples;
}

/**
 * Compute the log-likelihood of a K-means model given data X.
 *
 * @param X
 * @param N
 * @param y
 * @param Mu
 * @param K
 */
float computeLogLikelihood(
   __global const Vector2 *X, int N,
   __global const int *y,
   __global const Vector2 *Mu, int K)
{
   // compute within-class scatter
   float S = 0;

   for ( int k = 0; k < K; ++k )
   {
      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != k )
         {
            continue;
         }

         float dist = vectorDiffNorm(&X[i], &Mu[k]);

         S += dist * dist;
      }
   }

   return -S;
}

/**
 * Compute a K-means clustering model from a dataset.
 */
void computeKmeans(
   __global const Vector2 *X, int N, int K,
   __global int *y,
   float *logL,
   __global Vector2 *Mu,
   __global int *y_next)
{
   uint2 state = (get_global_id(0), get_global_id(1));

   // initialize means randomly from X
   for ( int k = 0; k < K; ++k )
   {
      int i = rand(&state) % N;
      Mu[k] = X[i];
   }

   // iterate K means until convergence
   while ( true )
   {
      // compute new labels
      for ( int i = 0; i < N; ++i )
      {
         // find k that minimizes norm(x_i - mu_k)
         int min_k = -1;
         float min_dist;

         for ( int k = 0; k < K; ++k )
         {
            float dist = vectorDiffNorm(&X[i], &Mu[k]);

            if ( min_k == -1 || dist < min_dist )
            {
               min_k = k;
               min_dist = dist;
            }
         }

         y_next[i] = min_k;
      }

      // check for convergence
      bool converged = true;

      for ( int i = 0; i < N; ++i )
      {
         if ( y[i] != y_next[i] )
         {
            converged = false;
            break;
         }
      }

      if ( converged )
      {
         break;
      }

      // update labels
      for ( int i = 0; i < N; ++i )
      {
         y[i] = y_next[i];
      }

      // update means
      for ( int k = 0; k < K; ++k )
      {
         // compute mu_k = mean of all x_i in cluster k
         int n_k = 0;

         vectorInitZero(&Mu[k]);

         for ( int i = 0; i < N; ++i )
         {
            if ( y[i] == k )
            {
               vectorAdd(&Mu[k], &X[i]);
               n_k++;
            }
         }

         vectorScale(&Mu[k], 1.0f / n_k);
      }
   }

   *logL = computeLogLikelihood(X, N, y, Mu, K);
}

/**
 * Compute the Bayes information criterion of a K-means model.
 *
 * @param K
 * @param logL
 * @param N
 * @param D
 */
float computeBIC(int K, float logL, int N, int D)
{
   int p = K * D;

   return log((float) N) * p - 2 * logL;
}

/**
 * Compute a block of K-means models given a block of gene pairs.
 *
 * For each gene pair, several models are computed and the best model
 * is selected according to a criterion (BIC). The selected K and the
 * resulting sample mask for each pair is returned.
 */
__kernel void computeKmeansBlock(
   __global const float *expressions, int size,
   __global const int2 *pairs,
   int minSamples, int minClusters, int maxClusters,
   __global Vector2 *work_X,
   __global int *work_y,
   __global Vector2 *work_Mu,
   __global int *work_ynext,
   __global int *result_K,
   __global int *result_labels)
{
   // initialize workspace variables
   int i = get_global_id(0);
   __global Vector2 *X = &work_X[i * size];
   __global int *y = &work_y[i * size];
   __global Vector2 *Mu = &work_Mu[i * maxClusters];
   __global int *y_next = &work_ynext[i * size];
   __global int *bestK = &result_K[i];
   __global int *bestLabels = &result_labels[i * size];

   // fetch data matrix X from expression matrix
   int numSamples = fetchData(expressions, size, pairs[i].x, pairs[i].y, X, bestLabels);

   // initialize output
   *bestK = 0;

   // make sure minimum number of related samples is reached
   if ( numSamples >= minSamples )
   {
      float bestValue = INFINITY;

      for ( int K = minClusters; K <= maxClusters; ++K )
      {
         // run each clustering model
         float logL;
         computeKmeans(X, numSamples, K, y, &logL, Mu, y_next);

         // evaluate model
         float value = computeBIC(K, logL, numSamples, 2);

         // save the best model
         if ( value < bestValue )
         {
            *bestK = K;
            bestValue = value;

            for ( int i = 0, j = 0; i < numSamples; ++i )
            {
               if ( bestLabels[i] != -1 )
               {
                  bestLabels[i] = y[j];
                  ++j;
               }
            }
         }
      }
   }
}


Delete getSuperK2_M.cl

Delete getSuperK2_M.cl

__kernel void getSuperK_M(
   __global uint* RSK_G, // Matrix of reads
   __global uint* counters, // Matrix of temporal output
   const unsigned int nr, // Number of reads
   const unsigned int r, // Read size
   const unsigned int k, // Kmer size
   const unsigned int m // m-mer size
   )
{
   uint x, y, yl, xl, p, start, offset, nmk, nm, min, a, b, c, d, idt, ts, nt, lsd, nkr, tmp, ls, start_tile, start_superk, nt1, nt2, nt3, nt4 = 0;
   bool flag;

   x = get_global_id(0);
   y = get_global_id(1);
   xl = get_local_id(0);
   yl = get_local_id(1);

  __local uint RSK[180]; // Vector to store reads, m-mers and superk-mers. This vector is overwritten in each stage, size: length of the read

  __local uint RCMT[7]; // Reverse complement of current m-mer of tile (32 bits), size: nt2
  __local uint MT[7]; // Current m-mer of tile (32 bits), size: Greater between nt2 and nt3
  __local uint counter, mp, nmp, minimizer; // minimizer position, new minimizer  position, minimizer

   nmk = k - m + 1;
   // Global to local
   // Reads in global memory to Read in local memory

   ts = get_local_size(0);
   nt1 = (r-1)/ts + 1; // nt: Number of tiles or number of sub-reads per read
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt1; i++) { // coalesced
        p=ts*i+x;
        if (p > r-1) {
          break;
        }
        RSK[p]=RSK_G[y*r+p];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // FUNCTION: GetM-mers
    // Read to decimal representation of m-mers
    lsd = get_local_size(0) / m; // lsd: Local space divisions
    nm = r - m + 1; // nm: Number of m-mers per read
    ts = ((nm-1)/lsd) + 1; // ts: Tile size for this process
    nt2 = ((nm-1)/ts) + 1 ; // nt2: Number of tiles for this process
    // a = (int) (pow(((double) 2, (double) (2*m)) - 1); // a:Mask (Sample: 63 for m = 4, to cancel all bits different to first 6)
    a = 4095; // Mask   2**(2m-2)cc

    if ((x < m*nt2) && (y<nr)) {
      // Parallel computing of the first m-mer of each tile

      idt = x/m; // Tile id
      start = ts*idt; // Position of the first base for each tile
      offset = x % m;
      p = (start) + (offset); // Position of the base corresponding a the thread
      MT[idt] = 0; MT Reset
      RCMT[idt] = 0; // RCMT Reset
      b = RSK[p]; // Each thread copies its base to b
      atomic_or(&MT[idt], (b << (m-(x%m)-1)*2)); // Calculating the first m-mer of each tile
      atomic_or(&RCMT[idt], (((~b) & 3) << (x%m)*2)); // Calculating the reverse complement of the first m-mer of each tile
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x<nt2) && (y<nr)) { // Finding the first canonical m-mer of each tile
      // The canonical m-mers of a read are saved in the same vector where the read was stored (RSK)
      /* Each canonical m-mer is saved in the position where was its last base (This avoid the over-write of bases that Will be used by another tile) */
      idt = x;
      start = ts*idt;
      if (MT[idt]<RCMT[idt]){
        RSK[start + m -1] = MT[idt];
      } else {
      	RSK[start + m -1] = RCMT[idt];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

      // Serial computation of the m-mers remaining of each tile

    if ((x<nt2) && (y<nr))	{
       idt = x; // Tile id
       start = ts*idt; // Position of the first base for each tile
       c = MT[idt]; // Copy to c the decimal value of the first m-mer of each tile
       d = RCMT[idt]; // Copy to d the decimal value of the reverse complement of the first m-mer of each tile

       for (int z=0; z<(ts-1); z++) {
       		if (start+m+z > r-1) {
            break;
          }
           b = RSK[start+m+z]; // Copy to b the last base of the m-mer that will be calculated (current m-mer)
           c = ((c & a) << 2)|b;  // Calculating the current m-mer from the previous m-mer and the new base read (b)
           d = ((d>>2) & (a)) | (((~b) & 3)<<((m-1)*2)); // Calculating the reverse complement of the current m-mer from the reverse complement of the previous m-mer and the new base read (b)
           if (c<d)	{  // Finding the canonical m-mer
             RSK[start+m+z]=c;
            } else {
             RSK[start+m+z]=d;
            }
           }
      }

      barrier(CLK_LOCAL_MEM_FENCE);

   // PROCESS: getSuperks
   // Canonical m-mers to minimizers

      nt = ((nm-1)/ts) + 1; // nt: Number of tiles or number of sub-reads per read

   /* Initialization */
   if (x==0) {
     counter = 0;
     mp = 0;
     nmp = 0xFFFFFFFF;
     minimizer = 0xFFFFFFFF;
   }

  barrier(CLK_LOCAL_MEM_FENCE);

  start_tile = 0; // Start of the currently evaluated zone
  start_superk = 0;
  if (k <= 36) {  // Choosing the tile size for the reduction algorithm used in this process
    nt3 = 6;
  } else if (k <= 64) {
    nt3 = 8;
  } else if (k <= 100) {
    nt3 = 10;
  } else {
    nt3 = 12;
  }

   // Evaluating the first k-mers of each read

/* To find the minimizer of the first k-mers of each reading a two-stage atomic reduction methodology is used: First, the canonical m-mers of the k-mer are grouped into a number of sets equal to nt3; for each set an atomic operation is performed to find the minimum canonical m-mer of each set. Secondly, the minimums found in the previous step are reduced to one minimum, which is the minimizer of the k-mer. */

  if (x<nmk) {
    p = x;
    b = RSK[p+m-1];  // Each thread copies its canonical m-mer to b
  }

  if (x < nt3){
    MT[x] = b;  // The first “nt3” canonical m-mers are copied to MT
  }

  barrier(CLK_LOCAL_MEM_FENCE);

  if (x >= nt3 && x<nmk) {
    atomic_min(&MT[x%nt3], b);
  }

  barrier(CLK_LOCAL_MEM_FENCE);

  if (x<nt3) {
    atomic_min(&minimizer, MT[x]);
  }

  barrier(CLK_LOCAL_MEM_FENCE);

  if ((x<nmk) && (b==minimizer)){
    atomic_max(&mp, p);
  }

  barrier(CLK_LOCAL_MEM_FENCE);

  // Processing the remaining k-mers

  while ((start_tile+nmk-1)<(nm-1)) {
    flag = false;  // Flag to indicate if the position of the current minimizer is equal to start_zone (Start of the last k-mer evaluated)
    if ((x<nmk) && (mp == start_tile)){
      // If the position of the current minimizer is equal to start_zone, the flag is set to true and start_zone is incremented by one
      flag = true;
      start_tile = start_tile + 1;
      if (p < start_tile) {
        p = p + nmk;
        b = RSK[p + m - 1];
      }
    }

    if ( (x==0) && flag ) {

       /* If the flag is true it means that the current minimizer is no longer part of the next k-mer to evaluate, as consequence there ends a super k-mer. The code below is to store the representation of the super k-mer in the vector RSK, additionally counter is incremented and star_superk and minimizer are set. */

      a = (start_tile + k - 1 - start_superk) & 0x000000FF;
      RSK[counter] = ((minimizer<<18) & 0xFFFC0000) | ((start_superk<<8) & 0x0003FF00) | a;
      counter++;
      start_superk  = start_tile;
      minimizer = 0xFFFFFFFF;
    }

     /* If the flag is true, the next k-mer to evaluate does not have minimizer of reference, therefore this k-mer is evaluated in the same way as the first k-mer.*/

    if ((x<nt3) && (flag)) {
      MT[x] = b;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x >= nt3) && (x<nmk) && flag) {
      atomic_min(&MT[x%nt3], b);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x < nt3) && flag) {
      atomic_min(&minimizer, MT[x]);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x<nmk) && flag && (b == minimizer)) {
      atomic_max(&mp, p);
    }

    /* If the flag is false it means that The current minimizer is in a different position to the start of the last k-mer evaluated, therefore it is possible that the following k-mers share the same minimizer */

   /* From all the next k-mers that have the possibility to share the same minimizer only the last one is evaluated: its canónical m-mers (different to the first) are compared with the current minimizer (its first canonical m-mer), if there are canonical m-mers lower than the current minimizer, the new minimizer will be the one with the lowest position. */

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x<nmk) && flag==false) {
      start_tile = mp;
      if (p<start_tile) {
        p = p + nmk;
        if ( p < (nm -1)) {
          b = RSK[p+m-1];
          if (b < minimizer) {
            atomic_min(&nmp, p);
          }
        }
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

     /* If a new minimizer is found, it means that a super k-mer ends. The code below is to save the representation of the super k-mer in the vector RSK, additionally counter is incremented and star_superk, minimizer, mp and nmp are set. */

    if ((x==0) && (flag==false) && (nmp != 0xFFFFFFFF)) {
      a = (nmp - start_superk + m - 1) & 0x000000FF;
      RSK[counter] = ((minimizer <<  18) & 0xFFFC0000) | ((start_superk << 8) & 0x0003FF00) | a;
      mp = nmp;
      minimizer = RSK[mp+m-1];
      counter++;
      start_superk = mp - nmk + 1;
      nmp = 0xFFFFFFFF;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

  }

   /* When each read is fully evaluated, it is necessary to store the representation of last super k-mer in the vector RSK  */

  if (x==0) {
    a = (nm - start_superk + m - 1) & 0x000000FF;
    RSK[counter] = ((minimizer << 18) & 0xFFFC0000) | ((start_superk<<8) & 0x0003FF00) | a;
    counter ++;
  }

  barrier(CLK_LOCAL_MEM_FENCE);

   // PROCESS: local2Global
    //Reads in local memory to Read in global memory

    ts = get_local_size(0);  // ts: Tile size, work group size
    nt4 = (r-1)/ts + 1; // nt: Number of tiles for this process

    if ((x<ts) && (x<counter) && (y<nr)) {
      for (int i=0; i<nt4; i++){  // coalesced
        p=ts*i+x;
        if (p > counter-1)
          {
            break;
          }
        RSK_G[y*r+p] = RSK[p];
      }
    }

   // Extract counters
    if ((x == 0) && (y<nr))
     {
             counters[y] = counter;
      }
}


change one of the RNG factors in prob_7 for lower discrepancy; this begs for automation

// source_main
	const int idx = (int)get_global_id(0);
	const int idy = (int)get_global_id(1);
	const int dimx = (int)get_global_size(0);
	const int dimy = (int)get_global_size(1);
#if OCL_QUIRK_0001
	const float3 cam0 = (float3)(src_d[0], src_d[1], src_d[ 2]);
	const float3 cam1 = (float3)(src_d[4], src_d[5], src_d[ 6]);
	const float3 cam2 = (float3)(src_d[8], src_d[9], src_d[10]);
	const float3 ray_origin = (float3)(src_d[12], src_d[13], src_d[14]);
	const float3 bbox_min   = (float3)(src_d[16], src_d[17], src_d[18]);
	const float3 bbox_max   = (float3)(src_d[20], src_d[21], src_d[22]);
#else
	const float3 cam0 = src_d[0].xyz;
	const float3 cam1 = src_d[1].xyz;
	const float3 cam2 = src_d[2].xyz;
	const float3 ray_origin = src_d[3].xyz;
	const float3 bbox_min   = src_d[4].xyz;
	const float3 bbox_max   = src_d[5].xyz;
#endif
	const struct BBox root_bbox = (struct BBox){ bbox_min, bbox_max };
	const float3 ray_direction =
		cam0 * ((idx * 2 - dimx) * (1.0f / dimx)) +
		cam1 * ((idy * 2 - dimy) * (1.0f / dimy)) +
		cam2;
	const float3 ray_rcpdir = clamp(1.f / ray_direction, -MAXFLOAT, MAXFLOAT);
	struct RayHit ray = (struct RayHit){ (struct Ray){ (float4)(ray_origin, as_float(-1U)), (float4)(ray_rcpdir, MAXFLOAT) } };
	uint result = traverse(get_octet(src_a, 0), src_b, src_c, &root_bbox, &ray.ray, &ray.hit);

	if (-1U != result) {
		const unsigned seed = get_global_id(0) + get_global_id(1) * get_global_size(0) + frame * get_global_size(1);
		const unsigned ri0 = xorshift(seed) * 42111 >> 8;
		const unsigned ri1 = xorshift(seed) * 41333 >> 8; // 43333
		const unsigned max_rand = (1U << 24) - 1;

		// cosine-weighted distribution
		const float r0 = ri0 * (1.f / max_rand); // decl (cos^2)
		const float r1 = ri1 * (M_PI / (1U << 23)); // azim
		const float sin_decl = sqrt(1.f - r0);
		const float cos_decl = sqrt(r0);
		float sin_azim;
		float cos_azim;
		sin_azim = sincos(r1, &cos_azim);

		// compute a bounce vector in some TBN space, in this case of an assumed normal along x-axis
		const float3 hemi = (float3)(cos_decl, cos_azim * sin_decl, sin_azim * sin_decl);

#if OCL_QUIRK_0002
		const uint a_mask = ray.hit.a_mask;
		const uint b_mask = ray.hit.b_mask;
		const float3 normal = b_mask ? (a_mask ? hemi.xyz : hemi.zxy) : hemi.yzx;
#else
		const uint a_mask = -ray.hit.a_mask;
		const uint b_mask = -ray.hit.b_mask;
		const float3 normal = select(hemi.yzx, select(hemi.zxy, hemi.xyz, (uint3)(a_mask)), (uint3)(b_mask));
#endif
		const int3 axis_sign = (int3)(0x80000000) & ray.hit.min_mask;
		const float dist = ray.ray.rcpdir.w;
		const float3 ray_rcpdir = clamp(1.f / as_float3(as_int3(normal) ^ axis_sign), -MAXFLOAT, MAXFLOAT);
		const struct Ray ray = (struct Ray){ (float4)(ray_origin + ray_direction * dist, as_float(result)), (float4)(ray_rcpdir, MAXFLOAT) };
		result = select(255, 16, occlude(get_octet(src_a, 0), src_b, src_c, &root_bbox, &ray));
	}
	else
		result = 0;

// source_main
	const int idx = (int)get_global_id(0);
	const int idy = (int)get_global_id(1);
	const int dimx = (int)get_global_size(0);
	const int dimy = (int)get_global_size(1);
#if OCL_QUIRK_0001
	const float3 cam0 = (float3)(src_d[0], src_d[1], src_d[ 2]);
	const float3 cam1 = (float3)(src_d[4], src_d[5], src_d[ 6]);
	const float3 cam2 = (float3)(src_d[8], src_d[9], src_d[10]);
	const float3 ray_origin = (float3)(src_d[12], src_d[13], src_d[14]);
	const float3 bbox_min   = (float3)(src_d[16], src_d[17], src_d[18]);
	const float3 bbox_max   = (float3)(src_d[20], src_d[21], src_d[22]);
#else
	const float3 cam0 = src_d[0].xyz;
	const float3 cam1 = src_d[1].xyz;
	const float3 cam2 = src_d[2].xyz;
	const float3 ray_origin = src_d[3].xyz;
	const float3 bbox_min   = src_d[4].xyz;
	const float3 bbox_max   = src_d[5].xyz;
#endif
	const struct BBox root_bbox = (struct BBox){ bbox_min, bbox_max };
	const float3 ray_direction =
		cam0 * ((idx * 2 - dimx) * (1.0f / dimx)) +
		cam1 * ((idy * 2 - dimy) * (1.0f / dimy)) +
		cam2;
	const float3 ray_rcpdir = clamp(1.f / ray_direction, -MAXFLOAT, MAXFLOAT);
	struct RayHit ray = (struct RayHit){ (struct Ray){ (float4)(ray_origin, as_float(-1U)), (float4)(ray_rcpdir, MAXFLOAT) } };
	uint result = traverse(get_octet(src_a, 0), src_b, src_c, &root_bbox, &ray.ray, &ray.hit);

	if (-1U != result) {
		const unsigned seed = get_global_id(0) + get_global_id(1) * get_global_size(0) + frame * get_global_size(1);
		const unsigned ri0 = xorshift(seed) * 42111 >> 8;
		const unsigned ri1 = xorshift(seed) * 43333 >> 8;
		const unsigned max_rand = (1U << 24) - 1;

		// cosine-weighted distribution
		const float r0 = ri0 * (1.f / max_rand); // decl (cos^2)
		const float r1 = ri1 * (M_PI / (1U << 23)); // azim
		const float sin_decl = sqrt(1.f - r0);
		const float cos_decl = sqrt(r0);
		float sin_azim;
		float cos_azim;
		sin_azim = sincos(r1, &cos_azim);

		// compute a bounce vector in some TBN space, in this case of an assumed normal along x-axis
		const float3 hemi = (float3)(cos_decl, cos_azim * sin_decl, sin_azim * sin_decl);

#if OCL_QUIRK_0002
		const uint a_mask = ray.hit.a_mask;
		const uint b_mask = ray.hit.b_mask;
		const float3 normal = b_mask ? (a_mask ? hemi.xyz : hemi.zxy) : hemi.yzx;
#else
		const uint a_mask = -ray.hit.a_mask;
		const uint b_mask = -ray.hit.b_mask;
		const float3 normal = select(hemi.yzx, select(hemi.zxy, hemi.xyz, (uint3)(a_mask)), (uint3)(b_mask));
#endif
		const int3 axis_sign = (int3)(0x80000000) & ray.hit.min_mask;
		const float dist = ray.ray.rcpdir.w;
		const float3 ray_rcpdir = clamp(1.f / as_float3(as_int3(normal) ^ axis_sign), -MAXFLOAT, MAXFLOAT);
		const struct Ray ray = (struct Ray){ (float4)(ray_origin + ray_direction * dist, as_float(result)), (float4)(ray_rcpdir, MAXFLOAT) };
		result = select(255, 16, occlude(get_octet(src_a, 0), src_b, src_c, &root_bbox, &ray));
	}
	else
		result = 0;


removed needless arithm_2_mat.cl file

removed needless arithm_2_mat.cl file

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Shengen Yan,yanshengen@gmail.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other oclMaterials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

/**************************************PUBLICFUNC*************************************/
#if defined (DOUBLE_SUPPORT)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics:enable
#pragma OPENCL EXTENSION cl_khr_global_int32_extended_atomics:enable
#define CV_PI   3.1415926535897932384626433832795

char round_char(double v){
    char v1=(char)v;
    return convert_char_sat(v+(v>=0 ? 0.5 : -0.5));
}
unsigned char round_uchar(double v){
    unsigned char v1=(unsigned char)v;
    return convert_uchar_sat(v+(v>=0 ? 0.5 : -0.5));
}
short round_short(double v){
    short v1=(short)v;
    return convert_short_sat(v+(v>=0 ? 0.5 : -0.5));
}
unsigned short round_ushort(double v){
    unsigned short v1=(unsigned short)v;
    return convert_ushort_sat(v+(v>=0 ? 0.5 : -0.5));
}
int round_int(double v){
    int v1=(int)v;
    return convert_int_sat(v+(v>=0 ? 0.5 : -0.5));
}

char round2_char(double v){
    char v1=(char)v;
    if((v-v1)==0.5&&v1%2==0)
        return v1;
    else
        return convert_char_sat(v+(v>=0 ? 0.5 : -0.5));
}
unsigned char round2_uchar(double v){
    unsigned char v1=(unsigned char)v;
    if((v-v1)==0.5&&v1%2==0)
        return v1;
    else
        return convert_uchar_sat(v+(v>=0 ? 0.5 : -0.5));
}
short round2_short(double v){
    short v1=(short)v;
    if((v-v1)==0.5&&v1%2==0)
        return v1;
    else
        return convert_short_sat(v+(v>=0 ? 0.5 : -0.5));
}
unsigned short round2_ushort(double v){
    unsigned short v1=(unsigned short)v;
    if((v-v1)==0.5&&v1%2==0)
        return v1;
    else
        return convert_ushort_sat(v+(v>=0 ? 0.5 : -0.5));
}
int round2_int(double v){
    int v1=(int)v;
    if((v-v1)==0.5&&v1%2==0)
        return v1;
    else
        return convert_int_sat(v+(v>=0 ? 0.5 : -0.5));
}

/*****************************************EXP***************************************/
__kernel void arithm_op_exp_5 (int rows,int cols,int srcStep,__global float *src1Mat,
                             __global float * dstMat,int channels)
{
    size_t x = get_global_id(0);
    size_t y = get_global_id(1);
    if (x < cols && y < rows)
    {
        size_t idx = y * ( srcStep >> 2 ) + x;
        dstMat[idx] = (float)exp((float)src1Mat[idx]);
    }
}
__kernel void arithm_op_exp_6 (int rows,int cols,int srcStep,__global double *src1Mat,
                             __global double * dstMat,int channels)
{
    size_t x = get_global_id(0);
    size_t y = get_global_id(1);
    if (x < cols && y < rows)
    {
        size_t idx = y * ( srcStep >> 3 ) + x;
        dstMat[idx] = exp(src1Mat[idx]);
    }
}

/*****************************************LOG***************************************/
__kernel void arithm_op_log_5 (int rows,int cols,int srcStep,__global float *src1Mat,
                             __global float * dstMat,int channels)
{
    size_t x = get_global_id(0);
    size_t y = get_global_id(1);
    if (x < cols && y < rows)
    {
        size_t idx = y * ( srcStep >> 2 ) + x;
        dstMat[idx] =(float) log((float)src1Mat[idx]);
    }
}
__kernel void arithm_op_log_6 (int rows,int cols,int srcStep,__global double *src1Mat,
                             __global double * dstMat,int channels)
{
    size_t x = get_global_id(0);
    size_t y = get_global_id(1);
    if (x < cols && y < rows)
    {
        size_t idx = y * ( srcStep >> 3 ) + x;
        dstMat[idx] = log(src1Mat[idx]);
    }
}