bug fixes

bug fixes

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#ifndef MIOPEN_TENSOR_DIMS
#define MIOPEN_TENSOR_DIMS 4
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              const int a_nstride,
                              const int a_cstride,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              const int b_cstride,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const float alpha0,
                              const float alpha1,
                              const float beta,
                              const int work_per_wg,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if INCR_WG == 1
    int o_n             = gid / b_c;
    int o_c             = gid % b_c;
    MIOPEN_TYPE operand = b_off[o_c * b_cstride] * alpha1;
    int a_sub_index     = o_n * a_nstride + o_c * a_cstride;
    int c_sub_index     = o_n * c_nstride + o_c * c_cstride;

    while(lid < work_per_wg)
    {
        int aindex    = a_sub_index + lid;
        int cindex    = c_sub_index + lid;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
    MIOPEN_TYPE operand = b_off[gid * b_cstride] * alpha1;
    int work_off        = work_per_wg / c_n;

    while(lid < work_per_wg)
    {
        int o_hw      = lid % work_off;
        int o_n       = lid / work_off;
        int aindex    = o_n * a_nstride + gid * a_cstride + o_hw;
        int cindex    = o_n * c_nstride + gid * c_cstride + o_hw;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#endif // INCR_WG
}

// DLOWELL : cutting out this section
#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  const int a_nstride,
                                  const int a_cstride,
                                  const int a_hstride,
                                  global MIOPEN_TYPE* b,
                                  const int b_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int b_cstride,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int b_hstride,
                                  global MIOPEN_TYPE* c,
                                  const int c_c,
                                  const int c_h,
#if FIRST_NOT_ONE <= 2
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
                                  const int c_cstride,
                                  const int c_hstride,
                                  const float alpha0,
                                  const float alpha1,
                                  const float beta,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid = get_global_id(0);
    // MIOPEN_TYPE operand = b[tid];

    int o_w = tid % c_w;
    int o_h = (tid / c_w) % c_h;
    int o_c = (tid / (c_w * c_h)) % c_c;
    int o_n = tid / (c_w * c_h * c_c);

    int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
    int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
    int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
    c_off[cindex] =
        MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], alpha1 * b_off[bindex]) + beta * c_off[cindex];

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    int o_h = gid % c_h;
    int o_c = (gid / c_h) % c_c;
    int o_n = gid / (c_c * c_h);

    int bindex          = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + lid;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + lid;
        c_off[cindex] = MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    int o_c = gid % c_c;
    int o_n = gid / c_c;

    int bindex          = o_n * b_nstride + o_c * b_cstride;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_h       = lid % c_h;
        int o_w       = lid / c_h;
        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b_off[gid * b_nstride] * alpha1;

    while(lid < work_per_wg)
    {
        int o_n       = gid;
        int o_c       = lid % c_c;
        int o_h       = (lid / c_c) % c_h;
        int o_w       = (lid / c_c) / c_h;
        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#endif
}

#endif

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_c_gid_off = (gid / b_w / b_h) % b_c;
    int o_n_gid_off = (gid / b_w / b_h) / b_c;

    int bindex =
        o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // if(gid>=b_nstride) return;
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
    int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);
    // printf("lid: %d, o_[h,d,c,n]_div: %d, %d, %d, %d\n",lid, o_h_div, o_d_div, o_c_div, o_n_div);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_d_gid_off = (gid / b_w / b_h) % b_d;
    int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
    int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

    int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                 o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
        int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;
        // printf("lid: %d, o_[w,h,d,c,n]: %d, %d, %d, %d, %d\n",lid, o_w, o_h, o_d, o_c, o_n);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;
        // printf("lid: %d, index: %d\n",lid, index);
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
    int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

    int o_h_gid_off = gid % b_h;
    int o_c_gid_off = (gid / b_h) % b_c;
    int o_n_gid_off = (gid / b_h) / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
        int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : (lid / o_c_div) / o_n_div;

        int aindex = o_n * a_nstride + o_c * a_cstride + o_h;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_h;

        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    int o_c_gid_off = gid % b_c;
    int o_n_gid_off = gid / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_c       = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
        int o_n       = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
        int aindex    = o_n * a_nstride + o_c;
        int cindex    = o_n * c_nstride + o_c;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }
}

// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
    // MIOPEN_TYPE operand = b[gid + Boffset];

    int o_n_gid_off     = gid % b_n;
    int bindex          = o_n_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
        c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
        lid += get_local_size(0);
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#ifndef MIOPEN_TENSOR_DIMS
#define MIOPEN_TENSOR_DIMS 4
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              const int a_nstride,
                              const int a_cstride,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              const int b_cstride,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const float alpha0,
                              const float alpha1,
                              const float beta,
                              const int work_per_wg,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if INCR_WG == 1
    int o_n             = gid / b_c;
    int o_c             = gid % b_c;
    MIOPEN_TYPE operand = b_off[o_c * b_cstride] * alpha1;
    int a_sub_index     = o_n * a_nstride + o_c * a_cstride;
    int c_sub_index     = o_n * c_nstride + o_c * c_cstride;

    while(lid < work_per_wg)
    {
        int aindex    = a_sub_index + lid;
        int cindex    = c_sub_index + lid;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
    MIOPEN_TYPE operand = b_off[gid * b_cstride] * alpha1;
    int work_off        = work_per_wg / c_n;

    while(lid < work_per_wg)
    {
        int o_hw      = lid % work_off;
        int o_n       = lid / work_off;
        int aindex    = o_n * a_nstride + gid * a_cstride + o_hw;
        int cindex    = o_n * c_nstride + gid * c_cstride + o_hw;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#endif // INCR_WG
}

// DLOWELL : cutting out this section
#if(FIRST_NOT_ONE < 4 && MIOPEN_TENSOR_DIMS == 4)

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  const int a_nstride,
                                  const int a_cstride,
                                  const int a_hstride,
                                  global MIOPEN_TYPE* b,
                                  const int b_nstride,
                                  const int b_cstride,
                                  const int b_hstride,
                                  global MIOPEN_TYPE* c,
                                  const int c_c,
                                  const int c_h,
#if FIRST_NOT_ONE <= 2
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
                                  const int c_cstride,
                                  const int c_hstride,
                                  const float alpha0,
                                  const float alpha1,
                                  const float beta,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid = get_global_id(0);
    // MIOPEN_TYPE operand = b[tid];

    int o_w = tid % c_w;
    int o_h = (tid / c_w) % c_h;
    int o_c = (tid / (c_w * c_h)) % c_c;
    int o_n = tid / (c_w * c_h * c_c);

    int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
    int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
    int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
    c_off[cindex] =
        MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], alpha1 * b_off[bindex]) + beta * c_off[cindex];

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    int o_h = gid % c_h;
    int o_c = (gid / c_h) % c_c;
    int o_n = gid / (c_c * c_h);

    int bindex          = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + lid;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + lid;
        c_off[cindex] = MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    int o_c = gid % c_c;
    int o_n = gid / c_c;

    int bindex          = o_n * b_nstride + o_c * b_cstride;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_h       = lid % c_h;
        int o_w       = lid / c_h;
        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid             = get_group_id(0);
    int lid             = get_local_id(0);
    MIOPEN_TYPE operand = b_off[gid * b_nstride] * alpha1;

    while(lid < work_per_wg)
    {
        int o_n       = gid;
        int o_c       = lid % c_c;
        int o_h       = (lid / c_c) % c_h;
        int o_w       = (lid / c_c) / c_h;
        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
#endif
}

#endif

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_c_gid_off = (gid / b_w / b_h) % b_c;
    int o_n_gid_off = (gid / b_w / b_h) / b_c;

    int bindex =
        o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        int aindex    = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int cindex    = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // if(gid>=b_nstride) return;
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
    int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);
    // printf("lid: %d, o_[h,d,c,n]_div: %d, %d, %d, %d\n",lid, o_h_div, o_d_div, o_c_div, o_n_div);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_d_gid_off = (gid / b_w / b_h) % b_d;
    int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
    int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

    int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                 o_h_gid_off * b_hstride + o_w_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
        int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;
        // printf("lid: %d, o_[w,h,d,c,n]: %d, %d, %d, %d, %d\n",lid, o_w, o_h, o_d, o_c, o_n);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;
        // printf("lid: %d, index: %d\n",lid, index);
        // c[index + Coffset] = MIOPEN_TENSOR_OP(a[index + Aoffset], operand);
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
    int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

    int o_h_gid_off = gid % b_h;
    int o_c_gid_off = (gid / b_h) % b_c;
    int o_n_gid_off = (gid / b_h) / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
        int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : (lid / o_c_div) / o_n_div;

        int aindex = o_n * a_nstride + o_c * a_cstride + o_h;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_h;

        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

        lid += get_local_size(0);
    }
}

// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    int o_c_gid_off = gid % b_c;
    int o_n_gid_off = gid / b_c;

    int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;

    while(lid < work_per_wg)
    {
        int o_c       = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
        int o_n       = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
        int aindex    = o_n * a_nstride + o_c;
        int cindex    = o_n * c_nstride + o_c;
        c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
        lid += get_local_size(0);
    }
}

// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const float alpha0,
                                const float alpha1,
                                const float beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
    // MIOPEN_TYPE operand = b[gid + Boffset];

    int o_n_gid_off     = gid % b_n;
    int bindex          = o_n_gid_off;
    MIOPEN_TYPE operand = b_off[bindex] * alpha1;
    while(lid < work_per_wg)
    {
        int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
        c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
        lid += get_local_size(0);
    }
}


cl cleanups

cl cleanups

// Doesn't return 0 so that rays can bounce without bumping.
float sphere_toi(
  const float3 eye,
  const float3 look,

  const float3 center,
  const float radius)
{
  // quadratic coefficients
  float a = dot(look, look);
  float b = 2 * dot(eye - center, look);
  float c = dot(center, center) + dot(eye, eye) - dot(eye, center) - radius * radius;

  // discriminant
  float d = b*b - 4*a*c;

  if (d < 0) {
    return HUGE_VALF;
  }

  float s1 = (sqrt(d) - b) / (2 * a);
  if (s1 <= 0) {
    s1 = HUGE_VALF;
  }

  float s2 = (-sqrt(d) - b) / (2 * a);
  if (s1 < s2 || s2 <= 0) {
    return s1;
  }

  return s2;
}

float3 rotate_vec(
  const float3 vec,
  const float3 axis)
{
  float3 r1 = {2*axis[0]*axis[0] - 1, 2*axis[0]*axis[1], 2*axis[0]*axis[2]};
  float3 r2 = {2*axis[0]*axis[1], 2*axis[1]*axis[1] - 1, 2*axis[1]*axis[2]};
  float3 r3 = {2*axis[0]*axis[2], 2*axis[1]*axis[2], 2*axis[2]*axis[2] - 1};

  float3 r = {dot(r1, vec), dot(r2, vec), dot(r3, vec)};
  return r;
}

typedef struct {
  float4 row1;
  float4 row2;
  float4 row3;
  float4 row4;
} mat4;

float4 vmult(mat4 m, float4 v) {
  return
    (float4)(
      dot(m.row1, v),
      dot(m.row2, v),
      dot(m.row3, v),
      dot(m.row4, v)
    );
}

mat4 screen_to_view(unsigned int w, unsigned int h, float fovy) {
  float aspect = (float)w / (float)h;

  float b = tan(fovy / 2);
  float a = 2 * b / h;
  b = -b;

  mat4 r;
  // We divide both x and y by h, to take aspect ratio into account.
  // An increase in image width will cause wider rays to be shot.
  r.row1 = (float4)(a, 0, 0, b * aspect);
  r.row2 = (float4)(0, a, 0, b);
  r.row3 = (float4)(0, 0, 1, 0);
  r.row4 = (float4)(0, 0, 0, 1);

  return r;
}

mat4 view_to_world(float3 eye, float3 look, float3 up) {
  float4 x = (float4)(cross(look, up), 0);
  float4 y = (float4)(up, 0);
  float4 z = (float4)(look, 0);
  float4 w = (float4)(eye, 1);

  // Change of basis

  mat4 r;
  r.row1.x = x.x;
  r.row2.x = x.y;
  r.row3.x = x.z;
  r.row4.x = x.w;

  r.row1.y = y.x;
  r.row2.y = y.y;
  r.row3.y = y.z;
  r.row4.y = y.w;

  r.row1.z = z.x;
  r.row2.z = z.y;
  r.row3.z = z.z;
  r.row4.z = z.w;

  r.row1.w = w.x;
  r.row2.w = w.y;
  r.row3.w = w.z;
  r.row4.w = w.w;

  return r;
}

typedef struct {
  float r;
  float g;
  float b;
} RGB;

RGB rgb(float3 xyz) {
  RGB r;
  r.r = xyz.x;
  r.g = xyz.y;
  r.b = xyz.z;
  return r;
}

typedef struct { float x, y, z } float3_parse;

float3 pack_float3(float3_parse f) {
  return (float3)(f.x, f.y, f.z);
}

typedef struct {
  float3_parse center;
  float radius;
  float3_parse color;
} Object;

float parse_float(__global const float** data) {
  float r = **data;
  ++*data;
  return r;
}

float3_parse parse_float3(__global const float** data) {
  float3_parse r;
  r.x = parse_float(data);
  r.y = parse_float(data);
  r.z = parse_float(data);
  return r;
}

Object parse_object(__global const float* data) {
  Object r;
  r.center = parse_float3(&data);
  r.radius = parse_float(&data);
  r.color = parse_float3(&data);
  return r;
}

float3 raytrace(
  float3 eye,
  float3 direction,

  __global const float* objects,
  const unsigned int num_objects
) {
  float toi = HUGE_VALF;
  float3 r = (float3)(0, 0, 0);

  for (unsigned int i = 0; i < num_objects; ++i) {
    Object object = parse_object((Object*)objects + i);
    float this_toi = sphere_toi(eye, direction, pack_float3(object.center), object.radius);

    if (this_toi >= toi) {
      continue;
    }

    toi = this_toi;
    r = pack_float3(object.color);
  }

  return r;
}

__kernel void render(
  const unsigned int window_width,
  const unsigned int window_height,

  const float fovy,
  float3 eye,
  const float3 look,
  const float3 up,

  __global const float* objects,
  const unsigned int num_objects,

  __global RGB * output)
{
  int id = get_global_id(0);

  const int x_pix = id % window_width;
  const int y_pix = id / window_width;

  float4 world_pos =
    vmult(view_to_world(eye, look, up),
    vmult(screen_to_view(window_width, window_height, fovy),
    (float4)(x_pix, y_pix, 1, 1)
  ));

  float3 ray = normalize((world_pos / world_pos.w).xyz - eye);

  output[id] = rgb(raytrace(eye, ray, objects, num_objects));
}

// Doesn't return 0 so that rays can bounce without bumping.
float sphere_toi(
  const float3 eye,
  const float3 look,

  const float3 center,
  const float radius)
{
  // quadratic coefficients
  float a = dot(look, look);
  float b = 2 * dot(eye - center, look);
  float c = dot(center, center) + dot(eye, eye) - dot(eye, center) - radius * radius;

  // discriminant
  float d = b*b - 4*a*c;

  if (d < 0) {
    return HUGE_VALF;
  }

  float s1 = (sqrt(d) - b) / (2 * a);
  if (s1 <= 0) {
    s1 = HUGE_VALF;
  }

  float s2 = (-sqrt(d) - b) / (2 * a);
  if (s1 < s2 || s2 <= 0) {
    return s1;
  }

  return s2;
}

float3 rotate_vec(
  const float3 vec,
  const float3 axis)
{
  float3 r1 = {2*axis[0]*axis[0] - 1, 2*axis[0]*axis[1], 2*axis[0]*axis[2]};
  float3 r2 = {2*axis[0]*axis[1], 2*axis[1]*axis[1] - 1, 2*axis[1]*axis[2]};
  float3 r3 = {2*axis[0]*axis[2], 2*axis[1]*axis[2], 2*axis[2]*axis[2] - 1};

  float3 r = {dot(r1, vec), dot(r2, vec), dot(r3, vec)};
  return r;
}

typedef struct {
  float4 row1;
  float4 row2;
  float4 row3;
  float4 row4;
} mat4;

float4 vmult(mat4 m, float4 v) {
  return
    (float4)(
      dot(m.row1, v),
      dot(m.row2, v),
      dot(m.row3, v),
      dot(m.row4, v)
    );
}

mat4 screen_to_view(unsigned int w, unsigned int h, float fovy) {
  float aspect = (float)w / (float)h;

  float b = tan(fovy / 2);
  float a = 2 * b / h;
  b = -b;

  mat4 r;
  // We divide both x and y by h, to take aspect ratio into account.
  // An increase in image width will cause wider rays to be shot.
  r.row1 = (float4)(a, 0, 0, b * aspect);
  r.row2 = (float4)(0, a, 0, b);
  r.row3 = (float4)(0, 0, 1, 0);
  r.row4 = (float4)(0, 0, 0, 1);

  return r;
}

mat4 view_to_world(float3 eye, float3 look, float3 up) {
  float4 x = (float4)(cross(look, up), 0);
  float4 y = (float4)(up, 0);
  float4 z = (float4)(look, 0);
  float4 w = (float4)(eye, 1);

  // Change of basis

  mat4 r;
  r.row1.x = x.x;
  r.row2.x = x.y;
  r.row3.x = x.z;
  r.row4.x = x.w;

  r.row1.y = y.x;
  r.row2.y = y.y;
  r.row3.y = y.z;
  r.row4.y = y.w;

  r.row1.z = z.x;
  r.row2.z = z.y;
  r.row3.z = z.z;
  r.row4.z = z.w;

  r.row1.w = w.x;
  r.row2.w = w.y;
  r.row3.w = w.z;
  r.row4.w = w.w;

  return r;
}

typedef struct {
  float r;
  float g;
  float b;
} RGB;

RGB rgb(float3 xyz) {
  RGB r;
  r.r = xyz.x;
  r.g = xyz.y;
  r.b = xyz.z;
  return r;
}

typedef struct { float x, y, z } float3_parse;

float3 pack_float3(float3_parse f) {
  return (float3)(f.x, f.y, f.z);
}

typedef struct {
  float3_parse center;
  float radius;
  float3_parse color;
} Object;

float parse_float(__global const float** data) {
  float r = **data;
  ++*data;
  return r;
}

float3_parse parse_float3(__global const float** data) {
  float3_parse r;
  r.x = parse_float(data);
  r.y = parse_float(data);
  r.z = parse_float(data);
  return r;
}

Object parse_object(__global const float* data) {
  Object r;
  r.center = parse_float3(&data);
  r.radius = parse_float(&data);
  r.color = parse_float3(&data);
  return r;
}

__kernel void render(
  const unsigned int window_width,
  const unsigned int window_height,

  const float fovy,
  float3 eye,
  const float3 look,
  const float3 up,

  __global const float* objects,
  const unsigned int num_objects,

  __global RGB * output)
{
  int id = get_global_id(0);

  const int x_pix = id % window_width;
  const int y_pix = id / window_width;

  float4 world_pos =
    vmult(view_to_world(eye, look, up),
    vmult(screen_to_view(window_width, window_height, fovy),
    (float4)(x_pix, y_pix, 1, 1)
  ));

  float3 ray = normalize((world_pos / world_pos.w).xyz - eye);

  float toi = HUGE_VALF;
  output[id] = rgb((float3)(0, 0, 0));

  for (unsigned int i = 0; i < num_objects; ++i) {
    Object object = parse_object(objects + i*sizeof(Object)/sizeof(float));
    float this_toi = sphere_toi(eye, ray, pack_float3(object.center), object.radius);

    if (this_toi >= toi) {
      continue;
    }

    toi = this_toi;
    output[id] = rgb(pack_float3(object.color));
  }
}


remove useless copying

remove useless copying


backward minor fix

backward minor fix

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

/*

This is a naive implementation.
The "sliding window" -based implementation is in MIOpenLRNFwd.cl file

*/

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannelBwd(const __global _FLOAT* top,
                          const __global _FLOAT* bot,
                          const __global _FLOAT* top_df,
                          const __global _FLOAT* scale,
                          __global _FLOAT* bot_df,
                          _FLOAT ratio, // 2. * alpha * beta / local_area
                          _FLOAT alpha,
                          _FLOAT beta)
{
    (void)ratio;
    __local _FLOAT top_df_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    __local _FLOAT ratio_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x          = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y          = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0    = get_local_id(0);
    int lcl_id1    = get_local_id(1);
    int ob         = get_global_id(2); // output * batch_sz
    int o          = ob / MLO_LRN_BATCH_SZ;
    int b          = ob - o * MLO_LRN_BATCH_SZ;
    int top_x      = x;
    int top_y      = y;
    int top_df_off = b * MLO_LRN_TOPDF_BATCH_STRIDE + o * MLO_LRN_TOPDF_CHANNEL_STRIDE;
    int scale_off  = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE;
    int bot_x      = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int bot_y      = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX;

    _FLOAT prv_exp_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    //		_FLOAT prv_top_df[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    // load top_diff and scale tiles
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_df_y_off = top_y_act * MLO_LRN_TOPDF_STRIDE;
        int scale_y_off  = top_y_act * MLO_LRN_SCALE_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act];
            _FLOAT scale_val  = scale[scale_off + scale_y_off + top_x_act];

            top_df_val = (invisibleX || invisibleY) ? 0 : top_df_val;
            scale_val  = (invisibleX || invisibleY) ? 1.f : scale_val;

            top_df_data[lcl_off_v + b_i] = top_df_val;
            ratio_data[lcl_off_v + b_i]  = scale_val;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // actual top_diffs and scales
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int lcl_off_v =
            (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_PAD + j) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT scale_ratio =
                ratio_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i];
            prv_exp_scale[j][i] = native_exp(-beta * native_log(scale_ratio));
            //				prv_exp_scale[j][i]= pow(scale, -beta);

            //				prv_top_df[j][i] = top_df_val;
        }
    }

    // read top and load ratio tile
    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE;
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_y_off = top_y_act * MLO_LRN_TOP_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_val = top[top_off + top_y_off + top_x_act];

            top_val = (invisibleX || invisibleY) ? 0 : top_val;

            _FLOAT top_df_val = top_df_data[lcl_off_v + b_i];

            _FLOAT scale_val = ratio_data[lcl_off_v + b_i];

            // scale val is not 0
            _FLOAT ratio_dta = (top_df_val * top_val) / scale_val;
            // replacing scale with ratio
            ratio_data[lcl_off_v + b_i] = ratio_dta;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // caculate bot diff
    _FLOAT prv_bot_diff[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int v_off_v = (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + j);
        int hstart  = y + v_off_v - MLO_LRN_PAD;
        int hend    = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_HEIGHT + MLO_LRN_PAD);

        // accum offset, vertical
        //			int lcl_a_off_v = v_off_v *  MLO_LRN_LCL_DATA_WIDTH;
        // value offset, vertical
        int lcl_v_off_v = (v_off_v + MLO_LRN_PAD) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT prv_ratio_accum = 0;
            int v_off_h            = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + i;

            int wstart = x + v_off_h - MLO_LRN_PAD;
            int wend   = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_WIDTH + MLO_LRN_PAD);

            int adj_area_size = (hend - hstart) * (wend - wstart);

            // accum offset, horiz
            int lcl_a_off_h = v_off_h;
            //	value offset, horiz
            int lcl_v_off_h = lcl_a_off_h + MLO_LRN_PAD;

            for(int k = 0; k < MLO_LRN_KERNEL_SZ; k++)
            {
                for(int l = 0; l < MLO_LRN_KERNEL_SZ; l++)
                {
                    prv_ratio_accum +=
                        ratio_data[(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l];
                }
            }

            _FLOAT top_df_val = top_df_data[lcl_v_off_v + lcl_v_off_h];
            _FLOAT bot_dta    = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * o +
                                 MLO_LRN_BOT_STRIDE * (y + v_off_v) + x + v_off_h];
            _FLOAT adj_ratio       = 2.f * alpha * beta / adj_area_size;
            _FLOAT prv_accum_ratio = adj_ratio * bot_dta * prv_ratio_accum;
            prv_bot_diff[j][i]     = prv_exp_scale[j][i] * top_df_val - prv_accum_ratio;
        }
    }

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; j++)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            if(bot_y + j < MLO_LRN_BOT_HEIGHT && bot_x + i < MLO_LRN_BOT_WIDTH)
            {

                bot_df[MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * o +
                       MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i] = prv_bot_diff[j][i];
            }
        }
    }
}

#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannelsBwd1(const __global _FLOAT* top,
                            const __global _FLOAT* bot,
                            const __global _FLOAT* top_df,
                            const __global _FLOAT* scale,
                            __global _FLOAT* bot_df,
                            _FLOAT ratio, // 2. * alpha * beta / local_area
                            _FLOAT alpha,
                            _FLOAT beta)
{

    (void)alpha;
    int x              = get_global_id(0); // channel x
    int y              = get_global_id(1); // channel y
    int b              = get_global_id(2); // batch
    _FLOAT accum_ratio = 0;
    _FLOAT top_df_in[MLO_LRN_KERNEL_SZ];
    _FLOAT scale_in[MLO_LRN_KERNEL_SZ];
    _FLOAT ratio_dta[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    int bot_df_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD; c_i++)
    {

        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];

        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];

#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
    }

    for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
    {
        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[c_o];

            _FLOAT exp_scale = native_exp(-beta * native_log(prv_scale));
            //					pow(prv_scale, -beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[c_o] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++)
    {

        _FLOAT prv_top_df_in =
            top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i +
                   MLO_LRN_TOPDF_STRIDE * y + x];
        _FLOAT prv_scale_in =
            scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                  MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        _FLOAT prv_ratio_dta = prv_top_df_in * top_dta / prv_scale_in;
#if MLO_LOW_CHNL_COUNT == 1
        prv_ratio_dta = (c_i < MLO_LRN_N_OUTPUTS) ? prv_ratio_dta : 0;
#endif

        accum_ratio = accum_ratio + prv_ratio_dta;

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

        top_df_in[MLO_LRN_KERNEL_SZ - 1] = prv_top_df_in;
        scale_in[MLO_LRN_KERNEL_SZ - 1]  = prv_scale_in;
        ratio_dta[MLO_LRN_KERNEL_SZ - 1] = prv_ratio_dta;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = native_exp(-beta * native_log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = native_exp(-beta * native_log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

/*

This is a naive implementation.
The "sliding window" -based implementation is in MIOpenLRNFwd.cl file

*/

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannelBwd(const __global _FLOAT* top,
                          const __global _FLOAT* bot,
                          const __global _FLOAT* top_df,
                          const __global _FLOAT* scale,
                          __global _FLOAT* bot_df,
                          _FLOAT ratio, // 2. * alpha * beta / local_area
                          _FLOAT alpha,
                          _FLOAT beta)
{
    (void)ratio;
    __local _FLOAT top_df_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    __local _FLOAT ratio_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x          = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y          = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0    = get_local_id(0);
    int lcl_id1    = get_local_id(1);
    int ob         = get_global_id(2); // output * batch_sz
    int o          = ob / MLO_LRN_BATCH_SZ;
    int b          = ob - o * MLO_LRN_BATCH_SZ;
    int top_x      = x;
    int top_y      = y;
    int top_df_off = b * MLO_LRN_TOPDF_BATCH_STRIDE + o * MLO_LRN_TOPDF_CHANNEL_STRIDE;
    int scale_off  = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE;
    int bot_x      = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int bot_y      = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX;

    _FLOAT prv_exp_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    //		_FLOAT prv_top_df[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    // load top_diff and scale tiles
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_df_y_off = top_y_act * MLO_LRN_TOPDF_STRIDE;
        int scale_y_off  = top_y_act * MLO_LRN_SCALE_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act];
            _FLOAT scale_val  = scale[scale_off + scale_y_off + top_x_act];

            top_df_val = (invisibleX || invisibleY) ? 0 : top_df_val;
            scale_val  = (invisibleX || invisibleY) ? 1.f : scale_val;

            top_df_data[lcl_off_v + b_i] = top_df_val;
            ratio_data[lcl_off_v + b_i]  = scale_val;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // actual top_diffs and scales
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int lcl_off_v =
            (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_PAD + j) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT scale_ratio =
                ratio_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i];
            prv_exp_scale[j][i] = native_exp(-beta * native_log(scale_ratio));
            //				prv_exp_scale[j][i]= pow(scale, -beta);

            //				prv_top_df[j][i] = top_df_val;
        }
    }

    // read top and load ratio tile
    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE;
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int top_y_act = top_y + b_j - MLO_LRN_PAD;

        bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT);

        top_y_act = (invisibleY) ? 0 : top_y_act;

        int top_y_off = top_y_act * MLO_LRN_TOP_STRIDE;

        int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH;

        for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0)
        {

            int top_x_act = top_x + b_i - MLO_LRN_PAD;

            bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH);

            top_x_act = (invisibleX) ? 0 : top_x_act;

            _FLOAT top_val = top[top_off + top_y_off + top_x_act];

            top_val = (invisibleX || invisibleY) ? 0 : top_val;

            _FLOAT top_df_val = top_df_data[lcl_off_v + b_i];

            _FLOAT scale_val = ratio_data[lcl_off_v + b_i];

            // scale val is not 0
            _FLOAT ratio_dta = (top_df_val * top_val) / scale_val;
            // replacing scale with ratio
            ratio_data[lcl_off_v + b_i] = ratio_dta;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // caculate bot diff
    _FLOAT prv_bot_diff[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        int v_off_v = (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + j);
        int hstart  = y + v_off_v - MLO_LRN_PAD;
        int hend    = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_HEIGHT + MLO_LRN_PAD);

        // accum offset, vertical
        //			int lcl_a_off_v = v_off_v *  MLO_LRN_LCL_DATA_WIDTH;
        // value offset, vertical
        int lcl_v_off_v = (v_off_v + MLO_LRN_PAD) * MLO_LRN_LCL_DATA_WIDTH;
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            _FLOAT prv_ratio_accum = 0;
            int v_off_h            = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + i;

            int wstart = x + v_off_h - MLO_LRN_PAD;
            int wend   = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_WIDTH + MLO_LRN_PAD);

            int adj_area_size = (hend - hstart) * (wend - wstart);

            // accum offset, horiz
            int lcl_a_off_h = v_off_h;
            //	value offset, horiz
            int lcl_v_off_h = lcl_a_off_h + MLO_LRN_PAD;

            for(int k = 0; k < MLO_LRN_KERNEL_SZ; k++)
            {
                for(int l = 0; l < MLO_LRN_KERNEL_SZ; l++)
                {
                    prv_ratio_accum +=
                        ratio_data[(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l];
                }
            }

            _FLOAT top_df_val = top_df_data[lcl_v_off_v + lcl_v_off_h];
            _FLOAT bot_dta    = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * o +
                                 MLO_LRN_BOT_STRIDE * (y + v_off_v) + x + v_off_h];
            _FLOAT adj_ratio       = 2.f * alpha * beta / adj_area_size;
            _FLOAT prv_accum_ratio = adj_ratio * bot_dta * prv_ratio_accum;
            prv_bot_diff[j][i]     = prv_exp_scale[j][i] * top_df_val - prv_accum_ratio;
        }
    }

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; j++)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++)
        {
            if(bot_y + j < MLO_LRN_BOT_HEIGHT && bot_x + i < MLO_LRN_BOT_WIDTH)
            {

                bot_df[MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * o +
                       MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i] = prv_bot_diff[j][i];
            }
        }
    }
}

#if(MLO_LRN_N_INPUTS + 2 * MLO_LRN_PAD - 1 < MLO_LRN_KERNEL_SZ || \
    MLO_LRN_N_OUTPUTS + 2 * MLO_LRN_PAD - 1 < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif

__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannelsBwd1(const __global _FLOAT* top,
                            const __global _FLOAT* bot,
                            const __global _FLOAT* top_df,
                            const __global _FLOAT* scale,
                            __global _FLOAT* bot_df,
                            _FLOAT ratio, // 2. * alpha * beta / local_area
                            _FLOAT alpha,
                            _FLOAT beta)
{

    (void)alpha;
    int x              = get_global_id(0); // channel x
    int y              = get_global_id(1); // channel y
    int b              = get_global_id(2); // batch
    _FLOAT accum_ratio = 0;
    _FLOAT top_df_in[MLO_LRN_KERNEL_SZ];
    _FLOAT scale_in[MLO_LRN_KERNEL_SZ];
    _FLOAT ratio_dta[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    int bot_df_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD; c_i++)
    {

        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];

        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];

#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
    }

    for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++)
    {
        top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b +
                                MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x];
        scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                              MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0;
#endif

        accum_ratio = accum_ratio + ratio_dta[c_i];
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[c_o];

            _FLOAT exp_scale = native_exp(-beta * native_log(prv_scale));
            //					pow(prv_scale, -beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[c_o] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++)
    {

        _FLOAT prv_top_df_in =
            top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i +
                   MLO_LRN_TOPDF_STRIDE * y + x];
        _FLOAT prv_scale_in =
            scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i +
                  MLO_LRN_SCALE_STRIDE * y + x];
        _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i +
                             MLO_LRN_TOP_STRIDE * y + x];
        _FLOAT prv_ratio_dta = prv_top_df_in * top_dta / prv_scale_in;
#if MLO_LOW_CHNL_COUNT == 1
        prv_ratio_dta = (c_i < MLO_LRN_N_OUTPUTS) ? prv_ratio_dta : 0;
#endif

        accum_ratio = accum_ratio + prv_ratio_dta;

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

        top_df_in[MLO_LRN_KERNEL_SZ - 1] = prv_top_df_in;
        scale_in[MLO_LRN_KERNEL_SZ - 1]  = prv_scale_in;
        ratio_dta[MLO_LRN_KERNEL_SZ - 1] = prv_ratio_dta;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = native_exp(-beta * native_log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum_ratio = accum_ratio - ratio_dta[0];

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            top_df_in[i] = top_df_in[i + 1];
            scale_in[i]  = scale_in[i + 1];
            ratio_dta[i] = ratio_dta[i + 1];
        }

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_INPUTS)
#endif
        {
            _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o +
                                 MLO_LRN_BOT_STRIDE * y + x];

            _FLOAT prv_scale = scale_in[MLO_LRN_PAD];

            _FLOAT exp_scale = native_exp(-beta * native_log(prv_scale));
            //				pow(prv_scale,-beta);

            _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio;

            _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio;

            bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o +
                         MLO_LRN_BOTDF_STRIDE * y + x;

            bot_df[bot_df_off] = out_val;
        }
    }
}


correlate.cl: fast GPU kernel

correlate.cl: fast GPU kernel

#define HADD(v) ((v).x+(v).y+(v).z+(v).w)

__attribute__((reqd_work_group_size(64,1,1)))
__kernel void correlate (
  __global float8 *correlation,
  const int corr_size,
  __constant float4 *base,
  __constant float4 *mask,
  const int sample_size,
  const int stride )
{
  float4 l_mults[24];
  float4 sums[8] = {0,0,0,0,0,0,0,0};
  int gid = get_global_id(0);
  int offset_y = gid*8 / corr_size;
  int offset_x = gid*8 - (offset_y*corr_size);
  int y = get_global_id(1);
  if (y < sample_size-offset_y) {
    int mask_idx = y*stride;
    int base_idx = (offset_y+y)*stride + offset_x;
    for (int i=0; i<8; i++) {
      l_mults[i] = base[base_idx+i] * mask[mask_idx+i];
    }
    for (int x=0; x < sample_size-offset_x; x+=16) {
      for (int i=8; i<24; i++) {
        l_mults[i] = base[base_idx+x+i] * mask[mask_idx+x+i];
      }
      for (int i=0; i<16; i++) {
        sums[0] += l_mults[i+0];
        sums[1] += l_mults[i+1];
        sums[2] += l_mults[i+2];
        sums[3] += l_mults[i+3];
        sums[4] += l_mults[i+4];
        sums[5] += l_mults[i+5];
        sums[6] += l_mults[i+6];
        sums[7] += l_mults[i+7];
      }
      for (int i=0; i<8; i++) {
        l_mults[i] = l_mults[i+16];
      }
    }
    correlation[gid] += (float8)(
      HADD(sums[0]),
      HADD(sums[1]),
      HADD(sums[2]),
      HADD(sums[3]),
      HADD(sums[4]),
      HADD(sums[5]),
      HADD(sums[6]),
      HADD(sums[7])
    );
  }
}

__attribute__((reqd_work_group_size(64,1,1)))
__kernel void correlate (
  __global float8 *correlation,
  const int corr_size,
  __constant float4 *base,
  __constant float4 *mask,
  const int sample_size,
  const int stride )
{
  float4 l_mask[24], l_base[24];
  float8 sum = 0;
  int gid = get_global_id(0);
  int offset_y = gid*8 / corr_size;
  int offset_x = gid*8 - (offset_y*corr_size);
  int y = get_global_id(1);
  if (y < sample_size-offset_y) {
    int mask_idx = y*stride;
    int base_idx = (offset_y+y)*stride + offset_x;
    for (int x=0; x < sample_size-offset_x; x+=16) {
      for (int i=0; i<24; i++) {
        l_base[i] = base[base_idx+x+i];
        l_mask[i] = mask[mask_idx+x+i];
      }
      for (int i=0; i<16; i++) {
        sum.s0 += dot(l_base[i+0], l_mask[i+0]);
        sum.s1 += dot(l_base[i+1], l_mask[i+1]);
        sum.s2 += dot(l_base[i+2], l_mask[i+2]);
        sum.s3 += dot(l_base[i+3], l_mask[i+3]);
        sum.s4 += dot(l_base[i+4], l_mask[i+4]);
        sum.s5 += dot(l_base[i+5], l_mask[i+5]);
        sum.s6 += dot(l_base[i+6], l_mask[i+6]);
        sum.s7 += dot(l_base[i+7], l_mask[i+7]);
      }
    }
    correlation[gid] += sum;
  }
}


OCL work around

OCL work around

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

// BUG!!!!
			barrier(CLK_LOCAL_MEM_FENCE);

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);

				int pool_size = (hend - hstart) * (wend - wstart);
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFFFFFFFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							int src_x = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0 + i;
							int src_y = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1 + j;
							//mask_private[k][l].x = src_x;
							//mask_private[k][l].y = src_y;
							mask_private[k][l] = src_x + src_y * MLO_POOLING_BOT_WIDTH;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

// BUG!!!!
#if MLO_POOLING_PAD0 > 0 || MLO_POOLING_PAD1  > 0
		barrier(CLK_LOCAL_MEM_FENCE);
#endif

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);

				int pool_size = (hend - hstart) * (wend - wstart);
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFFFFFFFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							int src_x = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0 + i;
							int src_y = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1 + j;
							//mask_private[k][l].x = src_x;
							//mask_private[k][l].y = src_y;
							mask_private[k][l] = src_x + src_y * MLO_POOLING_BOT_WIDTH;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}