fixed optensor kernel for fp16

fixed optensor kernel for fp16

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

#ifdef USE_FWD_BIAS

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const int work_per_wg,
                              const MIOPEN_TYPE alpha0,
                              const MIOPEN_TYPE alpha1,
                              const MIOPEN_TYPE beta,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset,
                              const int num_wg)
{
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int gid = get_group_id(0);
    int lid = get_local_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

#if INCR_WG == 1
        int o_n             = gid / b_c;
        int o_c             = gid % b_c;
        MIOPEN_TYPE operand = b_off[o_c] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid] * alpha1;
        int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_hw     = lid % work_off;
            int o_n      = lid / work_off;
            int index    = o_n * c_nstride + gid * c_cstride + o_hw;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

#endif

#ifdef USE_FWD_BIAS_GENERIC

__kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a,
                                     const int a_nstride,
                                     const int a_cstride,
                                     const int a_hstride,
                                     global MIOPEN_TYPE* b,
#if INCR_WG == 0
                                     UNUSED
#endif
                                     const int b_c,
                                     const int b_cstride,
                                     global MIOPEN_TYPE* c,
#if INCR_WG == 1
                                     UNUSED
#endif
                                     const int c_n,
                                     const int c_w,
                                     const int c_nstride,
                                     const int c_cstride,
                                     const int c_hstride,
                                     const MIOPEN_TYPE alpha0,
                                     const MIOPEN_TYPE alpha1,
                                     const MIOPEN_TYPE beta,
                                     const int work_per_wg,
                                     const long Aoffset,
                                     const long Boffset,
                                     const long Coffset,
                                     const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

#if INCR_WG == 1

        int o_c = gid % b_c;
        int o_n = gid / b_c;

        int bindex          = o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid * b_cstride] * alpha1;
        // int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_n    = lid % c_n;
            int o_h    = (lid / c_n) / c_w;
            int o_w    = (lid / c_n) % c_w;
            int aindex = o_n * a_nstride + gid * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + gid * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

#endif

// DLOWELL : cutting out this section
#ifdef USE_LEADING_ONES

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  global MIOPEN_TYPE* b,
                                  global MIOPEN_TYPE* c,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_c,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_h,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_cstride,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const MIOPEN_TYPE alpha0,
                                  const MIOPEN_TYPE alpha1,
                                  const MIOPEN_TYPE beta,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset,
                                  const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1

    int tid = get_global_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        MIOPEN_TYPE operand = b_off[tid] * alpha1;

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w;
        c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
    }
#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_c = gid % c_c;
        int o_n = gid / c_c;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = gid * c_nstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#endif
}

#endif

// DLOWELL : cutting out this section
#ifdef USE_LEADING_ONES_GENERIC

__kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a,
                                         const int a_nstride,
                                         const int a_cstride,
                                         const int a_hstride,
                                         global MIOPEN_TYPE* b,
                                         const int b_nstride,
#if FIRST_NOT_ONE == 0
                                         UNUSED
#endif
                                         const int b_cstride,
#if FIRST_NOT_ONE <= 1
                                         UNUSED
#endif
                                         const int b_hstride,
                                         global MIOPEN_TYPE* c,
                                         const int c_c,
#if FIRST_NOT_ONE == 1
                                         UNUSED
#endif
                                         const int c_h,
#if FIRST_NOT_ONE == 0 || FIRST_NOT_ONE == 2
                                         UNUSED
#endif
                                         const int c_w,
                                         const int c_nstride,
                                         const int c_cstride,
                                         const int c_hstride,
                                         const MIOPEN_TYPE alpha0,
                                         const MIOPEN_TYPE alpha1,
                                         const MIOPEN_TYPE beta,
#if FIRST_NOT_ONE == 3
                                         UNUSED
#endif
                                         const int work_per_wg,
                                         const long Aoffset,
                                         const long Boffset,
                                         const long Coffset,
                                         const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid = get_global_id(0);
    // MIOPEN_TYPE operand = b[tid];

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] =
            MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], alpha1 * b_off[bindex]) + beta * c_off[cindex];
    }

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        int bindex          = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + lid;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + lid;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_c = gid % c_c;
        int o_n = gid / c_c;

        int bindex          = o_n * b_nstride + o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid * b_nstride] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = lid % c_c;
            int o_h    = (lid / c_c) % c_h;
            int o_w    = (lid / c_c) / c_h;
            int aindex = gid * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = gid * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#endif
}

#endif

#ifdef USE_4D_TENSOR_GENERIC

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_c_gid_off = (gid / b_w / b_h) % b_c;
        int o_n_gid_off = (gid / b_w / b_h) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride +
                     o_w_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_5D_TENSOR_GENERIC
// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
        int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_d_gid_off = (gid / b_w / b_h) % b_d;
        int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
        int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                     o_h_gid_off * b_hstride + o_w_gid_off;

        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
            int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;

            int aindex =
                o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
            int cindex =
                o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_3D_TENSOR_GENERIC
// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid     = get_local_id(0);
        int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
        int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

        int o_h_gid_off = gid % b_h;
        int o_c_gid_off = (gid / b_h) % b_c;
        int o_n_gid_off = (gid / b_h) / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
            int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_2D_TENSOR_LITE
__kernel void Op2dTensorLite(const global MIOPEN_TYPE* a,
                             const int a_nstride,
                             const global MIOPEN_TYPE* b,
                             const int b_nstride,
                             global MIOPEN_TYPE* c,
                             const int c_nstride,
                             const MIOPEN_TYPE alpha0,
                             const MIOPEN_TYPE alpha1,
                             const MIOPEN_TYPE beta,
                             const long Aoffset,
                             const long Boffset,
                             const long Coffset)
{
    int gid0 = get_global_id(0);
    int gid1 = get_global_id(1);

    int a_index = gid1 * a_nstride + gid0 * RD_BLCK;
    int b_index = gid1 * b_nstride + gid0 * RD_BLCK;
    int c_index = gid1 * c_nstride + gid0 * RD_BLCK;

    MIOPEN_TYPE a_dat[RD_BLCK];
    MIOPEN_TYPE b_dat[RD_BLCK];
    MIOPEN_TYPE c_dat[RD_BLCK];

    *((READ_TYPE*)a_dat) = *((const global READ_TYPE*)(a + Aoffset + a_index));
    *((READ_TYPE*)b_dat) = *((const global READ_TYPE*)(b + Boffset + b_index));
#ifdef BETA
    *((READ_TYPE*)c_dat) = *((const global READ_TYPE*)(c + Coffset + c_index));
#endif

    for(int i = 0; i < RD_BLCK; ++i)
    {
        c_dat[i] = MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1)
#ifdef BETA
                   + beta * c_dat[i]
#endif
            ;
    }

    *((global READ_TYPE*)(c + Coffset + c_index)) = *((READ_TYPE*)c_dat);

#ifndef BETA
    (void)beta;
#endif
}

#endif

#ifdef USE_2D_TENSOR_GENERIC
// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid         = get_local_id(0);
        int o_c_gid_off = gid % b_c;
        int o_n_gid_off = gid / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
            int o_n    = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
            int aindex = o_n * a_nstride + o_c;
            int cindex = o_n * c_nstride + o_c;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_1D_TENSOR_GENERIC
// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        int o_n_gid_off     = gid % b_n;
        int bindex          = o_n_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;
        while(lid < work_per_wg)
        {
            int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
            c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_4D_TENSOR_LITE
// N - batch size
// C - # of maps
// H - map height
// W - map width
// TENS_LEN = (N*C*H*W);
// RD_BLCK = (TENS_LEN%4==0) ? 4 : (TENS_LEN%3==0)? 3 : (TENS_LEN%2==0)? 2 : 1;
// READ_TYPE = (RD_BLCK==4) ? "float4" : (RD_BLCK == 3) ? "float3" : (RD_BLC==2) ? "float2" :
// "float";
// local size = (256, 1, 1)
// global size = ((TENS_LEN/RD_BLCK), 1, 1)
__kernel void Op4dTensorLite(const global MIOPEN_TYPE* a,
                             const global MIOPEN_TYPE* b,
                             global MIOPEN_TYPE* c,
                             const MIOPEN_TYPE alpha0,
                             const MIOPEN_TYPE alpha1,
#ifndef BETA
                             UNUSED
#endif
                             const MIOPEN_TYPE beta,
                             const long Aoffset,
                             const long Boffset,
                             const long Coffset)
{
    int gid0 = get_global_id(0);

    int index = gid0 * RD_BLCK;

    MIOPEN_TYPE a_dat[RD_BLCK];
    MIOPEN_TYPE b_dat[RD_BLCK];
    MIOPEN_TYPE c_dat[RD_BLCK];

    *((READ_TYPE*)a_dat) = *((const global READ_TYPE*)(a + index + Aoffset));
    *((READ_TYPE*)b_dat) = *((const global READ_TYPE*)(b + index + Boffset));
#ifdef BETA
    *((READ_TYPE*)c_dat) = *((const global READ_TYPE*)(c + index + Coffset));
#endif

    for(int i = 0; i < RD_BLCK; ++i)
    {
        c_dat[i] = MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1)
#ifdef BETA
                   + beta * c_dat[i]
#endif
            ;
    }

    *((global READ_TYPE*)(c + index + Coffset)) = *((READ_TYPE*)c_dat);
}

#endif

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

#ifndef MIOPEN_TENSOR_OP
#define MIOPEN_TENSOR_OP miopenMul
#endif

#define UNUSED __attribute__((__unused__))

MIOPEN_TYPE miopenAdd(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a + b; }

MIOPEN_TYPE miopenMul(MIOPEN_TYPE a, MIOPEN_TYPE b) { return a * b; }

MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); }

MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); }

#ifdef USE_FWD_BIAS

__kernel void OpTensorFwdBias(global MIOPEN_TYPE* a,
                              global MIOPEN_TYPE* b,
#if INCR_WG == 0
                              UNUSED
#endif
                              const int b_c,
                              global MIOPEN_TYPE* c,
#if INCR_WG == 1
                              UNUSED
#endif
                              const int c_n,
                              const int c_nstride,
                              const int c_cstride,
                              const int work_per_wg,
                              const MIOPEN_TYPE alpha0,
                              const MIOPEN_TYPE alpha1,
                              const MIOPEN_TYPE beta,
                              const long Aoffset,
                              const long Boffset,
                              const long Coffset,
                              const int num_wg)
{
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int gid = get_group_id(0);
    int lid = get_local_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

#if INCR_WG == 1
        int o_n             = gid / b_c;
        int o_c             = gid % b_c;
        MIOPEN_TYPE operand = b_off[o_c] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid] * alpha1;
        int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_hw     = lid % work_off;
            int o_n      = lid / work_off;
            int index    = o_n * c_nstride + gid * c_cstride + o_hw;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

#endif

#ifdef USE_FWD_BIAS_GENERIC

__kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a,
                                     const int a_nstride,
                                     const int a_cstride,
                                     const int a_hstride,
                                     global MIOPEN_TYPE* b,
#if INCR_WG == 0
                                     UNUSED
#endif
                                     const int b_c,
                                     const int b_cstride,
                                     global MIOPEN_TYPE* c,
#if INCR_WG == 1
                                     UNUSED
#endif
                                     const int c_n,
                                     const int c_w,
                                     const int c_nstride,
                                     const int c_cstride,
                                     const int c_hstride,
                                     const MIOPEN_TYPE alpha0,
                                     const MIOPEN_TYPE alpha1,
                                     const MIOPEN_TYPE beta,
                                     const int work_per_wg,
                                     const long Aoffset,
                                     const long Boffset,
                                     const long Coffset,
                                     const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

#if INCR_WG == 1

        int o_c = gid % b_c;
        int o_n = gid / b_c;

        int bindex          = o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }

// each workgroup computes N*H*W for each C (bias-term)
// number of workgroups = c_c (b_c)
#elif INCR_WG == 0
        MIOPEN_TYPE operand = b_off[gid * b_cstride] * alpha1;
        // int work_off        = work_per_wg / c_n;

        while(lid < work_per_wg)
        {
            int o_n    = lid % c_n;
            int o_h    = (lid / c_n) / c_w;
            int o_w    = (lid / c_n) % c_w;
            int aindex = o_n * a_nstride + gid * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + gid * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
#endif // INCR_WG
    }
}

#endif

// DLOWELL : cutting out this section
#ifdef USE_LEADING_ONES

__kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a,
                                  global MIOPEN_TYPE* b,
                                  global MIOPEN_TYPE* c,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_c,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_h,
#if FIRST_NOT_ONE <= 1
                                  UNUSED
#endif
                                  const int c_w,
                                  const int c_nstride,
#if FIRST_NOT_ONE == 0
                                  UNUSED
#endif
                                  const int c_cstride,
#if FIRST_NOT_ONE == 3
                                  UNUSED
#endif
                                  const int work_per_wg,
                                  const MIOPEN_TYPE alpha0,
                                  const MIOPEN_TYPE alpha1,
                                  const MIOPEN_TYPE beta,
                                  const long Aoffset,
                                  const long Boffset,
                                  const long Coffset,
                                  const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1

    int tid = get_global_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        MIOPEN_TYPE operand = b_off[tid] * alpha1;

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w;
        c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
    }
#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + o_h * c_w + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        int o_c = gid % c_c;
        int o_n = gid / c_c;

        while(lid < work_per_wg)
        {
            int index    = o_n * c_nstride + o_c * c_cstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid] * alpha1;

        while(lid < work_per_wg)
        {
            int index    = gid * c_nstride + lid;
            c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index];
            lid += get_local_size(0);
        }
    }
#endif
}

#endif

// DLOWELL : cutting out this section
#ifdef USE_LEADING_ONES_GENERIC

__kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a,
                                         const int a_nstride,
                                         const int a_cstride,
                                         const int a_hstride,
                                         global MIOPEN_TYPE* b,
                                         const int b_nstride,
#if FIRST_NOT_ONE == 0
                                         UNUSED
#endif
                                         const int b_cstride,
#if FIRST_NOT_ONE <= 1
                                         UNUSED
#endif
                                         const int b_hstride,
                                         global MIOPEN_TYPE* c,
                                         const int c_c,
#if FIRST_NOT_ONE == 1
                                         UNUSED
#endif
                                         const int c_h,
#if FIRST_NOT_ONE == 0 || FIRST_NOT_ONE == 2
                                         UNUSED
#endif
                                         const int c_w,
                                         const int c_nstride,
                                         const int c_cstride,
                                         const int c_hstride,
                                         const MIOPEN_TYPE alpha0,
                                         const MIOPEN_TYPE alpha1,
                                         const MIOPEN_TYPE beta,
#if FIRST_NOT_ONE == 3
                                         UNUSED
#endif
                                         const int work_per_wg,
                                         const long Aoffset,
                                         const long Boffset,
                                         const long Coffset,
                                         const int num_wg)
{

    /* Special case for leading ones where the total no. of threads is the
     * inner_product of the tensor dims.  Each thread just updates one value
     */
    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;
#if FIRST_NOT_ONE == 3 // bitmap = 1,1,1,1
    int tid = get_global_id(0);
    // MIOPEN_TYPE operand = b[tid];

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; tid < num_wg; tid += MAX_NUM_WG)
    {

        int o_w = tid % c_w;
        int o_h = (tid / c_w) % c_h;
        int o_c = (tid / (c_w * c_h)) % c_c;
        int o_n = tid / (c_w * c_h * c_c);

        int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
        int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w;
        int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
        c_off[cindex] =
            MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], alpha1 * b_off[bindex]) + beta * c_off[cindex];
    }

#elif FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_h = gid % c_h;
        int o_c = (gid / c_h) % c_c;
        int o_n = gid / (c_c * c_h);

        int bindex          = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + lid;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + lid;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(alpha0 * a_off[aindex], operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#elif FIRST_NOT_ONE == 1 // bitmap = 1,1,0,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);
        int o_c = gid % c_c;
        int o_n = gid / c_c;

        int bindex          = o_n * b_nstride + o_c * b_cstride;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h    = lid / c_w;
            int o_w    = lid % c_w;
            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }

#elif FIRST_NOT_ONE == 0 // bitmap = 1,0,0,0
    int gid = get_group_id(0);

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        MIOPEN_TYPE operand = b_off[gid * b_nstride] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = lid % c_c;
            int o_h    = (lid / c_c) % c_h;
            int o_w    = (lid / c_c) / c_h;
            int aindex = gid * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = gid * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
#endif
}

#endif

#ifdef USE_4D_TENSOR_GENERIC

__kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_hstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // MIOPEN_TYPE operand = b[gid + Boffset];
    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_c_gid_off = (gid / b_w / b_h) % b_c;
        int o_n_gid_off = (gid / b_w / b_h) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off * b_hstride +
                     o_w_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_5D_TENSOR_GENERIC
// NCDHW
// (samples, color_depth, frames, width, height )
__kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                const int a_dstride,
                                const int a_hstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_d,
                                const int b_h,
                                const int b_w,
                                const int b_nstride,
                                const int b_cstride,
                                const int b_dstride,
                                const int b_hstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_d,
                                const int c_h,
                                const int c_w,
                                const int c_nstride,
                                const int c_cstride,
                                const int c_dstride,
                                const int c_hstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid = get_local_id(0);

        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_d_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_c_div = o_d_div * (bitmap & (1 << 2) ? 1 : c_d);
        int o_n_div = o_c_div * (bitmap & (1 << 3) ? 1 : c_c);

        int o_w_gid_off = gid % b_w;
        int o_h_gid_off = (gid / b_w) % b_h;
        int o_d_gid_off = (gid / b_w / b_h) % b_d;
        int o_c_gid_off = (gid / b_w / b_h / b_d) % b_c;
        int o_n_gid_off = (gid / b_w / b_h / b_d) / b_c;

        int bindex = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_d_gid_off * b_dstride +
                     o_h_gid_off * b_hstride + o_w_gid_off;

        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_d = (bitmap & (1 << 2)) ? o_d_gid_off : (lid / o_d_div) % c_d;
            int o_c = (bitmap & (1 << 3)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 4)) ? o_n_gid_off : lid / o_n_div;

            int aindex =
                o_n * a_nstride + o_c * a_cstride + o_d * a_dstride + o_h * a_hstride + o_w;
            int cindex =
                o_n * c_nstride + o_c * c_cstride + o_d * c_dstride + o_h * c_hstride + o_w;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_3D_TENSOR_GENERIC
// NCH
__kernel void Op3dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                const int a_cstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_h,
                                const int b_nstride,
                                const int b_cstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_h,
                                const int c_nstride,
                                const int c_cstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid     = get_local_id(0);
        int o_c_div = bitmap & (1 << 0) ? 1 : c_h;
        int o_n_div = o_c_div * (bitmap & (1 << 1) ? 1 : c_c);

        int o_h_gid_off = gid % b_h;
        int o_c_gid_off = (gid / b_h) % b_c;
        int o_n_gid_off = (gid / b_h) / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off * b_cstride + o_h_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_h = (bitmap & (1 << 0)) ? o_h_gid_off : lid % c_h;
            int o_c = (bitmap & (1 << 1)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 2)) ? o_n_gid_off : lid / o_n_div;

            int aindex = o_n * a_nstride + o_c * a_cstride + o_h;
            int cindex = o_n * c_nstride + o_c * c_cstride + o_h;

            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];

            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_2D_TENSOR_LITE
__kernel void Op2dTensorLite(const global MIOPEN_TYPE* a,
                             const int a_nstride,
                             const global MIOPEN_TYPE* b,
                             const int b_nstride,
                             global MIOPEN_TYPE* c,
                             const int c_nstride,
                             const float alpha0,
                             const float alpha1,
                             const float beta,
                             const long Aoffset,
                             const long Boffset,
                             const long Coffset)
{
    int gid0 = get_global_id(0);
    int gid1 = get_global_id(1);

    int a_index = gid1 * a_nstride + gid0 * RD_BLCK;
    int b_index = gid1 * b_nstride + gid0 * RD_BLCK;
    int c_index = gid1 * c_nstride + gid0 * RD_BLCK;

    float a_dat[RD_BLCK];
    float b_dat[RD_BLCK];
    float c_dat[RD_BLCK];

    *((READ_TYPE*)a_dat) = *((const global READ_TYPE*)(a + Aoffset + a_index));
    *((READ_TYPE*)b_dat) = *((const global READ_TYPE*)(b + Boffset + b_index));
#ifdef BETA
    *((READ_TYPE*)c_dat) = *((const global READ_TYPE*)(c + Coffset + c_index));
#endif

    for(int i = 0; i < RD_BLCK; ++i)
    {
        c_dat[i] = MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1)
#ifdef BETA
                   + beta * c_dat[i]
#endif
            ;
    }

    *((global READ_TYPE*)(c + Coffset + c_index)) = *((READ_TYPE*)c_dat);

#ifndef BETA
    (void)beta;
#endif
}

#endif

#ifdef USE_2D_TENSOR_GENERIC
// NC
__kernel void Op2dTensorGeneric(global MIOPEN_TYPE* a,
                                const int a_nstride,
                                global MIOPEN_TYPE* b,
                                const int b_c,
                                const int b_nstride,
                                global MIOPEN_TYPE* c,
                                const int c_c,
                                const int c_nstride,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    int o_n_div = bitmap & (1 << 0) ? 1 : c_c;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {

        int lid         = get_local_id(0);
        int o_c_gid_off = gid % b_c;
        int o_n_gid_off = gid / b_c;

        int bindex          = o_n_gid_off * b_nstride + o_c_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;

        while(lid < work_per_wg)
        {
            int o_c    = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c;
            int o_n    = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div;
            int aindex = o_n * a_nstride + o_c;
            int cindex = o_n * c_nstride + o_c;
            c_off[cindex] =
                MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex];
            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_1D_TENSOR_GENERIC
// N
__kernel void Op1dTensorGeneric(global MIOPEN_TYPE* a,
                                global MIOPEN_TYPE* b,
                                const int b_n,
                                global MIOPEN_TYPE* c,
                                const int c_n,
                                const MIOPEN_TYPE alpha0,
                                const MIOPEN_TYPE alpha1,
                                const MIOPEN_TYPE beta,
                                const unsigned int bitmap,
                                const int work_per_wg,
                                const long Aoffset,
                                const long Boffset,
                                const long Coffset,
                                const int num_wg)
{
    int gid = get_group_id(0);

    global MIOPEN_TYPE* a_off = a + Aoffset;
    global MIOPEN_TYPE* b_off = b + Boffset;
    global MIOPEN_TYPE* c_off = c + Coffset;

    // num_wg: the number of workgroups should be launched
    // MAX_NUM_WG: the maximum number of workgroups actually launched
    for(; gid < num_wg; gid += MAX_NUM_WG)
    {
        int lid             = get_local_id(0);
        int o_n_gid_off     = gid % b_n;
        int bindex          = o_n_gid_off;
        MIOPEN_TYPE operand = b_off[bindex] * alpha1;
        while(lid < work_per_wg)
        {
            int o_n    = (bitmap & (1 << 0)) ? o_n_gid_off : lid % c_n;
            c_off[o_n] = MIOPEN_TENSOR_OP(a_off[o_n] * alpha0, operand) + beta * c_off[o_n];
            lid += get_local_size(0);
        }
    }
}

#endif

#ifdef USE_4D_TENSOR_LITE
// N - batch size
// C - # of maps
// H - map height
// W - map width
// TENS_LEN = (N*C*H*W);
// RD_BLCK = (TENS_LEN%4==0) ? 4 : (TENS_LEN%3==0)? 3 : (TENS_LEN%2==0)? 2 : 1;
// READ_TYPE = (RD_BLCK==4) ? "float4" : (RD_BLCK == 3) ? "float3" : (RD_BLC==2) ? "float2" :
// "float";
// local size = (256, 1, 1)
// global size = ((TENS_LEN/RD_BLCK), 1, 1)
__kernel void Op4dTensorLite(const global MIOPEN_TYPE* a,
                             const global MIOPEN_TYPE* b,
                             global MIOPEN_TYPE* c,
                             const float alpha0,
                             const float alpha1,
#ifndef BETA
                             UNUSED
#endif
                             const float beta,
                             const long Aoffset,
                             const long Boffset,
                             const long Coffset)
{
    int gid0 = get_global_id(0);

    int index = gid0 * RD_BLCK;

    float a_dat[RD_BLCK];
    float b_dat[RD_BLCK];
    float c_dat[RD_BLCK];

    *((READ_TYPE*)a_dat) = *((const global READ_TYPE*)(a + index + Aoffset));
    *((READ_TYPE*)b_dat) = *((const global READ_TYPE*)(b + index + Boffset));
#ifdef BETA
    *((READ_TYPE*)c_dat) = *((const global READ_TYPE*)(c + index + Coffset));
#endif

    for(int i = 0; i < RD_BLCK; ++i)
    {
        c_dat[i] = MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1)
#ifdef BETA
                   + beta * c_dat[i]
#endif
            ;
    }

    *((global READ_TYPE*)(c + index + Coffset)) = *((READ_TYPE*)c_dat);
}

#endif


attenuation threshold

attenuation threshold

#include "cl/mwc64x/cl/mwc64x.cl"

// Doesn't return 0 so that rays can bounce without bumping.
float sphere_toi(
  const float3 eye,
  const float3 look,

  const float3 center,
  const float radius)
{
  // quadratic coefficients
  float a = dot(look, look);
  float b = 2 * dot(eye - center, look);
  float3 to_center = eye - center;
  float c = dot(to_center, to_center) - radius*radius;

  // discriminant
  float d = b*b - 4*a*c;

  if (d < 0) {
    return HUGE_VALF;
  }

  float s1 = (sqrt(d) - b) / (2 * a);
  if (s1 <= 0) {
    s1 = HUGE_VALF;
  }

  float s2 = (-sqrt(d) - b) / (2 * a);
  if (s1 < s2 || s2 <= 0) {
    return s1;
  }

  return s2;
}

typedef struct {
  float4 row1;
  float4 row2;
  float4 row3;
  float4 row4;
} mat4;

float4 vmult(mat4 m, float4 v) {
  return
    (float4)(
      dot(m.row1, v),
      dot(m.row2, v),
      dot(m.row3, v),
      dot(m.row4, v)
    );
}

mat4 screen_to_view(uint w, uint h, float fovy) {
  float aspect = (float)w / (float)h;

  float b = tan(fovy / 2);
  float a = 2 * b / h;
  b = -b;

  mat4 r;
  // We divide both x and y by h, to take aspect ratio into account.
  // An increase in image width will cause wider rays to be shot.
  r.row1 = (float4)(a, 0, 0, b * aspect);
  r.row2 = (float4)(0, a, 0, b);
  r.row3 = (float4)(0, 0, 1, 0);
  r.row4 = (float4)(0, 0, 0, 1);

  return r;
}

mat4 view_to_world(float3 eye, float3 look, float3 up) {
  float4 x = (float4)(cross(look, up), 0);
  float4 y = (float4)(up, 0);
  float4 z = (float4)(look, 0);
  float4 w = (float4)(eye, 1);

  // Change of basis

  mat4 r;
  r.row1.x = x.x;
  r.row2.x = x.y;
  r.row3.x = x.z;
  r.row4.x = x.w;

  r.row1.y = y.x;
  r.row2.y = y.y;
  r.row3.y = y.z;
  r.row4.y = y.w;

  r.row1.z = z.x;
  r.row2.z = z.y;
  r.row3.z = z.z;
  r.row4.z = z.w;

  r.row1.w = w.x;
  r.row2.w = w.y;
  r.row3.w = w.z;
  r.row4.w = w.w;

  return r;
}

typedef struct {
  float r;
  float g;
  float b;
} RGB;

RGB rgb(float3 xyz) {
  RGB r;
  r.r = xyz.x;
  r.g = xyz.y;
  r.b = xyz.z;
  return r;
}

typedef struct { float x, y, z } float3_parse;

#define MAKE_OBJECT(f3)  \
  struct {               \
    f3 center;           \
    float radius;        \
    f3 color;            \
    float diffuseness;   \
    float emittance;     \
    float reflectance;   \
    float transmittance; \
  }

typedef MAKE_OBJECT(float3) Object;
typedef MAKE_OBJECT(float3_parse) Object_parse;

float parse_float(__global const float** data) {
  float r = **data;
  ++*data;
  return r;
}

float3 parse_float3(__global const float** data) {
  float3 r;
  r.x = parse_float(data);
  r.y = parse_float(data);
  r.z = parse_float(data);
  return r;
}

Object parse_object(__global const float* data) {
  Object r;
  r.center        = parse_float3(&data);
  r.radius        = parse_float(&data);
  r.color         = parse_float3(&data);
  r.diffuseness   = parse_float(&data);
  r.emittance     = parse_float(&data);
  r.reflectance   = parse_float(&data);
  r.transmittance = parse_float(&data);
  return r;
}

typedef struct {
  float3 origin;
  float3 direction;
} Ray;

void raycast(
  Ray ray,

  __global const float* objects,
  const uint num_objects,

  float* toi,
  Object* collision
) {
  *toi = HUGE_VALF;

  for (uint i = 0; i < num_objects; ++i) {
    Object object = parse_object((Object_parse*)objects + i);
    float this_toi = sphere_toi(ray.origin, ray.direction, object.center, object.radius);

    if (this_toi >= *toi) {
      continue;
    }

    *toi = this_toi;
    *collision = object;
  }
}

float rand(mwc64x_state_t* rand_state) {
  return (float)MWC64X_NextUint(rand_state) / (float)UINT_MAX;
}

float3 perturb_frame(mwc64x_state_t* rand_state, const float max_angle, const float3* x, const float3* y, const float3* z) {
  float3 coeffs;
  const float c = cos(max_angle);
  coeffs.y = rand(rand_state) * (1 - c) + c;
  const float xz = sqrt(1 - coeffs.y * coeffs.y);
  const float azimuth = rand(rand_state) * 2 * 3.14;
  coeffs.x = cos(azimuth);
  coeffs.z = sin(azimuth);
  coeffs.x *= xz;
  coeffs.z *= xz;
  return coeffs.x * *x + coeffs.y * *y + coeffs.z * *z;
}

float3 perturb(mwc64x_state_t* rand_state, const float3* unperturbed, const float3* normal, const float max_angle) {
  const float3* y = unperturbed;
  // TODO: find z/x better when normal ~= unperturbed
  const float3 z = normalize(cross(*normal, *y));
  const float3 x = normalize(cross(z, *y));

  for (int i = 0; i < 8; ++i) {
    const float3 r = perturb_frame(rand_state, max_angle, &x, y, &z);
    if (dot(r, *normal) >= 0) {
      return r;
    }
  }

  // If we failed several times, we're probably almost perpendicular to the normal.
  // I think that's a pretty small area, so we can just forget the perturb here.
  return *unperturbed;
}

// If the ray is absorbed (i.e. there is no next path), this returns false.
bool pick_next_path(
  mwc64x_state_t* const rand_state,
  const Ray* const ray,
  const float3* const collision_point,
  const float3* const normal,
  const Object* const collided_object,
  float3* const new_direction,
  float3* const new_normal
) {
  float r = rand(rand_state);

  r -= collided_object->transmittance;
  if (r <= 0) {
    *new_direction = ray->direction;
    *new_normal = -*normal;
    return true;
  }

  r -= collided_object->reflectance;
  if (r <= 0) {
    const float cos_theta = dot(ray->direction, *normal);
    *new_direction = ray->direction - 2 * cos_theta * *normal;
    *new_normal = *normal;
    return true;
  }

  return false;
}

float3 pathtrace(
  Ray ray,
  const uint max_depth,
  const float min_attenuation,
  mwc64x_state_t* rand_state,
  __global const float* objects,
  const uint num_objects
) {
  float3 pixel_color = (float3)(0, 0, 0);
  float3 attenuation = (float3)(1, 1, 1);

  for (int i = 0; i < max_depth; ++i) {
    float toi;
    Object collided_object;

    raycast(ray, objects, num_objects, &toi, &collided_object);

    if (toi == HUGE_VALF) {
      break;
    }

    attenuation *= collided_object.color;

    if (attenuation.x < min_attenuation && attenuation.y < min_attenuation && attenuation.z < min_attenuation) {
      break;
    }

    pixel_color += attenuation * (float3)(collided_object.emittance);

    const float3 collision_point = ray.origin + toi*ray.direction;
    const float3 normal = (collision_point - collided_object.center) / collided_object.radius;

    float3 new_direction;
    float3 new_normal;
    if (!pick_next_path(rand_state, &ray, &collision_point, &normal, &collided_object, &new_direction, &new_normal)) {
      // Ray is absorbed.
      break;
    }

    const float max_scatter_angle = 3.14 * collided_object.diffuseness;

    // TODO: maybe instead of a max_scatter_angle, we could describe a probability distribution over the scatter angle.
    ray.direction = perturb(rand_state, &new_direction, &new_normal, max_scatter_angle);
    ray.origin = collision_point + 0.1f * ray.direction;
  }

  return pixel_color;
}

mwc64x_state_t init_rand_state(ulong random_seed) {
  mwc64x_state_t rand_state;
  rand_state.x = (uint)(random_seed & 0xFFFFFFFF);
  rand_state.c = (random_seed & 0xFFFFFFFF00000000) >> 32;
  return rand_state;
}

__kernel void render(
  const uint image_width,
  const uint image_height,

  const float fovy,
  float3 eye,
  const float3 look,
  const float3 up,

  ulong random_seed,

  __global const float* objects,
  const uint num_objects,

  __global RGB * output)
{
  const int id = get_global_id(0);

  const int x_pix = id % image_width;
  const int y_pix = id / image_width;

  const float4 world_pos =
    vmult(view_to_world(eye, look, up),
    vmult(screen_to_view(image_width, image_height, fovy),
    (float4)(x_pix, y_pix, 1, 1)
  ));

  Ray ray;
  ray.origin = eye;
  ray.direction = normalize((world_pos / world_pos.w).xyz - eye);

  random_seed = random_seed * id * id;
  mwc64x_state_t rand_state = init_rand_state(random_seed);
  MWC64X_Skip(&rand_state, 20);

  output[id] = rgb(pathtrace(ray, 20, 0.01, &rand_state, objects, num_objects));
}

#include "cl/mwc64x/cl/mwc64x.cl"

// Doesn't return 0 so that rays can bounce without bumping.
float sphere_toi(
  const float3 eye,
  const float3 look,

  const float3 center,
  const float radius)
{
  // quadratic coefficients
  float a = dot(look, look);
  float b = 2 * dot(eye - center, look);
  float3 to_center = eye - center;
  float c = dot(to_center, to_center) - radius*radius;

  // discriminant
  float d = b*b - 4*a*c;

  if (d < 0) {
    return HUGE_VALF;
  }

  float s1 = (sqrt(d) - b) / (2 * a);
  if (s1 <= 0) {
    s1 = HUGE_VALF;
  }

  float s2 = (-sqrt(d) - b) / (2 * a);
  if (s1 < s2 || s2 <= 0) {
    return s1;
  }

  return s2;
}

typedef struct {
  float4 row1;
  float4 row2;
  float4 row3;
  float4 row4;
} mat4;

float4 vmult(mat4 m, float4 v) {
  return
    (float4)(
      dot(m.row1, v),
      dot(m.row2, v),
      dot(m.row3, v),
      dot(m.row4, v)
    );
}

mat4 screen_to_view(uint w, uint h, float fovy) {
  float aspect = (float)w / (float)h;

  float b = tan(fovy / 2);
  float a = 2 * b / h;
  b = -b;

  mat4 r;
  // We divide both x and y by h, to take aspect ratio into account.
  // An increase in image width will cause wider rays to be shot.
  r.row1 = (float4)(a, 0, 0, b * aspect);
  r.row2 = (float4)(0, a, 0, b);
  r.row3 = (float4)(0, 0, 1, 0);
  r.row4 = (float4)(0, 0, 0, 1);

  return r;
}

mat4 view_to_world(float3 eye, float3 look, float3 up) {
  float4 x = (float4)(cross(look, up), 0);
  float4 y = (float4)(up, 0);
  float4 z = (float4)(look, 0);
  float4 w = (float4)(eye, 1);

  // Change of basis

  mat4 r;
  r.row1.x = x.x;
  r.row2.x = x.y;
  r.row3.x = x.z;
  r.row4.x = x.w;

  r.row1.y = y.x;
  r.row2.y = y.y;
  r.row3.y = y.z;
  r.row4.y = y.w;

  r.row1.z = z.x;
  r.row2.z = z.y;
  r.row3.z = z.z;
  r.row4.z = z.w;

  r.row1.w = w.x;
  r.row2.w = w.y;
  r.row3.w = w.z;
  r.row4.w = w.w;

  return r;
}

typedef struct {
  float r;
  float g;
  float b;
} RGB;

RGB rgb(float3 xyz) {
  RGB r;
  r.r = xyz.x;
  r.g = xyz.y;
  r.b = xyz.z;
  return r;
}

typedef struct { float x, y, z } float3_parse;

#define MAKE_OBJECT(f3)  \
  struct {               \
    f3 center;           \
    float radius;        \
    f3 color;            \
    float diffuseness;   \
    float emittance;     \
    float reflectance;   \
    float transmittance; \
  }

typedef MAKE_OBJECT(float3) Object;
typedef MAKE_OBJECT(float3_parse) Object_parse;

float parse_float(__global const float** data) {
  float r = **data;
  ++*data;
  return r;
}

float3 parse_float3(__global const float** data) {
  float3 r;
  r.x = parse_float(data);
  r.y = parse_float(data);
  r.z = parse_float(data);
  return r;
}

Object parse_object(__global const float* data) {
  Object r;
  r.center        = parse_float3(&data);
  r.radius        = parse_float(&data);
  r.color         = parse_float3(&data);
  r.diffuseness   = parse_float(&data);
  r.emittance     = parse_float(&data);
  r.reflectance   = parse_float(&data);
  r.transmittance = parse_float(&data);
  return r;
}

typedef struct {
  float3 origin;
  float3 direction;
} Ray;

void raycast(
  Ray ray,

  __global const float* objects,
  const uint num_objects,

  float* toi,
  Object* collision
) {
  *toi = HUGE_VALF;

  for (uint i = 0; i < num_objects; ++i) {
    Object object = parse_object((Object_parse*)objects + i);
    float this_toi = sphere_toi(ray.origin, ray.direction, object.center, object.radius);

    if (this_toi >= *toi) {
      continue;
    }

    *toi = this_toi;
    *collision = object;
  }
}

float rand(mwc64x_state_t* rand_state) {
  return (float)MWC64X_NextUint(rand_state) / (float)UINT_MAX;
}

float3 perturb_frame(mwc64x_state_t* rand_state, const float max_angle, const float3* x, const float3* y, const float3* z) {
  float3 coeffs;
  const float c = cos(max_angle);
  coeffs.y = rand(rand_state) * (1 - c) + c;
  const float xz = sqrt(1 - coeffs.y * coeffs.y);
  const float azimuth = rand(rand_state) * 2 * 3.14;
  coeffs.x = cos(azimuth);
  coeffs.z = sin(azimuth);
  coeffs.x *= xz;
  coeffs.z *= xz;
  return coeffs.x * *x + coeffs.y * *y + coeffs.z * *z;
}

float3 perturb(mwc64x_state_t* rand_state, const float3* unperturbed, const float3* normal, const float max_angle) {
  const float3* y = unperturbed;
  // TODO: find z/x better when normal ~= unperturbed
  const float3 z = normalize(cross(*normal, *y));
  const float3 x = normalize(cross(z, *y));

  for (int i = 0; i < 8; ++i) {
    const float3 r = perturb_frame(rand_state, max_angle, &x, y, &z);
    if (dot(r, *normal) >= 0) {
      return r;
    }
  }

  // If we failed several times, we're probably almost perpendicular to the normal.
  // I think that's a pretty small area, so we can just forget the perturb here.
  return *unperturbed;
}

// If the ray is absorbed (i.e. there is no next path), this returns false.
bool pick_next_path(
  mwc64x_state_t* const rand_state,
  const Ray* const ray,
  const float3* const collision_point,
  const float3* const normal,
  const Object* const collided_object,
  float3* const new_direction,
  float3* const new_normal
) {
  float r = rand(rand_state);

  r -= collided_object->transmittance;
  if (r <= 0) {
    *new_direction = ray->direction;
    *new_normal = -*normal;
    return true;
  }

  r -= collided_object->reflectance;
  if (r <= 0) {
    const float cos_theta = dot(ray->direction, *normal);
    *new_direction = ray->direction - 2 * cos_theta * *normal;
    *new_normal = *normal;
    return true;
  }

  return false;
}

float3 pathtrace(
  Ray ray,
  const uint max_depth,
  mwc64x_state_t* rand_state,
  __global const float* objects,
  const uint num_objects
) {
  float3 pixel_color = (float3)(0, 0, 0);
  float3 attenuation = (float3)(1, 1, 1);

  for (int i = 0; i < max_depth; ++i) {
    float toi;
    Object collided_object;

    raycast(ray, objects, num_objects, &toi, &collided_object);

    if (toi == HUGE_VALF) {
      break;
    }

    attenuation *= collided_object.color;

    pixel_color += attenuation * (float3)(collided_object.emittance);

    const float3 collision_point = ray.origin + toi*ray.direction;
    const float3 normal = (collision_point - collided_object.center) / collided_object.radius;

    float3 new_direction;
    float3 new_normal;
    if (!pick_next_path(rand_state, &ray, &collision_point, &normal, &collided_object, &new_direction, &new_normal)) {
      // Ray is absorbed.
      break;
    }

    const float max_scatter_angle = 3.14 * collided_object.diffuseness;

    // TODO: maybe instead of a max_scatter_angle, we could describe a probability distribution over the scatter angle.
    ray.direction = perturb(rand_state, &new_direction, &new_normal, max_scatter_angle);
    ray.origin = collision_point + 0.1f * ray.direction;
  }

  return pixel_color;
}

mwc64x_state_t init_rand_state(ulong random_seed) {
  mwc64x_state_t rand_state;
  rand_state.x = (uint)(random_seed & 0xFFFFFFFF);
  rand_state.c = (random_seed & 0xFFFFFFFF00000000) >> 32;
  return rand_state;
}

__kernel void render(
  const uint image_width,
  const uint image_height,

  const float fovy,
  float3 eye,
  const float3 look,
  const float3 up,

  ulong random_seed,

  __global const float* objects,
  const uint num_objects,

  __global RGB * output)
{
  const int id = get_global_id(0);

  const int x_pix = id % image_width;
  const int y_pix = id / image_width;

  const float4 world_pos =
    vmult(view_to_world(eye, look, up),
    vmult(screen_to_view(image_width, image_height, fovy),
    (float4)(x_pix, y_pix, 1, 1)
  ));

  Ray ray;
  ray.origin = eye;
  ray.direction = normalize((world_pos / world_pos.w).xyz - eye);

  random_seed = random_seed * id * id;
  mwc64x_state_t rand_state = init_rand_state(random_seed);
  MWC64X_Skip(&rand_state, 20);

  output[id] = rgb(pathtrace(ray, 20, &rand_state, objects, num_objects));
}


small bug fixes

small bug fixes

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define NBH_BITPOS         0x2
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

#define SIGMA_NEW			 0x1		  //0
#define SIGMA_OLD			 0x10		  //1
#define NBH					 0x20		  //2
#define RLC					 0x80		  //4
#define RLC_D_POSITION		 0x380        //6-9
#define PIXEL				 0x7FFF0000   //10-24
#define SIGN				 0x2000000    //25

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////////////////////////
	//0. read data into LDS
	int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1)&7)*CODEBLOCKY + getLocalId(1) );
	LOCAL int* statePtr = state + BOUNDARY + getLocalId(0);
	for (int i = 0; i < 4; ++i) {
		int pixel = read_imagei(channel, sampler, posIn).x;
		*statePtr = (abs(pixel) << PIXEL_START_BITPOS) | ((pixel << INPUT_TO_SIGN_SHIFT) & SIGN);
		posIn.y++;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between one and 32 - zero value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {
		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		for (int i = 0; i < CODEBLOCKY; ++i) {
			maxVal = max(maxVal, (*statePtr >> PIXEL_START_BITPOS)&0x7FFF);
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			 for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			 }
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	if (getLocalId(1) == 0) {
		//calculate global msb
		if (getLocalId(0) == 0) {
			int4 mx = (int4)(maxSigBit);

			LOCAL int* scratchPtr = msbScratch;
			for(int i=0; i < CODEBLOCKX; i+=4) {
				mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
			}
			maxSigBit = mx.x;
			maxSigBit = max(maxSigBit, mx.y);
			maxSigBit = max(maxSigBit, mx.z);
			maxSigBit = max(maxSigBit, mx.w);
			msbScratch[0] = maxSigBit;
		}
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	//Algorithm 13 Clean-up pass on the MSB
	//1: set sigmanew := bit value
	//2: set rlcNbh :=  SUM(sigmanew) of preceding column and upper position
	//3: if rlcNbh = 0 and y-dimension of current position is a multiple of 4
	//then
	//4: execute RLC operation
	//5: end if
	//6: if rlcNbh = 1 then
	//7: execute ZC operation
	//8: end if
	//9: if sigmanew = 1 then
	//10: execute SC operation
	//11: end if

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// sigma_new
	statePtr = state + (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;
	for (int i = 0; i < 4; ++i) {
		int current = *statePtr;
		*statePtr |= BIT(current);	 // set sigma_new
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// rlcNbh
	statePtr = state + (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;
	int top = statePtr[TOP];
	int leftTop = statePtr[LEFT_TOP];
	int left = statePtr[LEFT];
	int leftBottom = statePtr[LEFT_BOTTOM];

	for (int i = 0; i < 4; ++i) {
		int nbh = BIT(top) + BIT(leftTop) + BIT(left) + BIT(leftBottom);
		nbh = ((nbh | (~nbh + 1)) >> 27) * NBH;  // NBH if non-zero, otherwise zero
		*statePtr |= nbh;						// set nbh
		statePtr += STATE_BUFFER_STRIDE;
		top = statePtr[TOP];
		leftTop = left;
		left = leftBottom;
		leftBottom = statePtr[LEFT_BOTTOM];

	}
	localMemoryFence();

	// run CUP coding on MSB
	statePtr = state + (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;
    int current = statePtr[0];
	left = statePtr[LEFT];
	int right = statePtr[RIGHT];
	int currentPlusOne = statePtr[STATE_BUFFER_STRIDE];
	int currentPlusTwo = statePtr[STATE_BUFFER_STRIDE_X2];
	int currentPlusThree = statePtr[STATE_BUFFER_STRIDE_X3];

	//RLC
	// get information about both SIGMA_NEW and SIGMA_OLD from
	// left and right neighbours and current bit,
	// and use this to update the current RLC bit
	statePtr[0] |= (( left & SIGMA_NEW ) ||
						(left & SIGMA_OLD ) ||
							( current & SIGMA_NEW ) ||
							( current & SIGMA_OLD ) ||
							( right   & SIGMA_NEW ) ||
								( right   & SIGMA_OLD )) << RLC_BITPOS ;

	localMemoryFence();

	// get information about bit value from the left neighbour and update current RLC bit
	statePtr[0] |= BIT (left) << RLC_BITPOS;

	localMemoryFence();

	// get the information from RLC bits of successors
	//atomic_or(statePtr + (y&0xfffffffc - y)*STATE_BUFFER_STRIDE, *statePtr & RLC );

	localMemoryFence();

	int nbh = 0;

    //ZC
	if (BIT(current)) {
	//SC

	}
	localMemoryFence();
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

BPC Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define NBH_BITPOS         0x2
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

#define SIGMA_NEW			 0x1		  //0
#define SIGMA_OLD			 0x10		  //1
#define NBH					 0x20		  //2
#define RLC					 0x80		  //4
#define RLC_D_POSITION		 0x380        //6-9
#define PIXEL				 0x7FFF0000   //10-24
#define SIGN				 0x2000000    //25

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////////////////////////
	//0. read data into LDS
	int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1)&7)*CODEBLOCKY + getLocalId(1) );
	LOCAL int* statePtr = state + BOUNDARY + getLocalId(0);
	for (int i = 0; i < 4; ++i) {
		int pixel = read_imagei(channel, sampler, posIn).x;
		*statePtr = (abs(pixel) << PIXEL_START_BITPOS) | ((pixel << INPUT_TO_SIGN_SHIFT) & SIGN);
		posIn.y++;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between one and 32 - zero value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {
		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		for (int i = 0; i < CODEBLOCKY; ++i) {
			maxVal = max(maxVal, (*statePtr >> PIXEL_START_BITPOS)&0x7FFF);
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			 for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			 }
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	if (getLocalId(1) == 0) {
		//calculate global msb
		if (getLocalId(0) == 0) {
			int4 mx = (int4)(maxSigBit);

			LOCAL int* scratchPtr = msbScratch;
			for(int i=0; i < CODEBLOCKX; i+=4) {
				mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
			}
			maxSigBit = mx.x;
			maxSigBit = max(maxSigBit, mx.y);
			maxSigBit = max(maxSigBit, mx.z);
			maxSigBit = max(maxSigBit, mx.w);
			msbScratch[0] = maxSigBit;
		}
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	//Algorithm 13 Clean-up pass on the MSB
	//1: set sigmanew := bit value
	//2: set rlcNbh :=  SUM(sigmanew) of preceding column and upper position
	//3: if rlcNbh = 0 and y-dimension of current position is a multiple of 4
	//then
	//4: execute RLC operation
	//5: end if
	//6: if rlcNbh = 1 then
	//7: execute ZC operation
	//8: end if
	//9: if sigmanew = 1 then
	//10: execute SC operation
	//11: end if

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// sigma_new
	statePtr = state + (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;
	for (int i = 0; i < 4; ++i) {
		int current = *statePtr;
		*statePtr |= BIT(current);	 // set sigma_new
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// rlcNbh
	statePtr = state + (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;
	int top = statePtr[TOP];
	int leftTop = statePtr[LEFT_TOP];
	int left = statePtr[LEFT];
	int leftBottom = statePtr[LEFT_BOTTOM];

	for (int i = 0; i < 4; ++i) {
		int nbh = BIT(top) + BIT(leftTop) + BIT(left) + BIT(leftBottom);
		nbh = ((nbh | (~nbh + 1)) >> 27) * NBH;  // NBH if non-zero, otherwise zero
		*statePtr |= nbh;	 // set nbh
		statePtr += STATE_BUFFER_STRIDE;
		top = left;
		left = leftBottom;
		top = statePtr[TOP];
		leftBottom = statePtr[LEFT_BOTTOM];

	}
	localMemoryFence();

/*

	// run CUP coding on MSB
	statePtr = state + BOUNDARY + getLocalId(0);
	int y = 0;
    current = statePtr[0];
	int currentPlusOne = statePtr[STATE_BUFFER_STRIDE];
	for (int i = BOUNDARY; i < CODEBLOCKY+BOUNDARY; ++i) {
		 if (!(y&3) ) {
			//RLC

			// get information about both SIGMA_NEW and SIGMA_OLD from
			// left and right neighbours and current bit,
			// and use this to update the current RLC bit
#ifdef POO

			statePtr[0] |= (( left & SIGMA_NEW ) ||
								(left & SIGMA_OLD ) ||
								  ( current & SIGMA_NEW ) ||
								   ( current & SIGMA_OLD ) ||
								    ( right   & SIGMA_NEW ) ||
								      ( right   & SIGMA_OLD )) << RLC_BITPOS ;

			localMemoryFence();

			// get information about bit value from the left neighbour and update current RLC bit
			statePtr[0] |= BIT (statePtr[LEFT]) << RLC_BITPOS;

			localMemoryFence();

			// get the information from RLC bits of successors
			atomic_or(statePtr + (y&0xfffffffc - y)*STATE_BUFFER_STRIDE, *statePtr & RLC );

			localMemoryFence();

			int nbh = 0;
#endif
		 } else {
			//ZC

		 }
		 if (BIT(current)) {
			//SC

		 }
		 statePtr += STATE_BUFFER_STRIDE;
		 y++;

	}
	localMemoryFence();
#endif
*/
}