OpenCL implementation of RANECU PRNG

OpenCL implementation of RANECU PRNG

/******************************************************************************
 * @file     prngcl_ranecu.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of RANECU pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   F. James,
 *   "A review of pseudorandom number generators",
 *   Computer Physics Communications, 60 (1990), 329--344.
 *
 *   Pierre L'Ecuyer,
 *   "Efficient and Portable Combined Random Number Generators",
 *   Communications of the ACM, 31, 6 (1988), 742--751.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, 2014 Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_RANECU_CL
#define PRNGCL_RANECU_CL

#include "prngcl_common.cl"

#define RANECU_m_FP   (2147483648.0f)
#define RANECU_min    1
#define RANECU_max    2147483647
#define RANECU_min_FP (1.0/2147483648.0)
#define RANECU_max_FP (2147483647.0/2147483648.0)
#define RANECU_k      (4.656612873077392578125E-10) // 1/2^31

#define RANECU_left   (RANECU_min_FP+RANECU_k*RANECU_max_FP)
#define RANECU_right  (RANECU_max_FP+RANECU_k*RANECU_min_FP)

#define RANECU_twom31   (2147483648.0f)
#define RANECU_icons1   2147483563
#define RANECU_icons2   2147483399
#define RANECU_icons3   2147483562
#define RANECU_seedP11  53668
#define RANECU_seedP12  12211
#define RANECU_seedP13  40014
#define RANECU_seedP21  52774
#define RANECU_seedP22  3791
#define RANECU_seedP23  40692

//________________________________________________________________________________________________________ RANECU PRNG
__attribute__((always_inline)) void
ranecu_step(uint4* seed1,uint4* seed2,uint4* rnd)
{
	uint4 k = (*seed1) / ((uint4) RANECU_seedP11);
    uint4 RANECU_test_1 = ((uint4) RANECU_seedP13) * ((*seed1) - k * ((uint4) RANECU_seedP11));
    uint4 RANECU_test_2 = k * ((uint4) RANECU_seedP12);

    if (RANECU_test_1.x > RANECU_test_2.x) (*seed1).x = RANECU_test_1.x - RANECU_test_2.x; else (*seed1).x = RANECU_test_1.x - RANECU_test_2.x + RANECU_icons1;
    if (RANECU_test_1.y > RANECU_test_2.y) (*seed1).y = RANECU_test_1.y - RANECU_test_2.y; else (*seed1).y = RANECU_test_1.y - RANECU_test_2.y + RANECU_icons1;
    if (RANECU_test_1.z > RANECU_test_2.z) (*seed1).z = RANECU_test_1.z - RANECU_test_2.z; else (*seed1).z = RANECU_test_1.z - RANECU_test_2.z + RANECU_icons1;
    if (RANECU_test_1.w > RANECU_test_2.w) (*seed1).w = RANECU_test_1.w - RANECU_test_2.w; else (*seed1).w = RANECU_test_1.w - RANECU_test_2.w + RANECU_icons1;

	k = (*seed2) / ((uint4) RANECU_seedP21);

    RANECU_test_1 = ((uint4) RANECU_seedP23) * ((*seed2) - k * ((uint4) RANECU_seedP21));
    RANECU_test_2 = k * ((uint4) RANECU_seedP22);

    if (RANECU_test_1.x > RANECU_test_2.x) (*seed2).x = RANECU_test_1.x - RANECU_test_2.x; else (*seed2).x = RANECU_test_1.x - RANECU_test_2.x + RANECU_icons2;
    if (RANECU_test_1.y > RANECU_test_2.y) (*seed2).y = RANECU_test_1.y - RANECU_test_2.y; else (*seed2).y = RANECU_test_1.y - RANECU_test_2.y + RANECU_icons2;
    if (RANECU_test_1.z > RANECU_test_2.z) (*seed2).z = RANECU_test_1.z - RANECU_test_2.z; else (*seed2).z = RANECU_test_1.z - RANECU_test_2.z + RANECU_icons2;
    if (RANECU_test_1.w > RANECU_test_2.w) (*seed2).w = RANECU_test_1.w - RANECU_test_2.w; else (*seed2).w = RANECU_test_1.w - RANECU_test_2.w + RANECU_icons2;

    if ((*seed1).x > (*seed2).x) (*rnd).x = (*seed1).x - (*seed2).x; else (*rnd).x = (*seed1).x - (*seed2).x + RANECU_icons3;
    if ((*seed1).y > (*seed2).y) (*rnd).y = (*seed1).y - (*seed2).y; else (*rnd).y = (*seed1).y - (*seed2).y + RANECU_icons3;
    if ((*seed1).z > (*seed2).z) (*rnd).z = (*seed1).z - (*seed2).z; else (*rnd).z = (*seed1).z - (*seed2).z + RANECU_icons3;
    if ((*seed1).w > (*seed2).w) (*rnd).w = (*seed1).w - (*seed2).w; else (*rnd).w = (*seed1).w - (*seed2).w + RANECU_icons3;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double4
ranecu_step_double(uint4* seed1,uint4* seed2)
{
    hgpu_double4 result;
    uint4 rnd1, rnd2;
    uint4 sed1 = (*seed1);
    uint4 sed2 = (*seed2);
    ranecu_step(&sed1,&sed2,&rnd1);

#ifndef PRNG_SKIP_CHECK
    bool flag = true;
    while (flag) {
#endif

        ranecu_step(&sed1,&sed2,&rnd2);
        result = hgpu_uint4_to_double4(rnd1,rnd2,RANECU_min,RANECU_max,RANECU_k);

#ifndef PRNG_SKIP_CHECK
        if ((result.x>=RANECU_left) && (result.x<RANECU_right) &&
            (result.y>=RANECU_left) && (result.y<RANECU_right) &&
            (result.z>=RANECU_left) && (result.z<RANECU_right) &&
            (result.w>=RANECU_left) && (result.w<RANECU_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
#endif
    (*seed1) = sed1;
    (*seed2) = sed2;
    return result;
}
#endif

__kernel void
ranecu(__global uint4* seed_table,
       __global hgpu_float4* randoms,
          const uint N)
{
    uint giddst = GID;

    uint4 seed1 = seed_table[GID];
    uint4 seed2 = seed_table[GID + GID_SIZE];

#ifdef PRECISION_DOUBLE  // if double precision is defined
    uint4 rnd1, rnd2;
    hgpu_double4 result;
#else
    uint4 result;
    float4 normal = (float4) RANECU_twom31;
#endif

    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        result = ranecu_step_double(&seed1,&seed2);
        randoms[giddst] = result;
#else
        ranecu_step(&seed1,&seed2,&result);
        randoms[giddst] = hgpu_uint4_to_float4(result) / normal;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed1;
    seed_table[GID + GID_SIZE] = seed2;
}

#endif

/******************************************************************************
 * @file     prngcl_ranecu.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of RANECU pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   F. James,
 *   "A review of pseudorandom number generators",
 *   Computer Physics Communications, 60 (1990), 329--344.
 *
 *   Pierre L'Ecuyer,
 *   "Efficient and Portable Combined Random Number Generators",
 *   Communications of the ACM, 31, 6 (1988), 742--751.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_RANECU_CL
#define PRNGCL_RANECU_CL

#include "prngcl_common.cl"

#define RANECU_m_FP   (2147483648.0f)
#define RANECU_min    1
#define RANECU_max    2147483647
#define RANECU_min_FP (1.0/2147483648.0)
#define RANECU_max_FP (2147483647.0/2147483648.0)
#define RANECU_k      (4.656612873077392578125E-10) // 1/2^31

#define RANECU_left   (RANECU_min_FP+RANECU_k*RANECU_max_FP)
#define RANECU_right  (RANECU_max_FP+RANECU_k*RANECU_min_FP)

#define RANECU_twom31   (2147483648.0f)
#define RANECU_icons1   2147483563
#define RANECU_icons2   2147483399
#define RANECU_icons3   2147483562
#define RANECU_seedP11  53668
#define RANECU_seedP12  12211
#define RANECU_seedP13  40014
#define RANECU_seedP21  52774
#define RANECU_seedP22  3791
#define RANECU_seedP23  40692

//________________________________________________________________________________________________________ RANECU PRNG
__attribute__((always_inline)) void
ranecu_step(uint4* seed1,uint4* seed2,uint4* rnd)
{
	uint4 k = (*seed1) / ((uint4) RANECU_seedP11);
    uint4 RANECU_test_1 = ((uint4) RANECU_seedP13) * ((*seed1) - k * ((uint4) RANECU_seedP11));
    uint4 RANECU_test_2 = k * ((uint4) RANECU_seedP12);

    if (RANECU_test_1.x > RANECU_test_2.x) (*seed1).x = RANECU_test_1.x - RANECU_test_2.x; else (*seed1).x = RANECU_test_1.x - RANECU_test_2.x + RANECU_icons1;
    if (RANECU_test_1.y > RANECU_test_2.y) (*seed1).y = RANECU_test_1.y - RANECU_test_2.y; else (*seed1).y = RANECU_test_1.y - RANECU_test_2.y + RANECU_icons1;
    if (RANECU_test_1.z > RANECU_test_2.z) (*seed1).z = RANECU_test_1.z - RANECU_test_2.z; else (*seed1).z = RANECU_test_1.z - RANECU_test_2.z + RANECU_icons1;
    if (RANECU_test_1.w > RANECU_test_2.w) (*seed1).w = RANECU_test_1.w - RANECU_test_2.w; else (*seed1).w = RANECU_test_1.w - RANECU_test_2.w + RANECU_icons1;

	k = (*seed2) / ((uint4) RANECU_seedP21);

    RANECU_test_1 = ((uint4) RANECU_seedP23) * ((*seed2) - k * ((uint4) RANECU_seedP21));
    RANECU_test_2 = k * ((uint4) RANECU_seedP22);

    if (RANECU_test_1.x > RANECU_test_2.x) (*seed2).x = RANECU_test_1.x - RANECU_test_2.x; else (*seed2).x = RANECU_test_1.x - RANECU_test_2.x + RANECU_icons2;
    if (RANECU_test_1.y > RANECU_test_2.y) (*seed2).y = RANECU_test_1.y - RANECU_test_2.y; else (*seed2).y = RANECU_test_1.y - RANECU_test_2.y + RANECU_icons2;
    if (RANECU_test_1.z > RANECU_test_2.z) (*seed2).z = RANECU_test_1.z - RANECU_test_2.z; else (*seed2).z = RANECU_test_1.z - RANECU_test_2.z + RANECU_icons2;
    if (RANECU_test_1.w > RANECU_test_2.w) (*seed2).w = RANECU_test_1.w - RANECU_test_2.w; else (*seed2).w = RANECU_test_1.w - RANECU_test_2.w + RANECU_icons2;

    if ((*seed1).x > (*seed2).x) (*rnd).x = (*seed1).x - (*seed2).x; else (*rnd).x = (*seed1).x - (*seed2).x + RANECU_icons3;
    if ((*seed1).y > (*seed2).y) (*rnd).y = (*seed1).y - (*seed2).y; else (*rnd).y = (*seed1).y - (*seed2).y + RANECU_icons3;
    if ((*seed1).z > (*seed2).z) (*rnd).z = (*seed1).z - (*seed2).z; else (*rnd).z = (*seed1).z - (*seed2).z + RANECU_icons3;
    if ((*seed1).w > (*seed2).w) (*rnd).w = (*seed1).w - (*seed2).w; else (*rnd).w = (*seed1).w - (*seed2).w + RANECU_icons3;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double4
ranecu_step_double(uint4* seed1,uint4* seed2)
{
    hgpu_double4 result;
    uint4 rnd1, rnd2;
    uint4 sed1 = (*seed1);
    uint4 sed2 = (*seed2);
    bool flag = true;
    ranecu_step(&sed1,&sed2,&rnd1);
    while (flag) {
        ranecu_step(&sed1,&sed2,&rnd2);
        result = hgpu_uint4_to_double4(rnd1,rnd2,RANECU_min,RANECU_max,RANECU_k);
        if ((result.x>=RANECU_left) && (result.x<RANECU_right) &&
            (result.y>=RANECU_left) && (result.y<RANECU_right) &&
            (result.z>=RANECU_left) && (result.z<RANECU_right) &&
            (result.w>=RANECU_left) && (result.w<RANECU_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
    (*seed1) = sed1;
    (*seed2) = sed2;
    return result;
}
#endif

__kernel void
ranecu(__global uint4* seed_table,
       __global hgpu_float4* randoms,
          const uint N)
{
    uint giddst = GID;

    uint4 seed1 = seed_table[GID];
    uint4 seed2 = seed_table[GID + GID_SIZE];

#ifdef PRECISION_DOUBLE  // if double precision is defined
    uint4 rnd1, rnd2;
    hgpu_double4 result;
#else
    uint4 result;
    float4 normal = (float4) RANECU_twom31;
#endif

    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        result = ranecu_step_double(&seed1,&seed2);
        randoms[giddst] = result;
#else
        ranecu_step(&seed1,&seed2,&result);
        randoms[giddst] = hgpu_uint4_to_float4(result) / normal;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed1;
    seed_table[GID + GID_SIZE] = seed2;
}

#endif


Add experimental alpha blending for volume rendering

#include "liegroups.clh"

__kernel void populate_test_volume(
    __write_only image3d_t volume,
    float t) {
    const int3 vxl_coord = (int3) (get_global_id(0), get_global_id(1), get_global_id(2));

    const float3 volume_size = (float3) (
        get_image_width(volume),
        get_image_height(volume),
        get_image_depth(volume)
    );

    float3 vxl_coordf;
    vxl_coordf.x = (float) vxl_coord.x;
    vxl_coordf.y = (float) vxl_coord.y;
    vxl_coordf.z = (float) vxl_coord.z;

    const float3 world_pt = vxl_coordf - (volume_size / 2.0f);

    const int4 write_vxl_coord = (int4) (vxl_coord, 0);

    const float3 rcolor = 25.0f * world_pt / (dot(world_pt, world_pt));
    const float4 color = (float4) (rcolor, 1.0f);

    write_imagef(volume, write_vxl_coord, color);
}

// TODO: Volumetric Billboards
__kernel void render_volume(
    __read_only image2d_t ray_lut,
    __read_only image3d_t volume,
    __write_only image2d_t rendered_image,
    const struct clSE3 world_from_camera,
    const float scaling,
    const float slice_coord) {
    const sampler_t smp_ray = CLK_NORMALIZED_COORDS_FALSE |
                              CLK_ADDRESS_CLAMP |
                              CLK_FILTER_NEAREST;

    const sampler_t smp_volume = CLK_NORMALIZED_COORDS_FALSE |
                                 CLK_ADDRESS_CLAMP |
                                 CLK_FILTER_NEAREST;

    const int2 px_coord = (int2) (get_global_id(0), get_global_id(1));

    const float3 ray_dir_camera = read_imagef(ray_lut, smp_ray, px_coord).xyz;
    const float3 ray_dir_world = mul_so3(world_from_camera.rotation, ray_dir_camera);
    const float3 ray_origin_m = world_from_camera.translation;

    const float3 volume_dims_px = (float3) (
        get_image_width(volume),
        get_image_height(volume),
        get_image_depth(volume)
    );

    const float volume_size_m = 1.0f;
    const float volume_m_per_px = volume_size_m / volume_dims_px.x;

    // const float3 volume_origin_m = -(volume_m_per_px * volume_dims_px) / 2.0f;
    // const float3 volume_origin_m = (-2.0f, -2.0f, -2.0f);
    const float3 volume_origin_m = (float3) (0.0f, 0.0f, 0.0f);

    const float step_size = 0.05f;

    float3 integrated_color = (float3) (0.0f, 0.0f, 0.0f);
    float integrated_value = 0.0f;

    int mode = 0;
    int index = 2;
    float3 slice_diagonal_dir = (float3) (0.0f, 0.0f, 0.0f);

    const float cov = 0.8f;
    if (index == 0) {
       slice_diagonal_dir.x = 1.0f;
    } else if (index == 1) {
       slice_diagonal_dir.y = 1.0f;
    } else if (index == 2) {
       slice_diagonal_dir.z = 1.0f;
    }

    const float normalizer = 1.0f / sqrt(2.0f * M_PI_F * cov);

    // TODO: Consider an exponential step size
    /*for (float dist_m = 0.0f; dist_m < 10.0f; dist_m += step_size) {
        const float3 sample_pos_m = ray_origin_m + (dist_m * ray_dir_world);

        const float4 sample_pos_vx_frame = (float4) ((sample_pos_m - volume_origin_m) / volume_m_per_px, 1.0f);

        const float4 image_color = read_imagef(volume, smp_volume, sample_pos_vx_frame);
        integrated_color += image_color.xyz * step_size * 1.0f;
    }*/

    const float density = 0.1;

    for (float dist_m = 0.0f; dist_m < 10.0f; dist_m += step_size) {
        const float3 sample_pos_m = ray_origin_m + (dist_m * ray_dir_world);

        const float4 sample_pos_vx_frame = (float4) ((sample_pos_m - volume_origin_m) / volume_m_per_px, 1.0f);

        const float4 image_color = read_imagef(volume, smp_volume, sample_pos_vx_frame);
        // integrated_color += image_color.xyz * step_size * 1.0f;
        // integrated_value +=
        // integrated_color.w *= density;
        integrated_color.xyz += (density * image_color.xyz) * (1.0f - integrated_value);
        integrated_value += image_color.z * density;
        if (integrated_value > 1.0) {
            break;
        }
    }

    const float color_val = scaling * dot(slice_diagonal_dir, integrated_color);

    float3 color;
    if (mode == 0) {
        color = integrated_color;
    } else if (mode == 1) {
        color.x = step(0.0f, color_val) * color_val;
        color.y = 0.0f;
        color.z = -step(0.0f, -color_val) * color_val;
    }

    const float3 power_adjusted_color = pow(clamp(color.xyz, 0.0f, 1.0f), 0.45f);
    const float4 final_color = (float4) (power_adjusted_color, 1.0f);
    write_imagef(rendered_image, px_coord, 255.0f * final_color);
}

#include "liegroups.clh"

__kernel void populate_test_volume(
    __write_only image3d_t volume,
    float t) {
    const int3 vxl_coord = (int3) (get_global_id(0), get_global_id(1), get_global_id(2));

    const float3 volume_size = (float3) (
        get_image_width(volume),
        get_image_height(volume),
        get_image_depth(volume)
    );

    float3 vxl_coordf;
    vxl_coordf.x = (float) vxl_coord.x;
    vxl_coordf.y = (float) vxl_coord.y;
    vxl_coordf.z = (float) vxl_coord.z;

    const float3 world_pt = vxl_coordf - (volume_size / 2.0f);

    const int4 write_vxl_coord = (int4) (vxl_coord, 0);

    const float3 rcolor = 25.0f * world_pt / (dot(world_pt, world_pt));
    const float4 color = (float4) (rcolor, 1.0f);

    write_imagef(volume, write_vxl_coord, color);
}

// TODO: Volumetric Billboards
__kernel void render_volume(
    __read_only image2d_t ray_lut,
    __read_only image3d_t volume,
    __write_only image2d_t rendered_image,
    const struct clSE3 world_from_camera,
    const float scaling,
    const float slice_coord) {
    const sampler_t smp_ray = CLK_NORMALIZED_COORDS_FALSE |
                              CLK_ADDRESS_CLAMP |
                              CLK_FILTER_NEAREST;

    const sampler_t smp_volume = CLK_NORMALIZED_COORDS_FALSE |
                                 CLK_ADDRESS_CLAMP |
                                 CLK_FILTER_NEAREST;

    const int2 px_coord = (int2) (get_global_id(0), get_global_id(1));

    const float3 ray_dir_camera = read_imagef(ray_lut, smp_ray, px_coord).xyz;
    const float3 ray_dir_world = mul_so3(world_from_camera.rotation, ray_dir_camera);
    const float3 ray_origin_m = world_from_camera.translation;

    const float3 volume_dims_px = (float3) (
        get_image_width(volume),
        get_image_height(volume),
        get_image_depth(volume)
    );

    const float volume_size_m = 1.0f;
    const float volume_m_per_px = volume_size_m / volume_dims_px.x;

    // const float3 volume_origin_m = -(volume_m_per_px * volume_dims_px) / 2.0f;
    // const float3 volume_origin_m = (-2.0f, -2.0f, -2.0f);
    const float3 volume_origin_m = (float3) (0.0f, 0.0f, 0.0f);

    const float step_size = 0.05f;

    float3 integrated_color = (float3) (0.0f, 0.0f, 0.0f);
    float integrated_value = 0.0f;

    int mode = 0;
    int index = 2;
    float3 slice_diagonal_dir = (float3) (0.0f, 0.0f, 0.0f);

    const float cov = 0.8f;
    if (index == 0) {
       slice_diagonal_dir.x = 1.0f;
    } else if (index == 1) {
       slice_diagonal_dir.y = 1.0f;
    } else if (index == 2) {
       slice_diagonal_dir.z = 1.0f;
    }

    const float normalizer = 1.0f / sqrt(2.0f * M_PI_F * cov);

    // TODO: Consider an exponential step size
    for (float dist_m = 0.0f; dist_m < 10.0f; dist_m += step_size) {
        const float3 sample_pos_m = ray_origin_m + (dist_m * ray_dir_world);

        const float4 sample_pos_vx_frame = (float4) ((sample_pos_m - volume_origin_m) / volume_m_per_px, 1.0f);

        // const float error = dot(sample_pos_vx_frame.xyz, slice_diagonal_dir) - slice_coord;
        // const float mahalanobis_d = -error * error * cov;
        // const float weighting = normalizer * exp(0.5 * mahalanobis_d);

        const float4 image_color = read_imagef(volume, smp_volume, sample_pos_vx_frame);
        integrated_color += image_color.xyz * step_size * 1.0f;
        // integrated_color = min(integrated_color, error);
    }

    const float color_val = scaling * dot(slice_diagonal_dir, integrated_color);

    float3 color;
    if (mode == 0) {
        color = integrated_color;
    } else if (mode == 1) {
        color.x = step(0.0f, color_val) * color_val;
        color.y = 0.0f;
        color.z = -step(0.0f, -color_val) * color_val;
    }

    const float3 power_adjusted_color = pow(clamp(color.xyz, 0.0f, 1.0f), 0.45f);
    const float4 final_color = (float4) (power_adjusted_color, 1.0f);
    write_imagef(rendered_image, px_coord, 255.0f * final_color);
}


Performance improvements for backward convolutions in MLOpenConvUni kernel

It covers few thousand % perf difference between HSAIL and LC in
backward convolutions for 4 i32x79x341x5x10x32x38x166* tests and
in 3x224x224x11x11x64x55x55x128xNCHWxFP32x1. Both stacks improve
score.

Usual transformations done, such as adding restrict keyword to
pointer operands, switch to uint, avoid fp divisions in cases when
it is faster to use int etc. There is also a black magic around
inner result store loop, which is needed until LC learns to handle
register pressure in this case.

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#ifndef MLO_FILTER_STRIDE0
#define MLO_FILTER_STRIDE0 1
#endif
#ifndef MLO_FILTER_STRIDE1
#define MLO_FILTER_STRIDE1 1
#endif

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_TILE1*MLO_OUT_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_N_ALUTILES_TOTAL ((MLO_GRP_TILE0*MLO_GRP_TILE1)/(MLO_ALU_TILE_SZ))
#define MLO_N_ALUTILES_PERSTACK (MLO_N_ALUTILES_TOTAL/MLO_N_STACKS)
#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)
/*
#define MLO_N_OUT_TILES_PERSTACK (MLO_N_OUT_TILES*MLO_N_ALUTILES_PERSTACK)
#if MLO_N_OUT_TILES_PERSTACK > MLO_N_OUTPUTS
#undef MLO_N_OUT_TILES_PERSTACK
#define MLO_N_OUT_TILES_PERSTACK MLO_N_OUTPUTS
#endif
*/
#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  ((MLO_N_INPUTS+MLO_N_IN_TILES_PERSTACK-1)/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#if MLO_DIR_FORWARD == 1
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)  // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 - 1) * MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#else
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0) // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1 + MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#if MLO_DIR_FORWARD == 1
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 - 1) * MLO_FILTER_STRIDE1 + 1)
#else
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 +                        MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif

#define MLO_LCL_WEIGHTS 1

#define MLO_PADDING_SHIFT1 (MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1)
#define MLO_PADDING_SHIFT0 (MLO_FILTER_SIZE0 - MLO_FILTER_PAD0 - 1)

#if defined(__AMDGCN__)
extern uint __llvm_amdgcn_readfirstlane(uint) __asm("llvm.amdgcn.readfirstlane");
#define uniform(x) __llvm_amdgcn_readfirstlane(x)
#else
#define uniform(x) (x)
#endif

static inline uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
	return(r);
}

static inline uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

static inline void calculateXYPos(uint linPos, uint width, uint * __restrict x, uint * __restrict y)
{
	(*y) = (uint)((float)linPos * (1.0f / (float)width) + 0.00001f);
	(*x) = linPos - mul24((*y), width);
}

static inline uint calculateOffset(uint stride, uint x, uint y)
{
	uint ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(uint linPos,__local _FLOAT *lcl_data, int lcl_base, uint lcl_height, uint lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, uint gbl_height, uint gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug)
{
	uint x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	uint gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	int gbl_off = gbl_off0 + gbl_base;

	int l_x = x + lcl_x;
	int l_y = y + lcl_y;
	int lcl_off = lcl_base +
#if MLO_LARGE_MAP == 1
		linPos
#else
		mad24(l_y, lcl_stride, l_x);
#endif
	 ;

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(uint lcl_id, int size, int lcl_p_stride, __local _FLOAT *lcl_data, int lcl_base, uint lcl_height, uint lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, uint gbl_height, uint gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(uint i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(uint lcl_id, int lcl_p_stride,
					__local _FLOAT * lcl_data,
					int lcl_off, int lcl_size, uint lcl_height, uint lcl_width, int lcl_stride, int lcl_bot_y, int lcl_bot_x,
					const __global _FLOAT * gbl_data,
					int gbl_off, int gbl_size, uint gbl_height, uint glb_width, int gbl_stride, int gbl_bot_y, int gbl_bot_x,
					int buf_block_ind, int max_n_bufs, int lcl_n_bufs,
					bool debug)
{

	for(uint c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(uint o_map_base,
				int in_stg_off,
				__private _FLOAT * __restrict pvt_in_stage, __local _FLOAT * __restrict lcl_indata,
				__private _FLOAT * __restrict pvt_wei_stage, __local _FLOAT * __restrict lcl_wei,
				__private _FLOAT * __restrict pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		int in_stg_off1 = in_stg_off;
		for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			int wei_stg_base_off = mad24(o_map_base, (uint)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(uint)MLO_FILTER_SZ));
			int in_stg_off2 = in_stg_off1;
			for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#if MLO_DIR_FORWARD == 1
			for(uint k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
#else
			for(uint k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2 += (((k - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1%MLO_OUT_TILE1)) % MLO_FILTER_STRIDE1) ? 0 : MLO_IN_LCL_WIDTH)
			)
#endif
			{
				int k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(uint i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					int wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(uint i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] =  lcl_wei[wei_stg_off + i]; //(float)o_c/(float)MLO_N_OUT_TILES + (float)(i+k)/9;
					}

		// actual conv

					for( uint j = 0; j < MLO_OUT_TILE1; ++j)
					{
#if MLO_DIR_FORWARD == 0
						if (((j + k + 1 - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1 % MLO_FILTER_STRIDE1)) % MLO_FILTER_STRIDE1) == 0)
#endif
						for(uint i = 0; i < MLO_OUT_TILE0; ++i)
						{
							for(uint l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								int l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

#if MLO_DIR_FORWARD == 1
								pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								     += pvt_in_stage[j * MLO_PVT_IN_WIDTH * MLO_FILTER_STRIDE1 + i * MLO_FILTER_STRIDE0 + l] * pvt_wei_stage[l_act];
#else
								if (((i + l + 1 - MLO_PADDING_SHIFT0 + (MLO_FILTER_SIZE0 % MLO_FILTER_STRIDE0)) % MLO_FILTER_STRIDE0) == 0)
								{
									pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
										+= pvt_in_stage[(j / MLO_FILTER_STRIDE1) * MLO_PVT_IN_WIDTH + (i + l) / MLO_FILTER_STRIDE0] * pvt_wei_stage[l_act];
								}
#endif
							}
						}

					}

				} // for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

			} // for(uint k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenConvUni(
       const __global _FLOAT * __restrict in,
       const __global _FLOAT * __restrict weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
	  __global _FLOAT * __restrict out,
	   _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	uint grp_id0 = get_group_id(0);
#if MLO_N_OUT_TILE_BLOCKS0 & (MLO_N_OUT_TILE_BLOCKS0 - 1)
	uint y_tile_blk = iDiv(grp_id0, MLO_N_OUT_TILE_BLOCKS0);
	uint x_tile_blk = iMod(grp_id0, y_tile_blk, MLO_N_OUT_TILE_BLOCKS0);
#else
	uint y_tile_blk = grp_id0 / MLO_N_OUT_TILE_BLOCKS0;
	uint x_tile_blk = grp_id0 & (MLO_N_OUT_TILE_BLOCKS0 - 1);
#endif
	uint o_pack = get_group_id(1); // block of outputs
	uint b_pack = get_group_id(2); // batch block

	uint lcl_id = get_local_id(0);
#if MLO_ALUTILES_STACK_SZ & (MLO_ALUTILES_STACK_SZ - 1)
	uint stack = iDiv(lcl_id, MLO_ALUTILES_STACK_SZ);  // stack
	uint alu_stack_id = iMod(lcl_id, stack, MLO_ALUTILES_STACK_SZ);  // alu index in stack
#else
	uint stack = lcl_id / MLO_ALUTILES_STACK_SZ;  // stack
	uint alu_stack_id = lcl_id & (MLO_ALUTILES_STACK_SZ - 1);  // alu index in stack
#if MLO_ALUTILES_STACK_SZ >= 64
	stack = uniform(stack);
#endif
#endif
// ALU plane inside stack
#if MLO_ALU_TILE_SZ & (MLO_ALU_TILE_SZ - 1)
	uint alu_out_plane_id = iDiv(alu_stack_id, MLO_ALU_TILE_SZ);  // alu output plane index
	uint alu_out_id = iMod(alu_stack_id, alu_out_plane_id, MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
#else
	uint alu_out_plane_id = alu_stack_id / MLO_ALU_TILE_SZ;  // alu output plane index
	uint alu_out_id = alu_stack_id & (MLO_ALU_TILE_SZ - 1); // alu index inside an ALU output plane
#endif
// pos inside ALU tile
#if MLO_ALU_VTILE0 & (MLO_ALU_VTILE0 - 1)
	uint alu_tl1 = iDiv(alu_out_id, MLO_ALU_VTILE0);
	uint alu_tl0 = iMod(alu_out_id, alu_tl1, MLO_ALU_VTILE0);
#else
	uint alu_tl1 = alu_out_id / MLO_ALU_VTILE0;
	uint alu_tl0 = alu_out_id & (MLO_ALU_VTILE0 - 1);
#endif

	uint o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	uint o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	uint o_map = o_map_plane + o_map_base; // output map index per ALU plane
	uint b_index = b_pack * MLO_N_STACKS;

#if MLO_N_READ_PROCS & (MLO_N_READ_PROCS - 1)
	uint wave_id = iDiv(lcl_id, MLO_N_READ_PROCS);
	uint wave_lcl_id = iMod(lcl_id, wave_id, MLO_N_READ_PROCS);
#else
	uint wave_id = lcl_id / MLO_N_READ_PROCS;
	uint wave_lcl_id = lcl_id & (MLO_N_READ_PROCS - 1);
#if MLO_N_READ_PROCS >= 64
	wave_id = uniform(wave_id);
#endif
#endif

#if MLO_DIR_FORWARD == 1
	uint x_grp = x_tile_blk * MLO_IN_TILE0 * MLO_FILTER_STRIDE0;
	uint y_grp = y_tile_blk * MLO_IN_TILE1 * MLO_FILTER_STRIDE1;

	uint x_in_grp = x_grp - MLO_FILTER_PAD0;
	uint y_in_grp = y_grp - MLO_FILTER_PAD1;

	uint x_in_lcl = alu_tl0 * MLO_OUT_TILE0 * MLO_FILTER_STRIDE0;
	uint y_in_lcl = alu_tl1 * MLO_OUT_TILE1 * MLO_FILTER_STRIDE1;
#else
	uint x_grp = x_tile_blk * (MLO_IN_TILE0 / MLO_FILTER_STRIDE0);
	uint y_grp = y_tile_blk * (MLO_IN_TILE1 / MLO_FILTER_STRIDE1);
	uint x_in_grp = x_grp - (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE0);
	uint y_in_grp = y_grp - (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE1);
	uint x_in_lcl = alu_tl0 * (MLO_OUT_TILE0 / MLO_FILTER_STRIDE0);
	uint y_in_lcl = alu_tl1 * (MLO_OUT_TILE1 / MLO_FILTER_STRIDE1);
#endif

// base offset to read data from local input data
	uint in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    uint in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	uint wei_off = mul24(o_map_plane, (uint)(MLO_N_INPUTS * MLO_FILTER_SZ));
#else
	uint wei_off = mul24(o_map_plane,(uint)MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(uint i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(uint i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(uint ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		int in_lcl_off1 = 0;
		int in_off1 = in_off;
		for(uint i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			int in_off2 = in_off1;
			int in_lcl_off2 = in_lcl_off1;
			for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				uint elem_id = lcl_id;
				int lcl_p_stride = MLO_GRP_SZ0;
				int lcl_base = 0;
				int lcl_y = 0;
				int lcl_x = 0;
				int gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(uint i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
#if MLO_N_IN_TILES_PERSTACK & (MLO_N_IN_TILES_PERSTACK - 1)
			uint i_b = iDiv(i, MLO_N_IN_TILES_PERSTACK);
			uint i_c = iMod(i, i_b, MLO_N_IN_TILES_PERSTACK);
#else
			uint i_b = i / MLO_N_IN_TILES_PERSTACK;
			uint i_c = i & (MLO_N_IN_TILES_PERSTACK - 1);
#endif

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			int in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			int in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			uint elem_id = wave_lcl_id;
			int lcl_p_stride = MLO_N_READ_PROCS;
			int lcl_base = 0;
#if MLO_DIR_FORWARD == 1
			int lcl_y = MLO_FILTER_PAD1;
			int lcl_x = MLO_FILTER_PAD0;
#else
			int lcl_y = (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE0);
			int lcl_x = (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE1);
#endif
			int gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(uint i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
#if (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = iDiv(i, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			uint gbl_i = iMod(i, lcl_o, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
			uint gbl_we_off = wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i;
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);

			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[i] = wei;
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)
#if (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = iDiv(i, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			uint gbl_i = iMod(i, lcl_o, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
#if MLO_FILTER_SZ & (MLO_FILTER_SZ - 1)
			uint lcl_c = iDiv(gbl_i, MLO_FILTER_SZ);
			uint lcl_i = iMod(gbl_i, lcl_c, MLO_FILTER_SZ);
#else
			uint lcl_c = gbl_i / MLO_FILTER_SZ;
			uint lcl_i = gbl_i & (MLO_FILTER_SZ - 1);
#endif

			uint lcl_we_off = mad24(mad24(lcl_c, (uint)MLO_N_IN_TILES_PERSTACK, lcl_o), (uint)MLO_FILTER_SZ, lcl_i);
			uint gbl_we_off = mad24(mad24(lcl_o, (uint)MLO_N_OUTPUTS, lcl_c), (uint)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}
// write results out
#if MLO_DIR_FORWARD == 1
#if MLO_FILTER_STRIDE0 == 1
	int x_out_grp = x_grp;
#else
	int x_out_grp = x_tile_blk * MLO_IN_TILE0;
#endif
#if MLO_FILTER_STRIDE1 == 1
	int y_out_grp = y_grp;
#else
	int y_out_grp = y_tile_blk * MLO_IN_TILE1;
#endif
#else
	int x_out_grp = x_grp * MLO_FILTER_STRIDE0;
	int y_out_grp = y_grp * MLO_FILTER_STRIDE1;
#endif
	int x_out_lcl = alu_tl0 * MLO_OUT_TILE0;
	int y_out_lcl = alu_tl1 * MLO_OUT_TILE1;

    uint out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		int out_off1 = out_off;
		for(uint o = 0; o < MLO_N_OUT_TILES; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE)
		{
#if MLO_OUTPUTS_ALIGNED == 0
			if (o_map + o < MLO_N_OUTPUTS)
#endif
			{
// over output tile
				int out_off2 = out_off1;
#if MLO_OUT_TILE0 == 1
				for( int j = 0; j < MLO_OUT_TILE1 && y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT; ++j, out_off2 += MLO_OUT_STRIDE)
				{
					for(int i = 0; i < MLO_OUT_TILE0 && x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH && out_off2 + i < MLO_OUT_BATCH_STRIDE*MLO_BATCH_SZ; ++i)
					{
#else
				for( uint j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
				{
					if (y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT)
						for(uint i = 0; i < MLO_OUT_TILE0; ++i)
						{
							if (x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH && out_off2 + i < MLO_OUT_BATCH_STRIDE*MLO_BATCH_SZ)
#endif
								out[out_off2 + i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_TILE0 + i]
#if MLO_CONV_BIAS
									+ bias_val = bias[o_map + o];
#endif
									;

						}
				}
			}

		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#ifndef MLO_FILTER_STRIDE0
#define MLO_FILTER_STRIDE0 1
#endif
#ifndef MLO_FILTER_STRIDE1
#define MLO_FILTER_STRIDE1 1
#endif

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_TILE1*MLO_OUT_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_N_ALUTILES_TOTAL ((MLO_GRP_TILE0*MLO_GRP_TILE1)/(MLO_ALU_TILE_SZ))
#define MLO_N_ALUTILES_PERSTACK (MLO_N_ALUTILES_TOTAL/MLO_N_STACKS)
#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)
/*
#define MLO_N_OUT_TILES_PERSTACK (MLO_N_OUT_TILES*MLO_N_ALUTILES_PERSTACK)
#if MLO_N_OUT_TILES_PERSTACK > MLO_N_OUTPUTS
#undef MLO_N_OUT_TILES_PERSTACK
#define MLO_N_OUT_TILES_PERSTACK MLO_N_OUTPUTS
#endif
*/
#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  ((MLO_N_INPUTS+MLO_N_IN_TILES_PERSTACK-1)/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#if MLO_DIR_FORWARD == 1
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)  // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 - 1) * MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#else
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0) // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1 + MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#if MLO_DIR_FORWARD == 1
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 - 1) * MLO_FILTER_STRIDE1 + 1)
#else
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 +                        MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif

#define MLO_LCL_WEIGHTS 1

#define MLO_PADDING_SHIFT1 (MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1)
#define MLO_PADDING_SHIFT0 (MLO_FILTER_SIZE0 - MLO_FILTER_PAD0 - 1)

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void calculateXYPos(int linPos, int width, int *x, int *y)
{
	(*y) = (int)((float)linPos * (1.0f / (float)width) + 0.00001f);
	(*x) = linPos - mul24((*y), width);
}

static inline int calculateOffset(int stride, int x, int y)
{
	int ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(int linPos,__local _FLOAT *lcl_data, int lcl_base, int lcl_height, int lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, int gbl_height, int gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug)
{
	int x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	int gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	int gbl_off = gbl_off0 + gbl_base;

	int l_x = x + lcl_x;
	int l_y = y + lcl_y;
	int lcl_off = lcl_base +
#if MLO_LARGE_MAP == 1
		linPos
#else
		mad24(l_y, lcl_stride, l_x);
#endif
	 ;

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(int lcl_id, int size, int lcl_p_stride, __local _FLOAT *lcl_data, int lcl_base, int lcl_height, int lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, int gbl_height, int gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(int i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(int lcl_id, int lcl_p_stride,
					__local _FLOAT * lcl_data,
					int lcl_off, int lcl_size, int lcl_height, int lcl_width, int lcl_stride, int lcl_bot_y, int lcl_bot_x,
					const __global _FLOAT * gbl_data,
					int gbl_off, int gbl_size, int gbl_height, int glb_width, int gbl_stride, int gbl_bot_y, int gbl_bot_x,
					int buf_block_ind, int max_n_bufs, int lcl_n_bufs,
					bool debug)
{

	for(int c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(int o_map_base,
				int in_stg_off,
				__private _FLOAT *pvt_in_stage, __local _FLOAT * lcl_indata,
				__private _FLOAT *pvt_wei_stage, __local _FLOAT * lcl_wei,
				__private _FLOAT *pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		int in_stg_off1 = in_stg_off;
		for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			int wei_stg_base_off = mad24(o_map_base, (int)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(int)MLO_FILTER_SZ));
			int in_stg_off2 = in_stg_off1;
			for(int j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(int i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#if MLO_DIR_FORWARD == 1
			for(int k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
#else
			for(int k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2 += (((k - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1%MLO_OUT_TILE1)) % MLO_FILTER_STRIDE1) ? 0 : MLO_IN_LCL_WIDTH)
			)
#endif
			{
				int k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(int i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(int o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					int wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(int i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] =  lcl_wei[wei_stg_off + i]; //(float)o_c/(float)MLO_N_OUT_TILES + (float)(i+k)/9;
					}

		// actual conv

					for( int j = 0; j < MLO_OUT_TILE1; ++j)
					{
#if MLO_DIR_FORWARD == 0
						if (((j + k + 1 - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1 % MLO_FILTER_STRIDE1)) % MLO_FILTER_STRIDE1) == 0)
#endif
						for(int i = 0; i < MLO_OUT_TILE0; ++i)
						{
							for(int l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								int l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

#if MLO_DIR_FORWARD == 1
								pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								     += pvt_in_stage[j * MLO_PVT_IN_WIDTH * MLO_FILTER_STRIDE1 + i * MLO_FILTER_STRIDE0 + l] * pvt_wei_stage[l_act];
#else
								if (((i + l + 1 - MLO_PADDING_SHIFT0 + (MLO_FILTER_SIZE0 % MLO_FILTER_STRIDE0)) % MLO_FILTER_STRIDE0) == 0)
								{
									pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
										+= pvt_in_stage[(j / MLO_FILTER_STRIDE1) * MLO_PVT_IN_WIDTH + (i + l) / MLO_FILTER_STRIDE0] * pvt_wei_stage[l_act];
								}
#endif
							}
						}

					}

				} // for(int o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(int j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(int i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

			} // for(int k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenConvUni(
       const __global _FLOAT * in,
       const __global _FLOAT * weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * bias,
#endif
 	  __global _FLOAT *out,
	   _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	int grp_id0 = get_group_id(0);
	int y_tile_blk = iDiv(grp_id0, MLO_N_OUT_TILE_BLOCKS0);
	int x_tile_blk = iMod(grp_id0, y_tile_blk, MLO_N_OUT_TILE_BLOCKS0);
	int o_pack = get_group_id(1); // block of outputs
	int b_pack = get_group_id(2); // batch block

	int lcl_id = get_local_id(0);
	int stack = iDiv(lcl_id, MLO_ALUTILES_STACK_SZ);  // stack
	int alu_stack_id = iMod(lcl_id, stack, MLO_ALUTILES_STACK_SZ);  // alu index in stack
// ALU plane inside stack
	int alu_out_plane_id = iDiv(alu_stack_id, MLO_ALU_TILE_SZ);  // alu output plane index
	int alu_out_id = iMod(alu_stack_id, alu_out_plane_id, MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
// pos inside ALU tile
	int alu_tl1 = iDiv(alu_out_id, MLO_ALU_VTILE0);
	int alu_tl0 = iMod(alu_out_id, alu_tl1, MLO_ALU_VTILE0);

	int o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	int o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	int o_map = o_map_plane + o_map_base; // output map index per ALU plane
	int b_index = b_pack * MLO_N_STACKS;

	int wave_id = iDiv(lcl_id, MLO_N_READ_PROCS);
	int wave_lcl_id = iMod(lcl_id, wave_id, MLO_N_READ_PROCS);

#if MLO_DIR_FORWARD == 1
	int x_grp = x_tile_blk * MLO_IN_TILE0 * MLO_FILTER_STRIDE0;
	int y_grp = y_tile_blk * MLO_IN_TILE1 * MLO_FILTER_STRIDE1;

	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;

	int x_in_lcl = alu_tl0 * MLO_OUT_TILE0 * MLO_FILTER_STRIDE0;
	int y_in_lcl = alu_tl1 * MLO_OUT_TILE1 * MLO_FILTER_STRIDE1;
#else
	int x_grp = x_tile_blk * (MLO_IN_TILE0 / MLO_FILTER_STRIDE0);
	int y_grp = y_tile_blk * (MLO_IN_TILE1 / MLO_FILTER_STRIDE1);
	int x_in_grp = x_grp - (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE0);
	int y_in_grp = y_grp - (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE1);
	int x_in_lcl = alu_tl0 * (MLO_OUT_TILE0 / MLO_FILTER_STRIDE0);
	int y_in_lcl = alu_tl1 * (MLO_OUT_TILE1 / MLO_FILTER_STRIDE1);
#endif

// base offset to read data from local input data
	int in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    int in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	int wei_off = mul24(o_map_plane, MLO_N_INPUTS * MLO_FILTER_SZ);
#else
	int wei_off = mul24(o_map_plane,MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(int i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(int i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(int ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		int in_lcl_off1 = 0;
		int in_off1 = in_off;
		for(int i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			int in_off2 = in_off1;
			int in_lcl_off2 = in_lcl_off1;
			for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				int elem_id = lcl_id;
				int lcl_p_stride = MLO_GRP_SZ0;
				int lcl_base = 0;
				int lcl_y = 0;
				int lcl_x = 0;
				int gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(int i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
			int i_b = iDiv(i, MLO_N_IN_TILES_PERSTACK);
			int i_c = iMod(i, i_b, MLO_N_IN_TILES_PERSTACK);

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			int in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			int in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			int elem_id = wave_lcl_id;
			int lcl_p_stride = MLO_N_READ_PROCS;
			int lcl_base = 0;
#if MLO_DIR_FORWARD == 1
			int lcl_y = MLO_FILTER_PAD1;
			int lcl_x = MLO_FILTER_PAD0;
#else
			int lcl_y = (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE0);
			int lcl_x = (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE1);
#endif
			int gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(int i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
			int lcl_o = iDiv(i, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_i = iMod(i, lcl_o, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_we_off = wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i;
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);

			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[i] = wei;
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)
			int lcl_o = iDiv(i, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_i = iMod(i, lcl_o, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			int lcl_c = iDiv(gbl_i, MLO_FILTER_SZ);
			int lcl_i = iMod(gbl_i, lcl_c, MLO_FILTER_SZ);

			int lcl_we_off = mad24(mad24(lcl_c, (int)MLO_N_IN_TILES_PERSTACK, lcl_o), (int)MLO_FILTER_SZ, lcl_i);
			int gbl_we_off = mad24(mad24(lcl_o, (int)MLO_N_OUTPUTS, lcl_c), (int)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}
// write results out
#if MLO_DIR_FORWARD == 1
#if MLO_FILTER_STRIDE0 == 1
	int x_out_grp = x_grp;
#else
	int x_out_grp = x_tile_blk * MLO_IN_TILE0;
#endif
#if MLO_FILTER_STRIDE1 == 1
	int y_out_grp = y_grp;
#else
	int y_out_grp = y_tile_blk * MLO_IN_TILE1;
#endif
#else
	int x_out_grp = x_grp * MLO_FILTER_STRIDE0;
	int y_out_grp = y_grp * MLO_FILTER_STRIDE1;
#endif
	int x_out_lcl = alu_tl0 * MLO_OUT_TILE0;
	int y_out_lcl = alu_tl1 * MLO_OUT_TILE1;

    int out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		int out_off1 = out_off;
		for(int o = 0; o < MLO_N_OUT_TILES
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile

			int out_off2 = out_off1;
			for( int j = 0; j < MLO_OUT_TILE1 && y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				for(int i = 0; i < MLO_OUT_TILE0 && x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH && out_off2 + i < MLO_OUT_BATCH_STRIDE*MLO_BATCH_SZ; ++i)
				{
						out[out_off2 + i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_TILE0 + i]
#if MLO_CONV_BIAS
						+ bias_val = bias[o_map + o];
#endif
						 ;

				}
			}

		}
	}

}