Reduce vgpr usage

Reduce vgpr usage

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

/*************************************************************************
INCLUDES
**************************************************************************/
#include <../RadeonRays/src/kernels/CL/common.cl>
/*************************************************************************
EXTENSIONS
**************************************************************************/

/*************************************************************************
TYPE DEFINITIONS
**************************************************************************/
#define STARTIDX(x) (((int)(x.pmin.w)) >> 4)
#define NUMPRIMS(x) (((int)(x.pmin.w)) & 0xF)
#define LEAFNODE(x) (((x).pmin.w) != -1.f)
#define CHILD(x, i) ((int)x.bounds[i].pmax.w)
#define SHORT_STACK_SIZE 16
#define WAVEFRONT_SIZE 64

//#define GLOBAL_STACK

typedef struct
{
    union {
        struct
        {
            bbox lbound;
            bbox rbound;
        };

        struct
        {
            bbox bounds[2];
        };
    };

} FatBvhNode;

typedef struct
{
    // BVH structure
    __global FatBvhNode const *nodes;
    // Scene positional data
    __global float3 const *vertices;
    // Scene indices
    __global Face const *faces;
    // Shape IDs
    __global ShapeData const *shapes;
    // Extra data
    __global int const *extra;
} SceneData;

/*************************************************************************
HELPER FUNCTIONS
**************************************************************************/

/*************************************************************************
BVH FUNCTIONS
**************************************************************************/
//  intersect a ray with leaf BVH node
inline void IntersectLeafClosest(
    SceneData const *scenedata,
    int startidx,
    int numprims,
    ray const *r,       // ray to instersect
    Intersection *isect // Intersection structure
    )
{
    float3 v1, v2, v3;
    Face face;
    for (int i = 0; i < numprims; ++i)
    {
        face = scenedata->faces[startidx + i];
        v1 = scenedata->vertices[face.idx[0]];
        v2 = scenedata->vertices[face.idx[1]];
        v3 = scenedata->vertices[face.idx[2]];

#ifdef RR_RAY_MASK
        int shapemask = scenedata->shapes[face.shapeidx].mask;

        if (Ray_GetMask(r) & shapemask)
#endif
        {
            if (IntersectTriangle(r, v1, v2, v3, isect))
            {
                isect->primid = face.id;
                isect->shapeid = face.shapeidx;
            }
        }
    }
}

//  intersect a ray with leaf BVH node
inline bool IntersectLeafAny(
    SceneData const *scenedata,
    int startidx,
    int numprims,
    ray const *r // ray to instersect
    )
{
    float3 v1, v2, v3;
    Face face;

    bool hit = false;
    for (int i = 0; i < numprims; ++i)
    {
        face = scenedata->faces[startidx + i];
        v1 = scenedata->vertices[face.idx[0]];
        v2 = scenedata->vertices[face.idx[1]];
        v3 = scenedata->vertices[face.idx[2]];

#ifdef RR_RAY_MASK
        int shapemask = scenedata->shapes[face.shapeidx].mask;

        if (Ray_GetMask(r) & shapemask)
#endif
        {
            if (IntersectTriangleP(r, v1, v2, v3))
            {
                hit = true;
                break;
            }
        }
    }

    return hit;
}

inline bool intersect(
    SceneData const *scenedata,
    ray const *r,
    Intersection *isect,
    __global int *displacement,
    __global int *hash,
    int t)
{
    const float3 invdir = native_recip(r->d.xyz);
    isect->uvwt = make_float4(0.f, 0.f, 0.f, r->o.w);
    isect->shapeid = -1;
    isect->primid = -1;

    int bittrail = 0;
    int nodeidx = 1;
    int addr = 0;

    FatBvhNode node;

    while (addr > -1)
    {
        node = scenedata->nodes[addr];
        bool l0 = LEAFNODE(node.lbound);
        bool l1 = LEAFNODE(node.rbound);
        float d0 = -1.f;
        float d1 = -1.f;

        if (l0)
        {
            IntersectLeafClosest(scenedata, STARTIDX(node.lbound), NUMPRIMS(node.lbound), r, isect);
        }
        else
        {
            d0 = IntersectBoxF(r, invdir, node.lbound, isect->uvwt.w);
        }

        if (l1)
        {
            IntersectLeafClosest(scenedata, STARTIDX(node.rbound), NUMPRIMS(node.rbound), r, isect);
        }
        else
        {
            d1 = IntersectBoxF(r, invdir, node.rbound, isect->uvwt.w);
        }

        if (d0 > 0 && d1 > 0)
        {
            bittrail = bittrail << 1;
            nodeidx = nodeidx << 1;
            bittrail = bittrail ^ 0x1;

            if (d0 > d1)
            {
                addr = CHILD(node, 1);
                nodeidx = nodeidx ^ 0x1;
            }
            else
            {
                addr = CHILD(node, 0);
            }
            continue;
        }
        else if (d0 > 0)
        {
            bittrail = bittrail << 1;
            nodeidx = nodeidx << 1;
            addr = CHILD(node, 0);
            continue;
        }
        else if (d1 > 0)
        {
            bittrail = bittrail << 1;
            nodeidx = nodeidx << 1;
            nodeidx = nodeidx ^ 0x1;
            addr = CHILD(node, 1);
            continue;
        }

        if (bittrail == 0)
        {
            addr = -1;
            continue;
        }

        int num_levels = 31 - clz(bittrail & -bittrail);
        bittrail = (bittrail >> num_levels) ^ 0x1;
        nodeidx = (nodeidx >> num_levels) ^ 0x1;

        int d = displacement[nodeidx / t];
        addr = hash[d + (nodeidx & (t - 1))];
    }

    return isect->shapeid >= 0;
}

// intersect Ray against the whole BVH structure
inline bool occluded(SceneData const *scenedata, ray const *r)
{
    const float3 invdir = native_recip(r->d.xyz);

    int stack[32];
    int *sptr = stack;
    *sptr++ = -1;

    int idx = 0;
    FatBvhNode node;

    while (idx > -1)
    {
        while (idx > -1)
        {
            node = scenedata->nodes[idx];

            bool l0 = LEAFNODE(node.lbound);
            bool l1 = LEAFNODE(node.rbound);
            float d0 = -1.f;
            float d1 = -1.f;

            if (l0)
            {
                bool hit = IntersectLeafAny(scenedata, STARTIDX(node.lbound), NUMPRIMS(node.lbound), r);
                if (hit)
                {
                    return true;
                }
            }
            else
            {
                d0 = IntersectBoxF(r, invdir, node.lbound, r->o.w);
            }

            if (l1)
            {
                bool hit = IntersectLeafAny(scenedata, STARTIDX(node.rbound), NUMPRIMS(node.rbound), r);
                if (hit)
                {
                    return true;
                }
            }
            else
            {
                d1 = IntersectBoxF(r, invdir, node.rbound, r->o.w);
            }

            if (d0 > 0 && d1 > 0)
            {
                int deferred = -1;
                if (d0 > d1)
                {
                    idx = CHILD(node, 1);
                    deferred = CHILD(node, 0);
                }
                else
                {
                    idx = CHILD(node, 0);
                    deferred = CHILD(node, 1);
                }

                *sptr++ = deferred;
                continue;
            }
            else if (d0 > 0)
            {
                idx = CHILD(node, 0);
                continue;
            }
            else if (d1 > 0)
            {
                idx = CHILD(node, 1);
                continue;
            }

            idx = *--sptr;
        }
    }

    return false;
}

__attribute__((always_inline)) float IntersectTriangleLight(ray r, float3 v1, float3 v2, float3 v3, float t_max)
{
    const float3 e1 = v2 - v1;
    const float3 e2 = v3 - v1;
    const float3 s1 = cross(r.d.xyz, e2);
    const float invd = native_recip(dot(s1, e1));
    const float3 d = r.o.xyz - v1;
    const float b1 = dot(d, s1) * invd;
    const float3 s2 = cross(d, e1);
    const float b2 = dot(r.d.xyz, s2) * invd;
    const float temp = dot(e2, s2) * invd;

    if (b1 < 0.f || b1 > 1.f || b2 < 0.f || b1 + b2 > 1.f || temp < 0.f || temp > t_max)
    {
        return t_max;
    }
    else
    {
        return temp;
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
__kernel void
IntersectClosest(
    // Input
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    int numrays,                      // Number of rays to process
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global Intersection *hits // Hit datas
    )
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    SceneData scenedata =
        {
            nodes,
            vertices,
            faces,
            shapes,
            0};

    if (global_id < numrays)
    {
        // Fetch ray
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            Intersection isect;
            intersect(&scenedata, &r, &isect, displacement, hashmap, t);
            hits[global_id] = isect;
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
__kernel void
IntersectAny(
    // Input
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    int numrays,                      // Number of rays to process
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global int *hitresults // Hit results
    )
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    // Fill scene data
    SceneData scenedata =
        {
            nodes,
            vertices,
            faces,
            shapes,
            0};

    if (global_id < numrays)
    {
        // Fetch ray
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            // Calculate any intersection
            hitresults[global_id] = occluded(&scenedata, &r) ? 1 : -1;
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
// Version with range check
__kernel void
IntersectAnyRC(
    // Input
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    __global int const *numrays,      // Number of rays in the workload
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global int *hitresults // Hit results
    )
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    // Fill scene data
    SceneData scenedata =
        {
            nodes,
            vertices,
            faces,
            shapes,
            0};

    // Handle only working subset
    if (global_id < *numrays)
    {
        // Fetch ray
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            // Calculate any intersection
            hitresults[global_id] = occluded(&scenedata, &r) ? 1 : -1;
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
// Version with range check
__kernel void
IntersectClosestRC(
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    __global int const *numrays,      // Number of rays in the workload
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global Intersection *hits)
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    // Handle only working subset
    if (global_id < *numrays)
    {
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            float3 const invdir = native_recip(r.d.xyz);
            float3 const oxinvdir = -r.o.xyz * invdir;
            float t_max = r.o.w;

            int bittrail = 0;
            int nodeidx = 1;
            int addr = 0;
            int faceidx = -1;
            //int postponed = -1;

            while (addr > -1)
            {
                FatBvhNode node = nodes[addr];
                bool l0 = LEAFNODE(node.lbound);
                bool l1 = LEAFNODE(node.rbound);
                float d0 = -1.f;
                float d1 = -1.f;

                if (l0)
                {
                    Face face;
                    float3 v1, v2, v3;
                    face = faces[STARTIDX(node.lbound)];
                    v1 = vertices[face.idx[0]];
                    v2 = vertices[face.idx[1]];
                    v3 = vertices[face.idx[2]];
                    float f = IntersectTriangleLight(r, v1, v2, v3, t_max);
                    if (f < t_max)
                    {
                        faceidx = STARTIDX(node.lbound);
                        t_max = f;
                    }
                }
                else
                {
                    const float3 f = mad(node.lbound.pmax.xyz, invdir, oxinvdir);
                    const float3 n = mad(node.lbound.pmin.xyz, invdir, oxinvdir);

                    const float3 tmax = max(f, n);
                    const float3 tmin = min(f, n);

                    #ifndef AMD_MEDIA_OPS
                    const float t1 = min(min(tmax.x, min(tmax.y, tmax.z)), t_max);
                    const float t0 = max(max(tmin.x, max(tmin.y, tmin.z)), 0.f);
                    #else
                    const float t1 = min(amd_min3(tmax.x, tmax.y, tmax.z), t_max);
                    const float t0 = amd_max3(tmin.x, tmin.y, tmin.z);
                    #endif

                    if (t1 >= t0)
                    {
                        d0 = t0 > 0.f ? t0 : t1;
                    }
                }

                if (l1)
                {
                    Face face;
                    float3 v1, v2, v3;
                    face = faces[STARTIDX(node.rbound)];
                    v1 = vertices[face.idx[0]];
                    v2 = vertices[face.idx[1]];
                    v3 = vertices[face.idx[2]];
                    float f = IntersectTriangleLight(r, v1, v2, v3, t_max);
                    if (f < t_max)
                    {
                        faceidx = STARTIDX(node.rbound);
                        t_max = f;
                    }
                }
                else
                {
                    const float3 f = mad(node.rbound.pmax.xyz, invdir, oxinvdir);
                    const float3 n = mad(node.rbound.pmin.xyz, invdir, oxinvdir);

                    const float3 tmax = max(f, n);
                    const float3 tmin = min(f, n);

                    #ifndef AMD_MEDIA_OPS
                    const float t1 = min(min(tmax.x, min(tmax.y, tmax.z)), maxt);
                    const float t0 = max(max(tmin.x, max(tmin.y, tmin.z)), 0.f);
                    #else
                    const float t1 = min(amd_min3(tmax.x, tmax.y, tmax.z), t_max);
                    const float t0 = amd_max3(tmin.x, tmin.y, tmin.z);
                    #endif

                    if (t1 >= t0)
                    {
                        d1 = t0 > 0.f ? t0 : t1;
                    }
                }

                if (d0 > 0 && d1 > 0)
                {
                    bittrail = bittrail << 1;
                    nodeidx = nodeidx << 1;
                    bittrail = bittrail ^ 0x1;

                    if (d0 > d1)
                    {
                        addr = CHILD(node, 1);
                        nodeidx = nodeidx ^ 0x1;
                        //postponed = CHILD(node, 0);
                    }
                    else
                    {
                        addr = CHILD(node, 0);
                        //postponed =  CHILD(node, 1);
                    }
                    continue;
                }
                else if (d0 > 0)
                {
                    bittrail = bittrail << 1;
                    nodeidx = nodeidx << 1;
                    addr = CHILD(node, 0);
                    continue;
                }
                else if (d1 > 0)
                {
                    bittrail = bittrail << 1;
                    nodeidx = nodeidx << 1;
                    nodeidx = nodeidx ^ 0x1;
                    addr = CHILD(node, 1);
                    continue;
                }

                if (bittrail == 0)
                {
                    addr = -1;
                    continue;
                }

                int num_levels = 31 - clz(bittrail & -bittrail);
                bittrail = (bittrail >> num_levels) ^ 0x1;
                nodeidx = (nodeidx >> num_levels) ^ 0x1;

               // if (postponed != -1)
                //{
                    //addr = postponed;
                    //postponed = -1;
                    //continue;
                //}

                int d = displacement[nodeidx / t];
                addr = hashmap[d + (nodeidx & (t - 1))];
            }

            if (faceidx != -1)
            {
                Face face;
                float3 v1, v2, v3;
                face = faces[faceidx];
                hits[global_id].shapeid = face.shapeidx;
                hits[global_id].primid = face.id;
                v1 = vertices[face.idx[0]];
                v2 = vertices[face.idx[1]];
                v3 = vertices[face.idx[2]];

                const float3 e1 = v2 - v1;
                const float3 e2 = v3 - v1;
                const float3 s1 = cross(r.d.xyz, e2);
                const float  invd = native_recip(dot(s1, e1));
                const float3 d = r.o.xyz - v1;
                const float  b1 = dot(d, s1) * invd;
                const float3 s2 = cross(d, e1);
                const float  b2 = dot(r.d.xyz, s2) * invd;
                hits[global_id].uvwt = make_float4(b1, b2, 0.f, t_max);
            }
            else
            {
                hits[global_id].shapeid = -1;
                hits[global_id].primid = -1;
            }
        }
    }
}

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

/*************************************************************************
INCLUDES
**************************************************************************/
#include <../RadeonRays/src/kernels/CL/common.cl>
/*************************************************************************
EXTENSIONS
**************************************************************************/

/*************************************************************************
TYPE DEFINITIONS
**************************************************************************/
#define STARTIDX(x) (((int)(x.pmin.w)) >> 4)
#define NUMPRIMS(x) (((int)(x.pmin.w)) & 0xF)
#define LEAFNODE(x) (((x).pmin.w) != -1.f)
#define CHILD(x, i) ((int)x.bounds[i].pmax.w)
#define SHORT_STACK_SIZE 16
#define WAVEFRONT_SIZE 64

//#define GLOBAL_STACK

typedef struct
{
    union {
        struct
        {
            bbox lbound;
            bbox rbound;
        };

        struct
        {
            bbox bounds[2];
        };
    };

} FatBvhNode;

typedef struct
{
    // BVH structure
    __global FatBvhNode const *nodes;
    // Scene positional data
    __global float3 const *vertices;
    // Scene indices
    __global Face const *faces;
    // Shape IDs
    __global ShapeData const *shapes;
    // Extra data
    __global int const *extra;
} SceneData;

/*************************************************************************
HELPER FUNCTIONS
**************************************************************************/

/*************************************************************************
BVH FUNCTIONS
**************************************************************************/
//  intersect a ray with leaf BVH node
inline void IntersectLeafClosest(
    SceneData const *scenedata,
    int startidx,
    int numprims,
    ray const *r,       // ray to instersect
    Intersection *isect // Intersection structure
    )
{
    float3 v1, v2, v3;
    Face face;
    for (int i = 0; i < numprims; ++i)
    {
        face = scenedata->faces[startidx + i];
        v1 = scenedata->vertices[face.idx[0]];
        v2 = scenedata->vertices[face.idx[1]];
        v3 = scenedata->vertices[face.idx[2]];

#ifdef RR_RAY_MASK
        int shapemask = scenedata->shapes[face.shapeidx].mask;

        if (Ray_GetMask(r) & shapemask)
#endif
        {
            if (IntersectTriangle(r, v1, v2, v3, isect))
            {
                isect->primid = face.id;
                isect->shapeid = face.shapeidx;
            }
        }
    }
}

//  intersect a ray with leaf BVH node
inline bool IntersectLeafAny(
    SceneData const *scenedata,
    int startidx,
    int numprims,
    ray const *r // ray to instersect
    )
{
    float3 v1, v2, v3;
    Face face;

    bool hit = false;
    for (int i = 0; i < numprims; ++i)
    {
        face = scenedata->faces[startidx + i];
        v1 = scenedata->vertices[face.idx[0]];
        v2 = scenedata->vertices[face.idx[1]];
        v3 = scenedata->vertices[face.idx[2]];

#ifdef RR_RAY_MASK
        int shapemask = scenedata->shapes[face.shapeidx].mask;

        if (Ray_GetMask(r) & shapemask)
#endif
        {
            if (IntersectTriangleP(r, v1, v2, v3))
            {
                hit = true;
                break;
            }
        }
    }

    return hit;
}

inline bool intersect(
    SceneData const *scenedata,
    ray const *r,
    Intersection *isect,
    __global int *displacement,
    __global int *hash,
    int t)
{
    const float3 invdir = native_recip(r->d.xyz);
    isect->uvwt = make_float4(0.f, 0.f, 0.f, r->o.w);
    isect->shapeid = -1;
    isect->primid = -1;

    int bittrail = 0;
    int nodeidx = 1;
    int addr = 0;

    FatBvhNode node;

    while (addr > -1)
    {
        node = scenedata->nodes[addr];
        bool l0 = LEAFNODE(node.lbound);
        bool l1 = LEAFNODE(node.rbound);
        float d0 = -1.f;
        float d1 = -1.f;

        if (l0)
        {
            IntersectLeafClosest(scenedata, STARTIDX(node.lbound), NUMPRIMS(node.lbound), r, isect);
        }
        else
        {
            d0 = IntersectBoxF(r, invdir, node.lbound, isect->uvwt.w);
        }

        if (l1)
        {
            IntersectLeafClosest(scenedata, STARTIDX(node.rbound), NUMPRIMS(node.rbound), r, isect);
        }
        else
        {
            d1 = IntersectBoxF(r, invdir, node.rbound, isect->uvwt.w);
        }

        if (d0 > 0 && d1 > 0)
        {
            bittrail = bittrail << 1;
            nodeidx = nodeidx << 1;
            bittrail = bittrail ^ 0x1;

            if (d0 > d1)
            {
                addr = CHILD(node, 1);
                nodeidx = nodeidx ^ 0x1;
            }
            else
            {
                addr = CHILD(node, 0);
            }
            continue;
        }
        else if (d0 > 0)
        {
            bittrail = bittrail << 1;
            nodeidx = nodeidx << 1;
            addr = CHILD(node, 0);
            continue;
        }
        else if (d1 > 0)
        {
            bittrail = bittrail << 1;
            nodeidx = nodeidx << 1;
            nodeidx = nodeidx ^ 0x1;
            addr = CHILD(node, 1);
            continue;
        }

        if (bittrail == 0)
        {
            addr = -1;
            continue;
        }

        int num_levels = 31 - clz(bittrail & -bittrail);
        bittrail = (bittrail >> num_levels) ^ 0x1;
        nodeidx = (nodeidx >> num_levels) ^ 0x1;

        int d = displacement[nodeidx / t];
        addr = hash[d + (nodeidx & (t - 1))];
    }

    return isect->shapeid >= 0;
}

// intersect Ray against the whole BVH structure
inline bool occluded(SceneData const *scenedata, ray const *r)
{
    const float3 invdir = native_recip(r->d.xyz);

    int stack[32];
    int *sptr = stack;
    *sptr++ = -1;

    int idx = 0;
    FatBvhNode node;

    while (idx > -1)
    {
        while (idx > -1)
        {
            node = scenedata->nodes[idx];

            bool l0 = LEAFNODE(node.lbound);
            bool l1 = LEAFNODE(node.rbound);
            float d0 = -1.f;
            float d1 = -1.f;

            if (l0)
            {
                bool hit = IntersectLeafAny(scenedata, STARTIDX(node.lbound), NUMPRIMS(node.lbound), r);
                if (hit)
                {
                    return true;
                }
            }
            else
            {
                d0 = IntersectBoxF(r, invdir, node.lbound, r->o.w);
            }

            if (l1)
            {
                bool hit = IntersectLeafAny(scenedata, STARTIDX(node.rbound), NUMPRIMS(node.rbound), r);
                if (hit)
                {
                    return true;
                }
            }
            else
            {
                d1 = IntersectBoxF(r, invdir, node.rbound, r->o.w);
            }

            if (d0 > 0 && d1 > 0)
            {
                int deferred = -1;
                if (d0 > d1)
                {
                    idx = CHILD(node, 1);
                    deferred = CHILD(node, 0);
                }
                else
                {
                    idx = CHILD(node, 0);
                    deferred = CHILD(node, 1);
                }

                *sptr++ = deferred;
                continue;
            }
            else if (d0 > 0)
            {
                idx = CHILD(node, 0);
                continue;
            }
            else if (d1 > 0)
            {
                idx = CHILD(node, 1);
                continue;
            }

            idx = *--sptr;
        }
    }

    return false;
}

__attribute__((always_inline)) float IntersectTriangleLight(ray r, float3 v1, float3 v2, float3 v3, float t_max)
{
    const float3 e1 = v2 - v1;
    const float3 e2 = v3 - v1;
    const float3 s1 = cross(r.d.xyz, e2);
    const float invd = native_recip(dot(s1, e1));
    const float3 d = r.o.xyz - v1;
    const float b1 = dot(d, s1) * invd;
    const float3 s2 = cross(d, e1);
    const float b2 = dot(r.d.xyz, s2) * invd;
    const float temp = dot(e2, s2) * invd;

    if (b1 < 0.f || b1 > 1.f || b2 < 0.f || b1 + b2 > 1.f || temp < 0.f || temp > t_max)
    {
        return t_max;
    }
    else
    {
        return temp;
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
__kernel void
IntersectClosest(
    // Input
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    int numrays,                      // Number of rays to process
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global Intersection *hits // Hit datas
    )
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    SceneData scenedata =
        {
            nodes,
            vertices,
            faces,
            shapes,
            0};

    if (global_id < numrays)
    {
        // Fetch ray
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            Intersection isect;
            intersect(&scenedata, &r, &isect, displacement, hashmap, t);
            hits[global_id] = isect;
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
__kernel void
IntersectAny(
    // Input
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    int numrays,                      // Number of rays to process
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global int *hitresults // Hit results
    )
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    // Fill scene data
    SceneData scenedata =
        {
            nodes,
            vertices,
            faces,
            shapes,
            0};

    if (global_id < numrays)
    {
        // Fetch ray
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            // Calculate any intersection
            hitresults[global_id] = occluded(&scenedata, &r) ? 1 : -1;
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
// Version with range check
__kernel void
IntersectAnyRC(
    // Input
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    __global int const *numrays,      // Number of rays in the workload
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global int *hitresults // Hit results
    )
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    // Fill scene data
    SceneData scenedata =
        {
            nodes,
            vertices,
            faces,
            shapes,
            0};

    // Handle only working subset
    if (global_id < *numrays)
    {
        // Fetch ray
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            // Calculate any intersection
            hitresults[global_id] = occluded(&scenedata, &r) ? 1 : -1;
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
// Version with range check
__kernel void
IntersectClosestRC(
    __global FatBvhNode const *nodes, // BVH nodes
    __global float3 const *vertices,  // Scene positional data
    __global Face const *faces,       // Scene indices
    __global ShapeData const *shapes, // Shape data
    __global ray const *rays,         // Ray workload
    int offset,                       // Offset in rays array
    __global int const *numrays,      // Number of rays in the workload
    __global int const *displacement,
    __global int const *hashmap,
    int t,
    __global Intersection *hits)
{
    int global_id = get_global_id(0);
    int local_id = get_local_id(0);
    int group_id = get_group_id(0);

    // Handle only working subset
    if (global_id < *numrays)
    {
        ray r = rays[global_id];

        if (Ray_IsActive(&r))
        {
            float3 v1, v2, v3;
            Face face;
            float3 const invdir = native_recip(r.d.xyz);
            float3 const oxinvdir = -r.o.xyz * invdir;
            float t_max = r.o.w;

            int bittrail = 0;
            int nodeidx = 1;
            int addr = 0;
            int faceidx = -1;

            FatBvhNode node;
            while (addr > -1)
            {
                node = nodes[addr];
                bool l0 = LEAFNODE(node.lbound);
                bool l1 = LEAFNODE(node.rbound);
                float d0 = -1.f;
                float d1 = -1.f;

                if (l0)
                {
                    face = faces[STARTIDX(node.lbound)];
                    v1 = vertices[face.idx[0]];
                    v2 = vertices[face.idx[1]];
                    v3 = vertices[face.idx[2]];
                    float f = IntersectTriangleLight(r, v1, v2, v3, t_max);
                    if (f < t_max)
                    {
                        faceidx = STARTIDX(node.lbound);
                        t_max = f;
                    }
                }
                else
                {
                    const float3 f = mad(node.lbound.pmax.xyz, invdir, oxinvdir);
                    const float3 n = mad(node.lbound.pmin.xyz, invdir, oxinvdir);

                    const float3 tmax = max(f, n);
                    const float3 tmin = min(f, n);

                    #ifndef AMD_MEDIA_OPS
                    const float t1 = min(min(tmax.x, min(tmax.y, tmax.z)), t_max);
                    const float t0 = max(max(tmin.x, max(tmin.y, tmin.z)), 0.f);
                    #else
                    const float t1 = min(amd_min3(tmax.x, tmax.y, tmax.z), t_max);
                    const float t0 = amd_max3(tmin.x, tmin.y, tmin.z);
                    #endif

                    if (t1 >= t0)
                    {
                        d0 = t0 > 0.f ? t0 : t1;
                    }
                }

                if (l1)
                {
                    face = faces[STARTIDX(node.rbound)];
                    v1 = vertices[face.idx[0]];
                    v2 = vertices[face.idx[1]];
                    v3 = vertices[face.idx[2]];
                    float f = IntersectTriangleLight(r, v1, v2, v3, t_max);
                    if (f < t_max)
                    {
                        faceidx = STARTIDX(node.rbound);
                        t_max = f;
                    }
                }
                else
                {
                    const float3 f = mad(node.rbound.pmax.xyz, invdir, oxinvdir);
                    const float3 n = mad(node.rbound.pmin.xyz, invdir, oxinvdir);

                    const float3 tmax = max(f, n);
                    const float3 tmin = min(f, n);

                    #ifndef AMD_MEDIA_OPS
                    const float t1 = min(min(tmax.x, min(tmax.y, tmax.z)), maxt);
                    const float t0 = max(max(tmin.x, max(tmin.y, tmin.z)), 0.f);
                    #else
                    const float t1 = min(amd_min3(tmax.x, tmax.y, tmax.z), t_max);
                    const float t0 = amd_max3(tmin.x, tmin.y, tmin.z);
                    #endif

                    if (t1 >= t0)
                    {
                        d1 = t0 > 0.f ? t0 : t1;
                    }
                }

                if (d0 > 0 && d1 > 0)
                {
                    bittrail = bittrail << 1;
                    nodeidx = nodeidx << 1;
                    bittrail = bittrail ^ 0x1;

                    if (d0 > d1)
                    {
                        addr = CHILD(node, 1);
                        nodeidx = nodeidx ^ 0x1;
                    }
                    else
                    {
                        addr = CHILD(node, 0);
                    }
                    continue;
                }
                else if (d0 > 0)
                {
                    bittrail = bittrail << 1;
                    nodeidx = nodeidx << 1;
                    addr = CHILD(node, 0);
                    continue;
                }
                else if (d1 > 0)
                {
                    bittrail = bittrail << 1;
                    nodeidx = nodeidx << 1;
                    nodeidx = nodeidx ^ 0x1;
                    addr = CHILD(node, 1);
                    continue;
                }

                if (bittrail == 0)
                {
                    addr = -1;
                    continue;
                }

                int num_levels = 31 - clz(bittrail & -bittrail);
                bittrail = (bittrail >> num_levels) ^ 0x1;
                nodeidx = (nodeidx >> num_levels) ^ 0x1;

                int d = displacement[nodeidx / t];
                addr = hashmap[d + (nodeidx & (t - 1))];
            }

            if (faceidx != -1)
            {
                face = faces[faceidx];
                hits[global_id].shapeid = face.shapeidx;
                hits[global_id].primid = face.id;
                v1 = vertices[face.idx[0]];
                v2 = vertices[face.idx[1]];
                v3 = vertices[face.idx[2]];

                const float3 e1 = v2 - v1;
                const float3 e2 = v3 - v1;
                const float3 s1 = cross(r.d.xyz, e2);
                const float  invd = native_recip(dot(s1, e1));
                const float3 d = r.o.xyz - v1;
                const float  b1 = dot(d, s1) * invd;
                const float3 s2 = cross(d, e1);
                const float  b2 = dot(r.d.xyz, s2) * invd;
                hits[global_id].uvwt = make_float4(b1, b2, 0.f, t_max);
            }
            else
            {
                hits[global_id].shapeid = -1;
                hits[global_id].primid = -1;
            }
        }
    }
}


more test case for OpenCL 1.1 integer built-in macros

/* test OpenCL 1.1 Integet Built-in Functions (section 6.11.3) */
__kernel void compiler_integer_builtin() {
  int i = 0, i1 = -1, i2 = -2;
  unsigned u = 1, u1 = 2, u2 = 3;
  i = CHAR_MAX;
  i = abs(u);
  i = abs_diff(u1, u2);
  i = add_sat(i1, i2);
  i = hadd(i1, i2);
  i = rhadd(i1, i2);
  i = clz(i);
  i = clamp(i, i1, i2);
  i = mad_hi(i, i1, i2);
  i = mad_sat(i, i1, i2);
  i = max(i1, i2);
  i = min(i1, i2);
  i = mul_hi(i1, i2);
  i = rotate(i1, i2);
  i = sub_sat(i1, i2);
  long l = upsample(i, u);
  i = mad24(i, i1, i2);
  i = mul24(i1, i2);
}

/* test OpenCL 1.1 Integet Built-in Functions (section 6.11.3) */
__kernel void compiler_integer_builtin() {
  int i = 0, i1 = -1, i2 = -2;
  unsigned u = 1, u1 = 2, u2 = 3;
  i = abs(u);
  i = abs_diff(u1, u2);
  i = add_sat(i1, i2);
  i = hadd(i1, i2);
  i = rhadd(i1, i2);
  i = clz(i);
  i = clamp(i, i1, i2);
  i = mad_hi(i, i1, i2);
  i = mad_sat(i, i1, i2);
  i = max(i1, i2);
  i = min(i1, i2);
  i = mul_hi(i1, i2);
  i = rotate(i1, i2);
  i = sub_sat(i1, i2);
  long l = upsample(i, u);
  i = mad24(i, i1, i2);
  i = mul24(i1, i2);
}


generic uniformity

generic uniformity

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_HW_WAVE_ID_SETTING 1
// FIXME Conduct enabling from the host code.
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);

#define MLO_IN_LCL_WIDTH (MLO_IN_STRIDE + 2 * MLO_FILTER_PAD0)

static inline int getWaveId()
{
	int wave_id = 0;

#if MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_HSAIL==1
		wave_id = __hsail_get_dynwave_id();
		wave_id &= MLO_N_WAVES_MASK;
#elif MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_LC==1 && MLO_GRP_SZ1==1 && MLO_GRP_SZ2==1 && (MLO_GRP_SZ % (1 << MLO_LG2_WAVE_SZ))==0
		// (local_id/wavesize) has the same value in all workitems.
		// Make it scalar to enable scalarization optimizations.
		wave_id = __llvm_amdgcn_readfirstlane((uint)(get_local_id(0) >> MLO_LG2_WAVE_SZ));
		// Alternate implementation:
		//__asm__ ("v_readfirstlane_b32 %0, %1" : "=s" (wave_id) : "v" ((int)(get_local_id(0) >> MLO_LG2_WAVE_SZ)) );
#else
		wave_id = (get_local_id(0) >> MLO_LG2_WAVE_SZ);
#endif
	return(wave_id);
}

static inline int getWaveLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_WAVE_SZ) << MLO_LG2_WAVE_SZ);;
	return(lcl_wave_id);
}

static inline int getWaveLocalIds(int * lcl_wave_id1, int * lcl_wave_id0)
{
	int lcl_wave_id = getWaveLocalId();
	*lcl_wave_id1 = (lcl_wave_id >> MLO_LG2_WAVE_SZ0);
	*lcl_wave_id0 = lcl_wave_id - ((*lcl_wave_id1) << MLO_LG2_WAVE_SZ0);
	return(lcl_wave_id);
}

static inline int getLocalIds(int * lcl_id1, int * lcl_id0)
{
    int lcl_wave_id = getWaveLocalId();
	int wave_id = getWaveId();
	int lcl_id = (wave_id << MLO_LG2_WAVE_SZ) + lcl_wave_id; //get_local_id(0);
	*lcl_id1 = (lcl_id >> MLO_LG2ALU_EXTENT_X);
	*lcl_id0 = lcl_id - ((*lcl_id1) << MLO_LG2ALU_EXTENT_X);
	return(lcl_id);
}

/*********************************************************************************************************

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvD3x3_WSR0(
	const __global _FLOAT * in_ptr,
	const __global _FLOAT * restrict wei_ptr,
#if MLO_CONV_BIAS
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *out_ptr,
	_FLOAT alpha // LRU fusion
	)
{
// all other allocation should be before this one since we may exceed lcl mem range
	__local _FLOAT in_lcl[MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1)];

	__private _FLOAT pvt_accum[MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0];

	int grp_input_id = get_group_id(0); // tile id inside the input map
	int grp_gbl_offset = (grp_input_id == 0) ? 0 : (grp_input_id*MLO_OUT_EXTENT1 - 1)*MLO_IN_STRIDE;
	int gpr_lcl_off = (grp_input_id != 0) ? MLO_FILTER_PAD0 : MLO_IN_LCL_WIDTH*MLO_FILTER_PAD1 + MLO_FILTER_PAD0;

	int lcl_id1;
	int lcl_id0;
	int lcl_id = getLocalIds(&lcl_id1, &lcl_id0);
	grp_gbl_offset += lcl_id1*MLO_IN_STRIDE;
	gpr_lcl_off += lcl_id1*MLO_IN_LCL_WIDTH;

	int n_vert_reads = (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1);
	n_vert_reads -= (grp_input_id == 0 || grp_input_id == get_num_groups(0) - 1) ? 1 : 0;

// output
	int o_block = get_group_id(1);

	int weave_id = getWaveId();
	//assumption is MLO_N_OUTPUTS is multiple of MLO_N_LCL_OUT_MAPS*MLO_N_WAVES
	int o_base = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES + weave_id * MLO_N_LCL_OUT_MAPS;

	__private int weights_base_offsets[MLO_N_LCL_OUT_MAPS];

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
	{
		weights_base_offsets[i] = (o_base + i)*MLO_WEI_BATCH_STRIDE;
	}

// batch
	int b_block = get_group_id(2);
	// batch
	int b_base = b_block * MLO_N_LCL_BATCHS;

	grp_gbl_offset += b_base * MLO_IN_BATCH_STRIDE;

// padding
// ATT: padding == 1
	for (uint i = lcl_id; i < MLO_IN_LCL_WIDTH; i+=MLO_GRP_SZ)
	{
		in_lcl[i] = 0;
		in_lcl[i + MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + MLO_FILTER_PAD1)] = 0;
	}
	for (uint i = lcl_id; i < (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1); i += MLO_GRP_SZ)
	{
		in_lcl[i*MLO_IN_LCL_WIDTH] = 0;
		in_lcl[(i+1)*MLO_IN_LCL_WIDTH - 1] = 0;
	}

	// local read and relative output adresses and offsets
	int lcl_wave_id1;
	int lcl_wave_id0;
	getWaveLocalIds(&lcl_wave_id1, &lcl_wave_id0);
	int lcl_off1 = lcl_wave_id1 * MLO_OUT_TILE1;
	int lcl_off0 = lcl_wave_id0 * MLO_OUT_TILE0;
	int lcl_read_off = lcl_off1 * MLO_IN_LCL_WIDTH + lcl_off0;

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0; ++i)
	{
		pvt_accum[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0

	if (get_group_id(0) == 0 && lcl_id0 == 1)
	{
		printf("k:s: %d %d %d %d %d\n",
			getWaveId(),
			lcl_id,
			lcl_id1,
			grp_gbl_offset,
			gpr_lcl_off
			);
	}
#endif

	// outer loop over all input maps
	for (uint c = 0, wei_offset = 0;
		c < MLO_N_INPUTS;
		++c,
		grp_gbl_offset += MLO_IN_CHANNEL_STRIDE, wei_offset += MLO_WEI_CHANNEL_STRIDE)
	{
// not to overwrite
		barrier(CLK_LOCAL_MEM_FENCE);

		int gbl_scan_off = grp_gbl_offset;
		int lcl_scan_off = gpr_lcl_off;
		for (uint j = lcl_id1; j < n_vert_reads; j += MLO_ALU_EXTENT_Y, gbl_scan_off += MLO_IN_STRIDE*MLO_ALU_EXTENT_Y, lcl_scan_off += MLO_IN_LCL_WIDTH*MLO_ALU_EXTENT_Y)
		{
//			*(MLO_READ_TYPE*)&in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 1];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 2] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 2];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 3] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 3];
#if 0

			if (get_group_id(0) == 0 && j < 3 && lcl_id0 == 0)
			{
				printf("k:i: %d %d %d  %f %f %f\n",
					n_vert_reads,
					lcl_scan_off + lcl_id0*MLO_READ_UNIT,
					gbl_scan_off + lcl_id0*MLO_READ_UNIT,
					in_lcl[lcl_scan_off - 1], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1]
					);
			}
#endif
		}
// finish reading
		barrier(CLK_LOCAL_MEM_FENCE);

// read weights
		__private _FLOAT pvt_wei_stage[MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE0 * MLO_FILTER_SIZE0];
		for (uint j = 0; j < MLO_N_LCL_OUT_MAPS; ++j)
		{
			for (uint i = 0; i < MLO_WEI_CHANNEL_STRIDE; ++i)
			{
				int wei_gbl_off = weights_base_offsets[j] + wei_offset;
				pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = wei_ptr[wei_gbl_off + i];
			}
		}

// read data
		__private _FLOAT pvt_in_stage[(MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0)];

		int lcl_read_scan_off = lcl_read_off;
		for (uint j = 0; j < (MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1); ++j, lcl_read_scan_off += MLO_IN_LCL_WIDTH)
		{
			for (uint i = 0; i < (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0); ++i)
			{
				pvt_in_stage[j * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i]
					= in_lcl[lcl_read_scan_off + i];

			}

		}

		// actual conv

		for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc)
		{
			for (uint j = 0; j < MLO_OUT_TILE1; ++j)
			{
				for (uint i = 0; i < MLO_OUT_TILE0; ++i)
				{
					for (uint k = 0; k < MLO_FILTER_SIZE1; ++k)
					{
						for (uint l = 0; l < MLO_FILTER_SIZE0; ++l)
						{

							pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								+= pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l]
									* pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l];
#if 0

							if (get_group_id(0) == 0 && lcl_id == 0 && j == 0 && i == 0)
							{
								printf("k:c: %d %d %d  %f %f %f\n",
									oc,
									k,
									l,
									pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i],
									pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l],
									pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l]
								);
							}
#endif

						} //for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
					}

				} // for (int i = 0; i < MLO_OUT_TILE0; ++i)

			} // for (int j = 0; j < MLO_OUT_TILE1; ++j)

		}
	}

	// send it out
	// TODO: batch loop

	int grp_x = 0;
	int grp_y = grp_input_id*MLO_OUT_EXTENT1;
	int out_off = b_base*MLO_OUT_BATCH_STRIDE + o_base* MLO_OUT_CHANNEL_STRIDE + (grp_y + lcl_off1)*MLO_OUT_STRIDE + grp_x + lcl_off0;

//	int ib = 0;

//	out_off += ib*MLO_OUT_BATCH_STRIDE;

	int out_off1 = out_off;
	for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS
#if MLO_OUTPUTS_ALIGNED ==0
//		&& o + oc < MLO_N_OUTPUTS
#endif
		;
	++oc, out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
		int out_off2 = out_off1;
		for (uint j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
		{
			for (uint i = 0; i < MLO_OUT_TILE0 / MLO_READ_UNIT; ++i)
			{

				// WIDTH/HEIGHT
				*(__global MLO_READ_TYPE*)&out_ptr[out_off2 + i*MLO_READ_UNIT]
					= *(MLO_READ_TYPE*)&pvt_accum[/*ib*MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE_SZ + */(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i*MLO_READ_UNIT];
			}
		}
	}

}

/*********************************************************************************************************
reads weight sequentially, requiers rearanging kernel
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvD3x3_WSR1(
	const __global _FLOAT * in_ptr,
	const __global _FLOAT * restrict wei_ptr,
#if MLO_CONV_BIAS
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *out_ptr,
	_FLOAT alpha // LRU fusion
	)
{
// all other allocation should be before this one since we may exceed lcl mem range
	__local _FLOAT in_lcl[MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1)];

	__private _FLOAT pvt_accum[MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0];

	int grp_input_id = get_group_id(0); // tile id inside the input map
	int grp_gbl_offset = (grp_input_id == 0) ? 0 : (grp_input_id*MLO_OUT_EXTENT1 - 1)*MLO_IN_STRIDE;
	int gpr_lcl_off = (grp_input_id != 0) ? MLO_FILTER_PAD0 : MLO_IN_LCL_WIDTH*MLO_FILTER_PAD1 + MLO_FILTER_PAD0;

	int lcl_id1;
	int lcl_id0;
	int lcl_id = getLocalIds(&lcl_id1, &lcl_id0);
	grp_gbl_offset += lcl_id1*MLO_IN_STRIDE;
	gpr_lcl_off += lcl_id1*MLO_IN_LCL_WIDTH;

	int n_vert_reads = (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1);
	n_vert_reads -= (grp_input_id == 0 || grp_input_id == get_num_groups(0) - 1) ? 1 : 0;

// output
	int o_block = get_group_id(1);

	int weave_id = getWaveId();
	//assumption is MLO_N_OUTPUTS is multiple of MLO_N_LCL_OUT_MAPS*MLO_N_WAVES
	int o_base = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES + weave_id * MLO_N_LCL_OUT_MAPS;

	__private int weights_base_offsets;

	weights_base_offsets = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES*MLO_WEI_BATCH_STRIDE + weave_id*MLO_WEI_CHANNEL_STRIDE;

// batch
	int b_block = get_group_id(2);
	// batch
	int b_base = b_block * MLO_N_LCL_BATCHS;

	grp_gbl_offset += b_base * MLO_IN_BATCH_STRIDE;

// padding
// ATT: padding == 1
	for (int i = lcl_id; i < MLO_IN_LCL_WIDTH; i+=MLO_GRP_SZ)
	{
		in_lcl[i] = 0;
		in_lcl[i + MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + MLO_FILTER_PAD1)] = 0;
	}
	for (int i = lcl_id; i < (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1); i += MLO_GRP_SZ)
	{
		in_lcl[i*MLO_IN_LCL_WIDTH] = 0;
		in_lcl[(i+1)*MLO_IN_LCL_WIDTH - 1] = 0;
	}

	// local read and relative output adresses and offsets
	int lcl_wave_id1;
	int lcl_wave_id0;
	getWaveLocalIds(&lcl_wave_id1, &lcl_wave_id0);
	int lcl_off1 = lcl_wave_id1 * MLO_OUT_TILE1;
	int lcl_off0 = lcl_wave_id0 * MLO_OUT_TILE0;
	int lcl_read_off = lcl_off1 * MLO_IN_LCL_WIDTH + lcl_off0;

	for (int i = 0; i < MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0; ++i)
	{
		pvt_accum[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0

	if (get_group_id(0) == 0 && lcl_id0 == 1)
	{
		printf("k:s: %d %d %d %d %d\n",
			getWaveId(),
			lcl_id,
			lcl_id1,
			grp_gbl_offset,
			gpr_lcl_off
			);
	}
#endif

	// outer loop over all input maps
	for (int c = 0;
		c < MLO_N_INPUTS;
		++c,
		grp_gbl_offset += MLO_IN_CHANNEL_STRIDE, weights_base_offsets += MLO_N_LCL_OUT_MAPS*MLO_N_WAVES*MLO_WEI_CHANNEL_STRIDE)
	{
// not to overwrite
		barrier(CLK_LOCAL_MEM_FENCE);

		int gbl_scan_off = grp_gbl_offset;
		int lcl_scan_off = gpr_lcl_off;
		for (int j = lcl_id1; j < n_vert_reads; j += MLO_ALU_EXTENT_Y, gbl_scan_off += MLO_IN_STRIDE*MLO_ALU_EXTENT_Y, lcl_scan_off += MLO_IN_LCL_WIDTH*MLO_ALU_EXTENT_Y)
		{
//			*(MLO_READ_TYPE*)&in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 1];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 2] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 2];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 3] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 3];
#if 0

			if (get_group_id(0) == 0 && j < 3 && lcl_id0 == 0)
			{
				printf("k:i: %d %d %d  %f %f %f\n",
					n_vert_reads,
					lcl_scan_off + lcl_id0*MLO_READ_UNIT,
					gbl_scan_off + lcl_id0*MLO_READ_UNIT,
					in_lcl[lcl_scan_off - 1], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1]
					);
			}
#endif
		}
// finish reading
		barrier(CLK_LOCAL_MEM_FENCE);

// read weights
		__private _FLOAT pvt_wei_stage[MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE0 * MLO_FILTER_SIZE0];
		int wei_gbl_off  = weights_base_offsets;
		for (int j = 0; j < MLO_N_LCL_OUT_MAPS; ++j, wei_gbl_off+= MLO_N_WAVES*MLO_WEI_CHANNEL_STRIDE)
		{
			for (int i = 0; i < MLO_WEI_CHANNEL_STRIDE; ++i)
			{
				pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = wei_ptr[wei_gbl_off + i];
			}
		}

// read data
		__private _FLOAT pvt_in_stage[(MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0)];

		int lcl_read_scan_off = lcl_read_off;
		for (int j = 0; j < (MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1); ++j, lcl_read_scan_off += MLO_IN_LCL_WIDTH)
		{
			for (int i = 0; i < (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0); ++i)
			{
				pvt_in_stage[j * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i]
					= in_lcl[lcl_read_scan_off + i];

			}

		}

		// actual conv

		for (int oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc)
		{
			for (int j = 0; j < MLO_OUT_TILE1; ++j)
			{
				for (int i = 0; i < MLO_OUT_TILE0; ++i)
				{
					for (int k = 0; k < MLO_FILTER_SIZE1; ++k)
					{
						for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
						{

							pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								+= pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l]
									* pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l];
#if 0

							if (get_group_id(0) == 0 && lcl_id == 0 && j == 0 && i == 0)
							{
								printf("k:c: %d %d %d  %f %f %f\n",
									oc,
									k,
									l,
									pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i],
									pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l],
									pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l]
								);
							}
#endif

						} //for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
					}

				} // for (int i = 0; i < MLO_OUT_TILE0; ++i)

			} // for (int j = 0; j < MLO_OUT_TILE1; ++j)

		}
	}

	// send it out
	// TODO: batch loop

	int grp_x = 0;
	int grp_y = grp_input_id*MLO_OUT_EXTENT1;
	int out_off = b_base*MLO_OUT_BATCH_STRIDE + o_base* MLO_OUT_CHANNEL_STRIDE + (grp_y + lcl_off1)*MLO_OUT_STRIDE + grp_x + lcl_off0;

//	int ib = 0;

//	out_off += ib*MLO_OUT_BATCH_STRIDE;

	int out_off1 = out_off;
	for (int oc = 0; oc < MLO_N_LCL_OUT_MAPS
#if MLO_OUTPUTS_ALIGNED ==0
//		&& o + oc < MLO_N_OUTPUTS
#endif
		;
	++oc, out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
		int out_off2 = out_off1;
		for (int j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
		{
			for (int i = 0; i < MLO_OUT_TILE0 / MLO_READ_UNIT; ++i)
			{

				// WIDTH/HEIGHT
				*(__global MLO_READ_TYPE*)&out_ptr[out_off2 + i*MLO_READ_UNIT]
					= *(MLO_READ_TYPE*)&pvt_accum[/*ib*MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE_SZ + */(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i*MLO_READ_UNIT];
			}
		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_HW_WAVE_ID_SETTING 1
// FIXME Conduct enabling from the host code.
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);

#define MLO_IN_LCL_WIDTH (MLO_IN_STRIDE + 2 * MLO_FILTER_PAD0)

static inline int getWaveId()
{
	int wave_id = 0;
#if MLO_HW_WAVE_ID_SETTING
	wave_id = __hsail_get_dynwave_id();
	wave_id &= MLO_N_WAVES_MASK;
#else
	wave_id = (get_local_id(0) >> MLO_LG2_WAVE_SZ);
#endif
	return(wave_id);
}

static inline int getWaveLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_WAVE_SZ) << MLO_LG2_WAVE_SZ);;
	return(lcl_wave_id);
}

static inline int getWaveLocalIds(int * lcl_wave_id1, int * lcl_wave_id0)
{
	int lcl_wave_id = getWaveLocalId();
	*lcl_wave_id1 = (lcl_wave_id >> MLO_LG2_WAVE_SZ0);
	*lcl_wave_id0 = lcl_wave_id - ((*lcl_wave_id1) << MLO_LG2_WAVE_SZ0);
	return(lcl_wave_id);
}

static inline int getLocalIds(int * lcl_id1, int * lcl_id0)
{
    int lcl_wave_id = getWaveLocalId();
	int wave_id = getWaveId();
	int lcl_id = (wave_id << MLO_LG2_WAVE_SZ) + lcl_wave_id; //get_local_id(0);
	*lcl_id1 = (lcl_id >> MLO_LG2ALU_EXTENT_X);
	*lcl_id0 = lcl_id - ((*lcl_id1) << MLO_LG2ALU_EXTENT_X);
	return(lcl_id);
}

/*********************************************************************************************************

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvD3x3_WSR0(
	const __global _FLOAT * in_ptr,
	const __global _FLOAT * restrict wei_ptr,
#if MLO_CONV_BIAS
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *out_ptr,
	_FLOAT alpha // LRU fusion
	)
{
// all other allocation should be before this one since we may exceed lcl mem range
	__local _FLOAT in_lcl[MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1)];

	__private _FLOAT pvt_accum[MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0];

	int grp_input_id = get_group_id(0); // tile id inside the input map
	int grp_gbl_offset = (grp_input_id == 0) ? 0 : (grp_input_id*MLO_OUT_EXTENT1 - 1)*MLO_IN_STRIDE;
	int gpr_lcl_off = (grp_input_id != 0) ? MLO_FILTER_PAD0 : MLO_IN_LCL_WIDTH*MLO_FILTER_PAD1 + MLO_FILTER_PAD0;

	int lcl_id1;
	int lcl_id0;
	int lcl_id = getLocalIds(&lcl_id1, &lcl_id0);
	grp_gbl_offset += lcl_id1*MLO_IN_STRIDE;
	gpr_lcl_off += lcl_id1*MLO_IN_LCL_WIDTH;

	int n_vert_reads = (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1);
	n_vert_reads -= (grp_input_id == 0 || grp_input_id == get_num_groups(0) - 1) ? 1 : 0;

// output
	int o_block = get_group_id(1);

	int weave_id = getWaveId();
	//assumption is MLO_N_OUTPUTS is multiple of MLO_N_LCL_OUT_MAPS*MLO_N_WAVES
	int o_base = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES + weave_id * MLO_N_LCL_OUT_MAPS;

	__private int weights_base_offsets[MLO_N_LCL_OUT_MAPS];

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
	{
		weights_base_offsets[i] = (o_base + i)*MLO_WEI_BATCH_STRIDE;
	}

// batch
	int b_block = get_group_id(2);
	// batch
	int b_base = b_block * MLO_N_LCL_BATCHS;

	grp_gbl_offset += b_base * MLO_IN_BATCH_STRIDE;

// padding
// ATT: padding == 1
	for (uint i = lcl_id; i < MLO_IN_LCL_WIDTH; i+=MLO_GRP_SZ)
	{
		in_lcl[i] = 0;
		in_lcl[i + MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + MLO_FILTER_PAD1)] = 0;
	}
	for (uint i = lcl_id; i < (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1); i += MLO_GRP_SZ)
	{
		in_lcl[i*MLO_IN_LCL_WIDTH] = 0;
		in_lcl[(i+1)*MLO_IN_LCL_WIDTH - 1] = 0;
	}

	// local read and relative output adresses and offsets
	int lcl_wave_id1;
	int lcl_wave_id0;
	getWaveLocalIds(&lcl_wave_id1, &lcl_wave_id0);
	int lcl_off1 = lcl_wave_id1 * MLO_OUT_TILE1;
	int lcl_off0 = lcl_wave_id0 * MLO_OUT_TILE0;
	int lcl_read_off = lcl_off1 * MLO_IN_LCL_WIDTH + lcl_off0;

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0; ++i)
	{
		pvt_accum[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0

	if (get_group_id(0) == 0 && lcl_id0 == 1)
	{
		printf("k:s: %d %d %d %d %d\n",
			getWaveId(),
			lcl_id,
			lcl_id1,
			grp_gbl_offset,
			gpr_lcl_off
			);
	}
#endif

	// outer loop over all input maps
	for (uint c = 0, wei_offset = 0;
		c < MLO_N_INPUTS;
		++c,
		grp_gbl_offset += MLO_IN_CHANNEL_STRIDE, wei_offset += MLO_WEI_CHANNEL_STRIDE)
	{
// not to overwrite
		barrier(CLK_LOCAL_MEM_FENCE);

		int gbl_scan_off = grp_gbl_offset;
		int lcl_scan_off = gpr_lcl_off;
		for (uint j = lcl_id1; j < n_vert_reads; j += MLO_ALU_EXTENT_Y, gbl_scan_off += MLO_IN_STRIDE*MLO_ALU_EXTENT_Y, lcl_scan_off += MLO_IN_LCL_WIDTH*MLO_ALU_EXTENT_Y)
		{
//			*(MLO_READ_TYPE*)&in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 1];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 2] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 2];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 3] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 3];
#if 0

			if (get_group_id(0) == 0 && j < 3 && lcl_id0 == 0)
			{
				printf("k:i: %d %d %d  %f %f %f\n",
					n_vert_reads,
					lcl_scan_off + lcl_id0*MLO_READ_UNIT,
					gbl_scan_off + lcl_id0*MLO_READ_UNIT,
					in_lcl[lcl_scan_off - 1], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1]
					);
			}
#endif
		}
// finish reading
		barrier(CLK_LOCAL_MEM_FENCE);

// read weights
		__private _FLOAT pvt_wei_stage[MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE0 * MLO_FILTER_SIZE0];
		for (uint j = 0; j < MLO_N_LCL_OUT_MAPS; ++j)
		{
			for (uint i = 0; i < MLO_WEI_CHANNEL_STRIDE; ++i)
			{
				int wei_gbl_off = weights_base_offsets[j] + wei_offset;
				pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = wei_ptr[wei_gbl_off + i];
			}
		}

// read data
		__private _FLOAT pvt_in_stage[(MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0)];

		int lcl_read_scan_off = lcl_read_off;
		for (uint j = 0; j < (MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1); ++j, lcl_read_scan_off += MLO_IN_LCL_WIDTH)
		{
			for (uint i = 0; i < (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0); ++i)
			{
				pvt_in_stage[j * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i]
					= in_lcl[lcl_read_scan_off + i];

			}

		}

		// actual conv

		for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc)
		{
			for (uint j = 0; j < MLO_OUT_TILE1; ++j)
			{
				for (uint i = 0; i < MLO_OUT_TILE0; ++i)
				{
					for (uint k = 0; k < MLO_FILTER_SIZE1; ++k)
					{
						for (uint l = 0; l < MLO_FILTER_SIZE0; ++l)
						{

							pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								+= pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l]
									* pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l];
#if 0

							if (get_group_id(0) == 0 && lcl_id == 0 && j == 0 && i == 0)
							{
								printf("k:c: %d %d %d  %f %f %f\n",
									oc,
									k,
									l,
									pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i],
									pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l],
									pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l]
								);
							}
#endif

						} //for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
					}

				} // for (int i = 0; i < MLO_OUT_TILE0; ++i)

			} // for (int j = 0; j < MLO_OUT_TILE1; ++j)

		}
	}

	// send it out
	// TODO: batch loop

	int grp_x = 0;
	int grp_y = grp_input_id*MLO_OUT_EXTENT1;
	int out_off = b_base*MLO_OUT_BATCH_STRIDE + o_base* MLO_OUT_CHANNEL_STRIDE + (grp_y + lcl_off1)*MLO_OUT_STRIDE + grp_x + lcl_off0;

//	int ib = 0;

//	out_off += ib*MLO_OUT_BATCH_STRIDE;

	int out_off1 = out_off;
	for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS
#if MLO_OUTPUTS_ALIGNED ==0
//		&& o + oc < MLO_N_OUTPUTS
#endif
		;
	++oc, out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
		int out_off2 = out_off1;
		for (uint j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
		{
			for (uint i = 0; i < MLO_OUT_TILE0 / MLO_READ_UNIT; ++i)
			{

				// WIDTH/HEIGHT
				*(__global MLO_READ_TYPE*)&out_ptr[out_off2 + i*MLO_READ_UNIT]
					= *(MLO_READ_TYPE*)&pvt_accum[/*ib*MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE_SZ + */(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i*MLO_READ_UNIT];
			}
		}
	}

}

/*********************************************************************************************************
reads weight sequentially, requiers rearanging kernel
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvD3x3_WSR1(
	const __global _FLOAT * in_ptr,
	const __global _FLOAT * restrict wei_ptr,
#if MLO_CONV_BIAS
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *out_ptr,
	_FLOAT alpha // LRU fusion
	)
{
// all other allocation should be before this one since we may exceed lcl mem range
	__local _FLOAT in_lcl[MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1)];

	__private _FLOAT pvt_accum[MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0];

	int grp_input_id = get_group_id(0); // tile id inside the input map
	int grp_gbl_offset = (grp_input_id == 0) ? 0 : (grp_input_id*MLO_OUT_EXTENT1 - 1)*MLO_IN_STRIDE;
	int gpr_lcl_off = (grp_input_id != 0) ? MLO_FILTER_PAD0 : MLO_IN_LCL_WIDTH*MLO_FILTER_PAD1 + MLO_FILTER_PAD0;

	int lcl_id1;
	int lcl_id0;
	int lcl_id = getLocalIds(&lcl_id1, &lcl_id0);
	grp_gbl_offset += lcl_id1*MLO_IN_STRIDE;
	gpr_lcl_off += lcl_id1*MLO_IN_LCL_WIDTH;

	int n_vert_reads = (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1);
	n_vert_reads -= (grp_input_id == 0 || grp_input_id == get_num_groups(0) - 1) ? 1 : 0;

// output
	int o_block = get_group_id(1);

	int weave_id = getWaveId();
	//assumption is MLO_N_OUTPUTS is multiple of MLO_N_LCL_OUT_MAPS*MLO_N_WAVES
	int o_base = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES + weave_id * MLO_N_LCL_OUT_MAPS;

	__private int weights_base_offsets;

	weights_base_offsets = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES*MLO_WEI_BATCH_STRIDE + weave_id*MLO_WEI_CHANNEL_STRIDE;

// batch
	int b_block = get_group_id(2);
	// batch
	int b_base = b_block * MLO_N_LCL_BATCHS;

	grp_gbl_offset += b_base * MLO_IN_BATCH_STRIDE;

// padding
// ATT: padding == 1
	for (int i = lcl_id; i < MLO_IN_LCL_WIDTH; i+=MLO_GRP_SZ)
	{
		in_lcl[i] = 0;
		in_lcl[i + MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + MLO_FILTER_PAD1)] = 0;
	}
	for (int i = lcl_id; i < (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1); i += MLO_GRP_SZ)
	{
		in_lcl[i*MLO_IN_LCL_WIDTH] = 0;
		in_lcl[(i+1)*MLO_IN_LCL_WIDTH - 1] = 0;
	}

	// local read and relative output adresses and offsets
	int lcl_wave_id1;
	int lcl_wave_id0;
	getWaveLocalIds(&lcl_wave_id1, &lcl_wave_id0);
	int lcl_off1 = lcl_wave_id1 * MLO_OUT_TILE1;
	int lcl_off0 = lcl_wave_id0 * MLO_OUT_TILE0;
	int lcl_read_off = lcl_off1 * MLO_IN_LCL_WIDTH + lcl_off0;

	for (int i = 0; i < MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0; ++i)
	{
		pvt_accum[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0

	if (get_group_id(0) == 0 && lcl_id0 == 1)
	{
		printf("k:s: %d %d %d %d %d\n",
			getWaveId(),
			lcl_id,
			lcl_id1,
			grp_gbl_offset,
			gpr_lcl_off
			);
	}
#endif

	// outer loop over all input maps
	for (int c = 0;
		c < MLO_N_INPUTS;
		++c,
		grp_gbl_offset += MLO_IN_CHANNEL_STRIDE, weights_base_offsets += MLO_N_LCL_OUT_MAPS*MLO_N_WAVES*MLO_WEI_CHANNEL_STRIDE)
	{
// not to overwrite
		barrier(CLK_LOCAL_MEM_FENCE);

		int gbl_scan_off = grp_gbl_offset;
		int lcl_scan_off = gpr_lcl_off;
		for (int j = lcl_id1; j < n_vert_reads; j += MLO_ALU_EXTENT_Y, gbl_scan_off += MLO_IN_STRIDE*MLO_ALU_EXTENT_Y, lcl_scan_off += MLO_IN_LCL_WIDTH*MLO_ALU_EXTENT_Y)
		{
//			*(MLO_READ_TYPE*)&in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 1];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 2] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 2];
			in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 3] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + 3];
#if 0

			if (get_group_id(0) == 0 && j < 3 && lcl_id0 == 0)
			{
				printf("k:i: %d %d %d  %f %f %f\n",
					n_vert_reads,
					lcl_scan_off + lcl_id0*MLO_READ_UNIT,
					gbl_scan_off + lcl_id0*MLO_READ_UNIT,
					in_lcl[lcl_scan_off - 1], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1]
					);
			}
#endif
		}
// finish reading
		barrier(CLK_LOCAL_MEM_FENCE);

// read weights
		__private _FLOAT pvt_wei_stage[MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE0 * MLO_FILTER_SIZE0];
		int wei_gbl_off  = weights_base_offsets;
		for (int j = 0; j < MLO_N_LCL_OUT_MAPS; ++j, wei_gbl_off+= MLO_N_WAVES*MLO_WEI_CHANNEL_STRIDE)
		{
			for (int i = 0; i < MLO_WEI_CHANNEL_STRIDE; ++i)
			{
				pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = wei_ptr[wei_gbl_off + i];
			}
		}

// read data
		__private _FLOAT pvt_in_stage[(MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0)];

		int lcl_read_scan_off = lcl_read_off;
		for (int j = 0; j < (MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1); ++j, lcl_read_scan_off += MLO_IN_LCL_WIDTH)
		{
			for (int i = 0; i < (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0); ++i)
			{
				pvt_in_stage[j * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i]
					= in_lcl[lcl_read_scan_off + i];

			}

		}

		// actual conv

		for (int oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc)
		{
			for (int j = 0; j < MLO_OUT_TILE1; ++j)
			{
				for (int i = 0; i < MLO_OUT_TILE0; ++i)
				{
					for (int k = 0; k < MLO_FILTER_SIZE1; ++k)
					{
						for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
						{

							pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								+= pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l]
									* pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l];
#if 0

							if (get_group_id(0) == 0 && lcl_id == 0 && j == 0 && i == 0)
							{
								printf("k:c: %d %d %d  %f %f %f\n",
									oc,
									k,
									l,
									pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i],
									pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l],
									pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l]
								);
							}
#endif

						} //for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
					}

				} // for (int i = 0; i < MLO_OUT_TILE0; ++i)

			} // for (int j = 0; j < MLO_OUT_TILE1; ++j)

		}
	}

	// send it out
	// TODO: batch loop

	int grp_x = 0;
	int grp_y = grp_input_id*MLO_OUT_EXTENT1;
	int out_off = b_base*MLO_OUT_BATCH_STRIDE + o_base* MLO_OUT_CHANNEL_STRIDE + (grp_y + lcl_off1)*MLO_OUT_STRIDE + grp_x + lcl_off0;

//	int ib = 0;

//	out_off += ib*MLO_OUT_BATCH_STRIDE;

	int out_off1 = out_off;
	for (int oc = 0; oc < MLO_N_LCL_OUT_MAPS
#if MLO_OUTPUTS_ALIGNED ==0
//		&& o + oc < MLO_N_OUTPUTS
#endif
		;
	++oc, out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
		int out_off2 = out_off1;
		for (int j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
		{
			for (int i = 0; i < MLO_OUT_TILE0 / MLO_READ_UNIT; ++i)
			{

				// WIDTH/HEIGHT
				*(__global MLO_READ_TYPE*)&out_ptr[out_off2 + i*MLO_READ_UNIT]
					= *(MLO_READ_TYPE*)&pvt_accum[/*ib*MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE_SZ + */(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i*MLO_READ_UNIT];
			}
		}
	}

}