Removed TODO

Removed TODO

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

/*************************************************************************
INCLUDES
**************************************************************************/
#include <../RadeonRays/src/kernels/CL/common.cl>

/*************************************************************************
TYPE DEFINITIONS
**************************************************************************/

#define INVALID_ADDR 0xffffffffu
#define INTERNAL_NODE(node) ((node).addr0 != INVALID_ADDR)

#define GROUP_SIZE 64
#define STACK_SIZE 32
#define LDS_STACK_SIZE 16

// BVH node
typedef struct
{
    uint3 aabb01_min_or_v0;
    uint  addr0;

    uint3 aabb01_max_or_v1;
    uint  addr1_or_mesh_id;

    uint3 aabb23_min_or_v2;
    uint  addr2_or_prim_id;

    uint3 aabb23_max;
    uint  addr3;
} bvh_node;

#define mymin3(a, b, c) min(min((a), (b)), (c))
#define mymax3(a, b, c) max(max((a), (b)), (c))

INLINE half2 unpackFloat2x16(uint v)
{
    return (half2)
        (as_half(convert_ushort(v >> 16)),
         as_half(convert_ushort(v & 0xFFFF)));
}

INLINE half4 fast_intersect_bbox2(uint3 pmin, uint3 pmax, half3 invdir, half3 oxinvdir, float t_max)
{
    half2 pmin_x = unpackFloat2x16(pmin.x);
    half2 pmin_y = unpackFloat2x16(pmin.y);
    half2 pmin_z = unpackFloat2x16(pmin.z);
    half2 pmax_x = unpackFloat2x16(pmax.x);
    half2 pmax_y = unpackFloat2x16(pmax.y);
    half2 pmax_z = unpackFloat2x16(pmax.z);

    half2 f_x = fma(pmax_x, invdir.xx, oxinvdir.xx);
    half2 f_y = fma(pmax_y, invdir.yy, oxinvdir.yy);
    half2 f_z = fma(pmax_z, invdir.zz, oxinvdir.zz);

    half2 n_x = fma(pmin_x, invdir.xx, oxinvdir.xx);
    half2 n_y = fma(pmin_y, invdir.yy, oxinvdir.yy);
    half2 n_z = fma(pmin_z, invdir.zz, oxinvdir.zz);

    half2 t_max_x = max(f_x, n_x);
    half2 t_max_y = max(f_y, n_y);
    half2 t_max_z = max(f_z, n_z);

    half2 t_min_x = min(f_x, n_x);
    half2 t_min_y = min(f_y, n_y);
    half2 t_min_z = min(f_z, n_z);

    half2 t_zero = (half2)(0.0f, 0.0f);
    half2 t_max2 = (half2)(t_max, t_max);
    half2 t1 = min(mymin3(t_max_x, t_max_y, t_max_z), t_max2);
    half2 t0 = max(mymax3(t_min_x, t_min_y, t_min_z), t_zero);

    return (half4)(t0, t1);
}

void stack_push(
    __local uint *lds_stack,
    __private uint *lds_sptr,
    uint lds_stack_bottom,
    __global uint *stack,
    __private uint *sptr,
    uint idx)
{
    if (*lds_sptr - lds_stack_bottom >= LDS_STACK_SIZE)
    {
        for (int i = 1; i < LDS_STACK_SIZE; ++i)
        {
            stack[*sptr + i] = lds_stack[lds_stack_bottom + i];
        }

        *sptr = *sptr + LDS_STACK_SIZE;
        *lds_sptr = lds_stack_bottom + 1;
    }

    lds_stack[*lds_sptr] = idx;
    *lds_sptr = *lds_sptr + 1;
}

__attribute__((reqd_work_group_size(64, 1, 1)))
KERNEL void intersect_main(
    // Bvh nodes
    GLOBAL const bvh_node *restrict nodes,
    // Rays
    GLOBAL const ray *restrict rays,
    // Number of rays in rays buffer
    GLOBAL const int *restrict num_rays,
    // Stack memory
    GLOBAL uint *stack,
    // Hit data
    GLOBAL Intersection *hits)
{
    uint index = get_global_id(0);
    uint local_index = get_local_id(0);

    // Handle only working subset
    if (index < *num_rays)
    {
        const ray myRay = rays[index];

        if (ray_is_active(&myRay))
        {
            __local uint lds_stack[GROUP_SIZE * LDS_STACK_SIZE];

            // Precompute inverse direction and origin / dir for bbox testing
            const float3 invDir32 = safe_invdir(myRay);
            const half3 invDir = convert_half3(invDir32);
            const half3 oxInvDir = convert_half3(-myRay.o.xyz * invDir32);

            // Intersection parametric distance
            float closest_t = myRay.o.w;

            // Current node address
            uint addr = 0;
            // Current closest address
            uint closest_addr = INVALID_ADDR;

            uint stack_bottom = STACK_SIZE * index;
            uint sptr = stack_bottom;
            uint lds_stack_bottom = local_index * LDS_STACK_SIZE;
            uint lds_sptr = lds_stack_bottom;

            lds_stack[lds_sptr++] = INVALID_ADDR;

            while (addr != INVALID_ADDR)
            {
                const bvh_node node = nodes[addr];

                if (INTERNAL_NODE(node))
                {
                    half4 s01 = fast_intersect_bbox2(
                        node.aabb01_min_or_v0,
                        node.aabb01_max_or_v1,
                        invDir, oxInvDir, closest_t);
                    half4 s23 = fast_intersect_bbox2(
                        node.aabb23_min_or_v2,
                        node.aabb23_max,
                        invDir, oxInvDir, closest_t);

                    bool traverse_c0 = (s01.x <= s01.z);
                    bool traverse_c1 = (node.addr1_or_mesh_id != INVALID_ADDR) && (s01.y <= s01.w);
                    bool traverse_c2 = (s23.x <= s23.z);
                    bool traverse_c3 = (node.addr3 != INVALID_ADDR) && (s23.y <= s23.w);

                    if (traverse_c0 || traverse_c1 || traverse_c2 || traverse_c3)
                    {
                        uint a = INVALID_ADDR;
                        half d = 100000000.0f;

                        if (traverse_c0)
                        {
                            a = node.addr0;
                            d = s01.x;
                        }

                        if (traverse_c1)
                        {
                            if (a == INVALID_ADDR)
                                a = node.addr1_or_mesh_id;
                            else
                            {
                                uint topush = s01.y < d ? a : node.addr1_or_mesh_id;
                                d = min(s01.y, d);
                                a = topush == a ? node.addr1_or_mesh_id : a;
                                stack_push(lds_stack, &lds_sptr, lds_stack_bottom, stack, &sptr, topush);
                            }
                        }

                        if (traverse_c2)
                        {
                            if (a == INVALID_ADDR)
                                a = node.addr2_or_prim_id;
                            else
                            {
                                uint topush = s23.x < d ? a : node.addr2_or_prim_id;
                                d = min(s23.x, d);
                                a = topush == a ? node.addr2_or_prim_id : a;
                                stack_push(lds_stack, &lds_sptr, lds_stack_bottom, stack, &sptr, topush);
                            }
                        }

                        if (traverse_c3)
                        {
                            if (a == INVALID_ADDR)
                                a = node.addr3;
                            else
                            {
                                uint topush = s23.y < d ? a : node.addr3;
                                d = min(s23.y, d);
                                a = topush == a ? node.addr3 : a;
                                stack_push(lds_stack, &lds_sptr, lds_stack_bottom, stack, &sptr, topush);
                            }
                        }

                        addr = a;
                        continue;
                    }
                }
                else
                {
                    float t = fast_intersect_triangle(
                        myRay,
                        as_float3(node.aabb01_min_or_v0),
                        as_float3(node.aabb01_max_or_v1),
                        as_float3(node.aabb23_min_or_v2),
                        closest_t);

                    if (t < closest_t)
                    {
                        closest_t = t;
                        closest_addr = addr;
                    }
                }

                addr = lds_stack[--lds_sptr];

                if (addr == INVALID_ADDR && sptr > stack_bottom)
                {
                    sptr -= LDS_STACK_SIZE;
                    for (int i = 1; i < LDS_STACK_SIZE; ++i)
                    {
                        lds_stack[lds_stack_bottom + i] = stack[sptr + i];
                    }

                    lds_sptr = lds_stack_bottom + LDS_STACK_SIZE - 1;
                    addr = lds_stack[lds_sptr];
                }
            }

            // Check if we have found an intersection
            if (closest_addr != INVALID_ADDR)
            {
                // Calculate hit position
                const bvh_node node = nodes[closest_addr];
                const float3 p = myRay.o.xyz + closest_t * myRay.d.xyz;

                // Calculte barycentric coordinates
                const float2 uv = triangle_calculate_barycentrics(
                    p,
                    as_float3(node.aabb01_min_or_v0),
                    as_float3(node.aabb01_max_or_v1),
                    as_float3(node.aabb23_min_or_v2));

                // Update hit information
                hits[index].prim_id = node.addr2_or_prim_id;
                hits[index].shape_id = node.addr1_or_mesh_id;
                hits[index].uvwt = make_float4(uv.x, uv.y, 0.f, closest_t);
            }
            else
            {
                // Miss here
                hits[index].prim_id = INVALID_ADDR;
                hits[index].shape_id = INVALID_ADDR;
            }
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
KERNEL void occluded_main(
    // Bvh nodes
    GLOBAL const bvh_node *restrict nodes,
    // Rays
    GLOBAL const ray *restrict rays,
    // Number of rays in rays buffer
    GLOBAL const int *restrict num_rays,
    // Stack memory
    GLOBAL int *stack,
    // Hit results: 1 for hit and -1 for miss
    GLOBAL int *hits)
{
    // TODO: ... (gboisse)
}

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

/*************************************************************************
INCLUDES
**************************************************************************/
#include <../RadeonRays/src/kernels/CL/common.cl>

/*************************************************************************
TYPE DEFINITIONS
**************************************************************************/

#define INVALID_ADDR 0xffffffffu
#define INTERNAL_NODE(node) ((node).addr0 != INVALID_ADDR)

#define GROUP_SIZE 64
#define STACK_SIZE 32
#define LDS_STACK_SIZE 16

// BVH node
typedef struct
{
    uint3 aabb01_min_or_v0;
    uint  addr0;

    uint3 aabb01_max_or_v1;
    uint  addr1_or_mesh_id;

    uint3 aabb23_min_or_v2;
    uint  addr2_or_prim_id;

    uint3 aabb23_max;
    uint  addr3;
} bvh_node;

#define mymin3(a, b, c) min(min((a), (b)), (c))
#define mymax3(a, b, c) max(max((a), (b)), (c))

INLINE half2 unpackFloat2x16(uint v)
{
    return (half2)
        (as_half(convert_ushort(v >> 16)),
         as_half(convert_ushort(v & 0xFFFF)));
}

INLINE half4 fast_intersect_bbox2(uint3 pmin, uint3 pmax, half3 invdir, half3 oxinvdir, float t_max)
{
    half2 pmin_x = unpackFloat2x16(pmin.x);
    half2 pmin_y = unpackFloat2x16(pmin.y);
    half2 pmin_z = unpackFloat2x16(pmin.z);
    half2 pmax_x = unpackFloat2x16(pmax.x);
    half2 pmax_y = unpackFloat2x16(pmax.y);
    half2 pmax_z = unpackFloat2x16(pmax.z);

    half2 f_x = fma(pmax_x, invdir.xx, oxinvdir.xx);
    half2 f_y = fma(pmax_y, invdir.yy, oxinvdir.yy);
    half2 f_z = fma(pmax_z, invdir.zz, oxinvdir.zz);

    half2 n_x = fma(pmin_x, invdir.xx, oxinvdir.xx);
    half2 n_y = fma(pmin_y, invdir.yy, oxinvdir.yy);
    half2 n_z = fma(pmin_z, invdir.zz, oxinvdir.zz);

    half2 t_max_x = max(f_x, n_x);
    half2 t_max_y = max(f_y, n_y);
    half2 t_max_z = max(f_z, n_z);

    half2 t_min_x = min(f_x, n_x);
    half2 t_min_y = min(f_y, n_y);
    half2 t_min_z = min(f_z, n_z);

    half2 t_zero = (half2)(0.0f, 0.0f);
    half2 t_max2 = (half2)(t_max, t_max);
    half2 t1 = min(mymin3(t_max_x, t_max_y, t_max_z), t_max2);
    half2 t0 = max(mymax3(t_min_x, t_min_y, t_min_z), t_zero);

    return (half4)(t0, t1);
}

void stack_push(
    __local uint *lds_stack,
    __private uint *lds_sptr,
    uint lds_stack_bottom,
    __global uint *stack,
    __private uint *sptr,
    uint idx)
{
    if (*lds_sptr - lds_stack_bottom >= LDS_STACK_SIZE)
    {
        for (int i = 1; i < LDS_STACK_SIZE; ++i)
        {
            stack[*sptr + i] = lds_stack[lds_stack_bottom + i];
        }

        *sptr = *sptr + LDS_STACK_SIZE;
        *lds_sptr = lds_stack_bottom + 1;
    }

    lds_stack[*lds_sptr] = idx;
    *lds_sptr = *lds_sptr + 1;
}

// TODO: use GLOBAL, etc. macros (gboisse)

__attribute__((reqd_work_group_size(64, 1, 1)))
KERNEL void intersect_main(
    // Bvh nodes
    GLOBAL const bvh_node *restrict nodes,
    // Rays
    GLOBAL const ray *restrict rays,
    // Number of rays in rays buffer
    GLOBAL const int *restrict num_rays,
    // Stack memory
    GLOBAL uint *stack,
    // Hit data
    GLOBAL Intersection *hits)
{
    uint index = get_global_id(0);
    uint local_index = get_local_id(0);

    // Handle only working subset
    if (index < *num_rays)
    {
        const ray myRay = rays[index];

        if (ray_is_active(&myRay))
        {
            __local uint lds_stack[GROUP_SIZE * LDS_STACK_SIZE];

            // Precompute inverse direction and origin / dir for bbox testing
            const float3 invDir32 = safe_invdir(myRay);
            const half3 invDir = convert_half3(invDir32);
            const half3 oxInvDir = convert_half3(-myRay.o.xyz * invDir32);

            // Intersection parametric distance
            float closest_t = myRay.o.w;

            // Current node address
            uint addr = 0;
            // Current closest address
            uint closest_addr = INVALID_ADDR;

            uint stack_bottom = STACK_SIZE * index;
            uint sptr = stack_bottom;
            uint lds_stack_bottom = local_index * LDS_STACK_SIZE;
            uint lds_sptr = lds_stack_bottom;

            lds_stack[lds_sptr++] = INVALID_ADDR;

            while (addr != INVALID_ADDR)
            {
                const bvh_node node = nodes[addr];

                if (INTERNAL_NODE(node))
                {
                    half4 s01 = fast_intersect_bbox2(
                        node.aabb01_min_or_v0,
                        node.aabb01_max_or_v1,
                        invDir, oxInvDir, closest_t);
                    half4 s23 = fast_intersect_bbox2(
                        node.aabb23_min_or_v2,
                        node.aabb23_max,
                        invDir, oxInvDir, closest_t);

                    bool traverse_c0 = (s01.x <= s01.z);
                    bool traverse_c1 = (node.addr1_or_mesh_id != INVALID_ADDR) && (s01.y <= s01.w);
                    bool traverse_c2 = (s23.x <= s23.z);
                    bool traverse_c3 = (node.addr3 != INVALID_ADDR) && (s23.y <= s23.w);

                    if (traverse_c0 || traverse_c1 || traverse_c2 || traverse_c3)
                    {
                        uint a = INVALID_ADDR;
                        half d = 100000000.0f;

                        if (traverse_c0)
                        {
                            a = node.addr0;
                            d = s01.x;
                        }

                        if (traverse_c1)
                        {
                            if (a == INVALID_ADDR)
                                a = node.addr1_or_mesh_id;
                            else
                            {
                                uint topush = s01.y < d ? a : node.addr1_or_mesh_id;
                                d = min(s01.y, d);
                                a = topush == a ? node.addr1_or_mesh_id : a;
                                stack_push(lds_stack, &lds_sptr, lds_stack_bottom, stack, &sptr, topush);
                            }
                        }

                        if (traverse_c2)
                        {
                            if (a == INVALID_ADDR)
                                a = node.addr2_or_prim_id;
                            else
                            {
                                uint topush = s23.x < d ? a : node.addr2_or_prim_id;
                                d = min(s23.x, d);
                                a = topush == a ? node.addr2_or_prim_id : a;
                                stack_push(lds_stack, &lds_sptr, lds_stack_bottom, stack, &sptr, topush);
                            }
                        }

                        if (traverse_c3)
                        {
                            if (a == INVALID_ADDR)
                                a = node.addr3;
                            else
                            {
                                uint topush = s23.y < d ? a : node.addr3;
                                d = min(s23.y, d);
                                a = topush == a ? node.addr3 : a;
                                stack_push(lds_stack, &lds_sptr, lds_stack_bottom, stack, &sptr, topush);
                            }
                        }

                        addr = a;
                        continue;
                    }
                }
                else
                {
                    float t = fast_intersect_triangle(
                        myRay,
                        as_float3(node.aabb01_min_or_v0),
                        as_float3(node.aabb01_max_or_v1),
                        as_float3(node.aabb23_min_or_v2),
                        closest_t);

                    if (t < closest_t)
                    {
                        closest_t = t;
                        closest_addr = addr;
                    }
                }

                addr = lds_stack[--lds_sptr];

                if (addr == INVALID_ADDR && sptr > stack_bottom)
                {
                    sptr -= LDS_STACK_SIZE;
                    for (int i = 1; i < LDS_STACK_SIZE; ++i)
                    {
                        lds_stack[lds_stack_bottom + i] = stack[sptr + i];
                    }

                    lds_sptr = lds_stack_bottom + LDS_STACK_SIZE - 1;
                    addr = lds_stack[lds_sptr];
                }
            }

            // Check if we have found an intersection
            if (closest_addr != INVALID_ADDR)
            {
                // Calculate hit position
                const bvh_node node = nodes[closest_addr];
                const float3 p = myRay.o.xyz + closest_t * myRay.d.xyz;

                // Calculte barycentric coordinates
                const float2 uv = triangle_calculate_barycentrics(
                    p,
                    as_float3(node.aabb01_min_or_v0),
                    as_float3(node.aabb01_max_or_v1),
                    as_float3(node.aabb23_min_or_v2));

                // Update hit information
                hits[index].prim_id = node.addr2_or_prim_id;
                hits[index].shape_id = node.addr1_or_mesh_id;
                hits[index].uvwt = make_float4(uv.x, uv.y, 0.f, closest_t);
            }
            else
            {
                // Miss here
                hits[index].prim_id = INVALID_ADDR;
                hits[index].shape_id = INVALID_ADDR;
            }
        }
    }
}

__attribute__((reqd_work_group_size(64, 1, 1)))
KERNEL void occluded_main(
    // Bvh nodes
    GLOBAL const bvh_node *restrict nodes,
    // Rays
    GLOBAL const ray *restrict rays,
    // Number of rays in rays buffer
    GLOBAL const int *restrict num_rays,
    // Stack memory
    GLOBAL int *stack,
    // Hit results: 1 for hit and -1 for miss
    GLOBAL int *hits)
{
    // TODO: ... (gboisse)
}


Enabled avc_motion_estimation extension for Intel subgroup shuffling

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains two optimized matrix-multiplication kernels:
// - Kernel 0: inspired by the paper by Matsumoto et al. and the tutorial on
//   http://www.cedricnugteren.nl/tutorial.php
// - Kernel 1: inspired by a Qualcomm optimized GPU kernel with 2D register tiling
//   https://developer.qualcomm.com/blog/matrix-multiply-adreno-gpus-part-2-host-code-and-kernel
// Both are fully configurable (and tunable!) using many parameters. Both kernels support
// different data-types (SGEMM/DGEMM/CGEMM/ZGEMM/HGEMM) through a pre-processor define.
//
// For kernel 0 matrices are accessed as follows:
// A: [k*M + m], with 'k' ranging from 0:K and 'm' from 0:M (m,k,m)
// B: [k*N + n], with 'k' ranging from 0:K and 'n' from 0:N (n,k,n)
// C: [n*M + m], with 'n' ranging from 0:N and 'm' from 0:M (m,n,m)
// For kernel 1, both A and C are transposed w.r.t. the above
//
// Or as an image (assuming column-major)
//       K
//    o-------o
//    |       |
//  N | [B^T] |
//    |       |
//    o-------o
//        K               N
//    o-------o        o-----o
//  M |  [A]  |      M | [C] |
//    |       |        |     |
//    o-------o        o-----o
//
//
// This kernel is separated into multiple files. This is part 1 out of 4.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// Parameters set by the tuner or by the database. Here they are given a basic default value in case
// this kernel file is used outside of the CLBlast library.
#ifndef GEMMK
  #define GEMMK 0    // Kernel to choose: 0 regular, 1 with 2D register tiling
#endif
#ifndef MWG
  #define MWG 8      // Tile-size in dimension M (e.g. 64, 128)
#endif
#ifndef NWG
  #define NWG 8      // Tile-size in dimension N (e.g. 64, 128)
#endif
#ifndef KWG
  #define KWG 8      // Tile-size in dimension K (e.g. 8, 16)
#endif
#ifndef MDIMC
  #define MDIMC 8    // Threads per workgroup in M-dimension (e.g. 8, 16, 32)
#endif
#ifndef NDIMC
  #define NDIMC 8    // Threads per workgroup in N-dimension (e.g. 8, 16, 32)
#endif
#ifndef MDIMA
  #define MDIMA 8    // Re-shaped tile dimension of matrix A: KDIMA * MDIMA (kernel 0 only)
#endif
#ifndef NDIMB
  #define NDIMB 8    // Re-shaped tile dimension of matrix B: KDIMB * NDIMB (kernel 0 only)
#endif
#ifndef KWI
  #define KWI 1      // Unroll factor of the KWG loop (smaller or equal than KWG)
#endif
#ifndef VWM
  #define VWM 1      // Vector width of matrices A and C
#endif
#ifndef VWN
  #define VWN 1      // Vector width of matrix B
#endif
#ifndef STRM
  #define STRM 0     // Use strided access within a thread in the M-dimension (1) or not (0) (kernel 0 only)
#endif
#ifndef STRN
  #define STRN 0     // Use strided access within a thread in the N-dimension (1) or not (0) (kernel 0 only)
#endif
#ifndef SA
  #define SA 0       // Use local/shared memory to cache matrix A (1) or not (0) (kernel 0 only)
#endif
#ifndef SB
  #define SB 0       // Use local/shared memory to cache matrix B (1) or not (0) (kernel 0 only)
#endif
#ifndef KREG
  #define KREG 1     // Amount of register tiling in second dimension, multiple of VWN (kernel 1 only)
#endif

// Helper parameters based on the above tuning parameters
#define MWI (MWG/MDIMC)               // Work per work-item (M-dimension)
#define NWI (NWG/NDIMC)               // Work per work-item (N-dimension)
#define KDIMA ((MDIMC*NDIMC)/(MDIMA)) // Re-shaped tile dimension of matrix A: KDIMA * MDIMA
#define KDIMB ((MDIMC*NDIMC)/(NDIMB)) // Re-shaped tile dimension of matrix B: KDIMB * NDIMB
#define MWA (MWG/MDIMA)               // Amount of loads-per-thread for matrix A (M-dimension)
#define KWA (KWG/KDIMA)               // Amount of loads-per-thread for matrix A (K-dimension)
#define KWB (KWG/KDIMB)               // Amount of loads-per-thread for matrix B (K-dimension)
#define NWB (NWG/NDIMB)               // Amount of loads-per-thread for matrix B (N-dimension)

// Settings
#ifndef USE_VECTOR_MAD
  #define USE_VECTOR_MAD 0      // Unroll (0) or don't (1) unroll the vector MAD manually
#endif
#ifndef GLOBAL_MEM_FENCE
  #define GLOBAL_MEM_FENCE 0    // Global synchronisation barrier for potential better performance
#endif

#ifndef SUBGROUP_SHUFFLING_NVIDIA_PRE_VOLTA
  #define SUBGROUP_SHUFFLING_NVIDIA_PRE_VOLTA 0
#endif
#ifndef SUBGROUP_SHUFFLING_NVIDIA_POST_VOLTA
  #define SUBGROUP_SHUFFLING_NVIDIA_POST_VOLTA 0
#endif
#ifndef SUBGROUP_SHUFFLING_INTEL
  #define SUBGROUP_SHUFFLING_INTEL 0
#endif
#ifndef USE_SUBGROUP_SHUFFLING
  #define USE_SUBGROUP_SHUFFLING 0     // Optionally enables subgroup shuffling for Intel GPUs
#endif

// Intel subgroups (https://www.khronos.org/registry/OpenCL/extensions/intel/cl_intel_subgroups.html)
#if USE_SUBGROUP_SHUFFLING == 1 && SUBGROUP_SHUFFLING_INTEL == 1
  #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation: enable
  #define SUBGROUP_SIZE 8              // Assumes subgroup size is always 8 on Intel GPUs
#endif

// NVIDIA warps as subgroups using inline PTX (https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html)
#if USE_SUBGROUP_SHUFFLING == 1
  #if SUBGROUP_SHUFFLING_NVIDIA_PRE_VOLTA == 1 || SUBGROUP_SHUFFLING_NVIDIA_POST_VOLTA == 1
    #define SUBGROUP_SIZE 32            // Assumes subgroup size is always 32 on NVIDIA GPUs
  #endif
#endif

#if NWI != SUBGROUP_SIZE || MDIMC < SUBGROUP_SIZE
  #undef USE_SUBGROUP_SHUFFLING
  #define USE_SUBGROUP_SHUFFLING 0     // Disables subgroups in case the assumptions don't hold
#endif

// =================================================================================================

// Data-widths in dimension M
#if VWM == 1
    typedef real realM;
#elif VWM == 2
    typedef real2 realM;
#elif VWM == 4
    typedef real4 realM;
#elif VWM == 8
    typedef real8 realM;
#elif VWM == 16
    typedef real16 realM;
#endif

// Data-widths in dimension N
#if VWN == 1
    typedef real realN;
#elif VWN == 2
    typedef real2 realN;
#elif VWN == 4
    typedef real4 realN;
#elif VWN == 8
    typedef real8 realN;
#elif VWN == 16
    typedef real16 realN;
#endif

// =================================================================================================

// Initializes the accumulation registers to zero
INLINE_FUNC realM InitAccRegisters() {
  realM result;
  #if VWM == 1
    SetToZero(result);
  #elif VWM == 2
    SetToZero(result.x);
    SetToZero(result.y);
  #elif VWM == 4
    SetToZero(result.x);
    SetToZero(result.y);
    SetToZero(result.z);
    SetToZero(result.w);
  #elif VWM == 8
    SetToZero(result.s0);
    SetToZero(result.s1);
    SetToZero(result.s2);
    SetToZero(result.s3);
    SetToZero(result.s4);
    SetToZero(result.s5);
    SetToZero(result.s6);
    SetToZero(result.s7);
  #elif VWM == 16
    SetToZero(result.s0);
    SetToZero(result.s1);
    SetToZero(result.s2);
    SetToZero(result.s3);
    SetToZero(result.s4);
    SetToZero(result.s5);
    SetToZero(result.s6);
    SetToZero(result.s7);
    SetToZero(result.s8);
    SetToZero(result.s9);
    SetToZero(result.sA);
    SetToZero(result.sB);
    SetToZero(result.sC);
    SetToZero(result.sD);
    SetToZero(result.sE);
    SetToZero(result.sF);
  #endif
  return result;
}

// =================================================================================================

// Caches global off-chip memory into local (shared) memory on-chip. This function is specific for
// caching the A input matrix.
#if SA == 1
INLINE_FUNC void GlobalToLocalA(const __global realM* restrict agm, LOCAL_PTR realM* alm,
                                const int kSizeM, const int tid, const int kwg) {
  const int la0 = tid % MDIMA;
  const int la1 = tid / MDIMA;
  #pragma unroll
  for (int _mia = 0; _mia < MWA/VWM; _mia += 1) {
    #pragma unroll
    for (int _kia = 0; _kia < KWA; _kia += 1) {

      // Computes the indices based on strided/non-strided access
      #if STRM == 0
        int mg = _mia + la0*(MWA/VWM);
      #elif STRM == 1
        int mg = la0 + _mia*MDIMA;
      #endif

      // Computes the indices for the global memory
      int kg = _kia + la1*KWA;
      int idm = mg + GetGroupID0() * (MWG/VWM);
      int idk = kg + kwg;

      // Loads the data from global memory (not transposed) into the local memory
      alm[kg*(MWG/VWM) + mg] = agm[idk*(kSizeM/VWM) + idm];
    }
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
INLINE_FUNC void GlobalToLocalB(const __global realN* restrict bgm, LOCAL_PTR realN* blm,
                                const int kSizeN, const int tid, const int kwg) {
  const int lb0 = tid % NDIMB;
  const int lb1 = tid / NDIMB;
  #pragma unroll
  for (int _kib = 0; _kib < KWB; _kib += 1) {
    #pragma unroll
    for (int _nib = 0; _nib < NWB/VWN; _nib += 1) {

      // Computes the indices based on strided/non-strided access
      #if STRN == 0
        int ng = _nib + lb0*(NWB/VWN);
      #elif STRN == 1
        int ng = lb0 + _nib*NDIMB;
      #endif

      // Computes the indices for the global memory
      int kg = _kib + lb1*KWB;
      int idn = ng + GetGroupID1() * (NWG/VWN);
      int idk = kg + kwg;

      // Loads the data from global memory (transposed) into the local memory
      blm[kg*(NWG/VWN) + ng] = bgm[idk*(kSizeN/VWN) + idn];
    }
  }
}
#endif

// =================================================================================================

// Caches global off-chip memory directly into per-thread private memory (registers). This function
// is specific for caching the A input matrix.
#if SA == 0 && GEMMK == 0
INLINE_FUNC realM GlobalToPrivateA(const __global realM* restrict agm, const int _mi,
                                   const int kSizeM, const int idk, const int kwg) {
  // Computes the indices based on strided/non-strided access
  #if STRM == 0
    int mg = _mi + get_local_id(0)*(MWI/VWM);
  #elif STRM == 1
    int mg = get_local_id(0) + _mi*MDIMC;
  #endif

  // Computes the indices for the global memory
  int idm = mg + GetGroupID0() * (MWG/VWM);

  // Loads the data from global memory (not transposed) and stores into registers
  return agm[idk*(kSizeM/VWM) + idm];
}
#endif

// Same as above, but now for the B input matrix
#if SB == 0 && GEMMK == 0
INLINE_FUNC realN GlobalToPrivateB(const __global realN* restrict bgm, const int _ni,
                                   const int kSizeN, const int idk) {
  // Computes the indices based on strided/non-strided access
  #if STRN == 0
    int ng = _ni + get_local_id(1)*(NWI/VWN);
  #elif STRN == 1
    int ng = get_local_id(1) + _ni*NDIMC;
  #endif

  // Computes the indices for the global memory
  int idn = ng + GetGroupID1() * (NWG/VWN);

  // Loads the data from global memory (transposed) and stores into registers
  return bgm[idk*(kSizeN/VWN) + idn];
}
#endif

// =================================================================================================
#if GEMMK == 1

// Caches global off-chip memory directly into per-thread private memory (registers). This function
// is specific for caching the A input matrix for kernel 1.
INLINE_FUNC realN GlobalToPrivateA2D(const __global real* restrict a_ptr, const int tid_y, const int _ni,
                                     const int kSizeK, const int idk, const int _ki) {
  #if PRECISION == 3232 || PRECISION == 6464
    const int a_index = (tid_y * NWI + _ni) * (kSizeK / VWN) + idk / VWN + _ki;
    const __global realN* restrict agm = (const __global realN* restrict) a_ptr;
    return agm[a_index];
  #else
    const int a_index = (tid_y * NWI + _ni) * kSizeK + idk + _ki * VWN;
    #if VWN == 1
      return a_ptr[a_index];
    #elif VWN == 2
      return vload2(0, a_ptr + a_index);
    #elif VWN == 4
      return vload4(0, a_ptr + a_index);
    #elif VWN == 8
      return vload8(0, a_ptr + a_index);
    #elif VWN == 16
      return vload16(0, a_ptr + a_index);
    #endif
  #endif
}

// Same as above, but now for the B input matrix
INLINE_FUNC realM GlobalToPrivateB2D(const __global real* restrict b_ptr, const int tid_x, const int _mi,
                                     const int kSizeN, const int idk, const int _ki) {
  #if PRECISION == 3232 || PRECISION == 6464
    const int b_index = (idk + _ki) * (kSizeN / VWM) + tid_x * (MWI / VWM) + _mi;
    const __global realM* restrict bgm = (const __global realM* restrict) b_ptr;
    return bgm[b_index];
  #else
    const int b_index = (idk + _ki) * kSizeN + tid_x * MWI + _mi * VWM;
    #if VWM == 1
      return b_ptr[b_index];
    #elif VWM == 2
      return vload2(0, b_ptr + b_index);
    #elif VWM == 4
      return vload4(0, b_ptr + b_index);
    #elif VWM == 8
      return vload8(0, b_ptr + b_index);
    #elif VWM == 16
      return vload16(0, b_ptr + b_index);
    #endif
  #endif
}

#endif
// =================================================================================================

// Caches on-chip local memory into per-thread private memory (registers). This function is specific
// for caching the A input matrix.
#if SA == 1
INLINE_FUNC realM LocalToPrivateA(LOCAL_PTR realM* alm, const int _mi, const int kg) {
  #if STRM == 0
    int mg = _mi + get_local_id(0)*(MWI/VWM);
  #elif STRM == 1
    int mg = get_local_id(0) + _mi*MDIMC;
  #endif
  return alm[kg*(MWG/VWM) + mg];
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
INLINE_FUNC realN LocalToPrivateB(LOCAL_PTR realN* blm, const int _ni, const int kg) {
  #if STRN == 0
    int ng = _ni + get_local_id(1)*(NWI/VWN);
  #elif STRN == 1
    int ng = get_local_id(1) + _ni*NDIMC;
  #endif
  return blm[kg*(NWG/VWN) + ng];
}
#endif

)"
// End of the C++11 raw string literal

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains two optimized matrix-multiplication kernels:
// - Kernel 0: inspired by the paper by Matsumoto et al. and the tutorial on
//   http://www.cedricnugteren.nl/tutorial.php
// - Kernel 1: inspired by a Qualcomm optimized GPU kernel with 2D register tiling
//   https://developer.qualcomm.com/blog/matrix-multiply-adreno-gpus-part-2-host-code-and-kernel
// Both are fully configurable (and tunable!) using many parameters. Both kernels support
// different data-types (SGEMM/DGEMM/CGEMM/ZGEMM/HGEMM) through a pre-processor define.
//
// For kernel 0 matrices are accessed as follows:
// A: [k*M + m], with 'k' ranging from 0:K and 'm' from 0:M (m,k,m)
// B: [k*N + n], with 'k' ranging from 0:K and 'n' from 0:N (n,k,n)
// C: [n*M + m], with 'n' ranging from 0:N and 'm' from 0:M (m,n,m)
// For kernel 1, both A and C are transposed w.r.t. the above
//
// Or as an image (assuming column-major)
//       K
//    o-------o
//    |       |
//  N | [B^T] |
//    |       |
//    o-------o
//        K               N
//    o-------o        o-----o
//  M |  [A]  |      M | [C] |
//    |       |        |     |
//    o-------o        o-----o
//
//
// This kernel is separated into multiple files. This is part 1 out of 4.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// Parameters set by the tuner or by the database. Here they are given a basic default value in case
// this kernel file is used outside of the CLBlast library.
#ifndef GEMMK
  #define GEMMK 0    // Kernel to choose: 0 regular, 1 with 2D register tiling
#endif
#ifndef MWG
  #define MWG 8      // Tile-size in dimension M (e.g. 64, 128)
#endif
#ifndef NWG
  #define NWG 8      // Tile-size in dimension N (e.g. 64, 128)
#endif
#ifndef KWG
  #define KWG 8      // Tile-size in dimension K (e.g. 8, 16)
#endif
#ifndef MDIMC
  #define MDIMC 8    // Threads per workgroup in M-dimension (e.g. 8, 16, 32)
#endif
#ifndef NDIMC
  #define NDIMC 8    // Threads per workgroup in N-dimension (e.g. 8, 16, 32)
#endif
#ifndef MDIMA
  #define MDIMA 8    // Re-shaped tile dimension of matrix A: KDIMA * MDIMA (kernel 0 only)
#endif
#ifndef NDIMB
  #define NDIMB 8    // Re-shaped tile dimension of matrix B: KDIMB * NDIMB (kernel 0 only)
#endif
#ifndef KWI
  #define KWI 1      // Unroll factor of the KWG loop (smaller or equal than KWG)
#endif
#ifndef VWM
  #define VWM 1      // Vector width of matrices A and C
#endif
#ifndef VWN
  #define VWN 1      // Vector width of matrix B
#endif
#ifndef STRM
  #define STRM 0     // Use strided access within a thread in the M-dimension (1) or not (0) (kernel 0 only)
#endif
#ifndef STRN
  #define STRN 0     // Use strided access within a thread in the N-dimension (1) or not (0) (kernel 0 only)
#endif
#ifndef SA
  #define SA 0       // Use local/shared memory to cache matrix A (1) or not (0) (kernel 0 only)
#endif
#ifndef SB
  #define SB 0       // Use local/shared memory to cache matrix B (1) or not (0) (kernel 0 only)
#endif
#ifndef KREG
  #define KREG 1     // Amount of register tiling in second dimension, multiple of VWN (kernel 1 only)
#endif

// Helper parameters based on the above tuning parameters
#define MWI (MWG/MDIMC)               // Work per work-item (M-dimension)
#define NWI (NWG/NDIMC)               // Work per work-item (N-dimension)
#define KDIMA ((MDIMC*NDIMC)/(MDIMA)) // Re-shaped tile dimension of matrix A: KDIMA * MDIMA
#define KDIMB ((MDIMC*NDIMC)/(NDIMB)) // Re-shaped tile dimension of matrix B: KDIMB * NDIMB
#define MWA (MWG/MDIMA)               // Amount of loads-per-thread for matrix A (M-dimension)
#define KWA (KWG/KDIMA)               // Amount of loads-per-thread for matrix A (K-dimension)
#define KWB (KWG/KDIMB)               // Amount of loads-per-thread for matrix B (K-dimension)
#define NWB (NWG/NDIMB)               // Amount of loads-per-thread for matrix B (N-dimension)

// Settings
#ifndef USE_VECTOR_MAD
  #define USE_VECTOR_MAD 0      // Unroll (0) or don't (1) unroll the vector MAD manually
#endif
#ifndef GLOBAL_MEM_FENCE
  #define GLOBAL_MEM_FENCE 0    // Global synchronisation barrier for potential better performance
#endif

#ifndef SUBGROUP_SHUFFLING_NVIDIA_PRE_VOLTA
  #define SUBGROUP_SHUFFLING_NVIDIA_PRE_VOLTA 0
#endif
#ifndef SUBGROUP_SHUFFLING_NVIDIA_POST_VOLTA
  #define SUBGROUP_SHUFFLING_NVIDIA_POST_VOLTA 0
#endif
#ifndef SUBGROUP_SHUFFLING_INTEL
  #define SUBGROUP_SHUFFLING_INTEL 0
#endif
#ifndef USE_SUBGROUP_SHUFFLING
  #define USE_SUBGROUP_SHUFFLING 0     // Optionally enables subgroup shuffling for Intel GPUs
#endif

// Intel subgroups (https://www.khronos.org/registry/OpenCL/extensions/intel/cl_intel_subgroups.txt)
#if USE_SUBGROUP_SHUFFLING == 1 && SUBGROUP_SHUFFLING_INTEL == 1
  #define SUBGROUP_SIZE 8              // Assumes subgroup size is always 8 on Intel GPUs
#endif

// NVIDIA warps as subgroups using inline PTX (https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html)
#if USE_SUBGROUP_SHUFFLING == 1
  #if SUBGROUP_SHUFFLING_NVIDIA_PRE_VOLTA == 1 || SUBGROUP_SHUFFLING_NVIDIA_POST_VOLTA == 1
    #define SUBGROUP_SIZE 32            // Assumes subgroup size is always 32 on NVIDIA GPUs
  #endif
#endif

#if NWI != SUBGROUP_SIZE || MDIMC < SUBGROUP_SIZE
  #undef USE_SUBGROUP_SHUFFLING
  #define USE_SUBGROUP_SHUFFLING 0     // Disables subgroups in case the assumptions don't hold
#endif

// =================================================================================================

// Data-widths in dimension M
#if VWM == 1
    typedef real realM;
#elif VWM == 2
    typedef real2 realM;
#elif VWM == 4
    typedef real4 realM;
#elif VWM == 8
    typedef real8 realM;
#elif VWM == 16
    typedef real16 realM;
#endif

// Data-widths in dimension N
#if VWN == 1
    typedef real realN;
#elif VWN == 2
    typedef real2 realN;
#elif VWN == 4
    typedef real4 realN;
#elif VWN == 8
    typedef real8 realN;
#elif VWN == 16
    typedef real16 realN;
#endif

// =================================================================================================

// Initializes the accumulation registers to zero
INLINE_FUNC realM InitAccRegisters() {
  realM result;
  #if VWM == 1
    SetToZero(result);
  #elif VWM == 2
    SetToZero(result.x);
    SetToZero(result.y);
  #elif VWM == 4
    SetToZero(result.x);
    SetToZero(result.y);
    SetToZero(result.z);
    SetToZero(result.w);
  #elif VWM == 8
    SetToZero(result.s0);
    SetToZero(result.s1);
    SetToZero(result.s2);
    SetToZero(result.s3);
    SetToZero(result.s4);
    SetToZero(result.s5);
    SetToZero(result.s6);
    SetToZero(result.s7);
  #elif VWM == 16
    SetToZero(result.s0);
    SetToZero(result.s1);
    SetToZero(result.s2);
    SetToZero(result.s3);
    SetToZero(result.s4);
    SetToZero(result.s5);
    SetToZero(result.s6);
    SetToZero(result.s7);
    SetToZero(result.s8);
    SetToZero(result.s9);
    SetToZero(result.sA);
    SetToZero(result.sB);
    SetToZero(result.sC);
    SetToZero(result.sD);
    SetToZero(result.sE);
    SetToZero(result.sF);
  #endif
  return result;
}

// =================================================================================================

// Caches global off-chip memory into local (shared) memory on-chip. This function is specific for
// caching the A input matrix.
#if SA == 1
INLINE_FUNC void GlobalToLocalA(const __global realM* restrict agm, LOCAL_PTR realM* alm,
                                const int kSizeM, const int tid, const int kwg) {
  const int la0 = tid % MDIMA;
  const int la1 = tid / MDIMA;
  #pragma unroll
  for (int _mia = 0; _mia < MWA/VWM; _mia += 1) {
    #pragma unroll
    for (int _kia = 0; _kia < KWA; _kia += 1) {

      // Computes the indices based on strided/non-strided access
      #if STRM == 0
        int mg = _mia + la0*(MWA/VWM);
      #elif STRM == 1
        int mg = la0 + _mia*MDIMA;
      #endif

      // Computes the indices for the global memory
      int kg = _kia + la1*KWA;
      int idm = mg + GetGroupID0() * (MWG/VWM);
      int idk = kg + kwg;

      // Loads the data from global memory (not transposed) into the local memory
      alm[kg*(MWG/VWM) + mg] = agm[idk*(kSizeM/VWM) + idm];
    }
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
INLINE_FUNC void GlobalToLocalB(const __global realN* restrict bgm, LOCAL_PTR realN* blm,
                                const int kSizeN, const int tid, const int kwg) {
  const int lb0 = tid % NDIMB;
  const int lb1 = tid / NDIMB;
  #pragma unroll
  for (int _kib = 0; _kib < KWB; _kib += 1) {
    #pragma unroll
    for (int _nib = 0; _nib < NWB/VWN; _nib += 1) {

      // Computes the indices based on strided/non-strided access
      #if STRN == 0
        int ng = _nib + lb0*(NWB/VWN);
      #elif STRN == 1
        int ng = lb0 + _nib*NDIMB;
      #endif

      // Computes the indices for the global memory
      int kg = _kib + lb1*KWB;
      int idn = ng + GetGroupID1() * (NWG/VWN);
      int idk = kg + kwg;

      // Loads the data from global memory (transposed) into the local memory
      blm[kg*(NWG/VWN) + ng] = bgm[idk*(kSizeN/VWN) + idn];
    }
  }
}
#endif

// =================================================================================================

// Caches global off-chip memory directly into per-thread private memory (registers). This function
// is specific for caching the A input matrix.
#if SA == 0 && GEMMK == 0
INLINE_FUNC realM GlobalToPrivateA(const __global realM* restrict agm, const int _mi,
                                   const int kSizeM, const int idk, const int kwg) {
  // Computes the indices based on strided/non-strided access
  #if STRM == 0
    int mg = _mi + get_local_id(0)*(MWI/VWM);
  #elif STRM == 1
    int mg = get_local_id(0) + _mi*MDIMC;
  #endif

  // Computes the indices for the global memory
  int idm = mg + GetGroupID0() * (MWG/VWM);

  // Loads the data from global memory (not transposed) and stores into registers
  return agm[idk*(kSizeM/VWM) + idm];
}
#endif

// Same as above, but now for the B input matrix
#if SB == 0 && GEMMK == 0
INLINE_FUNC realN GlobalToPrivateB(const __global realN* restrict bgm, const int _ni,
                                   const int kSizeN, const int idk) {
  // Computes the indices based on strided/non-strided access
  #if STRN == 0
    int ng = _ni + get_local_id(1)*(NWI/VWN);
  #elif STRN == 1
    int ng = get_local_id(1) + _ni*NDIMC;
  #endif

  // Computes the indices for the global memory
  int idn = ng + GetGroupID1() * (NWG/VWN);

  // Loads the data from global memory (transposed) and stores into registers
  return bgm[idk*(kSizeN/VWN) + idn];
}
#endif

// =================================================================================================
#if GEMMK == 1

// Caches global off-chip memory directly into per-thread private memory (registers). This function
// is specific for caching the A input matrix for kernel 1.
INLINE_FUNC realN GlobalToPrivateA2D(const __global real* restrict a_ptr, const int tid_y, const int _ni,
                                     const int kSizeK, const int idk, const int _ki) {
  #if PRECISION == 3232 || PRECISION == 6464
    const int a_index = (tid_y * NWI + _ni) * (kSizeK / VWN) + idk / VWN + _ki;
    const __global realN* restrict agm = (const __global realN* restrict) a_ptr;
    return agm[a_index];
  #else
    const int a_index = (tid_y * NWI + _ni) * kSizeK + idk + _ki * VWN;
    #if VWN == 1
      return a_ptr[a_index];
    #elif VWN == 2
      return vload2(0, a_ptr + a_index);
    #elif VWN == 4
      return vload4(0, a_ptr + a_index);
    #elif VWN == 8
      return vload8(0, a_ptr + a_index);
    #elif VWN == 16
      return vload16(0, a_ptr + a_index);
    #endif
  #endif
}

// Same as above, but now for the B input matrix
INLINE_FUNC realM GlobalToPrivateB2D(const __global real* restrict b_ptr, const int tid_x, const int _mi,
                                     const int kSizeN, const int idk, const int _ki) {
  #if PRECISION == 3232 || PRECISION == 6464
    const int b_index = (idk + _ki) * (kSizeN / VWM) + tid_x * (MWI / VWM) + _mi;
    const __global realM* restrict bgm = (const __global realM* restrict) b_ptr;
    return bgm[b_index];
  #else
    const int b_index = (idk + _ki) * kSizeN + tid_x * MWI + _mi * VWM;
    #if VWM == 1
      return b_ptr[b_index];
    #elif VWM == 2
      return vload2(0, b_ptr + b_index);
    #elif VWM == 4
      return vload4(0, b_ptr + b_index);
    #elif VWM == 8
      return vload8(0, b_ptr + b_index);
    #elif VWM == 16
      return vload16(0, b_ptr + b_index);
    #endif
  #endif
}

#endif
// =================================================================================================

// Caches on-chip local memory into per-thread private memory (registers). This function is specific
// for caching the A input matrix.
#if SA == 1
INLINE_FUNC realM LocalToPrivateA(LOCAL_PTR realM* alm, const int _mi, const int kg) {
  #if STRM == 0
    int mg = _mi + get_local_id(0)*(MWI/VWM);
  #elif STRM == 1
    int mg = get_local_id(0) + _mi*MDIMC;
  #endif
  return alm[kg*(MWG/VWM) + mg];
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
INLINE_FUNC realN LocalToPrivateB(LOCAL_PTR realN* blm, const int _ni, const int kg) {
  #if STRN == 0
    int ng = _ni + get_local_id(1)*(NWI/VWN);
  #elif STRN == 1
    int ng = get_local_id(1) + _ni*NDIMC;
  #endif
  return blm[kg*(NWG/VWN) + ng];
}
#endif

)"
// End of the C++11 raw string literal

// =================================================================================================


replaced % by %%, because they can get removed by sprintf.

/**
 * Copyright 2014 University of Bremen, Institute for Artificial Intelligence
 * Author: Thiemo Wiedemeyer <wiedemeyer@cs.uni-bremen.de>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Rotation
#define r00 %0.14e
#define r01 %0.14e
#define r02 %0.14e
#define r10 %0.14e
#define r11 %0.14e
#define r12 %0.14e
#define r20 %0.14e
#define r21 %0.14e
#define r22 %0.14e

// Translation
#define tx %0.14e
#define ty %0.14e
#define tz %0.14e

// Camera parameter upscaled depth
#define fxD %.14e
#define fyD %.14e
#define cxD %.14e
#define cyD %.14e
#define fxDInv (1.0f / fxD)
#define fyDInv (1.0f / fyD)

// Camera parameter color
#define fxC %.14e
#define fyC %.14e
#define cxC %.14e
#define cyC %.14e

// Clipping distances
#define zNear %d
#define zFar %d
#define zDist %.14e

// Size color image
#define heightC %d
#define widthC %d

// Size depth image
#define heightD %d
#define widthD %d

// Size raw depth image
#define heightR %d
#define widthR %d

// Set render buffer to zero
void kernel setZero(global ushort* rendered, global float *selDist){
  const uint i = get_global_id(0);
  rendered[i] = 0;
  selDist[i] = 10;
}

// Calculate 3d point, project them to color camera coordinate system and create rendered depth image
void kernel project(global const ushort *depth, global int4 *idx, global ushort *zImg, global float4 *dists, global float *selDist, global ushort *rendered){
  const uint i = get_global_id(0);

  const int xD = i %% widthD;
  const int yD = i / widthD;

  const ushort d = depth[i];

  // Projection matrix coloumns
  const float4 projX = (float4)(r00, r01, r02, tx);
  const float4 projY = (float4)(r10, r11, r12, ty);
  const float4 projZ = (float4)(r20, r21, r22, tz);

  // Compute 3D point
  const float z = d / 1000.0f;
  const float4 point = (float4)((xD - cxD) * fxDInv * z, (yD - cyD) * fyDInv * z, z, 1.0f);

  // Rotate and translate
  const float3 projected = (float3)(dot(point, projX), dot(point, projY), dot(point, projZ));

  const float invZ = 1.0f / projected.z;

  // Compute projected image coordinates
  const float x = (fxD * projected.x) * invZ + cxD;
  const float y = (fyD * projected.y) * invZ + cyD;
  const short xL = (short)floor(x);
  const short yL = (short)floor(y);
  const short xH = xL + 1;
  const short yH = yL + 1;

  const float4 distXY = pown((float4)(x - xL, xH - x, y - yL, yH - y), 2);
  const float4 dist2 = (float4)(distXY.s0 + distXY.s2, distXY.s1 + distXY.s2, distXY.s0 + distXY.s3, distXY.s1 + distXY.s3);

  const int yLI = yL * widthD;
  const int yHI = yH * widthD;
  int4 indices = (int4)(yLI + xL, yLI + xH, yHI + xL, yHI + xH);

  const ushort zI = (ushort)(projected.z * 1000.0f);

  // Check if depth is valid and projection is inside the image
  if(xL < 0)
  {
    indices.s0 = indices.s2 = -1;
  }
  if(yL < 0)
  {
    indices.s0 = indices.s1 = -1;
  }
  if(xH >= widthD)
  {
    indices.s1 = indices.s3 = -1;
  }
  if(yH >= heightD)
  {
    indices.s2 = indices.s3 = -1;
  }
  if(indices.s0 >= 0)
  {
    selDist[indices.s0] = dist2.s0;
    rendered[indices.s0] = zI;
  }
  if(indices.s1 >= 0)
  {
    selDist[indices.s1] = dist2.s1;
    rendered[indices.s1] = zI;
  }
  if(indices.s2 >= 0)
  {
    selDist[indices.s2] = dist2.s2;
    rendered[indices.s2] = zI;
  }
  if(indices.s3 >= 0)
  {
    selDist[indices.s3] = dist2.s3;
    rendered[indices.s3] = zI;
  }

  idx[i] = indices;
  dists[i] = dist2;
  zImg[i] = zI;
}

// update rendered depth image to make sure nearest depth values are used
void kernel render(global const int4 *idx, global const ushort *zImg, global const float4 *dists, global float *selDist, global ushort *rendered){
  const uint i = get_global_id(0);

  const int4 index = idx[i];
  const ushort zI = zImg[i];
  const ushort thres = 0.01 * zI;
  const ushort zIThres = zI + thres;
  const float4 dist2 = dists[i];

  ushort zRen;
  zRen = rendered[index.s0];
  if(index.s0 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s0] > dist2.s0) || zRen > zIThres))
  {
    selDist[index.s0] = dist2.s0;
    rendered[index.s0] = zI;
  }
  zRen = rendered[index.s1];
  if(index.s1 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s1] > dist2.s1) || zRen > zIThres))
  {
    selDist[index.s1] = dist2.s1;
    rendered[index.s1] = zI;
  }
  zRen = rendered[index.s2];
  if(index.s2 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s2] > dist2.s2) || zRen > zIThres))
  {
    selDist[index.s2] = dist2.s2;
    rendered[index.s2] = zI;
  }
  zRen = rendered[index.s3];
  if(index.s3 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s3] > dist2.s3) || zRen > zIThres))
  {
    selDist[index.s3] = dist2.s3;
    rendered[index.s3] = zI;
  }
}

// Calculate 3d point, project them to color camera coordinate system and create rendered depth image
/*void kernel project(global const unsigned short *depth, global int *idx, global unsigned short *zImg, global unsigned short *rendered){
  const uint i = get_global_id(0);

  const int xD = i %% widthD;
  const int yD = i / widthD;

  const ushort d = depth[i];

  // Projection matrix coloumns
  const float4 projX = (float4)(r00, r01, r02, tx);
  const float4 projY = (float4)(r10, r11, r12, ty);
  const float4 projZ = (float4)(r20, r21, r22, tz);

  // Compute 3D point
  const float z = d / 1000.0f;
  const float4 point = (float4)((xD - cxD) * fxDInv * z, (yD - cyD) * fyDInv * z, z, 1.0f);

  // Rotate and translate
  const float3 projected = (float3)(dot(point, projX), dot(point, projY), dot(point, projZ));

  const float invZ = 1.0f / projected.z;

  // Compute projected image coordinates
  const short xI = (short)((fxD * projected.x) * invZ + cxD);
  const short yI = (short)((fyD * projected.y) * invZ + cyD);

  // Check if depth is valid and projection is inside the image
  if(d >= zNear && d <= zFar && xI >= 0 && xI < widthD && yI >= 0 && yI < heightD)
  {
    const ushort zI = (ushort)(projected.z * 1000.0f);
    const int index = xI + yI * widthD;

    idx[i] = index;
    zImg[i] = zI;

    // Store in rendered image. Due to concurrency value can get overridden by other workers
    if(rendered[index] == 0 || zI < rendered[index])
    {
      rendered[index] = zI;
    }
  }
  else
  {
    idx[i] = -1;
    zImg[i] = 0;
  }
}

// update rendered depth image to make sure nearest depth values are used
void kernel render(global const int *idx, global const ushort *zImg, global ushort *rendered){
  const uint i = get_global_id(0);

  const int index = idx[i];
  const ushort zI = zImg[i];

  if(index >= 0)
  {
    if(zI < rendered[index])
    {
      rendered[index] = zI;
    }
  }
}*/

// remap depth image
void kernel remapDepth(global const ushort *in, global ushort *out, global const float *mapX, global const float *mapY)
{
  const uint i = get_global_id(0);

  const float x = mapX[i];
  const float y = mapY[i];
  const int xL = (int)floor(x);
  const int xH = xL + 1;
  const int yL = (int)floor(y);
  const int yH = yL + 1;

  if(xL < 0 || yL < 0 || xH >= widthR || yH >= heightR)
  {
    out[i] = 0;
    return;
  }

  const uint iLT = yL * widthR + xL;
  const uint iRT = iLT + 1;
  const uint iLB = iLT + widthR;
  const uint iRB = iLB + 1;

  const float4 p = (float4)(in[iLT], in[iRT], in[iLB], in[iRB]);
  int4 valid = isgreaterequal(p, (float4)(1));
  int count = abs(valid.s0 + valid.s1 + valid.s2 + valid.s3);

  if(count < 3)
  {
    out[i] = 0;
    return;
  }

  const float avg = (p.s0 + p.s1 + p.s2 + p.s3) / count;
  const float thres = 0.01 * avg;
  valid = isless(fabs(p - avg), (float4)(thres));
  count = abs(valid.s0 + valid.s1 + valid.s2 + valid.s3);

  if(count < 3)
  {
    out[i] = 0;
    return;
  }

  const float4 distXY = pown((float4)(x - xL, xH - x, y - yL, yH - y), 2);
  const float4 tmp = (float4)(sqrt(2.0));
  const float4 dist2 = (float4)(distXY.s0 + distXY.s2, distXY.s1 + distXY.s2, distXY.s0 + distXY.s3, distXY.s1 + distXY.s3);
  const float4 dist = select((float4)(0), tmp - sqrt(dist2), valid);
  const float sum = dist.s0 + dist.s1 + dist.s2 + dist.s3;

  out[i] = (dot(p, dist) / sum) + 0.5;
}

/**
 * Copyright 2014 University of Bremen, Institute for Artificial Intelligence
 * Author: Thiemo Wiedemeyer <wiedemeyer@cs.uni-bremen.de>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Rotation
#define r00 %0.14e
#define r01 %0.14e
#define r02 %0.14e
#define r10 %0.14e
#define r11 %0.14e
#define r12 %0.14e
#define r20 %0.14e
#define r21 %0.14e
#define r22 %0.14e

// Translation
#define tx %0.14e
#define ty %0.14e
#define tz %0.14e

// Camera parameter upscaled depth
#define fxD %.14e
#define fyD %.14e
#define cxD %.14e
#define cyD %.14e
#define fxDInv (1.0f / fxD)
#define fyDInv (1.0f / fyD)

// Camera parameter color
#define fxC %.14e
#define fyC %.14e
#define cxC %.14e
#define cyC %.14e

// Clipping distances
#define zNear %d
#define zFar %d
#define zDist %.14e

// Size color image
#define heightC %d
#define widthC %d

// Size depth image
#define heightD %d
#define widthD %d

// Size raw depth image
#define heightR %d
#define widthR %d

// Set render buffer to zero
void kernel setZero(global ushort* rendered, global float *selDist){
  const uint i = get_global_id(0);
  rendered[i] = 0;
  selDist[i] = 10;
}

// Calculate 3d point, project them to color camera coordinate system and create rendered depth image
void kernel project(global const ushort *depth, global int4 *idx, global ushort *zImg, global float4 *dists, global float *selDist, global ushort *rendered){
  const uint i = get_global_id(0);

  const int xD = i % widthD;
  const int yD = i / widthD;

  const ushort d = depth[i];

  // Projection matrix coloumns
  const float4 projX = (float4)(r00, r01, r02, tx);
  const float4 projY = (float4)(r10, r11, r12, ty);
  const float4 projZ = (float4)(r20, r21, r22, tz);

  // Compute 3D point
  const float z = d / 1000.0f;
  const float4 point = (float4)((xD - cxD) * fxDInv * z, (yD - cyD) * fyDInv * z, z, 1.0f);

  // Rotate and translate
  const float3 projected = (float3)(dot(point, projX), dot(point, projY), dot(point, projZ));

  const float invZ = 1.0f / projected.z;

  // Compute projected image coordinates
  const float x = (fxD * projected.x) * invZ + cxD;
  const float y = (fyD * projected.y) * invZ + cyD;
  const short xL = (short)floor(x);
  const short yL = (short)floor(y);
  const short xH = xL + 1;
  const short yH = yL + 1;

  const float4 distXY = pown((float4)(x - xL, xH - x, y - yL, yH - y), 2);
  const float4 dist2 = (float4)(distXY.s0 + distXY.s2, distXY.s1 + distXY.s2, distXY.s0 + distXY.s3, distXY.s1 + distXY.s3);

  const int yLI = yL * widthD;
  const int yHI = yH * widthD;
  int4 indices = (int4)(yLI + xL, yLI + xH, yHI + xL, yHI + xH);

  const ushort zI = (ushort)(projected.z * 1000.0f);

  // Check if depth is valid and projection is inside the image
  if(xL < 0)
  {
    indices.s0 = indices.s2 = -1;
  }
  if(yL < 0)
  {
    indices.s0 = indices.s1 = -1;
  }
  if(xH >= widthD)
  {
    indices.s1 = indices.s3 = -1;
  }
  if(yH >= heightD)
  {
    indices.s2 = indices.s3 = -1;
  }
  if(indices.s0 >= 0)
  {
    selDist[indices.s0] = dist2.s0;
    rendered[indices.s0] = zI;
  }
  if(indices.s1 >= 0)
  {
    selDist[indices.s1] = dist2.s1;
    rendered[indices.s1] = zI;
  }
  if(indices.s2 >= 0)
  {
    selDist[indices.s2] = dist2.s2;
    rendered[indices.s2] = zI;
  }
  if(indices.s3 >= 0)
  {
    selDist[indices.s3] = dist2.s3;
    rendered[indices.s3] = zI;
  }

  idx[i] = indices;
  dists[i] = dist2;
  zImg[i] = zI;
}

// update rendered depth image to make sure nearest depth values are used
void kernel render(global const int4 *idx, global const ushort *zImg, global const float4 *dists, global float *selDist, global ushort *rendered){
  const uint i = get_global_id(0);

  const int4 index = idx[i];
  const ushort zI = zImg[i];
  const ushort thres = 0.01 * zI;
  const ushort zIThres = zI + thres;
  const float4 dist2 = dists[i];

  ushort zRen;
  zRen = rendered[index.s0];
  if(index.s0 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s0] > dist2.s0) || zRen > zIThres))
  {
    selDist[index.s0] = dist2.s0;
    rendered[index.s0] = zI;
  }
  zRen = rendered[index.s1];
  if(index.s1 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s1] > dist2.s1) || zRen > zIThres))
  {
    selDist[index.s1] = dist2.s1;
    rendered[index.s1] = zI;
  }
  zRen = rendered[index.s2];
  if(index.s2 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s2] > dist2.s2) || zRen > zIThres))
  {
    selDist[index.s2] = dist2.s2;
    rendered[index.s2] = zI;
  }
  zRen = rendered[index.s3];
  if(index.s3 >= 0 && ((abs_diff(zRen, zI) < thres && selDist[index.s3] > dist2.s3) || zRen > zIThres))
  {
    selDist[index.s3] = dist2.s3;
    rendered[index.s3] = zI;
  }
}

// Calculate 3d point, project them to color camera coordinate system and create rendered depth image
/*void kernel project(global const unsigned short *depth, global int *idx, global unsigned short *zImg, global unsigned short *rendered){
  const uint i = get_global_id(0);

  const int xD = i % widthD;
  const int yD = i / widthD;

  const ushort d = depth[i];

  // Projection matrix coloumns
  const float4 projX = (float4)(r00, r01, r02, tx);
  const float4 projY = (float4)(r10, r11, r12, ty);
  const float4 projZ = (float4)(r20, r21, r22, tz);

  // Compute 3D point
  const float z = d / 1000.0f;
  const float4 point = (float4)((xD - cxD) * fxDInv * z, (yD - cyD) * fyDInv * z, z, 1.0f);

  // Rotate and translate
  const float3 projected = (float3)(dot(point, projX), dot(point, projY), dot(point, projZ));

  const float invZ = 1.0f / projected.z;

  // Compute projected image coordinates
  const short xI = (short)((fxD * projected.x) * invZ + cxD);
  const short yI = (short)((fyD * projected.y) * invZ + cyD);

  // Check if depth is valid and projection is inside the image
  if(d >= zNear && d <= zFar && xI >= 0 && xI < widthD && yI >= 0 && yI < heightD)
  {
    const ushort zI = (ushort)(projected.z * 1000.0f);
    const int index = xI + yI * widthD;

    idx[i] = index;
    zImg[i] = zI;

    // Store in rendered image. Due to concurrency value can get overridden by other workers
    if(rendered[index] == 0 || zI < rendered[index])
    {
      rendered[index] = zI;
    }
  }
  else
  {
    idx[i] = -1;
    zImg[i] = 0;
  }
}

// update rendered depth image to make sure nearest depth values are used
void kernel render(global const int *idx, global const ushort *zImg, global ushort *rendered){
  const uint i = get_global_id(0);

  const int index = idx[i];
  const ushort zI = zImg[i];

  if(index >= 0)
  {
    if(zI < rendered[index])
    {
      rendered[index] = zI;
    }
  }
}*/

// remap depth image
void kernel remapDepth(global const ushort *in, global ushort *out, global const float *mapX, global const float *mapY)
{
  const uint i = get_global_id(0);

  const float x = mapX[i];
  const float y = mapY[i];
  const int xL = (int)floor(x);
  const int xH = xL + 1;
  const int yL = (int)floor(y);
  const int yH = yL + 1;

  if(xL < 0 || yL < 0 || xH >= widthR || yH >= heightR)
  {
    out[i] = 0;
    return;
  }

  const uint iLT = yL * widthR + xL;
  const uint iRT = iLT + 1;
  const uint iLB = iLT + widthR;
  const uint iRB = iLB + 1;

  const float4 p = (float4)(in[iLT], in[iRT], in[iLB], in[iRB]);
  int4 valid = isgreaterequal(p, (float4)(1));
  int count = abs(valid.s0 + valid.s1 + valid.s2 + valid.s3);

  if(count < 3)
  {
    out[i] = 0;
    return;
  }

  const float avg = (p.s0 + p.s1 + p.s2 + p.s3) / count;
  const float thres = 0.01 * avg;
  valid = isless(fabs(p - avg), (float4)(thres));
  count = abs(valid.s0 + valid.s1 + valid.s2 + valid.s3);

  if(count < 3)
  {
    out[i] = 0;
    return;
  }

  const float4 distXY = pown((float4)(x - xL, xH - x, y - yL, yH - y), 2);
  const float4 tmp = (float4)(sqrt(2.0));
  const float4 dist2 = (float4)(distXY.s0 + distXY.s2, distXY.s1 + distXY.s2, distXY.s0 + distXY.s3, distXY.s1 + distXY.s3);
  const float4 dist = select((float4)(0), tmp - sqrt(dist2), valid);
  const float sum = dist.s0 + dist.s1 + dist.s2 + dist.s3;

  out[i] = (dot(p, dist) / sum) + 0.5;
}


mem boundary checks in pops kernel

mem boundary checks in pops kernel

/*

TODO proper doc explaining memory/computation model

i_block (begin of block)
coords
pops

global work size: full length + fake length to be divisible by 64
  -> reference
  -> ref-line to private memory
  -> index in resulting vector (pops)
local work size:  64
  -> block to compute against
  -> read block into local mem in parallel by workgroup

 */

__kernel void
pops(const unsigned int i_block,
     const unsigned int n_rows,
     const unsigned int n_cols,
     __global const float* coords,
     const float rad2,
     __global unsigned int* pops
     __local float* tmp_block) {

  float row_i[32];
  unsigned int i_global = get_global_id(0);
  unsigned int i_local = get_local_id(0);
  unsigned int i_block_global = i_block + i_local;
  unsigned int n_local = min(get_local_size(0), (n_cols - i_block));
  unsigned int j,k;
  unsigned int tmp_pops = 0;
  float dist2, tmp;
  if (i_global < n_rows) {
    // copy row i_global to local memory for fast retrieval
    for (k=0; k < n_cols; ++k) {
      row_i[k] = coords[i_global*n_cols+k];
    }
    // copy block of reference coords to local memory for fast retrieval
    for (k=0; k < n_cols; ++k) {
      tmp_block[i_local*n_cols+k] = coords[i_block_global*n_cols+k];
    }
    // sync workgroup
    barrier(CLK_LOCAL_MEM_FENCE);
    // compute squared distances of i to reference structures
    // in block and compare them to rad2
    for (j=0; j < n_local; ++j) {
      dist2 = 0.0f;
      for (k=0; k < n_cols; ++k) {
        tmp = row_i[k] - tmp_block[j*n_cols+k];
        dist2 += tmp * tmp;
      }
      if (dist2 <= rad2) {
        ++tmp_pops;
      }
    }
    pops[i_global] += tmp_pops;
  } else {
    return;
  }
}

/*

i_block (begin of block)
coords
pops

global work size: full length + fake length to be divisible by 64
  -> reference
  -> ref-line to private memory
  -> index in resulting vector (pops)
local work size:  64
  -> block to compute against
  -> read block into local mem in parallel by workgroup

 */

__kernel void
pops(const unsigned int i_block,
     const unsigned int n_rows,
     const unsigned int n_cols,
     __global const float* coords,
     const float rad2,
     __global unsigned int* pops
     __local float* tmp_block) {

  float row_i[32];
  unsigned int i_global = get_global_id(0);
  unsigned int i_local = get_local_id(0);
  unsigned int n_local = get_local_size(0);
  unsigned int j,k;
  unsigned int tmp_pops = 0;
  float dist2, tmp;
  if (i_global < n_rows) {
    // copy row i_global to local memory for fast retrieval
    for (k=0; k < n_cols; ++k) {
      row_i[k] = coords[i_global*n_cols+k];
    }
    // copy block with reference coords to local memory for fast retrieval
    for (k=0; k < n_cols; ++k) {
      tmp_block[i_local*n_cols+k] = coords[(i_block+i_local)*n_cols+k];
    }
    // sync workgroup
    barrier(CLK_LOCAL_MEM_FENCE);
    // compute squared distances of i to reference structures
    // in block and compare them to rad2
    for (j=0; j < n_local; ++j) {
      dist2 = 0.0f;
      for (k=0; k < n_cols; ++k) {
        tmp = row_i[k] - tmp_block[j*n_cols+k];
        dist2 += tmp * tmp;
      }
      if (dist2 <= rad2) {
        ++tmp_pops;
      }
    }
    pops[i_global] += tmp_pops;
  } else {
    return;
  }
}