[cuda][builtin] Fix x and y mixup

[cuda][builtin] Fix x and y mixup

__kernel void
pocl_abs_f32 (global const float *__restrict input,
              global float *__restrict output)
{
  size_t i = get_global_id (0);
  output[i] = fabs (input[i]);
}

__kernel void
org_khronos_openvx_scale_image_nn_u8 (
    global const unsigned char *__restrict input,
    global unsigned char *__restrict output, float width_scale,
    float height_scale, int input_width, int input_height)
{
  int x = get_global_id (0);
  int y = get_global_id (1);
  int z = get_global_id (2);
  int num_planes = get_global_size (2);
  // printf("x=%d,y=%d",x,y);

  float x_src = ((float)x + 0.5f) * width_scale - 0.5f;
  float y_src = ((float)y + 0.5f) * height_scale - 0.5f;
  // printf("x_s=%f,y_s=%f",x_src,y_src);
  float x_min = floor (x_src);
  float y_min = floor (y_src);
  // printf("x_m=%f,y_m=%f",x_min,y_min);

  int x1 = (int)x_min;
  int y1 = (int)y_min;
  // printf("x1=%d,y1=%d",x1,y1);

  if (x_src - x_min >= 0.5f)
    x1++;
  if (y_src - y_min >= 0.5f)
    y1++;

  unsigned char result = input[num_planes * (x1 * input_width + y1) + z];
  // printf("x1=%d,y1=%d",x1,y1);

  // printf("result=%d",result);

  output[num_planes * (x * get_global_size (0) + y) + z] = result;

  // for(int i =0;i<9;i++) {
  //  printf("input value %d\n",input[i]);
  //}
}

__kernel void
org_khronos_openvx_scale_image_bl_u8 (
    global const unsigned char *__restrict input,
    global unsigned char *__restrict output, float width_scale,
    float height_scale, int input_width, int input_height)
{
  size_t planes = get_global_size (2);
  size_t x = get_global_id (0);
  size_t y = get_global_id (1);
  size_t plane = get_global_id (2);

  // printf("x=%d,y=%d",x,y);
  int input_size = input_width * input_height;
  int output_size = get_global_size (0) * get_global_size (1);

  float x_src = ((float)x + 0.5f) * width_scale - 0.5f;
  float y_src = ((float)y + 0.5f) * height_scale - 0.5f;
  // printf("x_s=%f,y_s=%f",x_src,y_src);
  int x_min = floor (x_src);
  int y_min = floor (y_src);
  // printf("x_m=%f,y_m=%f",x_min,y_min);

  float s = x_src - x_min;
  float t = y_src - y_min;

  float result
      = (1 - s) * (1 - t)
            * input[planes * (y_min * input_width + x_min) + plane]
        + s * (1 - t)
              * input[planes * ((y_min + 1) * input_width + x_min) + plane]
        + (1 - s) * t
              * input[planes * (y_min * input_width + x_min + 1) + plane]
        + s * t
              * input[planes * ((y_min + 1) * input_width + x_min + 1)
                      + plane];
  // printf("x1=%d,y1=%d",x1,y1);

  // printf("result=%d",result);

  output[planes * (y * get_global_size (0) + x) + plane]
      = (unsigned char)result;

  // printf("%hhu", (unsigned char)result);
  // for(int i =0;i<9;i++) {
  //  printf("input value %d\n",input[i]);
  //}
}

__kernel void
pocl_add_i8 (global const signed char *__restrict a,
             global const signed char *__restrict b,
             global signed char *__restrict output)
{
  size_t x = get_global_id (0);

  output[x] = a[x] + b[x];
}

__kernel void
pocl_abs_f32 (global const float *__restrict input,
              global float *__restrict output)
{
  size_t i = get_global_id (0);
  output[i] = fabs (input[i]);
}

__kernel void
org_khronos_openvx_scale_image_nn_u8 (
    global const unsigned char *__restrict input,
    global unsigned char *__restrict output, float width_scale,
    float height_scale, int input_width, int input_height)
{
  int x = get_global_id (0);
  int y = get_global_id (1);
  int z = get_global_id (2);
  int num_planes = get_global_size (2);
  // printf("x=%d,y=%d",x,y);

  float x_src = ((float)x + 0.5f) * width_scale - 0.5f;
  float y_src = ((float)y + 0.5f) * height_scale - 0.5f;
  // printf("x_s=%f,y_s=%f",x_src,y_src);
  float x_min = floor (x_src);
  float y_min = floor (y_src);
  // printf("x_m=%f,y_m=%f",x_min,y_min);

  int x1 = (int)x_min;
  int y1 = (int)y_min;
  // printf("x1=%d,y1=%d",x1,y1);

  if (x_src - x_min >= 0.5f)
    x1++;
  if (y_src - y_min >= 0.5f)
    y1++;

  unsigned char result = input[num_planes * (x1 * input_width + y1) + z];
  // printf("x1=%d,y1=%d",x1,y1);

  // printf("result=%d",result);

  output[num_planes * (x * get_global_size (0) + y) + z] = result;

  // for(int i =0;i<9;i++) {
  //  printf("input value %d\n",input[i]);
  //}
}

__kernel void
org_khronos_openvx_scale_image_bl_u8 (
    global const unsigned char *__restrict input,
    global unsigned char *__restrict output, float width_scale,
    float height_scale, int input_width, int input_height)
{
  size_t planes = get_global_size (2);
  size_t x = get_global_id (0);
  size_t y = get_global_id (1);
  size_t plane = get_global_id (2);

  // printf("x=%d,y=%d",x,y);
  int input_size = input_width * input_height;
  int output_size = get_global_size (0) * get_global_size (1);

  float x_src = ((float)x + 0.5f) * width_scale - 0.5f;
  float y_src = ((float)y + 0.5f) * height_scale - 0.5f;
  // printf("x_s=%f,y_s=%f",x_src,y_src);
  int x_min = floor (x_src);
  int y_min = floor (y_src);
  // printf("x_m=%f,y_m=%f",x_min,y_min);

  float s = x_src - x_min;
  float t = y_src - y_min;

  float result
      = (1 - s) * (1 - t)
            * input[planes * (x_min * input_width + y_min) + plane]
        + s * (1 - t)
              * input[planes * ((x_min + 1) * input_width + y_min) + plane]
        + (1 - s) * t
              * input[planes * (x_min * input_width + y_min + 1) + plane]
        + s * t
              * input[planes * ((x_min + 1) * input_width + y_min + 1)
                      + plane];
  // printf("x1=%d,y1=%d",x1,y1);

  // printf("result=%d",result);

  output[planes * (x * get_global_size (0) + y) + plane]
      = (unsigned char)result;

  // printf("%hhu", (unsigned char)result);
  // for(int i =0;i<9;i++) {
  //  printf("input value %d\n",input[i]);
  //}
}

__kernel void
pocl_add_i8 (global const signed char *__restrict a,
             global const signed char *__restrict b,
             global signed char *__restrict output)
{
  size_t x = get_global_id (0);

  output[x] = a[x] + b[x];
}


BUGFIX: Hotfix for cast in opencl backend

/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

#define __add(lhs, rhs) (lhs) + (rhs)
#define __sub(lhs, rhs) (lhs) - (rhs)
#define __mul(lhs, rhs) (lhs) * (rhs)
#define __div(lhs, rhs) (lhs) / (rhs)
#define __and(lhs, rhs) (lhs) && (rhs)
#define __or(lhs, rhs) (lhs) || (rhs)

#define __lt(lhs, rhs) (lhs) < (rhs)
#define __gt(lhs, rhs) (lhs) > (rhs)
#define __le(lhs, rhs) (lhs) <= (rhs)
#define __ge(lhs, rhs) (lhs) >= (rhs)
#define __eq(lhs, rhs) (lhs) == (rhs)
#define __neq(lhs, rhs) (lhs) != (rhs)

#define __conj(in) (in)
#define __real(in) (in)
#define __imag(in) (0)
#define __abs(in) abs(in)
#define __abs2(in) (in) * (in)

#define __crealf(in) ((in).x)
#define __cimagf(in) ((in).y)
#define __cabsf2(in) ((in).x * (in).x + (in).y * (in).y)
#define __cabsf(in) sqrt(__cabsf2(in))

#define __creal(in) ((in).x)
#define __cimag(in) ((in).y)
#define __cabs2(in) ((in).x * (in).x + (in).y * (in).y)
#define __cabs(in) sqrt(__cabs2(in))

float2 __cconjf(float2 in)
{
    float2 out = {in.x, -in.y};
    return out;
}

float2 __caddf(float2 lhs, float2 rhs)
{
    float2 out = {lhs.x + rhs.x, lhs.y + rhs.y};
    return out;
}

float2 __csubf(float2 lhs, float2 rhs)
{
    float2 out = {lhs.x - rhs.x, lhs.y - rhs.y};
    return out;
}

float2 __cmulf(float2 lhs, float2 rhs)
{
    float2 out;
    out.x = lhs.x * rhs.x - lhs.y * rhs.y;
    out.y = lhs.x * rhs.y + lhs.y * rhs.x;
    return out;
}

// FIXME: overflow / underflow issues
float2 __cdivf(float2 lhs, float2 rhs)
{
    float2 out;
    float den = (rhs.x * rhs.x + rhs.y * rhs.y);
    float2 num = __cmulf(lhs, __cconjf(rhs));

    out.x = num.x / den;
    out.y = num.y / den;

    return out;
}

#define __candf(lhs, rhs) __cabsf2(lhs) && __cabsf2(rhs)
#define __cand(lhs, rhs) __cabs2(lhs) && __cabs2(rhs)

#define __corf(lhs, rhs) __cabsf2(lhs) || __cabsf2(rhs)
#define __cor(lhs, rhs) __cabs2(lhs) || __cabs2(rhs)

#define __ceqf(lhs, rhs) (((lhs).x == (rhs).x) && ((lhs).y == (rhs).y))
#define __cneqf(lhs, rhs) !__ceqf((lhs), (rhs))
#define __cltf(lhs, rhs) (__cabsf2(lhs) < __cabsf2(rhs))
#define __clef(lhs, rhs) (__cabsf2(lhs) <= __cabsf2(rhs))
#define __cgtf(lhs, rhs) (__cabsf2(lhs) > __cabsf2(rhs))
#define __cgef(lhs, rhs) (__cabsf2(lhs) >= __cabsf2(rhs))

#define __ceq(lhs, rhs) (((lhs).x == (rhs).x) && ((lhs).y == (rhs).y))
#define __cneq(lhs, rhs) !__ceq((lhs), (rhs))
#define __clt(lhs, rhs) (__cabs2(lhs) < __cabs2(rhs))
#define __cle(lhs, rhs) (__cabs2(lhs) <= __cabs2(rhs))
#define __cgt(lhs, rhs) (__cabs2(lhs) > __cabs2(rhs))
#define __cge(lhs, rhs) (__cabs2(lhs) >= __cabs2(rhs))

float2 __cminf(float2 lhs, float2 rhs)
{
    return __abs2(lhs) < __abs2(rhs) ? lhs : rhs;
}

float2 __cmaxf(float2 lhs, float2 rhs)
{
    return __abs2(lhs) > __abs2(rhs) ? lhs : rhs;
}

float2 __cplx2f(float lhs, float rhs)
{
    float2 out = {lhs, rhs};
    return out;
}

float2 __convert_cfloat(float in)
{
    float2 out = {in, 0};
    return out;
}

#define __convert_char(val) (char)(convert_char((val)) != 0)

#define fpow(lhs, rhs) pow((lhs), (rhs))

#define frem(lhs, rhs) remainder((lhs), (rhs))

#define iszero(a) ((a) == 0)

#ifdef USE_DOUBLE

float2  __convert_z2c(double2 in) { float2  out = {in.x, in.y}; return out; }

double2 __cconj(double2 in)
{
    double2 out = {in.x, -in.y};
    return out;
}

double2 __cadd(double2 lhs, double2 rhs)
{
    double2 out = {lhs.x + rhs.x, lhs.y + rhs.y};
    return out;
}

double2 __csub(double2 lhs, double2 rhs)
{
    double2 out = {lhs.x - rhs.x, lhs.y - rhs.y};
    return out;
}

double2 __cmul(double2 lhs, double2 rhs)
{
    double2 out;
    out.x = lhs.x * rhs.x - lhs.y * rhs.y;
    out.y = lhs.x * rhs.y + lhs.y * rhs.x;
    return out;
}

double2 __cdiv(double2 lhs, double2 rhs)
{
    double2 out;
    double den = (rhs.x * rhs.x + rhs.y * rhs.y);
    double2 num = __cmul(lhs, __cconj(rhs));

    out.x = num.x / den;
    out.y = num.y / den;
    return out;
}

double2 __cmin(double2 lhs, double2 rhs)
{
    return __abs2(lhs) < __abs2(rhs) ? lhs : rhs;
}

double2 __cmax(double2 lhs, double2 rhs)
{
    return __abs2(lhs) > __abs2(rhs) ? lhs : rhs;
}

double2 __cplx2(double lhs, double rhs)
{
    double2 out = {lhs, rhs};
    return out;
}

double2 __convert_cdouble(double in)
{
    double2 out = {in, 0};
    return out;
}

double2 __convert_c2z(float2  in) { double2 out = {in.x, in.y}; return out; }

#endif // USE_DOUBLE

/*******************************************************
 * Copyright (c) 2014, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

#define __add(lhs, rhs) (lhs) + (rhs)
#define __sub(lhs, rhs) (lhs) - (rhs)
#define __mul(lhs, rhs) (lhs) * (rhs)
#define __div(lhs, rhs) (lhs) / (rhs)
#define __and(lhs, rhs) (lhs) && (rhs)
#define __or(lhs, rhs) (lhs) || (rhs)

#define __lt(lhs, rhs) (lhs) < (rhs)
#define __gt(lhs, rhs) (lhs) > (rhs)
#define __le(lhs, rhs) (lhs) <= (rhs)
#define __ge(lhs, rhs) (lhs) >= (rhs)
#define __eq(lhs, rhs) (lhs) == (rhs)
#define __neq(lhs, rhs) (lhs) != (rhs)

#define __conj(in) (in)
#define __real(in) (in)
#define __imag(in) (0)
#define __abs(in) abs(in)
#define __abs2(in) (in) * (in)

#define __crealf(in) ((in).x)
#define __cimagf(in) ((in).y)
#define __cabsf2(in) ((in).x * (in).x + (in).y * (in).y)
#define __cabsf(in) sqrt(__cabsf2(in))

#define __creal(in) ((in).x)
#define __cimag(in) ((in).y)
#define __cabs2(in) ((in).x * (in).x + (in).y * (in).y)
#define __cabs(in) sqrt(__cabs2(in))

float2 __cconjf(float2 in)
{
    float2 out = {in.x, -in.y};
    return out;
}

float2 __caddf(float2 lhs, float2 rhs)
{
    float2 out = {lhs.x + rhs.x, lhs.y + rhs.y};
    return out;
}

float2 __csubf(float2 lhs, float2 rhs)
{
    float2 out = {lhs.x - rhs.x, lhs.y - rhs.y};
    return out;
}

float2 __cmulf(float2 lhs, float2 rhs)
{
    float2 out;
    out.x = lhs.x * rhs.x - lhs.y * rhs.y;
    out.y = lhs.x * rhs.y + lhs.y * rhs.x;
    return out;
}

// FIXME: overflow / underflow issues
float2 __cdivf(float2 lhs, float2 rhs)
{
    float2 out;
    float den = (rhs.x * rhs.x + rhs.y * rhs.y);
    float2 num = __cmulf(lhs, __cconjf(rhs));

    out.x = num.x / den;
    out.y = num.y / den;

    return out;
}

#define __candf(lhs, rhs) __cabsf2(lhs) && __cabsf2(rhs)
#define __cand(lhs, rhs) __cabs2(lhs) && __cabs2(rhs)

#define __corf(lhs, rhs) __cabsf2(lhs) || __cabsf2(rhs)
#define __cor(lhs, rhs) __cabs2(lhs) || __cabs2(rhs)

#define __ceqf(lhs, rhs) (((lhs).x == (rhs).x) && ((lhs).y == (rhs).y))
#define __cneqf(lhs, rhs) !__ceqf((lhs), (rhs))
#define __cltf(lhs, rhs) (__cabsf2(lhs) < __cabsf2(rhs))
#define __clef(lhs, rhs) (__cabsf2(lhs) <= __cabsf2(rhs))
#define __cgtf(lhs, rhs) (__cabsf2(lhs) > __cabsf2(rhs))
#define __cgef(lhs, rhs) (__cabsf2(lhs) >= __cabsf2(rhs))

#define __ceq(lhs, rhs) (((lhs).x == (rhs).x) && ((lhs).y == (rhs).y))
#define __cneq(lhs, rhs) !__ceq((lhs), (rhs))
#define __clt(lhs, rhs) (__cabs2(lhs) < __cabs2(rhs))
#define __cle(lhs, rhs) (__cabs2(lhs) <= __cabs2(rhs))
#define __cgt(lhs, rhs) (__cabs2(lhs) > __cabs2(rhs))
#define __cge(lhs, rhs) (__cabs2(lhs) >= __cabs2(rhs))

float2 __cminf(float2 lhs, float2 rhs)
{
    return __abs2(lhs) < __abs2(rhs) ? lhs : rhs;
}

float2 __cmaxf(float2 lhs, float2 rhs)
{
    return __abs2(lhs) > __abs2(rhs) ? lhs : rhs;
}

float2 __cplx2f(float lhs, float rhs)
{
    float2 out = {lhs, rhs};
    return out;
}

float2 __convert_cfloat(float in)
{
    float2 out = {in, 0};
    return out;
}

#define __convert_char(val) (char)(convert_char((val)) != 0)

#define fpow(lhs, rhs) pow((lhs), (rhs))

#define frem(lhs, rhs) remainder((lhs), (rhs))

#define iszero(a) ((a) == 0)

#ifdef USE_DOUBLE

#define __convert_cdouble(in) ___convert_cdouble((double)in)

float2  __convert_z2c(double2 in) { float2  out = {in.x, in.y}; return out; }

double2 __cconj(double2 in)
{
    double2 out = {in.x, -in.y};
    return out;
}

double2 __cadd(double2 lhs, double2 rhs)
{
    double2 out = {lhs.x + rhs.x, lhs.y + rhs.y};
    return out;
}

double2 __csub(double2 lhs, double2 rhs)
{
    double2 out = {lhs.x - rhs.x, lhs.y - rhs.y};
    return out;
}

double2 __cmul(double2 lhs, double2 rhs)
{
    double2 out;
    out.x = lhs.x * rhs.x - lhs.y * rhs.y;
    out.y = lhs.x * rhs.y + lhs.y * rhs.x;
    return out;
}

double2 __cdiv(double2 lhs, double2 rhs)
{
    double2 out;
    double den = (rhs.x * rhs.x + rhs.y * rhs.y);
    double2 num = __cmul(lhs, __cconj(rhs));

    out.x = num.x / den;
    out.y = num.y / den;
    return out;
}

double2 __cmin(double2 lhs, double2 rhs)
{
    return __abs2(lhs) < __abs2(rhs) ? lhs : rhs;
}

double2 __cmax(double2 lhs, double2 rhs)
{
    return __abs2(lhs) > __abs2(rhs) ? lhs : rhs;
}

double2 __cplx2(double lhs, double rhs)
{
    double2 out = {lhs, rhs};
    return out;
}

double2 __convert_cdouble(double in)
{
    double2 out = {in, 0};
    return out;
}

double2 __convert_c2z(float2  in) { double2 out = {in.x, in.y}; return out; }

#endif // USE_DOUBLE


Refactored the GEMV kernel

Refactored the GEMV kernel

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the Xgemv kernel for matrix-vector multiplication.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Parameters set by the tuner or by the database. Here they are given a basic default value in case
// this kernel file is used outside of the CLBlast library.
#ifndef WGS
  #define WGS 64     // The local work-group size
#endif
#ifndef WPT
  #define WPT 1      // The amount of work-per-thread
#endif
#ifndef VW
  #define VW 1       // Vector width of vectors X and Y
#endif

// =================================================================================================

// The multiply-add function for the main part (divisable by WGS)
inline void MatrixVectorMain(const __global real* restrict agm, __local real* xlm, real acc[WPT],
                             const int gid, const int w, const int kwg,
                             const int a_ld, const int a_offset, const int a_transposed) {
  if (a_transposed == 0) { // Not transposed
    #pragma unroll
    for (int kl=0; kl<WGS; ++kl) {
      const int k = kwg + kl;
      MultiplyAdd(acc[w], agm[gid + a_ld*k + a_offset], xlm[kl]);
    }
  }
  else { // Transposed
    #pragma unroll
    for (int kl=0; kl<WGS; ++kl) {
      const int k = kwg + kl;
      MultiplyAdd(acc[w], agm[k + a_ld*gid + a_offset], xlm[kl]);
    }
  }
}

// The multiply-add function for the remainder part (not divisable by WGS)
inline void MatrixVectorRemainder(const __global real* restrict agm,
                                  const __global real* restrict xgm, real acc[WPT],
                                  const int gid, const int w, const int n_floor, const int n,
                                  const int a_ld, const int a_offset, const int a_transposed,
                                  const int x_inc, const int x_offset) {
  if (a_transposed == 0) { // Not transposed
    #pragma unroll
    for (int k=n_floor; k<n; ++k) {
      MultiplyAdd(acc[w], agm[gid + a_ld*k + a_offset], xgm[k*x_inc + x_offset]);
    }
  }
  else { // Transposed
    #pragma unroll
    for (int k=n_floor; k<n; ++k) {
      MultiplyAdd(acc[w], agm[k + a_ld*gid + a_offset], xgm[k*x_inc + x_offset]);
    }
  }
}

// =================================================================================================

// The gemv kernel
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void Xgemv(const int m, const int n, const real alpha, const real beta,
                    const int a_transposed,
                    const __global real* restrict agm, const int a_offset, const int a_ld,
                    const __global real* restrict xgm, const int x_offset, const int x_inc,
                    __global real* ygm, const int y_offset, const int y_inc) {

  // Local memory for the vector X
  __local real xlm[WGS];

  // Initializes the accumulation register
  real acc[WPT];
  #pragma unroll
  for (int w=0; w<WPT; ++w) {
    SetToZero(acc[w]);
  }

  // Divides the work in a main and tail section
  const int n_tail = n % WGS;
  const int n_floor = n - n_tail;

  // Loops over work-group sized portions of the work
  for (int kwg=0; kwg<n_floor; kwg+=WGS) {

    // Loads the vector X into local memory
    const int lid = get_local_id(0);
    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];

    // Synchronizes all threads in a workgroup
    barrier(CLK_LOCAL_MEM_FENCE);

    // Loops over the work per thread, and checks whether in bounds
    #pragma unroll
    for (int w=0; w<WPT; ++w) {
      const int gid = w*get_global_size(0) + get_global_id(0);
      if (gid < m) {
        MatrixVectorMain(agm, xlm, acc, gid, w, kwg, a_ld, a_offset, a_transposed);
      }
    }

    // Synchronizes all threads in a workgroup
    barrier(CLK_LOCAL_MEM_FENCE);
  }

  // Loops over the work per thread, and checks whether in bounds
  #pragma unroll
  for (int w=0; w<WPT; ++w) {
    const int gid = w*get_global_size(0) + get_global_id(0);
    if (gid < m) {
      MatrixVectorRemainder(agm, xgm, acc, gid, w, n_floor, n,
                            a_ld, a_offset, a_transposed, x_inc, x_offset);

      // Stores the final result
      AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, ygm[gid*y_inc + y_offset]);
    }
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)";

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the Xgemv kernel for matrix-vector multiplication.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Parameters set by the tuner or by the database. Here they are given a basic default value in case
// this kernel file is used outside of the CLBlast library.
#ifndef WGS
  #define WGS 64     // The local work-group size
#endif
#ifndef WPT
  #define WPT 1      // The amount of work-per-thread
#endif
#ifndef VW
  #define VW 1       // Vector width of vectors X and Y
#endif

// =================================================================================================

// The gemv kernel
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void Xgemv(const int m, const int n, const real alpha, const real beta,
                    const int a_transposed,
                    const __global real* restrict agm, const int a_offset, const int a_ld,
                    const __global real* restrict xgm, const int x_offset, const int x_inc,
                    __global real* ygm, const int y_offset, const int y_inc) {

  // Local memory for the vector X
  __local real xlm[WGS];

  // Initializes the accumulation register
  real acc[WPT];
  #pragma unroll
  for (int w=0; w<WPT; ++w) {
    SetToZero(acc[w]);
  }

  // Divides the work in a main and tail section
  const int n_tail = n % WGS;
  const int n_floor = n - n_tail;

  // Loops over work-group sized portions of the work
  for (int kwg=0; kwg<n_floor; kwg+=WGS) {

    // Loads the vector X into local memory
    const int lid = get_local_id(0);
    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];

    // Synchronizes all threads in a workgroup
    barrier(CLK_LOCAL_MEM_FENCE);

    // Loops over the work per thread
    #pragma unroll
    for (int w=0; w<WPT; ++w) {
      const int gid = w*get_global_size(0) + get_global_id(0);

      // Checks whether this thread is within bounds
      // Note: placed here because of the synchronisation barriers
      if (gid < m) {

        // Main multiply-add computation (regular)
        if (a_transposed == 0) {
          #pragma unroll
          for (int kl=0; kl<WGS; ++kl) {
            const int k = kwg + kl;
            MultiplyAdd(acc[w], agm[gid + a_ld*k + a_offset], xlm[kl]);
          }
        }

        // Main multiply-add computation (transposed)
        else {
          #pragma unroll
          for (int kl=0; kl<WGS; ++kl) {
            const int k = kwg + kl;
            MultiplyAdd(acc[w], agm[k + a_ld*gid + a_offset], xlm[kl]);
          }
        }
      }
    }

    // Synchronizes all threads in a workgroup
    barrier(CLK_LOCAL_MEM_FENCE);
  }

  // Loops over the work per thread
  #pragma unroll
  for (int w=0; w<WPT; ++w) {
    const int gid = w*get_global_size(0) + get_global_id(0);

    // Checks whether this thread is within bounds
    if (gid < m) {

      // Multiply-add computation for the remaining tail (regular)
      if (a_transposed == 0) {
        #pragma unroll
        for (int k=n_floor; k<n; ++k) {
          MultiplyAdd(acc[w], agm[gid + a_ld*k + a_offset], xgm[k*x_inc + x_offset]);
        }
      }

      // Multiply-add computation for the remaining tail (transposed)
      else {
        #pragma unroll
        for (int k=n_floor; k<n; ++k) {
          MultiplyAdd(acc[w], agm[k + a_ld*gid + a_offset], xgm[k*x_inc + x_offset]);
        }
      }

      // Stores the final result
      AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, ygm[gid*y_inc + y_offset]);
    }
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)";

// =================================================================================================


force same gamma

force same gamma

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const half black_level = 42.0;

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.82717181, -0.31231438, 0.07307673),
  (half3)(-0.5743977, 1.36858544, -0.53183455),
  (half3)(-0.25277411, -0.05627105, 1.45875782),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.0025;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 9 - 100*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb, int ggain) {
  half3 ret = (0,0,0);
  half cpx = 0.01; //clamp(0.01h, 0.05h, cpxb + cpxk * min(10, ggain));
  ret += (half)rgb.x * color_correction[0];
  ret += (half)rgb.y * color_correction[1];
  ret += (half)rgb.z * color_correction[2];
  ret.x = mf(ret.x, cpx);
  ret.y = mf(ret.y, cpx);
  ret.z = mf(ret.z, cpx);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

half val_from_10(const uchar * source, int gx, int gy) {
  // parse 10bit
  int start = gy * FRAME_STRIDE + (5 * (gx / 4));
  int offset = gx % 4;
  uint major = (uint)source[start + offset] << 2;
  uint minor = (source[start + 4] >> (2 * offset)) & 3;
  half pv = (half)(major + minor);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv *= 0.00101833h; // /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        uint ggain
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb, (int)ggain);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);

}

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const half black_level = 42.0;

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.82717181, -0.31231438, 0.07307673),
  (half3)(-0.5743977, 1.36858544, -0.53183455),
  (half3)(-0.25277411, -0.05627105, 1.45875782),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.0025;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 9 - 100*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb, int ggain) {
  half3 ret = (0,0,0);
  half cpx = clamp(0.01h, 0.05h, cpxb + cpxk * min(10, ggain));
  ret += (half)rgb.x * color_correction[0];
  ret += (half)rgb.y * color_correction[1];
  ret += (half)rgb.z * color_correction[2];
  ret.x = mf(ret.x, cpx);
  ret.y = mf(ret.y, cpx);
  ret.z = mf(ret.z, cpx);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

half val_from_10(const uchar * source, int gx, int gy) {
  // parse 10bit
  int start = gy * FRAME_STRIDE + (5 * (gx / 4));
  int offset = gx % 4;
  uint major = (uint)source[start + offset] << 2;
  uint minor = (source[start + 4] >> (2 * offset)) & 3;
  half pv = (half)(major + minor);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv *= 0.00101833h; // /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        uint ggain
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb, (int)ggain);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);

}


Update nn_fpga.cl

Update nn_fpga.cl

Modified kernel for easy readability

#define WORK_GROUP_SIZE 256
#define GLOBAL_SIZE 32768
global float dist[GLOBAL_SIZE]; //on-chip global buffer

// This computes the distance of all points in d_locations from the given
// longitude and latitude, and returns it in the on-chip global buffer
// dist.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE, 1, 1)))
void distance_calc(__global float2 *d_locations,
                     const float lat,
		     const float lng) {
    __attribute__((xcl_pipeline_workitems)) {

  int globalId = get_global_id(0);
  float lat_tmp, lng_tmp, dist_lat, dist_lng;

  lat_tmp = d_locations[globalId].x;
  lng_tmp = d_locations[globalId].y;

  dist_lat = lat-lat_tmp;
  dist_lng = lng-lng_tmp;

  //squared euclidean distance calculation
  dist[globalId] = (dist_lat*dist_lat) + (dist_lng*dist_lng);
  }
}

// This computes the resultsCount nearest neighbors, one per work item.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE,1,1)))
void NearestNeighbor (__global float *d_distances,
                      __global int *indices,
                        const int resultsCount) {
  __local float dmin1; {
  int localId = get_local_id(0);
  if (localId < resultsCount) {
  float dist1;
  float dmin = MAXFLOAT;
  int count = 0;

  if (localId == 0) {
  dmin1 = 0.0f;}

  __attribute__((xcl_pipeline_loop))
  for (int k = 0; k < WORK_GROUP_SIZE; k++) {
  dist1 = dist[k];
  if (dist1 < dmin && dist1 > dmin1) {
  dmin = dist1;
  count = k;}
  if (k == WORK_GROUP_SIZE-1) {
  dmin1 = dmin;
  indices[localId] = count;
  d_distances[localId] = dmin;}
      }
    }
  }
}

#define WORK_GROUP_SIZE 256
#define GLOBAL_SIZE 32768
global float dist[GLOBAL_SIZE]; //on-chip global buffer

// This computes the distance of all points in d_locations from the given
// longitude and latitude, and returns it in the on-chip global buffer
// dist.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE, 1, 1)))
void distance_calc(__global float2 *d_locations,
                     const float lat,
		     const float lng) {
    __attribute__((xcl_pipeline_workitems)) {

  int globalId = get_global_id(0);
  float lat_tmp, lng_tmp, dist_lat, dist_lng;

  lat_tmp = d_locations[globalId].x;
  lng_tmp = d_locations[globalId].y;

  dist_lat = lat-lat_tmp;
  dist_lng = lng-lng_tmp;

  //squared euclidean distance calculation
  dist[globalId] = (dist_lat*dist_lat) + (dist_lng*dist_lng);
  }
}

// This computes the resultsCount nearest neighbors, one per work item.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE,1,1)))
void NearestNeighbor (__global float *d_distances,
                      __global int *indices,
                      const int resultsCount) {
  int localId = get_local_id(0);
  if (localId < resultsCount) {
    int index1 = localId;
    float val,tempDist;
    __attribute__((xcl_pipeline_loop))
    for (int k=localId; k < WORK_GROUP_SIZE; k++) {
      val = dist[k];
      if (val < dist[index1]) index1 = k;
    }

    // swap distances
    tempDist = dist[localId];
    dist[localId] = dist[index1];
    dist[index1] = tempDist;

    // first WG used to ensure that the indices of the neighbors are preserved
    if (get_group_id(0)==0) {
      d_distances[localId] = dist[localId];
      indices[localId] = index1;
    }
  }
}


Remove prototypes of removed functions

Remove prototypes of removed functions

/******************************************************************************
 * function: merge_lit_by_val_chids                                          *
 * input:                                                                     *
 * output:                                                                    *
 ******************************************************************************/

/**
 * High level function:
 * size_t merge_lit_by_val_chids(global uint* input, global uint* chids,
 *                                global uint* lits, size_t li, size_t work_size);
 */

/**
 * Steps
 * - create heads
 * - lazy_segmented_scan
 * - stream compaction to reduce all that have a 0 in heads?
 * (There is probably a more efficient solution)
 */

kernel void create_heads(global uint* keys_high, global uint* keys_low,
                         global uint* heads) {
  uint idx = get_global_id(0);
  heads[idx] = (idx == 0) ||
               (keys_high[idx] != keys_high[idx - 1]) ||
               (keys_low [idx] != keys_low [idx - 1]);
  /*
  heads[idx] = 0;
  if (idx > 0) {
    if ((keys_high[idx] != keys_high[idx - 1]) ||
        (keys_low [idx] != keys_low [idx - 1]))
    heads[idx] = 1;
  } else {
    heads[idx] = 1;
  }
  */
}

// somewhat inefficient segmented scan (not really a scan as it stores the
// accumulated value at the beginning of the segment)
kernel void lazy_segmented_scan(global uint* heads, global uint* data) {
  uint idx = get_global_id(0);
  uint maximum = get_global_size(0);
  if (heads[idx] != 0) {
    uint val = data[idx];
    uint curr = idx + 1;
    while (heads[curr] == 0 && curr < maximum) {
      val |= data[curr]; // OR operation
      curr += 1;
    }
    data[idx] = val;
  }
}

/******************************************************************************
 * function: merge_lit_by_val_chids                                          *
 * input:                                                                     *
 * output:                                                                    *
 ******************************************************************************/

/**
 * High level function:
 * size_t merge_lit_by_val_chids(global uint* input, global uint* chids,
 *                                global uint* lits, size_t li, size_t work_size);
 */

// prototypes
uint sumReduce128(local uint* arr);
uint compactSIMDPrefixSum(local const uint* dsData, local const uint* dsValid,
                         local uint* dsCompact,    local uint* dsLocalIndex);
uint exclusivePrescan128(local const uint* in, local uint* outAndTemp);

/**
 * Steps
 * - create heads
 * - lazy_segmented_scan
 * - stream compaction to reduce all that have a 0 in heads?
 * (There is probably a more efficient solution)
 */

kernel void create_heads(global uint* keys_high, global uint* keys_low,
                         global uint* heads) {
  uint idx = get_global_id(0);
  heads[idx] = (idx == 0) ||
               (keys_high[idx] != keys_high[idx - 1]) ||
               (keys_low [idx] != keys_low [idx - 1]);
  /*
  heads[idx] = 0;
  if (idx > 0) {
    if ((keys_high[idx] != keys_high[idx - 1]) ||
        (keys_low [idx] != keys_low [idx - 1]))
    heads[idx] = 1;
  } else {
    heads[idx] = 1;
  }
  */
}

// somewhat inefficient segmented scan (not really a scan as it stores the
// accumulated value at the beginning of the segment)
kernel void lazy_segmented_scan(global uint* heads, global uint* data) {
  uint idx = get_global_id(0);
  uint maximum = get_global_size(0);
  if (heads[idx] != 0) {
    uint val = data[idx];
    uint curr = idx + 1;
    while (heads[curr] == 0 && curr < maximum) {
      val |= data[curr]; // OR operation
      curr += 1;
    }
    data[idx] = val;
  }
}