Adding an explicit triple to this test to get it to pass all build bots.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@222837 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s

// test that we generate address space casts everywhere we need conversions of
// pointers to different address spaces

void test(global int *arg_glob, generic int *arg_gen) {
  int var_priv;
  arg_gen = arg_glob; // implicit cast global -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
  arg_gen = &var_priv; // implicit cast with obtaining adr, private -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32* %var_priv to i32 addrspace(4)*
  arg_glob = (global int *)arg_gen; // explicit cast
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(4)* %{{[0-9]+}} to i32 addrspace(1)*
  global int *var_glob =
      (global int *)arg_glob; // explicit cast in the same address space
  // CHECK-NOT: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(1)*
  var_priv = arg_gen - arg_glob; // arithmetic operation
  // CHECK: %sub.ptr.lhs.cast = ptrtoint i32 addrspace(4)* %6 to i64
  // CHECK: %sub.ptr.rhs.cast = ptrtoint i32 addrspace(1)* %7 to i64
  var_priv = arg_gen > arg_glob; // comparison
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
}

// RUN: %clang_cc1 %s -O0 -ffake-address-space-map -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s

// test that we generate address space casts everywhere we need conversions of
// pointers to different address spaces

void test(global int *arg_glob, generic int *arg_gen) {
  int var_priv;
  arg_gen = arg_glob; // implicit cast global -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
  arg_gen = &var_priv; // implicit cast with obtaining adr, private -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32* %var_priv to i32 addrspace(4)*
  arg_glob = (global int *)arg_gen; // explicit cast
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(4)* %{{[0-9]+}} to i32 addrspace(1)*
  global int *var_glob =
      (global int *)arg_glob; // explicit cast in the same address space
  // CHECK-NOT: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(1)*
  var_priv = arg_gen - arg_glob; // arithmetic operation
  // CHECK: %sub.ptr.lhs.cast = ptrtoint i32 addrspace(4)* %6 to i64
  // CHECK: %sub.ptr.rhs.cast = ptrtoint i32 addrspace(1)* %7 to i64
  var_priv = arg_gen > arg_glob; // comparison
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
}


Moved work_group_reduction_sum_horizontal to ClojureCL's reductions.cl

#ifndef REAL
#define REAL float
#endif

#ifndef WGS
#define WGS 256
#endif

#ifndef WGSm
#define WGSm 16
#endif

#ifndef WGSn
#define WGSn 16
#endif

#ifndef TS
#define TS 32
#endif

#ifndef WPT
#define WPT 4
#endif

#define RTS (TS/WPT)

//||||||||||||||||| CLBuffer |||||||||||||||||||||||||||||||||||||||||||||||||||

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void equals_vector (__global uint* eq_flag,
                             __global const REAL* x, const uint offset_x,
                             const uint stride_x,
                             __global const REAL* y, const uint offset_y,
                             const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    if ((x[ix] != y[iy])){
        eq_flag[0]++;
    }
}

__kernel void equals_matrix (__global uint* eq_flag,
                             __global const REAL* a, const uint offset_a,
                             const uint ld_a,
                             __global const REAL* b, const uint offset_b,
                             const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    if ((a[ia] != b[ib])){
        eq_flag[0]++;
    }
}

//|||||||||||||||||       BLAS 1       |||||||||||||||||||||||||||||||||||||||||

// ================ Embarassingly parallel kernels =============================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void swap (__global REAL* x, const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    REAL temp = x[ix];
    x[ix] = y[iy];
    y[iy] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void copy (__global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    y[offset_y + get_global_id(0) * stride_y] =
        x[offset_x + get_global_id(0) * stride_x];
}

__kernel void copy_matrix (__global const REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    b[ib] = a[ia];
}

__kernel void swap_matrix (__global REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    REAL temp = a[ia];
    a[ia] = b[ia];
    b[ib] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void scal (const REAL alpha, __global REAL* x,
                    const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    x[ix] = alpha * x[ix];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpy (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y,
                    const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + y[iy];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpby (const REAL alpha, __global const REAL* x,
                     uint offset_x, const uint stride_x,
                     const REAL beta, __global REAL* y,
                     const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + beta * y[iy];
}

// ================= Reduction kernels ========================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduction (__global double* acc) {
    double sum = work_group_reduction_sum(acc[get_global_id(0)]);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== Dot product ==============================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void dot_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x,
                          __global const REAL* y,
                          const uint offset_y, const uint stride_y) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    REAL py = y[offset_y + get_global_id(0) * stride_y];
    double sum = work_group_reduction_sum((double)(px * py));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== asum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void asum_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)fabs(px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== sum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)px);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== nrm2 =====================================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void nrm2_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)(px * px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================ Max reduction ==============================================
inline void work_group_reduction_imax (__global uint* iacc,
                                       __global double* vacc,
                                       uint const ind, const double val) {

    uint local_id = get_local_id(0);
    uint local_size = get_local_size(0);

    __local uint liacc[WGS];
    __local double lvacc[WGS];
    liacc[local_id] = ind;
    lvacc[local_id] = val;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    uint index = ind;
    double value = val;

    uint i = local_size;
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_id == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            double other_value = lvacc[local_id + i + 1];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i + 1];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        if (local_id < i) {
            double other_value = lvacc[local_id + i];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    if(local_id == 0) {
        uint group_id = get_group_id(0);
        iacc[group_id] = index;
        vacc[group_id] = value;
    }

}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduction (__global uint* iacc, __global double* vacc) {
    uint gid = get_global_id(0);
    work_group_reduction_imax(iacc, vacc, iacc[gid], (double)(vacc[gid]));
}

// ================== iamax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void iamax_reduce (__global uint* iacc, __global double* vacc,
                            __global const REAL* x,
                            const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(fabs(x[ix])));
}

// ================== imax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(x[ix]));
}

// ================== imin reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imin_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(-x[ix]));
}

// ||||||||||||||||       BLAS 2      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMV =====================================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void sum_reduction_horizontal (__global REAL* acc) {

    uint ia = get_global_size(0) * get_global_id(1) + get_global_id(0);
    work_group_reduction_sum_horizontal(acc, acc[ia]);
}

// ========================= gemv ==============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gemv_reduce (__global REAL* acc,
                           const REAL alpha, __global const REAL* a,
                           const uint offset_a, const uint ld_a,
                           __global const REAL* x, const uint offset_x,
                           const uint stride_x) {

    uint ia = offset_a + ld_a * get_global_id(1) + get_local_id(0);
    uint ix = offset_x + get_global_id(1) * stride_x;

    work_group_reduction_sum_horizontal(acc, alpha * a[ia] * x[ix]);
}

// ========================== gerk =============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gerk (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global const REAL* y,
                    const uint offset_y, const uint stride_y,
                    __global REAL* a, const uint offset_a, const uint ld_a) {

    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(1) * stride_y;
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    a[ia] += alpha * x[ix] * y[iy];
}

// ||||||||||||||||       BLAS 3      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMM =====================================================

inline uint globalize (const uint tile_size, const uint tile_id,
                       const uint id){
    return tile_id * tile_size + id;
}

// ========================= gemm ==============================================

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled (const REAL alpha, __global const REAL* a,
                          const uint offset_a, const uint ld_a,
                          __global const REAL* b,
                          const uint offset_b, const uint ld_b,
                          const REAL beta, __global REAL* c,
                          const uint offset_c, const uint ld_c,
                          const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    // Elements that are in partial m-tiles and n-tiles need to be
    // loaded, but only if they exist.
    const bool load_row = c_row < m;
    bool load_col[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
        load_col[w] = c_col + w * RTS < n;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {

        const uint tile_row = TS * t + row;
        const uint tile_col = TS * t + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    // Compute partial k-tiles.
    const uint rest_k = k - tc * TS;
    if (0 < rest_k) {

        const uint tile_row = TS * tc + row;
        const uint tile_col = TS * tc + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < rest_k; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

    }

    //Only the elements that exist in partial c-tiles should be stored.
#pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const bool store = load_row && load_col[w];
        if (store) {
            const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
            c[ic] = alpha * acc[w] + beta * c[ic];
        }
    }

}

// Simpler version that requires dimensions that fit tiles

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled_fit (const REAL alpha, __global const REAL* a,
                              const uint offset_a, const uint ld_a,
                              __global const REAL* b,
                              const uint offset_b, const uint ld_b,
                              const REAL beta, __global REAL* c,
                              const uint offset_c, const uint ld_c,
                              const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {
        for (uint w = 0; w < WPT; w++) {
            const uint tile_row = TS * t + row;
            const uint tile_col = TS * t + col;
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = a[ia];
            b_tile[col + w * RTS][row] = b[ib];
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
        c[ic] = alpha * acc[w] + beta * c[ic];
    }

}

#ifndef REAL
#define REAL float
#endif

#ifndef WGS
#define WGS 256
#endif

#ifndef WGSm
#define WGSm 16
#endif

#ifndef WGSn
#define WGSn 16
#endif

#ifndef TS
#define TS 32
#endif

#ifndef WPT
#define WPT 4
#endif

#define RTS (TS/WPT)

//||||||||||||||||| CLBuffer |||||||||||||||||||||||||||||||||||||||||||||||||||

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void equals_vector (__global uint* eq_flag,
                             __global const REAL* x, const uint offset_x,
                             const uint stride_x,
                             __global const REAL* y, const uint offset_y,
                             const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    if ((x[ix] != y[iy])){
        eq_flag[0]++;
    }
}

__kernel void equals_matrix (__global uint* eq_flag,
                             __global const REAL* a, const uint offset_a,
                             const uint ld_a,
                             __global const REAL* b, const uint offset_b,
                             const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    if ((a[ia] != b[ib])){
        eq_flag[0]++;
    }
}

//|||||||||||||||||       BLAS 1       |||||||||||||||||||||||||||||||||||||||||

// ================ Embarassingly parallel kernels =============================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void swap (__global REAL* x, const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    REAL temp = x[ix];
    x[ix] = y[iy];
    y[iy] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void copy (__global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    y[offset_y + get_global_id(0) * stride_y] =
        x[offset_x + get_global_id(0) * stride_x];
}

__kernel void copy_matrix (__global const REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    b[ib] = a[ia];
}

__kernel void swap_matrix (__global REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    REAL temp = a[ia];
    a[ia] = b[ia];
    b[ib] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void scal (const REAL alpha, __global REAL* x,
                    const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    x[ix] = alpha * x[ix];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpy (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y,
                    const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + y[iy];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpby (const REAL alpha, __global const REAL* x,
                     uint offset_x, const uint stride_x,
                     const REAL beta, __global REAL* y,
                     const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + beta * y[iy];
}

// ================= Reduction kernels ========================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduction (__global double* acc) {
    double sum = work_group_reduction_sum(acc[get_global_id(0)]);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== Dot product ==============================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void dot_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x,
                          __global const REAL* y,
                          const uint offset_y, const uint stride_y) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    REAL py = y[offset_y + get_global_id(0) * stride_y];
    double sum = work_group_reduction_sum((double)(px * py));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== asum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void asum_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)fabs(px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== sum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)px);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== nrm2 =====================================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void nrm2_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)(px * px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================ Max reduction ==============================================
inline void work_group_reduction_imax (__global uint* iacc,
                                       __global double* vacc,
                                       uint const ind, const double val) {

    uint local_id = get_local_id(0);
    uint local_size = get_local_size(0);

    __local uint liacc[WGS];
    __local double lvacc[WGS];
    liacc[local_id] = ind;
    lvacc[local_id] = val;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    uint index = ind;
    double value = val;

    uint i = local_size;
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_id == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            double other_value = lvacc[local_id + i + 1];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i + 1];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        if (local_id < i) {
            double other_value = lvacc[local_id + i];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    if(local_id == 0) {
        uint group_id = get_group_id(0);
        iacc[group_id] = index;
        vacc[group_id] = value;
    }

}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduction (__global uint* iacc, __global double* vacc) {
    uint gid = get_global_id(0);
    work_group_reduction_imax(iacc, vacc, iacc[gid], (double)(vacc[gid]));
}

// ================== iamax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void iamax_reduce (__global uint* iacc, __global double* vacc,
                            __global const REAL* x,
                            const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(fabs(x[ix])));
}

// ================== imax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(x[ix]));
}

// ================== imin reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imin_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(-x[ix]));
}

// ||||||||||||||||       BLAS 2      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMV =====================================================

inline void work_group_reduction_sum_horizontal
(__global REAL* acc, const REAL value) {

    uint local_row = get_local_id(0);
    uint local_col = get_local_id(1);

    __local REAL lacc[WGSm][WGSn];
    lacc[local_row][local_col] = value;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    REAL pacc = value;
    uint i = get_local_size(1);
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_col == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            pacc += lacc[local_row][local_col + i + 1];
        }
        if (local_col < i) {
            pacc += lacc[local_row][local_col + i];
            lacc[local_row][local_col] = pacc;
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    if(local_col == 0) {
        acc[get_global_size(0) * get_group_id(1) + get_global_id(0)] = pacc;
    }
}

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void sum_reduction_horizontal (__global REAL* acc) {

    uint ia = get_global_size(0) * get_global_id(1) + get_global_id(0);
    work_group_reduction_sum_horizontal(acc, acc[ia]);
}

// ========================= gemv ==============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gemv_reduce (__global REAL* acc,
                           const REAL alpha, __global const REAL* a,
                           const uint offset_a, const uint ld_a,
                           __global const REAL* x, const uint offset_x,
                           const uint stride_x) {

    uint ia = offset_a + ld_a * get_global_id(1) + get_local_id(0);
    uint ix = offset_x + get_global_id(1) * stride_x;

    work_group_reduction_sum_horizontal(acc, alpha * a[ia] * x[ix]);
}

// ========================== gerk =============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gerk (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global const REAL* y,
                    const uint offset_y, const uint stride_y,
                    __global REAL* a, const uint offset_a, const uint ld_a) {

    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(1) * stride_y;
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    a[ia] += alpha * x[ix] * y[iy];
}

// ||||||||||||||||       BLAS 3      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMM =====================================================

inline uint globalize (const uint tile_size, const uint tile_id,
                       const uint id){
    return tile_id * tile_size + id;
}

// ========================= gemm ==============================================

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled (const REAL alpha, __global const REAL* a,
                          const uint offset_a, const uint ld_a,
                          __global const REAL* b,
                          const uint offset_b, const uint ld_b,
                          const REAL beta, __global REAL* c,
                          const uint offset_c, const uint ld_c,
                          const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    // Elements that are in partial m-tiles and n-tiles need to be
    // loaded, but only if they exist.
    const bool load_row = c_row < m;
    bool load_col[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
        load_col[w] = c_col + w * RTS < n;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {

        const uint tile_row = TS * t + row;
        const uint tile_col = TS * t + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    // Compute partial k-tiles.
    const uint rest_k = k - tc * TS;
    if (0 < rest_k) {

        const uint tile_row = TS * tc + row;
        const uint tile_col = TS * tc + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < rest_k; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

    }

    //Only the elements that exist in partial c-tiles should be stored.
#pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const bool store = load_row && load_col[w];
        if (store) {
            const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
            c[ic] = alpha * acc[w] + beta * c[ic];
        }
    }

}

// Simpler version that requires dimensions that fit tiles

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled_fit (const REAL alpha, __global const REAL* a,
                              const uint offset_a, const uint ld_a,
                              __global const REAL* b,
                              const uint offset_b, const uint ld_b,
                              const REAL beta, __global REAL* c,
                              const uint offset_c, const uint ld_c,
                              const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {
        for (uint w = 0; w < WPT; w++) {
            const uint tile_row = TS * t + row;
            const uint tile_col = TS * t + col;
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = a[ia];
            b_tile[col + w * RTS][row] = b[ib];
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
        c[ic] = alpha * acc[w] + beta * c[ic];
    }

}


Improvement on first super-kmer

Improvement on first super-kmer

__kernel void getSuperK_M(
   __global uint* R2M_G, // Matrix of reads
   __global uint* counters, // Matrix of temporal output
   __global uint* TMP, // Matrix of temporal output
   const unsigned int nr, // Number of reads
   const unsigned int r, // Read size
   const unsigned int k, // Kmer size
   const unsigned int m // m-mer size
   )
{
   uint x, y, yl, xl, p, start, offset, nmk, nm, min, a, b, c, idt, ts, nt, lsd, nkr, tmp, ls, d = 0;
   bool flag;

   x = get_global_id(0);
   y = get_global_id(1);
   xl = get_local_id(0);
   yl = get_local_id(1);
   ls = get_local_size(0);

  __local uint R2M_L[180]; // Vector of a read and super k-mers (32 bits) , len = lenght of reads
  __local uint mR_10[180-4+1]; // Vector containing mmers, len = lenght of reads - minimizer size + 1
  __local uint counter; // Size of mmers
  __local uint MT[9]; // Position of minimizer in each tile
  __local uint PMT[9]; // Current minimizer in each tile

  if (x==0) {
    counter = 0;
  }

   // Global to local
   ts = ls;
   nt = (r-1)/ts + 1; // nt: Number of tiles or number of sub-reads per read
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++) { // coalesced
        p=ts*i+x;
        if (p > r-1) {
          break;
        }
        R2M_L[p]=R2M_G[y*r+p];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // FUNCTION: GetM-mers
    // Read to decimal representation of m-mers
    lsd = ls / m;
    nm = r - m + 1; // nm: Number of m-mers per read
    ts = ((nm-1)/lsd) + 1; // ts: Tile size
    nt = ((nm-1)/ts) + 1 ; // nt: Number of tiles or number of sub-reads per read
    a = 63; // a:Mask , same as: 2^(2m-2)-1  m=4

    // rewrited from here
    if ((x < m*nt) && (y<nr)) {
      // Cómputo en pllo del primer m-mer de cada tile
      idt = x/m;
      start = ts*idt;
      offset = x % m;
      p = (start) + (offset);
      mR_10[start] = 0;
      b = R2M_L [p];
      atomic_or(&mR_10[start], (b<<(m-(x%m)-1)*2));
    }
    barrier(CLK_LOCAL_MEM_FENCE);

      // Cómputo en serie del resto de m-mers de cada tile

      if ((x%m == 0) && (x < m*nt) && (y<nr)) {
        c = mR_10[start];
        for (int z=0; z<(ts-1); z++) {
          if (start+m+z > r-1) {
            break;
          }
          c = (c & a) << 2;
          b = R2M_L[start+m+z];
          c = c | b;
          mR_10[start+1+z]=c;
        }
      }
    barrier(CLK_LOCAL_MEM_FENCE);

   // Clear R2M_L
   ts = ls;  // ts: Tile size, work group size
   nt = (r-1)/ts + 1; // nt: Number of tiles
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++)  // coalesced
        {
          p=ts*i+x;
          if (p > r-1)
            { break; }
          R2M_L[p] = 0;
        }
    }

   nt = ((nm-1)/ts) + 1; // nt: Number of tiles or number of sub-reads per read

   // Extract MR_10 debuggin
   if ((x<ts) && (y<nr)){
     for (int i=0; i<nt; i++) { // coalesced
       p=ts*i+x;
       if (p > nm-1) {
         break;
       }
       TMP[y*nm+p]=mR_10[p];
     }
   }

    // FUNCTION: GetSK_M
    // m-mers to minimizers
    nmk = k-m+1; // nm: Number of m-mers per k-mers
    nkr = r - k + 1;
    lsd = ls/nmk;
    ts = (nkr-1)/lsd + 1; // ts: Tile size
    nt = (nkr-1)/ts + 1; // nt: Number of tiles
    if (x < nt){
      MT[x] = 0xFFFFFFFF;
      PMT[x] = 0xFFFFFFF;
    }
    PMT[0] = 0;

    barrier(CLK_LOCAL_MEM_FENCE);

  if ((x<(nt*nmk )) && (y<nr)){
  //if ((x>=(5*nmk)) && (y<nr) && (x<(6*nmk))){
      idt = x/nmk;
      start = ts*idt;
      offset = x%nmk;
      p = start + offset;
      // Cómputo en pllo del min del primer k-mer de cada tile
      b = mR_10[p];
      atomic_min(&MT[idt],b); // Atomic min - se obtiene el min del primer k-mer de cada tile
}

barrier(CLK_LOCAL_MEM_FENCE);

 if ((x < (nt*nmk)) && (y<nr)) {
   if (b == MT[idt]) {
     if (idt == 0){
        atomic_max(&PMT[idt],p);
        } else {
        atomic_min(&PMT[idt],p);
      };
   }
 }
barrier(CLK_LOCAL_MEM_FENCE);

for (int z=0; z < ts ; z++){ // Cómputo en serie del min del resto de k-mers de cada tile

    if ((start + nmk + z) > (r -1)){
      break;
    }

    if ((x%nmk - z%nmk) == 0)
    {
            b = mR_10[start + nmk + z]; // Se lee el último m-mer del k-mer a evaluar
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x < (nt*nmk)) && (y<nr))
    {
      flag = false;
      if (PMT[idt] != (start + z))
      {
        flag = true;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ( (x < (nt*nmk)) && (y<nr) ) {
        if (flag){
          if ((x%nmk - z%nmk) == 0){
            if (b < MT[idt]){

              // Atomic OR - end of super k-mers
              atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));
              printf ("P1: PMT[idt]: %d,  R2M_L[PMT[idt]]: %d, b: %d, MT[idt]: %d, (start + z + k - 1): %d, z: %d, start: %d\n",
                      PMT[idt], R2M_L[PMT[idt]], b, MT[idt], start + z + k - 1, z, start);

              PMT[idt] = start+nmk+z;

              MT[idt] = b;
              // Atomic OR - start of super k-mers
              atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));
              printf ("P2: PMT[idt]: %d,  R2M_L[PMT[idt]]: %d, b: %d, MT[idt]: %d, (start + z + k - 1)<<12: %d, z: %d, start: %d\n",
                      PMT[idt], R2M_L[PMT[idt]], b, MT[idt], (start + z + 1) << 12, z, start);
            }
          }
        }
      }
      if ((x < (nt*nmk)) && (y<nr))  {
        if (!flag) {
          if (x%nmk == 0){
            // Atomic OR - end of super k-mer
            atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));

          }
          MT[idt] = 0xFFFFFFFF;
        }
      }
        barrier(CLK_LOCAL_MEM_FENCE);

        if ((x < (nt*nmk)) && (y<nr))
      	{
          if (!flag)
          {
              atomic_min(&MT[idt], b);
          }
        }
          barrier(CLK_LOCAL_MEM_FENCE);

          if ((x < (nt*nmk)) && (y<nr))
          {
    			     if (!flag)
               {
    	            if (b == MT[idt])
                    {
                      PMT[idt] = 0xFFFFFFFF;
                      atomic_min(&PMT[idt],p);
                    }
               }
    		  }

        barrier(CLK_LOCAL_MEM_FENCE);
      if ((x < (nt*nmk)) && (y<nr) )
        {
  			if (!flag)
          {
          if (p == PMT[idt]) {

            if (x%nmk <= z%nmk){
              PMT[idt] = start + ((z/nmk)+1)*nmk + (x%nmk);
            } else {
              PMT[idt] = start + (z/nmk)*nmk +(x%nmk);
            }
            // Atomic OR - start of super k-mers
            atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));

          }

         }
       }
     }
    barrier(CLK_LOCAL_MEM_FENCE);

      if (x == (nt*nmk - 1)){
        // Atomic OR - end of super k-mer
        atomic_or(&R2M_L[PMT[idt]], (r-1));
      }
      barrier(CLK_LOCAL_MEM_FENCE);

    /* Compact output
    lsd = ls/m; // lsd: Local space divisions
    nm=r-m+1; // nm: Number of m-mers per read
    ts=(nm-1)/lsd + 1; // ts: Tile size
    nt=(nm-1)/ts + 1; // // nt: Number of tiles
    if ((x<ts) && (x<nm) && (y<nr)) {
   		for (int i=0; i<nt; i++) {
        p=ts*i+x;
        if (p > nm-1)
          { break; }
        b = R2M_L[p];
        if (b != 0)
        {
    	     c = atom_inc(&counter); // Atomic increment
           a = ((b & 0x00000FFF) - ((b >> 12) & 0x00000FFF) + 1) & 0x000000FF; // a is the size
           R2M_L[c] =  ((mR_10[p] << 20) & 0xFFF00000) | ((b >> 4) & 0x000FFF00) | a;
        }
      }
    } */
    barrier(CLK_LOCAL_MEM_FENCE);

    //   //FUNCTION: Local2Global
    //   //Reads in local memory to Read in global memory
    ts = ls;  // ts: Tile size, work group size
    nt = (r-1)/ts + 1; // nt: Number of tiles

    if ((x<ts) && (x<r) && (y<nr)) {
      for (int i=0; i<nt; i++){  // coalesced
        p=ts*i+x;
        if (p > nm-1)
          {
            R2M_G[y*r+p] = 0;
            break;
          }
        R2M_G[y*r+p] = R2M_L[p];
      }
    }
   // Extract counters
    if ((x == 0) && (y<nr))
     {
             counters[y] = counter;
      }
}

__kernel void getSuperK_M(
   __global uint* R2M_G, // Matrix of reads
   __global uint* counters, // Matrix of temporal output
   __global uint* TMP, // Matrix of temporal output
   const unsigned int nr, // Number of reads
   const unsigned int r, // Read size
   const unsigned int k, // Kmer size
   const unsigned int m // m-mer size
   )
{
   uint x, y, yl, xl, p, start, offset, nmk, nm, min, a, b, c, idt, ts, nt, lsd, nkr, tmp, ls, d = 0;
   bool flag;

   x = get_global_id(0);
   y = get_global_id(1);
   xl = get_local_id(0);
   yl = get_local_id(1);
   ls = get_local_size(0);

  __local uint R2M_L[180]; // Vector of a read and super k-mers (32 bits) , len = lenght of reads
  __local uint mR_10[180-4+1]; // Vector containing mmers, len = lenght of reads - minimizer size + 1
  __local uint counter; // Size of mmers
  __local uint MT[9]; // Position of minimizer in each tile
  __local uint PMT[9]; // Current minimizer in each tile

  if (x==0) {
    counter = 0;
  }

   // Global to local
   ts = ls;
   nt = (r-1)/ts + 1; // nt: Number of tiles or number of sub-reads per read
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++) { // coalesced
        p=ts*i+x;
        if (p > r-1) {
          break;
        }
        R2M_L[p]=R2M_G[y*r+p];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // FUNCTION: GetM-mers
    // Read to decimal representation of m-mers
    lsd = ls / m;
    nm = r - m + 1; // nm: Number of m-mers per read
    ts = ((nm-1)/lsd) + 1; // ts: Tile size
    nt = ((nm-1)/ts) + 1 ; // nt: Number of tiles or number of sub-reads per read
    a = 63; // a:Mask , same as: 2^(2m-2)-1  m=4

    // rewrited from here
    if ((x < m*nt) && (y<nr)) {
      // Cómputo en pllo del primer m-mer de cada tile
      idt = x/m;
      start = ts*idt;
      offset = x % m;
      p = (start) + (offset);
      mR_10[start] = 0;
      b = R2M_L [p];
      atomic_or(&mR_10[start], (b<<(m-(x%m)-1)*2));
    }
    barrier(CLK_LOCAL_MEM_FENCE);

      // Cómputo en serie del resto de m-mers de cada tile

      if ((x%m == 0) && (x < m*nt) && (y<nr)) {
        c = mR_10[start];
        for (int z=0; z<(ts-1); z++) {
          if (start+m+z > r-1) {
            break;
          }
          c = (c & a) << 2;
          b = R2M_L[start+m+z];
          c = c | b;
          mR_10[start+1+z]=c;
        }
      }
    barrier(CLK_LOCAL_MEM_FENCE);

   // Clear R2M_L
   ts = ls;  // ts: Tile size, work group size
   nt = (r-1)/ts + 1; // nt: Number of tiles
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++)  // coalesced
        {
          p=ts*i+x;
          if (p > r-1)
            { break; }
          R2M_L[p] = 0;
        }
    }

   nt = ((nm-1)/ts) + 1; // nt: Number of tiles or number of sub-reads per read

   // Extract MR_10 debuggin
   if ((x<ts) && (y<nr)){
     for (int i=0; i<nt; i++) { // coalesced
       p=ts*i+x;
       if (p > nm-1) {
         break;
       }
       TMP[y*nm+p]=mR_10[p];
     }
   }

    // FUNCTION: GetSK_M
    // m-mers to minimizers
    nmk = k-m+1; // nm: Number of m-mers per k-mers
    nkr = r - k + 1;
    lsd = ls/nmk;
    ts = (nkr-1)/lsd + 1; // ts: Tile size
    nt = (nkr-1)/ts + 1; // nt: Number of tiles
    if (x < nt){
      MT[x] = 0xFFFFFFFF;
      PMT[x] = 0xFFFFFFF;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

  if ((x<(nt*nmk )) && (y<nr)){
  //if ((x>=(5*nmk)) && (y<nr) && (x<(6*nmk))){
      idt = x/nmk;
      start = ts*idt;
      offset = x%nmk;
      p = start + offset;
      // Cómputo en pllo del min del primer k-mer de cada tile
      b = mR_10[p];
      atomic_min(&MT[idt],b); // Atomic min - se obtiene el min del primer k-mer de cada tile
}

barrier(CLK_LOCAL_MEM_FENCE);

 if ((x < (nt*nmk)) && (y<nr)) {
   if (b == MT[idt]) {
    // if (idt == 0){
        atomic_min(&PMT[idt],p);
      //  } else {
    //    atomic_min(&PMT[idt],p);
    //  };
   }
 }
barrier(CLK_LOCAL_MEM_FENCE);

for (int z=0; z < ts ; z++){ // Cómputo en serie del min del resto de k-mers de cada tile

    if ((start + nmk + z) > (r -1)){
      break;
    }

    if ((x%nmk - z%nmk) == 0)
    {
            b = mR_10[start + nmk + z]; // Se lee el último m-mer del k-mer a evaluar
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x < (nt*nmk)) && (y<nr))
    {
      flag = false;
      if (PMT[idt] != (start + z))
      {
        flag = true;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ( (x < (nt*nmk)) && (y<nr) ) {
        if (flag){
          if ((x%nmk - z%nmk) == 0){
            if (b < MT[idt]){

              // Atomic OR - end of super k-mers
              atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));
              printf ("P1: PMT[idt]: %d,  R2M_L[PMT[idt]]: %d, b: %d, MT[idt]: %d, (start + z + k - 1): %d, z: %d, start: %d\n",
                      PMT[idt], R2M_L[PMT[idt]], b, MT[idt], start + z + k - 1, z, start);

              PMT[idt] = start+nmk+z;

              MT[idt] = b;
              // Atomic OR - start of super k-mers
              atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));
              printf ("P2: PMT[idt]: %d,  R2M_L[PMT[idt]]: %d, b: %d, MT[idt]: %d, (start + z + k - 1)<<12: %d, z: %d, start: %d\n",
                      PMT[idt], R2M_L[PMT[idt]], b, MT[idt], (start + z + 1) << 12, z, start);
            }
          }
        }
      }
      if ((x < (nt*nmk)) && (y<nr))  {
        if (!flag) {
          if (x%nmk == 0){
            // Atomic OR - end of super k-mer
            atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));

          }
          MT[idt] = 0xFFFFFFFF;
        }
      }
        barrier(CLK_LOCAL_MEM_FENCE);

        if ((x < (nt*nmk)) && (y<nr))
      	{
          if (!flag)
          {
              atomic_min(&MT[idt], b);
          }
        }
          barrier(CLK_LOCAL_MEM_FENCE);

          if ((x < (nt*nmk)) && (y<nr))
          {
    			     if (!flag)
               {
    	            if (b == MT[idt])
                    {
                      PMT[idt] = 0xFFFFFFFF;
                      atomic_min(&PMT[idt],p);
                    }
               }
    		  }

        barrier(CLK_LOCAL_MEM_FENCE);
      if ((x < (nt*nmk)) && (y<nr) )
        {
  			if (!flag)
          {
          if (p == PMT[idt]) {

            if (x%nmk <= z%nmk){
              PMT[idt] = start + ((z/nmk)+1)*nmk + (x%nmk);
            } else {
              PMT[idt] = start + (z/nmk)*nmk +(x%nmk);
            }
            // Atomic OR - start of super k-mers
            atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));

          }

         }
       }
     }
    barrier(CLK_LOCAL_MEM_FENCE);

      if (x == (nt*nmk - 1)){
        // Atomic OR - end of super k-mer
        atomic_or(&R2M_L[PMT[idt]], (r-1));
      }
      barrier(CLK_LOCAL_MEM_FENCE);

    // Compact output
    lsd = ls/m; // lsd: Local space divisions
    nm=r-m+1; // nm: Number of m-mers per read
    ts=(nm-1)/lsd + 1; // ts: Tile size
    nt=(nm-1)/ts + 1; // // nt: Number of tiles
    if ((x<ts) && (x<nm) && (y<nr)) {
   		for (int i=0; i<nt; i++) {
        p=ts*i+x;
        if (p > nm-1)
          { break; }
        b = R2M_L[p];
        if (b != 0)
        {
    	     c = atom_inc(&counter); // Atomic increment
           a = ((b & 0x00000FFF) - ((b >> 12) & 0x00000FFF) + 1) & 0x000000FF; // a is the size
           R2M_L[c] =  ((mR_10[p] << 20) & 0xFFF00000) | ((b >> 4) & 0x000FFF00) | a;
        }
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    //   //FUNCTION: Local2Global
    //   //Reads in local memory to Read in global memory
    ts = ls;  // ts: Tile size, work group size
    nt = (r-1)/ts + 1; // nt: Number of tiles

    if ((x<ts) && (x<r) && (y<nr)) {
      for (int i=0; i<nt; i++){  // coalesced
        p=ts*i+x;
        if (p > nm-1)
          {
            R2M_G[y*r+p] = 0;
            break;
          }
        R2M_G[y*r+p] = R2M_L[p];
      }
    }
   // Extract counters
    if ((x == 0) && (y<nr))
     {
             counters[y] = counter;
      }
}


little cleanup

little cleanup


Force double precision.

Force double precision.

So on the default number of n (chosen for timing) the calculation breaks down
because by default floats are (of course) single precision.

This patch fixes that.

#!/usr/bin/env clisp
; PI Common LISP example.

(defvar n 5000000)

; Set double precision.
(setf *read-default-float-format* 'double-float)

; This likely only works in GNU Common LISP.
(if (> (list-length *args*) 0)
  (setq n (parse-integer (car *args*)))
)

(format t "Calculating PI using~%")
(format t "  ~a slices~%" n)
(format t "  1 process~%")

; get-internal-real-time gets time from some arbitrary, implementation
; dependent starting point in some fraction of a second.
(defvar start (get-internal-real-time))

(defvar sum 0.0)
(defvar st (/ 1.0 n))
(defvar x 0.0)

(loop for i from 1 to n do
; In Common LISP we have to enclose multi-line dos in a block.
  (block inner
    (setq x (* (- i 0.5) st))
    (setq sum (+ sum (/ 4.0 (+ 1.0 (* x x)))))
  )
)

(defvar p (* sum st))

(defvar stop (get-internal-real-time))

; get-internal-real-time is in some platform specific unit which we can find in
; internal-time-units-per-second
(defvar runt (/ (- stop start) internal-time-units-per-second))

(format t "Obtained value of PI: ~a~%" p)

; We have to print the time with ~f to force a float output.
(format t "Time taken: ~f seconds~%" runt)

#!/usr/bin/env clisp
; PI Common LISP example.

(defvar n 5000000)

; This likely only works in GNU Common LISP.
(if (> (list-length *args*) 0)
  (setq n (parse-integer (car *args*)))
)

(format t "Calculating PI using~%")
(format t "  ~a slices~%" n)
(format t "  1 process~%")

; get-internal-real-time gets time from some arbitrary, implementation
; dependent starting point in some fraction of a second.
(defvar start (get-internal-real-time))

(defvar sum 0.0)
(defvar st (/ 1.0 n))
(defvar x 0.0)

(loop for i from 1 to n do
; In Common LISP we have to enclose multi-line dos in a block.
  (block inner
    (setq x (* (- i 0.5) st))
    (setq sum (+ sum (/ 4.0 (+ 1 (* x x)))))
  )
)

(defvar p (* sum st))

(defvar stop (get-internal-real-time))

; get-internal-real-time is in some platform specific unit which we can find in
; internal-time-units-per-second
(defvar runt (/ (- stop start) internal-time-units-per-second))

(format t "Obtained value of PI: ~a~%" p)

; We have to print the time with ~f to force a float output.
(format t "Time taken: ~f seconds~%" runt)


Fix kernel crash on nvidia, caused by float4 alignemtn error, see https://github.com/clMathLibraries/clBLAS/issues/108 for more details

/* ************************************************************************
 * Copyright 2013 Advanced Micro Devices, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ************************************************************************/

// Column-Major Case

static const char *ger_C_kernel = "
#ifdef DOUBLE_PRECISION
    #ifdef cl_khr_fp64
    #pragma OPENCL EXTENSION cl_khr_fp64 : enable
    #else
    #pragma OPENCL EXTENSION cl_amd_fp64 : enable
    #endif
#endif

#define BH %BH_DEF
#define BW %BW_DEF

__kernel void %PREFIXger_C_kernel( __global %TYPE const* restrict _X, __global %TYPE const* restrict _Y, __global %TYPE* _A,
				uint M, uint N, uint offx, int incx, uint offy, int incy, uint offa, uint lda,
				%TYPE alpha, int doConj )
{
	__global %TYPE* A;
	__global %TYPE const* restrict X;
	__global %TYPE const* restrict Y;

	A = _A + offa;
	X = _X + offx;
	Y = _Y + offy;

	if ( incx < 0 ) // Goto end of vector
	{
		X = X + ( M - 1) * abs(incx);
	}

	if ( incy < 0 ) // Goto end of vector
	{
		Y = Y + ( N - 1) * abs(incy);
	}

	// create local memory
  __local %TYPE%V localXV[ BH ];
  __local %TYPE *localX = (__local %TYPE *)localXV;
	__local %TYPE localY[ BW ];

	uint lID = get_local_id( 0 );
	uint gID = get_group_id( 0 );

	uint tIDy = lID & ( BH-1 );  //get y coordinate of a thread in 1D workgroup
	uint tIDx = lID / BH;        //get x coordinate of a thread in !D workgroup
    uint nBlocksX = (( N + BW - 1) / BW );
    uint nBlocksY = (( M + BH * %V - 1 ) / ( BH * %V ));

	uint gIDy = gID % nBlocksY;	//get y coordinate of a workgroup in 1D grid
    uint gIDx = gID / nBlocksY;	// get x coordinate of a workgroup in a 1D grid

    uint row = (( BH * gIDy)+  tIDy) * %V;
    uint col = (( BW * gIDx)+  tIDx);

    if( (gIDx != (nBlocksX-1)) && (gIDy != (nBlocksY-1)) )       // Completely vector blocks
    {
        //populate local memory
        for( int i = lID; i< ( BH * %V); i+= get_local_size(0) )
        {
            int idx = i + ( gIDy * BH * %V);
            localX[ i ] = *(X + (idx * incx));
        }

        for( int i = lID; i< BW; i+= get_local_size(0) )
        {
            int idx = i + ( gIDx * BW);
            localY[ i ] = *(Y + (idx * incy));
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        %TYPE%V prevA, temp;
        %TYPE yReg = localY[  tIDx ];
        %TYPE%V xReg = *(__local %TYPE%V*)(&localX[ tIDy * %V]);

        prevA = %VLOAD( 0, ( A + col*lda + row ) );
        %CONJUGATE(doConj, yReg);
        %VMUL( temp, xReg, alpha );
        %VMAD( prevA, temp, yReg);
        %VSTORE( prevA, 0 , ( A + col*lda + row ) );

    }
    else                            // Border blocks in both X & Y direction
    {
    	//populate local memory
        for( int i = lID; i< ( BH * %V); i+= get_local_size(0) )
        {
    		int idx = i + ( gIDy * BH * %V);
           	if ( idx < M )
    		{
    			localX[ i ] = *(X + (idx * incx));
        	}
    	}

        for( int i = lID; i< BW; i+= get_local_size(0) )
        {
    		int idx = i + ( gIDx * BW);
    		if ( idx < N)
    		{
           		localY[ i ] = *(Y + (idx * incy));
        	}
    	}
    	barrier(CLK_LOCAL_MEM_FENCE);

    	uint gTIDx = (gIDx * BW) + tIDx;
        if ( gTIDx < N)  // if whithin last column
    	{
    		if( (row + %V - 1) < M )  // if the next V rows are still within M, then do vector math
	    	{
    			%TYPE%V prevA, temp;
    			%TYPE yReg = localY[  tIDx ];
    			%TYPE%V xReg = *(__local %TYPE%V*)(&localX[ tIDy * %V]);

	    		prevA = %VLOAD( 0, ( A + col*lda + row ) );
		    	%CONJUGATE(doConj, yReg);
			    %VMUL( temp, xReg, alpha );
    			%VMAD( prevA, temp, yReg);
    			%VSTORE( prevA, 0 , ( A + col*lda + row ) );

    		}
	    	else if( row < M  )  //else do scalar multiplication
		    {
    			%TYPE xRegS, yReg, prevA, temp;
    			for( int i=row; i<M; i++ )
    			{
    				yReg  = localY[ tIDx ];
    				xRegS = localX[ (tIDy * %V) + (i-row) ];
    				prevA = A[ col*lda + i];
	    			%CONJUGATE(doConj, yReg);
    				%MUL( temp, xRegS, alpha );
    				%MAD( prevA, temp, yReg );
	    			A[ col*lda + i ] = prevA;
		    	}
		    }
	    }
    }
}
\n";

//Row major kernel

static const char *ger_R_kernel = "
#ifdef DOUBLE_PRECISION
    #ifdef cl_khr_fp64
    #pragma OPENCL EXTENSION cl_khr_fp64 : enable
    #else
    #pragma OPENCL EXTENSION cl_amd_fp64 : enable
    #endif
#endif

#define BH %BH_DEF
#define BW %BW_DEF

__kernel void %PREFIXger_R_kernel( __global %TYPE const* restrict _X, __global %TYPE const* restrict _Y, __global %TYPE* _A,
				uint M, uint N, uint offx, int incx, uint offy, int incy, uint offa, uint lda,
				%TYPE alpha, int doConj )
{
	__global %TYPE* A;
	__global %TYPE const* restrict X;
	__global %TYPE const* restrict Y;

	A = _A + offa;
	X = _X + offx;
	Y = _Y + offy;

	if ( incx < 0 ) // Goto end of vector
	{
		X = X + ( M - 1) * abs(incx);
	}

	if ( incy < 0 ) // Goto end of vector
	{
		Y = Y + ( N - 1) * abs(incy);
	}

    __local %TYPE localX[ BH ];
    __local %TYPE%V localYV[ BW ];
    __local %TYPE *localY = (__local %TYPE *)localYV;

    uint lID = get_local_id( 0 );
    uint gID = get_group_id( 0 );

    uint tIDy = lID / BW;
    uint tIDx = lID & ( BW - 1);
    uint nBlocksY = (( M + BH - 1) / BH );
    uint nBlocksX = (( N + BW * %V - 1 ) / ( BW * %V ));

    uint gIDy = gID / nBlocksX;
    uint gIDx = gID % nBlocksX;

    uint row = (( BH * gIDy)+  tIDy);
    uint col = (( BW * gIDx)+  tIDx) * %V;

    if( (gIDy != (nBlocksY-1)) && (gIDx != (nBlocksX-1)) )         // Perfectly vector blocks
    {
        //populate local memory
        for( int i = lID; i< ( BW * %V); i+= get_local_size(0) )
        {
            int idx = i + ( gIDx * BW * %V);
            localY[ i ] = *(Y + (idx * incy));
        }

        for( int i = lID; i< BH; i+= get_local_size(0) )
        {
            int idx = i + ( gIDy * BH);
            localX[ i ] = *(X + (idx * incx));
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        %TYPE%V prevA, temp;
        %TYPE xReg = localX[ tIDy ];
        %TYPE%V yRegS = *(__local %TYPE%V*)(&localY[ tIDx * %V]);

        prevA = %VLOAD( 0, ( A + row*lda + col ) );
        %CONJUGATE(doConj, yRegS);
        %VMUL( temp, yRegS, alpha );
        %VMAD( prevA, temp, xReg );
        %VSTORE( prevA, 0 , ( A + row*lda + col ) );
    }
    else
    {
        //populate local memory
        for( int i = lID; i< ( BW * %V); i+= get_local_size(0) )
        {
            int idx = i + ( gIDx * BW * %V);
            if ( idx < N)
            {
                localY[ i ] = *(Y + (idx * incy));
            }
        }

        for( int i = lID; i< BH; i+= get_local_size(0) )
        {
            int idx = i + ( gIDy * BH);
            if ( idx < M)
            {
                localX[ i ] = *(X + (idx * incx));
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        uint gTIDy = gIDy * BH + tIDy;
        if ( gTIDy < M)
        {
            if( (col + %V - 1) < N )
            {
                %TYPE%V prevA, temp;
                %TYPE xReg = localX[ tIDy ];
                %TYPE%V yRegS = *(__local %TYPE%V*)(&localY[ tIDx * %V]);

                prevA = %VLOAD( 0, ( A + row*lda + col ) );
                %CONJUGATE(doConj, yRegS);
    			%VMUL( temp, yRegS, alpha );
    			%VMAD( prevA, temp, xReg );
                %VSTORE( prevA, 0 , ( A + row*lda + col ) );

            }
            else if( col < N  )
            {
                %TYPE xReg, yRegS, prevA, temp;
                for( int i=col; i<N; i++ )
                {
                    yRegS = localY[ (tIDx * %V) + (i-col) ];
                    xReg  = localX[ tIDy ];
                    prevA = A[ row*lda + i];
                    %CONJUGATE(doConj, yRegS);
    				%MUL( temp, yRegS, alpha );
    				%MAD( prevA, temp, xReg );
    				A[ row*lda + i ] = prevA;
                }
            }
        }
    }
}
\n";

/* ************************************************************************
 * Copyright 2013 Advanced Micro Devices, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ************************************************************************/

// Column-Major Case

static const char *ger_C_kernel = "
#ifdef DOUBLE_PRECISION
    #ifdef cl_khr_fp64
    #pragma OPENCL EXTENSION cl_khr_fp64 : enable
    #else
    #pragma OPENCL EXTENSION cl_amd_fp64 : enable
    #endif
#endif

#define BH %BH_DEF
#define BW %BW_DEF

__kernel void %PREFIXger_C_kernel( __global %TYPE const* restrict _X, __global %TYPE const* restrict _Y, __global %TYPE* _A,
				uint M, uint N, uint offx, int incx, uint offy, int incy, uint offa, uint lda,
				%TYPE alpha, int doConj )
{
	__global %TYPE* A;
	__global %TYPE const* restrict X;
	__global %TYPE const* restrict Y;

	A = _A + offa;
	X = _X + offx;
	Y = _Y + offy;

	if ( incx < 0 ) // Goto end of vector
	{
		X = X + ( M - 1) * abs(incx);
	}

	if ( incy < 0 ) // Goto end of vector
	{
		Y = Y + ( N - 1) * abs(incy);
	}

	// create local memory
	__local %TYPE localX[ BH * %V ];
	__local %TYPE localY[ BW ];

	uint lID = get_local_id( 0 );
	uint gID = get_group_id( 0 );

	uint tIDy = lID & ( BH-1 );  //get y coordinate of a thread in 1D workgroup
	uint tIDx = lID / BH;        //get x coordinate of a thread in !D workgroup
    uint nBlocksX = (( N + BW - 1) / BW );
    uint nBlocksY = (( M + BH * %V - 1 ) / ( BH * %V ));

	uint gIDy = gID % nBlocksY;	//get y coordinate of a workgroup in 1D grid
    uint gIDx = gID / nBlocksY;	// get x coordinate of a workgroup in a 1D grid

    uint row = (( BH * gIDy)+  tIDy) * %V;
    uint col = (( BW * gIDx)+  tIDx);

    if( (gIDx != (nBlocksX-1)) && (gIDy != (nBlocksY-1)) )       // Completely vector blocks
    {
        //populate local memory
        for( int i = lID; i< ( BH * %V); i+= get_local_size(0) )
        {
            int idx = i + ( gIDy * BH * %V);
            localX[ i ] = *(X + (idx * incx));
        }

        for( int i = lID; i< BW; i+= get_local_size(0) )
        {
            int idx = i + ( gIDx * BW);
            localY[ i ] = *(Y + (idx * incy));
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        %TYPE%V prevA, temp;
        %TYPE yReg = localY[  tIDx ];
        %TYPE%V xReg = *(__local %TYPE%V*)(&localX[ tIDy * %V]);

        prevA = %VLOAD( 0, ( A + col*lda + row ) );
        %CONJUGATE(doConj, yReg);
        %VMUL( temp, xReg, alpha );
        %VMAD( prevA, temp, yReg);
        %VSTORE( prevA, 0 , ( A + col*lda + row ) );

    }
    else                            // Border blocks in both X & Y direction
    {
    	//populate local memory
        for( int i = lID; i< ( BH * %V); i+= get_local_size(0) )
        {
    		int idx = i + ( gIDy * BH * %V);
           	if ( idx < M )
    		{
    			localX[ i ] = *(X + (idx * incx));
        	}
    	}

        for( int i = lID; i< BW; i+= get_local_size(0) )
        {
    		int idx = i + ( gIDx * BW);
    		if ( idx < N)
    		{
           		localY[ i ] = *(Y + (idx * incy));
        	}
    	}
    	barrier(CLK_LOCAL_MEM_FENCE);

    	uint gTIDx = (gIDx * BW) + tIDx;
        if ( gTIDx < N)  // if whithin last column
    	{
    		if( (row + %V - 1) < M )  // if the next V rows are still within M, then do vector math
	    	{
    			%TYPE%V prevA, temp;
    			%TYPE yReg = localY[  tIDx ];
    			%TYPE%V xReg = *(__local %TYPE%V*)(&localX[ tIDy * %V]);

	    		prevA = %VLOAD( 0, ( A + col*lda + row ) );
		    	%CONJUGATE(doConj, yReg);
			    %VMUL( temp, xReg, alpha );
    			%VMAD( prevA, temp, yReg);
    			%VSTORE( prevA, 0 , ( A + col*lda + row ) );

    		}
	    	else if( row < M  )  //else do scalar multiplication
		    {
    			%TYPE xRegS, yReg, prevA, temp;
    			for( int i=row; i<M; i++ )
    			{
    				yReg  = localY[ tIDx ];
    				xRegS = localX[ (tIDy * %V) + (i-row) ];
    				prevA = A[ col*lda + i];
	    			%CONJUGATE(doConj, yReg);
    				%MUL( temp, xRegS, alpha );
    				%MAD( prevA, temp, yReg );
	    			A[ col*lda + i ] = prevA;
		    	}
		    }
	    }
    }
}
\n";

//Row major kernel

static const char *ger_R_kernel = "
#ifdef DOUBLE_PRECISION
    #ifdef cl_khr_fp64
    #pragma OPENCL EXTENSION cl_khr_fp64 : enable
    #else
    #pragma OPENCL EXTENSION cl_amd_fp64 : enable
    #endif
#endif

#define BH %BH_DEF
#define BW %BW_DEF

__kernel void %PREFIXger_R_kernel( __global %TYPE const* restrict _X, __global %TYPE const* restrict _Y, __global %TYPE* _A,
				uint M, uint N, uint offx, int incx, uint offy, int incy, uint offa, uint lda,
				%TYPE alpha, int doConj )
{
	__global %TYPE* A;
	__global %TYPE const* restrict X;
	__global %TYPE const* restrict Y;

	A = _A + offa;
	X = _X + offx;
	Y = _Y + offy;

	if ( incx < 0 ) // Goto end of vector
	{
		X = X + ( M - 1) * abs(incx);
	}

	if ( incy < 0 ) // Goto end of vector
	{
		Y = Y + ( N - 1) * abs(incy);
	}

    __local %TYPE localX[ BH ];
    __local %TYPE localY[ BW * %V ];

    uint lID = get_local_id( 0 );
    uint gID = get_group_id( 0 );

    uint tIDy = lID / BW;
    uint tIDx = lID & ( BW - 1);
    uint nBlocksY = (( M + BH - 1) / BH );
    uint nBlocksX = (( N + BW * %V - 1 ) / ( BW * %V ));

    uint gIDy = gID / nBlocksX;
    uint gIDx = gID % nBlocksX;

    uint row = (( BH * gIDy)+  tIDy);
    uint col = (( BW * gIDx)+  tIDx) * %V;

    if( (gIDy != (nBlocksY-1)) && (gIDx != (nBlocksX-1)) )         // Perfectly vector blocks
    {
        //populate local memory
        for( int i = lID; i< ( BW * %V); i+= get_local_size(0) )
        {
            int idx = i + ( gIDx * BW * %V);
            localY[ i ] = *(Y + (idx * incy));
        }

        for( int i = lID; i< BH; i+= get_local_size(0) )
        {
            int idx = i + ( gIDy * BH);
            localX[ i ] = *(X + (idx * incx));
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        %TYPE%V prevA, temp;
        %TYPE xReg = localX[ tIDy ];
        %TYPE%V yRegS = *(__local %TYPE%V*)(&localY[ tIDx * %V]);

        prevA = %VLOAD( 0, ( A + row*lda + col ) );
        %CONJUGATE(doConj, yRegS);
        %VMUL( temp, yRegS, alpha );
        %VMAD( prevA, temp, xReg );
        %VSTORE( prevA, 0 , ( A + row*lda + col ) );
    }
    else
    {
        //populate local memory
        for( int i = lID; i< ( BW * %V); i+= get_local_size(0) )
        {
            int idx = i + ( gIDx * BW * %V);
            if ( idx < N)
            {
                localY[ i ] = *(Y + (idx * incy));
            }
        }

        for( int i = lID; i< BH; i+= get_local_size(0) )
        {
            int idx = i + ( gIDy * BH);
            if ( idx < M)
            {
                localX[ i ] = *(X + (idx * incx));
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        uint gTIDy = gIDy * BH + tIDy;
        if ( gTIDy < M)
        {
            if( (col + %V - 1) < N )
            {
                %TYPE%V prevA, temp;
                %TYPE xReg = localX[ tIDy ];
                %TYPE%V yRegS = *(__local %TYPE%V*)(&localY[ tIDx * %V]);

                prevA = %VLOAD( 0, ( A + row*lda + col ) );
                %CONJUGATE(doConj, yRegS);
    			%VMUL( temp, yRegS, alpha );
    			%VMAD( prevA, temp, xReg );
                %VSTORE( prevA, 0 , ( A + row*lda + col ) );

            }
            else if( col < N  )
            {
                %TYPE xReg, yRegS, prevA, temp;
                for( int i=col; i<N; i++ )
                {
                    yRegS = localY[ (tIDx * %V) + (i-col) ];
                    xReg  = localX[ tIDy ];
                    prevA = A[ row*lda + i];
                    %CONJUGATE(doConj, yRegS);
    				%MUL( temp, yRegS, alpha );
    				%MAD( prevA, temp, xReg );
    				A[ row*lda + i ] = prevA;
                }
            }
        }
    }
}
\n";