bugfix in gemm_kernel_oland.cl

bugfix in gemm_kernel_oland.cl

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

typedef union GPtr {
    __global float *f;
    __global double *d;
    __global float2 *f2v;
    __global double2 *d2v;
    __global float4 *f4v;
    __global double4 *d4v;
    __global float8 *f8v;
    __global double8 *d8v;
    __global float16 *f16v;
    __global double16 *d16v;
} GPtr;

typedef union LPtr {
    __local float *f;
    __local double *d;
    __local float2 *f2v;
    __local double2 *d2v;
    __local float4 *f4v;
    __local double4 *d4v;
    __local float8 *f8v;
    __local double8 *d8v;
    __local float16 *f16v;
    __local double16 *d16v;
} LPtr;

typedef union PPtr {
    float *f;
    double *d;
    float2 *f2v;
    double2 *d2v;
    float4 *f4v;
    double4 *d4v;
    float8 *f8v;
    double8 *d8v;
    float16 *f16v;
    double16 *d16v;
} PPtr;

__attribute__((reqd_work_group_size(8, 8, 1)))
void __kernel
sgemm_kernel( const uint M, const uint N, const uint K, const float alpha, const __global float8 *restrict A, const __global float8 *restrict B, __global float8 *C )
{
    float8 a0, a1, a2, a3;
    float8 b0, b1, b2, b3;
    float8 c0, c1, c2, c3, c4, c5, c6, c7;

    uint4 coord = 0u; /* contains coordB, coordA, k */

    uint lda = M / 8;
    uint ldb = N / 8;
    uint ldc = M;

    A += (uint) get_global_id(0);

    uint get_group_id_1 = (get_group_id(0) + get_group_id(1))% get_num_groups(1);
    uint get_global_id_1 = get_group_id_1 * get_local_size(1) + get_local_id(1);

    uint kif = ( (N % 512) != 0);

    get_global_id_1 = (kif*(uint)get_global_id(1)) + ((1-kif)*get_global_id_1);

    B += get_global_id_1;

    coord.y = 8u * (uint)get_global_id(0);
    coord.x = 8u * (uint)get_global_id_1;

    c0 = 0;
    c1 = 0;
    c2 = 0;
    c3 = 0;
    c4 = 0;
    c5 = 0;
    c6 = 0;
    c7 = 0;

    for (uint k1 = 0; k1 < K; k1 += 4) {
        /* -- Tiles multiplier -- */
        b0 = B[0];
        b1 = B[ldb];
        b2 = B[(ldb << 1)];
        b3 = B[mad24(3u, ldb, 0u)];

        a0 = A[0];
        a1 = A[lda];
        a2 = A[(lda << 1)];
        a3 = A[mad24(3u, lda, 0u)];

        c0 = mad(a0, b0.s0, c0);
        c1 = mad(a0, b0.s1, c1);
        c2 = mad(a0, b0.s2, c2);
        c3 = mad(a0, b0.s3, c3);
        c4 = mad(a0, b0.s4, c4);
        c5 = mad(a0, b0.s5, c5);
        c6 = mad(a0, b0.s6, c6);
        c7 = mad(a0, b0.s7, c7);

        c0 = mad(a1, b1.s0, c0);
        c1 = mad(a1, b1.s1, c1);
        c2 = mad(a1, b1.s2, c2);
        c3 = mad(a1, b1.s3, c3);
        c4 = mad(a1, b1.s4, c4);
        c5 = mad(a1, b1.s5, c5);
        c6 = mad(a1, b1.s6, c6);
        c7 = mad(a1, b1.s7, c7);

        c0 = mad(a2, b2.s0, c0);
        c1 = mad(a2, b2.s1, c1);
        c2 = mad(a2, b2.s2, c2);
        c3 = mad(a2, b2.s3, c3);
        c4 = mad(a2, b2.s4, c4);
        c5 = mad(a2, b2.s5, c5);
        c6 = mad(a2, b2.s6, c6);
        c7 = mad(a2, b2.s7, c7);

        c0 = mad(a3, b3.s0, c0);
        c1 = mad(a3, b3.s1, c1);
        c2 = mad(a3, b3.s2, c2);
        c3 = mad(a3, b3.s3, c3);
        c4 = mad(a3, b3.s4, c4);
        c5 = mad(a3, b3.s5, c5);
        c6 = mad(a3, b3.s6, c6);
        c7 = mad(a3, b3.s7, c7);

        A += (lda << 2);
        B += (ldb << 2);
        /* ---------------------- */
    }

    GPtr uC;

    uC.f8v = C + (coord.x * ldc + coord.y)/8;

    uint ldc3 = ldc >> 3;

    __global float8 *pC = uC.f8v;

    pC[0] 				= mad(c0, alpha, 0);
    pC[ldc3] 				= mad(c1, alpha, 0);
    pC[(ldc >> 2)] 			= mad(c2, alpha, 0);
    pC[mad24(3u, ldc3, 0u)] 		= mad(c3, alpha, 0);
    pC[(ldc >> 1)] 			= mad(c4, alpha, 0);
    pC[mad24(5u, ldc3, 0u)] 		= mad(c5, alpha, 0);
    pC[mad24(6u, ldc3, 0u)] 		= mad(c6, alpha, 0);
    pC[mad24(7u, ldc3, 0u)] 		= mad(c7, alpha, 0);
}

__attribute__((reqd_work_group_size(8, 8, 1)))
void __kernel
 cgemm_kernel( uint M, uint N, uint K, const float2 alpha, const __global float8 *restrict A, const __global float8 *restrict B, __global float8 *C)
{
    float8 a0, a1, a2, a3;
    float8 b0, b1, b2, b3;
    float8 c0, c1, c2, c3;
    uint4 coord = 0u; /* contains coordB, coordA, k */

    uint lda = M/4;
    uint ldb = N/4;
    uint ldc = M;

    uint kif;
    uint get_group_id_1;
    uint get_global_id_1;

    A += (uint)get_global_id(0);
    get_group_id_1 = (get_group_id(0) + get_group_id(1))% get_num_groups(1);
    get_global_id_1 = get_group_id_1 * get_local_size(1) + get_local_id(1);

    kif = (N % 256 != 0);
    get_global_id_1 = (kif*(uint)get_global_id(1)) + ((1-kif)*get_global_id_1);

    B += get_global_id_1;
    coord.y = 4u * (uint)get_global_id(0);
    coord.x = 4u * (uint)get_global_id_1;

    c0 = 0;
    c1 = 0;
    c2 = 0;
    c3 = 0;

    for (uint k1 = 0; k1 < K; k1 += 4) {
        /* -- Tiles multiplier -- */
        b0 = B[0];
        b1 = B[ldb];
        b2 = B[(ldb << 1)];
        b3 = B[mad24(3u, ldb, 0u)];

        a0 = A[0];
        a1 = A[lda];
        a2 = A[(lda << 1)];
        a3 = A[mad24(3u, lda, 0u)];

        c0.s01 = mad(b0.s01, a0.s0, c0.s01);
        c0.s01 = mad(b0.s10, (float2)(-a0.s1, a0.s1), c0.s01);
        c1.s01 = mad(b0.s23, a0.s0, c1.s01);
        c1.s01 = mad(b0.s32, (float2)(-a0.s1, a0.s1), c1.s01);
        c2.s01 = mad(b0.s45, a0.s0, c2.s01);
        c2.s01 = mad(b0.s54, (float2)(-a0.s1, a0.s1), c2.s01);
        c3.s01 = mad(b0.s67, a0.s0, c3.s01);
        c3.s01 = mad(b0.s76, (float2)(-a0.s1, a0.s1), c3.s01);
        c0.s23 = mad(b0.s01, a0.s2, c0.s23);
        c0.s23 = mad(b0.s10, (float2)(-a0.s3, a0.s3), c0.s23);
        c1.s23 = mad(b0.s23, a0.s2, c1.s23);
        c1.s23 = mad(b0.s32, (float2)(-a0.s3, a0.s3), c1.s23);
        c2.s23 = mad(b0.s45, a0.s2, c2.s23);
        c2.s23 = mad(b0.s54, (float2)(-a0.s3, a0.s3), c2.s23);
        c3.s23 = mad(b0.s67, a0.s2, c3.s23);
        c3.s23 = mad(b0.s76, (float2)(-a0.s3, a0.s3), c3.s23);
        c0.s45 = mad(b0.s01, a0.s4, c0.s45);
        c0.s45 = mad(b0.s10, (float2)(-a0.s5, a0.s5), c0.s45);
        c1.s45 = mad(b0.s23, a0.s4, c1.s45);
        c1.s45 = mad(b0.s32, (float2)(-a0.s5, a0.s5), c1.s45);
        c2.s45 = mad(b0.s45, a0.s4, c2.s45);
        c2.s45 = mad(b0.s54, (float2)(-a0.s5, a0.s5), c2.s45);
        c3.s45 = mad(b0.s67, a0.s4, c3.s45);
        c3.s45 = mad(b0.s76, (float2)(-a0.s5, a0.s5), c3.s45);
        c0.s67 = mad(b0.s01, a0.s6, c0.s67);
        c0.s67 = mad(b0.s10, (float2)(-a0.s7, a0.s7), c0.s67);
        c1.s67 = mad(b0.s23, a0.s6, c1.s67);
        c1.s67 = mad(b0.s32, (float2)(-a0.s7, a0.s7), c1.s67);
        c2.s67 = mad(b0.s45, a0.s6, c2.s67);
        c2.s67 = mad(b0.s54, (float2)(-a0.s7, a0.s7), c2.s67);
        c3.s67 = mad(b0.s67, a0.s6, c3.s67);
        c3.s67 = mad(b0.s76, (float2)(-a0.s7, a0.s7), c3.s67);

        c0.s01 = mad(b1.s01, a1.s0, c0.s01);
        c0.s01 = mad(b1.s10, (float2)(-a1.s1, a1.s1), c0.s01);
        c1.s01 = mad(b1.s23, a1.s0, c1.s01);
        c1.s01 = mad(b1.s32, (float2)(-a1.s1, a1.s1), c1.s01);
        c2.s01 = mad(b1.s45, a1.s0, c2.s01);
        c2.s01 = mad(b1.s54, (float2)(-a1.s1, a1.s1), c2.s01);
        c3.s01 = mad(b1.s67, a1.s0, c3.s01);
        c3.s01 = mad(b1.s76, (float2)(-a1.s1, a1.s1), c3.s01);
        c0.s23 = mad(b1.s01, a1.s2, c0.s23);
        c0.s23 = mad(b1.s10, (float2)(-a1.s3, a1.s3), c0.s23);
        c1.s23 = mad(b1.s23, a1.s2, c1.s23);
        c1.s23 = mad(b1.s32, (float2)(-a1.s3, a1.s3), c1.s23);
        c2.s23 = mad(b1.s45, a1.s2, c2.s23);
        c2.s23 = mad(b1.s54, (float2)(-a1.s3, a1.s3), c2.s23);
        c3.s23 = mad(b1.s67, a1.s2, c3.s23);
        c3.s23 = mad(b1.s76, (float2)(-a1.s3, a1.s3), c3.s23);
        c0.s45 = mad(b1.s01, a1.s4, c0.s45);
        c0.s45 = mad(b1.s10, (float2)(-a1.s5, a1.s5), c0.s45);
        c1.s45 = mad(b1.s23, a1.s4, c1.s45);
        c1.s45 = mad(b1.s32, (float2)(-a1.s5, a1.s5), c1.s45);
        c2.s45 = mad(b1.s45, a1.s4, c2.s45);
        c2.s45 = mad(b1.s54, (float2)(-a1.s5, a1.s5), c2.s45);
        c3.s45 = mad(b1.s67, a1.s4, c3.s45);
        c3.s45 = mad(b1.s76, (float2)(-a1.s5, a1.s5), c3.s45);
        c0.s67 = mad(b1.s01, a1.s6, c0.s67);
        c0.s67 = mad(b1.s10, (float2)(-a1.s7, a1.s7), c0.s67);
        c1.s67 = mad(b1.s23, a1.s6, c1.s67);
        c1.s67 = mad(b1.s32, (float2)(-a1.s7, a1.s7), c1.s67);
        c2.s67 = mad(b1.s45, a1.s6, c2.s67);
        c2.s67 = mad(b1.s54, (float2)(-a1.s7, a1.s7), c2.s67);
        c3.s67 = mad(b1.s67, a1.s6, c3.s67);
        c3.s67 = mad(b1.s76, (float2)(-a1.s7, a1.s7), c3.s67);

        c0.s01 = mad(b2.s01, a2.s0, c0.s01);
        c0.s01 = mad(b2.s10, (float2)(-a2.s1, a2.s1), c0.s01);
        c1.s01 = mad(b2.s23, a2.s0, c1.s01);
        c1.s01 = mad(b2.s32, (float2)(-a2.s1, a2.s1), c1.s01);
        c2.s01 = mad(b2.s45, a2.s0, c2.s01);
        c2.s01 = mad(b2.s54, (float2)(-a2.s1, a2.s1), c2.s01);
        c3.s01 = mad(b2.s67, a2.s0, c3.s01);
        c3.s01 = mad(b2.s76, (float2)(-a2.s1, a2.s1), c3.s01);
        c0.s23 = mad(b2.s01, a2.s2, c0.s23);
        c0.s23 = mad(b2.s10, (float2)(-a2.s3, a2.s3), c0.s23);
        c1.s23 = mad(b2.s23, a2.s2, c1.s23);
        c1.s23 = mad(b2.s32, (float2)(-a2.s3, a2.s3), c1.s23);
        c2.s23 = mad(b2.s45, a2.s2, c2.s23);
        c2.s23 = mad(b2.s54, (float2)(-a2.s3, a2.s3), c2.s23);
        c3.s23 = mad(b2.s67, a2.s2, c3.s23);
        c3.s23 = mad(b2.s76, (float2)(-a2.s3, a2.s3), c3.s23);
        c0.s45 = mad(b2.s01, a2.s4, c0.s45);
        c0.s45 = mad(b2.s10, (float2)(-a2.s5, a2.s5), c0.s45);
        c1.s45 = mad(b2.s23, a2.s4, c1.s45);
        c1.s45 = mad(b2.s32, (float2)(-a2.s5, a2.s5), c1.s45);
        c2.s45 = mad(b2.s45, a2.s4, c2.s45);
        c2.s45 = mad(b2.s54, (float2)(-a2.s5, a2.s5), c2.s45);
        c3.s45 = mad(b2.s67, a2.s4, c3.s45);
        c3.s45 = mad(b2.s76, (float2)(-a2.s5, a2.s5), c3.s45);
        c0.s67 = mad(b2.s01, a2.s6, c0.s67);
        c0.s67 = mad(b2.s10, (float2)(-a2.s7, a2.s7), c0.s67);
        c1.s67 = mad(b2.s23, a2.s6, c1.s67);
        c1.s67 = mad(b2.s32, (float2)(-a2.s7, a2.s7), c1.s67);
        c2.s67 = mad(b2.s45, a2.s6, c2.s67);
        c2.s67 = mad(b2.s54, (float2)(-a2.s7, a2.s7), c2.s67);
        c3.s67 = mad(b2.s67, a2.s6, c3.s67);
        c3.s67 = mad(b2.s76, (float2)(-a2.s7, a2.s7), c3.s67);

        c0.s01 = mad(b3.s01, a3.s0, c0.s01);
        c0.s01 = mad(b3.s10, (float2)(-a3.s1, a3.s1), c0.s01);
        c1.s01 = mad(b3.s23, a3.s0, c1.s01);
        c1.s01 = mad(b3.s32, (float2)(-a3.s1, a3.s1), c1.s01);
        c2.s01 = mad(b3.s45, a3.s0, c2.s01);
        c2.s01 = mad(b3.s54, (float2)(-a3.s1, a3.s1), c2.s01);
        c3.s01 = mad(b3.s67, a3.s0, c3.s01);
        c3.s01 = mad(b3.s76, (float2)(-a3.s1, a3.s1), c3.s01);
        c0.s23 = mad(b3.s01, a3.s2, c0.s23);
        c0.s23 = mad(b3.s10, (float2)(-a3.s3, a3.s3), c0.s23);
        c1.s23 = mad(b3.s23, a3.s2, c1.s23);
        c1.s23 = mad(b3.s32, (float2)(-a3.s3, a3.s3), c1.s23);
        c2.s23 = mad(b3.s45, a3.s2, c2.s23);
        c2.s23 = mad(b3.s54, (float2)(-a3.s3, a3.s3), c2.s23);
        c3.s23 = mad(b3.s67, a3.s2, c3.s23);
        c3.s23 = mad(b3.s76, (float2)(-a3.s3, a3.s3), c3.s23);
        c0.s45 = mad(b3.s01, a3.s4, c0.s45);
        c0.s45 = mad(b3.s10, (float2)(-a3.s5, a3.s5), c0.s45);
        c1.s45 = mad(b3.s23, a3.s4, c1.s45);
        c1.s45 = mad(b3.s32, (float2)(-a3.s5, a3.s5), c1.s45);
        c2.s45 = mad(b3.s45, a3.s4, c2.s45);
        c2.s45 = mad(b3.s54, (float2)(-a3.s5, a3.s5), c2.s45);
        c3.s45 = mad(b3.s67, a3.s4, c3.s45);
        c3.s45 = mad(b3.s76, (float2)(-a3.s5, a3.s5), c3.s45);
        c0.s67 = mad(b3.s01, a3.s6, c0.s67);
        c0.s67 = mad(b3.s10, (float2)(-a3.s7, a3.s7), c0.s67);
        c1.s67 = mad(b3.s23, a3.s6, c1.s67);
        c1.s67 = mad(b3.s32, (float2)(-a3.s7, a3.s7), c1.s67);
        c2.s67 = mad(b3.s45, a3.s6, c2.s67);
        c2.s67 = mad(b3.s54, (float2)(-a3.s7, a3.s7), c2.s67);
        c3.s67 = mad(b3.s67, a3.s6, c3.s67);
        c3.s67 = mad(b3.s76, (float2)(-a3.s7, a3.s7), c3.s67);

        A += (lda << 2);
        B += (ldb << 2);
        /* ---------------------- */
    }

    GPtr uC;

    uC.f2v = C + (coord.x * ldc + coord.y)/4;

    __global float8 *pC = uC.f8v;

    float8 tempC0, tempC1, tempC2, tempC3;

    tempC0.s01 = alpha * c0.s0 + alpha.s10 * (float2)(-c0.s1, c0.s1);
    tempC0.s23 = alpha * c0.s2 + alpha.s10 * (float2)(-c0.s3, c0.s3);
    tempC0.s45 = alpha * c0.s4 + alpha.s10 * (float2)(-c0.s5, c0.s5);
    tempC0.s67 = alpha * c0.s6 + alpha.s10 * (float2)(-c0.s7, c0.s7);
    tempC1.s01 = alpha * c1.s0 + alpha.s10 * (float2)(-c1.s1, c1.s1);
    tempC1.s23 = alpha * c1.s2 + alpha.s10 * (float2)(-c1.s3, c1.s3);
    tempC1.s45 = alpha * c1.s4 + alpha.s10 * (float2)(-c1.s5, c1.s5);
    tempC1.s67 = alpha * c1.s6 + alpha.s10 * (float2)(-c1.s7, c1.s7);
    tempC2.s01 = alpha * c2.s0 + alpha.s10 * (float2)(-c2.s1, c2.s1);
    tempC2.s23 = alpha * c2.s2 + alpha.s10 * (float2)(-c2.s3, c2.s3);
    tempC2.s45 = alpha * c2.s4 + alpha.s10 * (float2)(-c2.s5, c2.s5);
    tempC2.s67 = alpha * c2.s6 + alpha.s10 * (float2)(-c2.s7, c2.s7);
    tempC3.s01 = alpha * c3.s0 + alpha.s10 * (float2)(-c3.s1, c3.s1);
    tempC3.s23 = alpha * c3.s2 + alpha.s10 * (float2)(-c3.s3, c3.s3);
    tempC3.s45 = alpha * c3.s4 + alpha.s10 * (float2)(-c3.s5, c3.s5);
    tempC3.s67 = alpha * c3.s6 + alpha.s10 * (float2)(-c3.s7, c3.s7);

    pC[0] = tempC0;
    pC[(ldc >> 2)] = tempC1;
    pC[(ldc >> 1)] = tempC2;
    pC[mad24(3u, (ldc >> 2), 0u)] = tempC3;
}

__attribute__((reqd_work_group_size(8, 8, 1)))
void __kernel
dgemm_kernel( const uint M, const uint N, const uint K, const double alpha, const __global double4 *restrict A, const __global double4 *restrict B, __global double4 *C )
{
    double4 a0, a1, a2, a3;
    double4 b0, b1, b2, b3;
    double4 c0, c1, c2, c3;

    uint4 coord = 0u; /* contains coordB, coordA, k */

    uint lda = M / 4;
    uint ldb = N / 4;
    uint ldc = M;

    A += (uint) get_global_id(0);

    uint get_group_id_1 = (get_group_id(0) + get_group_id(1))% get_num_groups(1);
    uint get_global_id_1 = get_group_id_1 * get_local_size(1) + get_local_id(1);

    uint kif = ( (N % 256) != 0);

    get_global_id_1 = (kif*(uint)get_global_id(1)) + ((1-kif)*get_global_id_1);

    B += get_global_id_1;

    coord.y = 4u * (uint)get_global_id(0);
    coord.x = 4u * (uint)get_global_id_1;

    c0 = 0;
    c1 = 0;
    c2 = 0;
    c3 = 0;

    for (uint k1 = 0; k1 < K; k1 += 4) {
        /* -- Tiles multiplier -- */
        b0 = B[0];
        b1 = B[ldb];
        b2 = B[(ldb << 1)];
        b3 = B[mad24(3u, ldb, 0u)];

        a0 = A[0];
        a1 = A[lda];
        a2 = A[(lda << 1)];
        a3 = A[mad24(3u, lda, 0u)];

        c0 = mad(a0, b0.s0, c0);
        c1 = mad(a0, b0.s1, c1);
        c2 = mad(a0, b0.s2, c2);
        c3 = mad(a0, b0.s3, c3);

        c0 = mad(a1, b1.s0, c0);
        c1 = mad(a1, b1.s1, c1);
        c2 = mad(a1, b1.s2, c2);
        c3 = mad(a1, b1.s3, c3);

        c0 = mad(a2, b2.s0, c0);
        c1 = mad(a2, b2.s1, c1);
        c2 = mad(a2, b2.s2, c2);
        c3 = mad(a2, b2.s3, c3);

        c0 = mad(a3, b3.s0, c0);
        c1 = mad(a3, b3.s1, c1);
        c2 = mad(a3, b3.s2, c2);
        c3 = mad(a3, b3.s3, c3);

        A += (lda << 2);
        B += (ldb << 2);
        /* ---------------------- */
    }

    GPtr uC;

    uC.d4v = C + (coord.x * ldc + coord.y)/4;

    __global double4 *pC = uC.d4v;

    pC[0] 				= mad(c0, alpha, 0);
    pC[(ldc >> 2)] 			= mad(c1, alpha, 0);
    pC[(ldc >> 1)] 			= mad(c2, alpha, 0);
    pC[mad24(3u,(ldc >> 2), 0u)] 	= mad(c3, alpha, 0);

}

typedef union GPtr {
    __global float *f;
    __global double *d;
    __global float2 *f2v;
    __global double2 *d2v;
    __global float4 *f4v;
    __global double4 *d4v;
    __global float8 *f8v;
    __global double8 *d8v;
    __global float16 *f16v;
    __global double16 *d16v;
} GPtr;

typedef union LPtr {
    __local float *f;
    __local double *d;
    __local float2 *f2v;
    __local double2 *d2v;
    __local float4 *f4v;
    __local double4 *d4v;
    __local float8 *f8v;
    __local double8 *d8v;
    __local float16 *f16v;
    __local double16 *d16v;
} LPtr;

typedef union PPtr {
    float *f;
    double *d;
    float2 *f2v;
    double2 *d2v;
    float4 *f4v;
    double4 *d4v;
    float8 *f8v;
    double8 *d8v;
    float16 *f16v;
    double16 *d16v;
} PPtr;

__attribute__((reqd_work_group_size(8, 8, 1)))
void __kernel
sgemm_kernel( const uint M, const uint N, const uint K, const float alpha, const __global float8 *restrict A, const __global float8 *restrict B, __global float8 *C )
{
    float8 a0, a1, a2, a3;
    float8 b0, b1, b2, b3;
    float8 c0, c1, c2, c3, c4, c5, c6, c7;

    uint4 coord = 0u; /* contains coordB, coordA, k */

    uint lda = M / 8;
    uint ldb = N / 8;
    uint ldc = M;

    A += (uint) get_global_id(0);

    uint get_group_id_1 = (get_group_id(0) + get_group_id(1))% get_num_groups(1);
    uint get_global_id_1 = get_group_id_1 * get_local_size(1) + get_local_id(1);

    uint kif = ( (N % 512) != 0);

    get_global_id_1 = (kif*(uint)get_global_id(1)) + ((1-kif)*get_global_id_1);

    B += get_global_id_1;

    coord.y = 8u * (uint)get_global_id(0);
    coord.x = 8u * (uint)get_global_id_1;

    c0 = 0;
    c1 = 0;
    c2 = 0;
    c3 = 0;
    c4 = 0;
    c5 = 0;
    c6 = 0;
    c7 = 0;

    for (uint k1 = 0; k1 < K; k1 += 4) {
        /* -- Tiles multiplier -- */
        b0 = B[0];
        b1 = B[ldb];
        b2 = B[(ldb << 1)];
        b3 = B[mad24(3u, ldb, 0u)];

        a0 = A[0];
        a1 = A[lda];
        a2 = A[(lda << 1)];
        a3 = A[mad24(3u, lda, 0u)];

        c0 = mad(a0, b0.s0, c0);
        c1 = mad(a0, b0.s1, c1);
        c2 = mad(a0, b0.s2, c2);
        c3 = mad(a0, b0.s3, c3);
        c4 = mad(a0, b0.s4, c4);
        c5 = mad(a0, b0.s5, c5);
        c6 = mad(a0, b0.s6, c6);
        c7 = mad(a0, b0.s7, c7);

        c0 = mad(a1, b1.s0, c0);
        c1 = mad(a1, b1.s1, c1);
        c2 = mad(a1, b1.s2, c2);
        c3 = mad(a1, b1.s3, c3);
        c4 = mad(a1, b1.s4, c4);
        c5 = mad(a1, b1.s5, c5);
        c6 = mad(a1, b1.s6, c6);
        c7 = mad(a1, b1.s7, c7);

        c0 = mad(a2, b2.s0, c0);
        c1 = mad(a2, b2.s1, c1);
        c2 = mad(a2, b2.s2, c2);
        c3 = mad(a2, b2.s3, c3);
        c4 = mad(a2, b2.s4, c4);
        c5 = mad(a2, b2.s5, c5);
        c6 = mad(a2, b2.s6, c6);
        c7 = mad(a2, b2.s7, c7);

        c0 = mad(a3, b3.s0, c0);
        c1 = mad(a3, b3.s1, c1);
        c2 = mad(a3, b3.s2, c2);
        c3 = mad(a3, b3.s3, c3);
        c4 = mad(a3, b3.s4, c4);
        c5 = mad(a3, b3.s5, c5);
        c6 = mad(a3, b3.s6, c6);
        c7 = mad(a3, b3.s7, c7);

        A += (lda << 2);
        B += (ldb << 2);
        /* ---------------------- */
    }

    GPtr uC;

    uC.f8v = C + (coord.x * ldc + coord.y)/8;

    uint ldc3 = ldc >> 3;

    __global float8 *pC = uC.f8v;

    pC[0] 				= mad(c0, alpha, 0);
    pC[ldc3] 				= mad(c1, alpha, 0);
    pC[(ldc >> 2)] 			= mad(c2, alpha, 0);
    pC[mad24(3u, ldc3, 0u)] 		= mad(c3, alpha, 0);
    pC[(ldc >> 1)] 			= mad(c4, alpha, 0);
    pC[mad24(5u, ldc3, 0u)] 		= mad(c5, alpha, 0);
    pC[mad24(6u, ldc3, 0u)] 		= mad(c6, alpha, 0);
    pC[mad24(7u, ldc3, 0u)] 		= mad(c7, alpha, 0);
}

__attribute__((reqd_work_group_size(8, 8, 1)))
void __kernel
 cgemm_kernel( uint M, uint N, uint K, const float2 alpha, const __global float8 *restrict A, const __global float8 *restrict B, __global float8 *C)
{
    float8 a0, a1, a2, a3;
    float8 b0, b1, b2, b3;
    float8 c0, c1, c2, c3;
    uint4 coord = 0u; /* contains coordB, coordA, k */

    uint lda = M/4;
    uint ldb = N/4;
    uint ldc = M;

    uint kif;
    uint get_group_id_1;
    uint get_global_id_1;

    A += (uint)get_global_id(0);
    get_group_id_1 = (get_group_id(0) + get_group_id(1))% get_num_groups(1);
    get_global_id_1 = get_group_id_1 * get_local_size(1) + get_local_id(1);

    kif = (N % 256 != 0);
    get_global_id_1 = (kif*(uint)get_global_id(1)) + ((1-kif)*get_global_id_1);

    B += get_global_id_1;
    coord.y = 4u * (uint)get_global_id(0);
    coord.x = 4u * (uint)get_global_id_1;

    c0 = 0;
    c1 = 0;
    c2 = 0;
    c3 = 0;

    for (uint k1 = 0; k1 < K; k1 += 4) {
        /* -- Tiles multiplier -- */
        b0 = B[0];
        b1 = B[ldb];
        b2 = B[(ldb << 1)];
        b3 = B[mad24(3u, ldb, 0u)];

        a0 = A[0];
        a1 = A[lda];
        a2 = A[(lda << 1)];
        a3 = A[mad24(3u, lda, 0u)];

        c0.s01 = mad(b0.s01, a0.s0, c0.s01);
        c0.s01 = mad(b0.s10, (float2)(-a0.s1, a0.s1), c0.s01);
        c1.s01 = mad(b0.s23, a0.s0, c1.s01);
        c1.s01 = mad(b0.s32, (float2)(-a0.s1, a0.s1), c1.s01);
        c2.s01 = mad(b0.s45, a0.s0, c2.s01);
        c2.s01 = mad(b0.s54, (float2)(-a0.s1, a0.s1), c2.s01);
        c3.s01 = mad(b0.s67, a0.s0, c3.s01);
        c3.s01 = mad(b0.s76, (float2)(-a0.s1, a0.s1), c3.s01);
        c0.s23 = mad(b0.s01, a0.s2, c0.s23);
        c0.s23 = mad(b0.s10, (float2)(-a0.s3, a0.s3), c0.s23);
        c1.s23 = mad(b0.s23, a0.s2, c1.s23);
        c1.s23 = mad(b0.s32, (float2)(-a0.s3, a0.s3), c1.s23);
        c2.s23 = mad(b0.s45, a0.s2, c2.s23);
        c2.s23 = mad(b0.s54, (float2)(-a0.s3, a0.s3), c2.s23);
        c3.s23 = mad(b0.s67, a0.s2, c3.s23);
        c3.s23 = mad(b0.s76, (float2)(-a0.s3, a0.s3), c3.s23);
        c0.s45 = mad(b0.s01, a0.s4, c0.s45);
        c0.s45 = mad(b0.s10, (float2)(-a0.s5, a0.s5), c0.s45);
        c1.s45 = mad(b0.s23, a0.s4, c1.s45);
        c1.s45 = mad(b0.s32, (float2)(-a0.s5, a0.s5), c1.s45);
        c2.s45 = mad(b0.s45, a0.s4, c2.s45);
        c2.s45 = mad(b0.s54, (float2)(-a0.s5, a0.s5), c2.s45);
        c3.s45 = mad(b0.s67, a0.s4, c3.s45);
        c3.s45 = mad(b0.s76, (float2)(-a0.s5, a0.s5), c3.s45);
        c0.s67 = mad(b0.s01, a0.s6, c0.s67);
        c0.s67 = mad(b0.s10, (float2)(-a0.s7, a0.s7), c0.s67);
        c1.s67 = mad(b0.s23, a0.s6, c1.s67);
        c1.s67 = mad(b0.s32, (float2)(-a0.s7, a0.s7), c1.s67);
        c2.s67 = mad(b0.s45, a0.s6, c2.s67);
        c2.s67 = mad(b0.s54, (float2)(-a0.s7, a0.s7), c2.s67);
        c3.s67 = mad(b0.s67, a0.s6, c3.s67);
        c3.s67 = mad(b0.s76, (float2)(-a0.s7, a0.s7), c3.s67);

        c0.s01 = mad(b1.s01, a1.s0, c0.s01);
        c0.s01 = mad(b1.s10, (float2)(-a1.s1, a1.s1), c0.s01);
        c1.s01 = mad(b1.s23, a1.s0, c1.s01);
        c1.s01 = mad(b1.s32, (float2)(-a1.s1, a1.s1), c1.s01);
        c2.s01 = mad(b1.s45, a1.s0, c2.s01);
        c2.s01 = mad(b1.s54, (float2)(-a1.s1, a1.s1), c2.s01);
        c3.s01 = mad(b1.s67, a1.s0, c3.s01);
        c3.s01 = mad(b1.s76, (float2)(-a1.s1, a1.s1), c3.s01);
        c0.s23 = mad(b1.s01, a1.s2, c0.s23);
        c0.s23 = mad(b1.s10, (float2)(-a1.s3, a1.s3), c0.s23);
        c1.s23 = mad(b1.s23, a1.s2, c1.s23);
        c1.s23 = mad(b1.s32, (float2)(-a1.s3, a1.s3), c1.s23);
        c2.s23 = mad(b1.s45, a1.s2, c2.s23);
        c2.s23 = mad(b1.s54, (float2)(-a1.s3, a1.s3), c2.s23);
        c3.s23 = mad(b1.s67, a1.s2, c3.s23);
        c3.s23 = mad(b1.s76, (float2)(-a1.s3, a1.s3), c3.s23);
        c0.s45 = mad(b1.s01, a1.s4, c0.s45);
        c0.s45 = mad(b1.s10, (float2)(-a1.s5, a1.s5), c0.s45);
        c1.s45 = mad(b1.s23, a1.s4, c1.s45);
        c1.s45 = mad(b1.s32, (float2)(-a1.s5, a1.s5), c1.s45);
        c2.s45 = mad(b1.s45, a1.s4, c2.s45);
        c2.s45 = mad(b1.s54, (float2)(-a1.s5, a1.s5), c2.s45);
        c3.s45 = mad(b1.s67, a1.s4, c3.s45);
        c3.s45 = mad(b1.s76, (float2)(-a1.s5, a1.s5), c3.s45);
        c0.s67 = mad(b1.s01, a1.s6, c0.s67);
        c0.s67 = mad(b1.s10, (float2)(-a1.s7, a1.s7), c0.s67);
        c1.s67 = mad(b1.s23, a1.s6, c1.s67);
        c1.s67 = mad(b1.s32, (float2)(-a1.s7, a1.s7), c1.s67);
        c2.s67 = mad(b1.s45, a1.s6, c2.s67);
        c2.s67 = mad(b1.s54, (float2)(-a1.s7, a1.s7), c2.s67);
        c3.s67 = mad(b1.s67, a1.s6, c3.s67);
        c3.s67 = mad(b1.s76, (float2)(-a1.s7, a1.s7), c3.s67);

        c0.s01 = mad(b2.s01, a2.s0, c0.s01);
        c0.s01 = mad(b2.s10, (float2)(-a2.s1, a2.s1), c0.s01);
        c1.s01 = mad(b2.s23, a2.s0, c1.s01);
        c1.s01 = mad(b2.s32, (float2)(-a2.s1, a2.s1), c1.s01);
        c2.s01 = mad(b2.s45, a2.s0, c2.s01);
        c2.s01 = mad(b2.s54, (float2)(-a2.s1, a2.s1), c2.s01);
        c3.s01 = mad(b2.s67, a2.s0, c3.s01);
        c3.s01 = mad(b2.s76, (float2)(-a2.s1, a2.s1), c3.s01);
        c0.s23 = mad(b2.s01, a2.s2, c0.s23);
        c0.s23 = mad(b2.s10, (float2)(-a2.s3, a2.s3), c0.s23);
        c1.s23 = mad(b2.s23, a2.s2, c1.s23);
        c1.s23 = mad(b2.s32, (float2)(-a2.s3, a2.s3), c1.s23);
        c2.s23 = mad(b2.s45, a2.s2, c2.s23);
        c2.s23 = mad(b2.s54, (float2)(-a2.s3, a2.s3), c2.s23);
        c3.s23 = mad(b2.s67, a2.s2, c3.s23);
        c3.s23 = mad(b2.s76, (float2)(-a2.s3, a2.s3), c3.s23);
        c0.s45 = mad(b2.s01, a2.s4, c0.s45);
        c0.s45 = mad(b2.s10, (float2)(-a2.s5, a2.s5), c0.s45);
        c1.s45 = mad(b2.s23, a2.s4, c1.s45);
        c1.s45 = mad(b2.s32, (float2)(-a2.s5, a2.s5), c1.s45);
        c2.s45 = mad(b2.s45, a2.s4, c2.s45);
        c2.s45 = mad(b2.s54, (float2)(-a2.s5, a2.s5), c2.s45);
        c3.s45 = mad(b2.s67, a2.s4, c3.s45);
        c3.s45 = mad(b2.s76, (float2)(-a2.s5, a2.s5), c3.s45);
        c0.s67 = mad(b2.s01, a2.s6, c0.s67);
        c0.s67 = mad(b2.s10, (float2)(-a2.s7, a2.s7), c0.s67);
        c1.s67 = mad(b2.s23, a2.s6, c1.s67);
        c1.s67 = mad(b2.s32, (float2)(-a2.s7, a2.s7), c1.s67);
        c2.s67 = mad(b2.s45, a2.s6, c2.s67);
        c2.s67 = mad(b2.s54, (float2)(-a2.s7, a2.s7), c2.s67);
        c3.s67 = mad(b2.s67, a2.s6, c3.s67);
        c3.s67 = mad(b2.s76, (float2)(-a2.s7, a2.s7), c3.s67);

        c0.s01 = mad(b3.s01, a3.s0, c0.s01);
        c0.s01 = mad(b3.s10, (float2)(-a3.s1, a3.s1), c0.s01);
        c1.s01 = mad(b3.s23, a3.s0, c1.s01);
        c1.s01 = mad(b3.s32, (float2)(-a3.s1, a3.s1), c1.s01);
        c2.s01 = mad(b3.s45, a3.s0, c2.s01);
        c2.s01 = mad(b3.s54, (float2)(-a3.s1, a3.s1), c2.s01);
        c3.s01 = mad(b3.s67, a3.s0, c3.s01);
        c3.s01 = mad(b3.s76, (float2)(-a3.s1, a3.s1), c3.s01);
        c0.s23 = mad(b3.s01, a3.s2, c0.s23);
        c0.s23 = mad(b3.s10, (float2)(-a3.s3, a3.s3), c0.s23);
        c1.s23 = mad(b3.s23, a3.s2, c1.s23);
        c1.s23 = mad(b3.s32, (float2)(-a3.s3, a3.s3), c1.s23);
        c2.s23 = mad(b3.s45, a3.s2, c2.s23);
        c2.s23 = mad(b3.s54, (float2)(-a3.s3, a3.s3), c2.s23);
        c3.s23 = mad(b3.s67, a3.s2, c3.s23);
        c3.s23 = mad(b3.s76, (float2)(-a3.s3, a3.s3), c3.s23);
        c0.s45 = mad(b3.s01, a3.s4, c0.s45);
        c0.s45 = mad(b3.s10, (float2)(-a3.s5, a3.s5), c0.s45);
        c1.s45 = mad(b3.s23, a3.s4, c1.s45);
        c1.s45 = mad(b3.s32, (float2)(-a3.s5, a3.s5), c1.s45);
        c2.s45 = mad(b3.s45, a3.s4, c2.s45);
        c2.s45 = mad(b3.s54, (float2)(-a3.s5, a3.s5), c2.s45);
        c3.s45 = mad(b3.s67, a3.s4, c3.s45);
        c3.s45 = mad(b3.s76, (float2)(-a3.s5, a3.s5), c3.s45);
        c0.s67 = mad(b3.s01, a3.s6, c0.s67);
        c0.s67 = mad(b3.s10, (float2)(-a3.s7, a3.s7), c0.s67);
        c1.s67 = mad(b3.s23, a3.s6, c1.s67);
        c1.s67 = mad(b3.s32, (float2)(-a3.s7, a3.s7), c1.s67);
        c2.s67 = mad(b3.s45, a3.s6, c2.s67);
        c2.s67 = mad(b3.s54, (float2)(-a3.s7, a3.s7), c2.s67);
        c3.s67 = mad(b3.s67, a3.s6, c3.s67);
        c3.s67 = mad(b3.s76, (float2)(-a3.s7, a3.s7), c3.s67);

        A += (lda << 2);
        B += (ldb << 2);
        /* ---------------------- */
    }

    GPtr uC;

    uC.f2v = C + (coord.x * ldc + coord.y)/4;

    __global float8 *pC = uC.f8v;

    float8 tempC0, tempC1, tempC2, tempC3;

    tempC0.s01 = alpha * c0.s0 + alpha.s10 * (float2)(-c0.s1, c0.s1);
    tempC0.s23 = alpha * c0.s2 + alpha.s10 * (float2)(-c0.s3, c0.s3);
    tempC0.s45 = alpha * c0.s4 + alpha.s10 * (float2)(-c0.s5, c0.s5);
    tempC0.s67 = alpha * c0.s6 + alpha.s10 * (float2)(-c0.s7, c0.s7);
    tempC1.s01 = alpha * c1.s0 + alpha.s10 * (float2)(-c1.s1, c1.s1);
    tempC1.s23 = alpha * c1.s2 + alpha.s10 * (float2)(-c1.s3, c1.s3);
    tempC1.s45 = alpha * c1.s4 + alpha.s10 * (float2)(-c1.s5, c1.s5);
    tempC1.s67 = alpha * c1.s6 + alpha.s10 * (float2)(-c1.s7, c1.s7);
    tempC2.s01 = alpha * c2.s0 + alpha.s10 * (float2)(-c2.s1, c2.s1);
    tempC2.s23 = alpha * c2.s2 + alpha.s10 * (float2)(-c2.s3, c2.s3);
    tempC2.s45 = alpha * c2.s4 + alpha.s10 * (float2)(-c2.s5, c2.s5);
    tempC2.s67 = alpha * c2.s6 + alpha.s10 * (float2)(-c2.s7, c2.s7);
    tempC3.s01 = alpha * c3.s0 + alpha.s10 * (float2)(-c3.s1, c3.s1);
    tempC3.s23 = alpha * c3.s2 + alpha.s10 * (float2)(-c3.s3, c3.s3);
    tempC3.s45 = alpha * c3.s4 + alpha.s10 * (float2)(-c3.s5, c3.s5);
    tempC3.s67 = alpha * c3.s6 + alpha.s10 * (float2)(-c3.s7, c3.s7);

    pC[0] = tempC0;
    pC[(ldc >> 2)] = tempC1;
    pC[(ldc >> 1)] = tempC2;
    pC[mad24(3u, (ldc >> 2), 0u)] = tempC3;
}

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

__attribute__((reqd_work_group_size(8, 8, 1)))
void __kernel
dgemm_kernel( const uint M, const uint N, const uint K, const double alpha, const __global double4 *restrict A, const __global double4 *restrict B, __global double4 *C )
{
    double4 a0, a1, a2, a3;
    double4 b0, b1, b2, b3;
    double4 c0, c1, c2, c3;

    uint4 coord = 0u; /* contains coordB, coordA, k */

    uint lda = M / 4;
    uint ldb = N / 4;
    uint ldc = M;

    A += (uint) get_global_id(0);

    uint get_group_id_1 = (get_group_id(0) + get_group_id(1))% get_num_groups(1);
    uint get_global_id_1 = get_group_id_1 * get_local_size(1) + get_local_id(1);

    uint kif = ( (N % 256) != 0);

    get_global_id_1 = (kif*(uint)get_global_id(1)) + ((1-kif)*get_global_id_1);

    B += get_global_id_1;

    coord.y = 4u * (uint)get_global_id(0);
    coord.x = 4u * (uint)get_global_id_1;

    c0 = 0;
    c1 = 0;
    c2 = 0;
    c3 = 0;

    for (uint k1 = 0; k1 < K; k1 += 4) {
        /* -- Tiles multiplier -- */
        b0 = B[0];
        b1 = B[ldb];
        b2 = B[(ldb << 1)];
        b3 = B[mad24(3u, ldb, 0u)];

        a0 = A[0];
        a1 = A[lda];
        a2 = A[(lda << 1)];
        a3 = A[mad24(3u, lda, 0u)];

        c0 = mad(a0, b0.s0, c0);
        c1 = mad(a0, b0.s1, c1);
        c2 = mad(a0, b0.s2, c2);
        c3 = mad(a0, b0.s3, c3);

        c0 = mad(a1, b1.s0, c0);
        c1 = mad(a1, b1.s1, c1);
        c2 = mad(a1, b1.s2, c2);
        c3 = mad(a1, b1.s3, c3);

        c0 = mad(a2, b2.s0, c0);
        c1 = mad(a2, b2.s1, c1);
        c2 = mad(a2, b2.s2, c2);
        c3 = mad(a2, b2.s3, c3);

        c0 = mad(a3, b3.s0, c0);
        c1 = mad(a3, b3.s1, c1);
        c2 = mad(a3, b3.s2, c2);
        c3 = mad(a3, b3.s3, c3);

        A += (lda << 2);
        B += (ldb << 2);
        /* ---------------------- */
    }

    GPtr uC;

    uC.d4v = C + (coord.x * ldc + coord.y)/4;

    __global double4 *pC = uC.d4v;

    pC[0] 				= mad(c0, alpha, 0);
    pC[(ldc >> 2)] 			= mad(c1, alpha, 0);
    pC[(ldc >> 1)] 			= mad(c2, alpha, 0);
    pC[mad24(3u,(ldc >> 2), 0u)] 	= mad(c3, alpha, 0);

}


Don't use subscript access for OpenCL vectors (fixes #10)

uint16 arrange_padding_block(ulong padding_specifier, uint4 padding_block_ending);
void sha1_compress(__private uint* h, uint16 w);

__constant uint PADDING_CHUNKS[16] = {
    0x20202020, 0x20202009, 0x20200920, 0x20200909,
    0x20092020, 0x20092009, 0x20090920, 0x20090909,
    0x09202020, 0x09202009, 0x09200920, 0x09200909,
    0x09092020, 0x09092009, 0x09090920, 0x09090909,
};

__kernel void scatter_padding_and_find_match(
    __global uint* desired_prefix_data,
    __global uint* desired_prefix_mask,
    __global uint* h,
    __global uint16* dynamic_blocks,
    size_t num_dynamic_blocks,
    __global uint* successful_match_receiver
) {
    uint finalized_hash[5] = {h[0], h[1], h[2], h[3], h[4]};
    sha1_compress(
        finalized_hash,
        arrange_padding_block(
            get_global_id(0),
            dynamic_blocks[0].sCDEF
        )
    );
    for (size_t i = 1; i < num_dynamic_blocks; i++) {
        sha1_compress(finalized_hash, dynamic_blocks[i]);
    }

    if (
        (finalized_hash[0] & desired_prefix_mask[0]) == desired_prefix_data[0] &&
        (finalized_hash[1] & desired_prefix_mask[1]) == desired_prefix_data[1] &&
        (finalized_hash[2] & desired_prefix_mask[2]) == desired_prefix_data[2] &&
        (finalized_hash[3] & desired_prefix_mask[3]) == desired_prefix_data[3] &&
        (finalized_hash[4] & desired_prefix_mask[4]) == desired_prefix_data[4]
    ) {
        atomic_cmpxchg(successful_match_receiver, UINT_MAX, get_global_id(0) % get_global_size(0));
    }
}

uint16 arrange_padding_block(ulong padding_specifier, uint4 padding_block_ending) {
    return (uint16)(
        PADDING_CHUNKS[(padding_specifier >> 4) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 0) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 12) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 8) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 20) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 16) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 28) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 24) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 36) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 32) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 40) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 44) & 0xf],
        padding_block_ending.s0,
        padding_block_ending.s1,
        padding_block_ending.s2,
        padding_block_ending.s3
    );
}

/*
The sha1 implementation below is mostly adapted from hashcat (https://hashcat.net/hashcat).

The MIT License (MIT)

Copyright (c) 2015-2020 Jens Steube

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
#define SHA1_F0o(x,y,z) (bitselect((z), (y), (x)))
#define SHA1_F2o(x,y,z) (bitselect((x), (y), ((x) ^ (z))))

#define SHA1_STEP_S(f,a,b,c,d,e,x) \
{ \
    e += x + f(b, c, d) + K + rotate(a, 5u); \
    b = rotate(b, 30u); \
}

void sha1_compress(__private uint* h, uint16 w) {
    uint a = h[0];
    uint b = h[1];
    uint c = h[2];
    uint d = h[3];
    uint e = h[4];

    uint w0_t = w.s0;
    uint w1_t = w.s1;
    uint w2_t = w.s2;
    uint w3_t = w.s3;
    uint w4_t = w.s4;
    uint w5_t = w.s5;
    uint w6_t = w.s6;
    uint w7_t = w.s7;
    uint w8_t = w.s8;
    uint w9_t = w.s9;
    uint wa_t = w.sA;
    uint wb_t = w.sB;
    uint wc_t = w.sC;
    uint wd_t = w.sD;
    uint we_t = w.sE;
    uint wf_t = w.sF;

    #define K 0x5a827999

    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, w0_t);
    SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, w1_t);
    SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, w2_t);
    SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, w3_t);
    SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, w4_t);
    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, w5_t);
    SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, w6_t);
    SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, w7_t);
    SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, w8_t);
    SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, w9_t);
    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, wa_t);
    SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, wb_t);
    SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, wc_t);
    SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, wd_t);
    SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, we_t);
    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, w3_t);

    #undef K
    #define K 0x6ed9eba1

    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w7_t);
    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, wb_t);
    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w3_t);
    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w7_t);

    #undef K
    #define K 0x8f1bbcdc

    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, wb_t);
    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, w3_t);
    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, w7_t);
    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, wb_t);

    #undef K
    #define K 0xca62c1d6

    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w3_t);
    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w7_t);
    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, wb_t);
    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, wf_t);

    #undef K

    h[0] += a;
    h[1] += b;
    h[2] += c;
    h[3] += d;
    h[4] += e;
}

uint16 arrange_padding_block(ulong padding_specifier, uint4 padding_block_ending);
void sha1_compress(__private uint* h, uint16 w);

__constant uint PADDING_CHUNKS[16] = {
    0x20202020, 0x20202009, 0x20200920, 0x20200909,
    0x20092020, 0x20092009, 0x20090920, 0x20090909,
    0x09202020, 0x09202009, 0x09200920, 0x09200909,
    0x09092020, 0x09092009, 0x09090920, 0x09090909,
};

__kernel void scatter_padding_and_find_match(
    __global uint* desired_prefix_data,
    __global uint* desired_prefix_mask,
    __global uint* h,
    __global uint16* dynamic_blocks,
    size_t num_dynamic_blocks,
    __global uint* successful_match_receiver
) {
    uint finalized_hash[5] = {h[0], h[1], h[2], h[3], h[4]};
    sha1_compress(
        finalized_hash,
        arrange_padding_block(
            get_global_id(0),
            dynamic_blocks[0].sCDEF
        )
    );
    for (size_t i = 1; i < num_dynamic_blocks; i++) {
        sha1_compress(finalized_hash, dynamic_blocks[i]);
    }

    if (
        (finalized_hash[0] & desired_prefix_mask[0]) == desired_prefix_data[0] &&
        (finalized_hash[1] & desired_prefix_mask[1]) == desired_prefix_data[1] &&
        (finalized_hash[2] & desired_prefix_mask[2]) == desired_prefix_data[2] &&
        (finalized_hash[3] & desired_prefix_mask[3]) == desired_prefix_data[3] &&
        (finalized_hash[4] & desired_prefix_mask[4]) == desired_prefix_data[4]
    ) {
        atomic_cmpxchg(successful_match_receiver, UINT_MAX, get_global_id(0) % get_global_size(0));
    }
}

uint16 arrange_padding_block(ulong padding_specifier, uint4 padding_block_ending) {
    return (uint16)(
        PADDING_CHUNKS[(padding_specifier >> 4) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 0) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 12) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 8) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 20) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 16) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 28) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 24) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 36) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 32) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 40) & 0xf],
        PADDING_CHUNKS[(padding_specifier >> 44) & 0xf],
        padding_block_ending[0],
        padding_block_ending[1],
        padding_block_ending[2],
        padding_block_ending[3]
    );
}

/*
The sha1 implementation below is mostly adapted from hashcat (https://hashcat.net/hashcat).

The MIT License (MIT)

Copyright (c) 2015-2020 Jens Steube

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
#define SHA1_F0o(x,y,z) (bitselect((z), (y), (x)))
#define SHA1_F2o(x,y,z) (bitselect((x), (y), ((x) ^ (z))))

#define SHA1_STEP_S(f,a,b,c,d,e,x) \
{ \
    e += x + f(b, c, d) + K + rotate(a, 5u); \
    b = rotate(b, 30u); \
}

void sha1_compress(__private uint* h, uint16 w) {
    uint a = h[0];
    uint b = h[1];
    uint c = h[2];
    uint d = h[3];
    uint e = h[4];

    uint w0_t = w[0];
    uint w1_t = w[1];
    uint w2_t = w[2];
    uint w3_t = w[3];
    uint w4_t = w[4];
    uint w5_t = w[5];
    uint w6_t = w[6];
    uint w7_t = w[7];
    uint w8_t = w[8];
    uint w9_t = w[9];
    uint wa_t = w[10];
    uint wb_t = w[11];
    uint wc_t = w[12];
    uint wd_t = w[13];
    uint we_t = w[14];
    uint wf_t = w[15];

    #define K 0x5a827999

    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, w0_t);
    SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, w1_t);
    SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, w2_t);
    SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, w3_t);
    SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, w4_t);
    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, w5_t);
    SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, w6_t);
    SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, w7_t);
    SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, w8_t);
    SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, w9_t);
    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, wa_t);
    SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, wb_t);
    SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, wc_t);
    SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, wd_t);
    SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, we_t);
    SHA1_STEP_S(SHA1_F0o, a, b, c, d, e, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F0o, e, a, b, c, d, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F0o, d, e, a, b, c, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F0o, c, d, e, a, b, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F0o, b, c, d, e, a, w3_t);

    #undef K
    #define K 0x6ed9eba1

    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w7_t);
    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, wb_t);
    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w3_t);
    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w7_t);

    #undef K
    #define K 0x8f1bbcdc

    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, wb_t);
    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, w3_t);
    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F2o, a, b, c, d, e, w7_t);
    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F2o, e, a, b, c, d, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F2o, d, e, a, b, c, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F2o, c, d, e, a, b, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F2o, b, c, d, e, a, wb_t);

    #undef K
    #define K 0xca62c1d6

    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, wf_t);
    w0_t = rotate((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w0_t);
    w1_t = rotate((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w1_t);
    w2_t = rotate((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w2_t);
    w3_t = rotate((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w3_t);
    w4_t = rotate((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w4_t);
    w5_t = rotate((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, w5_t);
    w6_t = rotate((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, w6_t);
    w7_t = rotate((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, w7_t);
    w8_t = rotate((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, w8_t);
    w9_t = rotate((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, w9_t);
    wa_t = rotate((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, wa_t);
    wb_t = rotate((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP_S(SHA1_F1, a, b, c, d, e, wb_t);
    wc_t = rotate((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP_S(SHA1_F1, e, a, b, c, d, wc_t);
    wd_t = rotate((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP_S(SHA1_F1, d, e, a, b, c, wd_t);
    we_t = rotate((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP_S(SHA1_F1, c, d, e, a, b, we_t);
    wf_t = rotate((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP_S(SHA1_F1, b, c, d, e, a, wf_t);

    #undef K

    h[0] += a;
    h[1] += b;
    h[2] += c;
    h[3] += d;
    h[4] += e;
}


revised particle management

revised particle management

Cleaned up the comments.
Fixed particles wrapping around.

/*
    Move the particles
    Copyright (c) 2013-2014, Randall Maas

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.

    Inpsired by Cameron Beccario, https://github.com/cambecc/earth
*/

#define M_PI   3.1415926535897932384626433832795f

// --- Randomize particle location -----------------------
/**Returns a random number
   @param seed The Seed to the random number generator
   @param max  The max number to return
   @returns A random number
 */
float random(__global ulong* seed, uint max)
{
    *seed = (*seed * 0x5DEECE66DL + 0xBL) & ((1L << 48) - 1);
    uint result = (*seed) >> 16;
    return result % max;
}

/** Find a random location for a particle
    @param numXBins The number of bins in the x axis
    @param numYBins The number of bins in the y axis
    @param seed     The random number seed
    @returns A {float2} vector
*/
float2 particleRandomize(
                       int numXBins, int numYBins,
                       __global ulong* seed
                       )
{
    float m =random(seed, (numXBins*numYBins));
    float yy=(int)(m/numXBins);
    float xx = m - numXBins*yy;

    float c = (cos(M_PI*yy/numYBins)+1.0)*0.5;
    c*=c;

    return (float2){xx,(numYBins-1)*(1.0-c)};
}

/** This randomizes a single particle
    @param positions   Set of vectors for each particle
    @param positions_t The set of new vectors for each particle
    @param numXBins The number of bins in the x axis
    @param numYBins The number of bins in the y axis
    @param dataGrid The grid of vectors (mapped to an array)
    @param seed     The random number seed
 */
__kernel void particleInit( __global float2*   vertex   // position of each particle
                          , int numXBins, int  numYBins    // The size of the data grid
                          , __global ulong*    seed        // A randomizer
                          )
{
    // The global id of the work item.  (the index i)
    int idx = get_global_id(0);
    // Particle isn't visible, but it still moves through the field.
    // particle has escaped the grid, never to return...
    float2 position = particleRandomize(numXBins, numYBins, seed);

    // Path from (x,y) to (xt,yt) is visible, so add this particle to the appropriate draw bucket.
    // The positions are actually vertices (2)
    int idx2 = 2*idx;
    vertex[idx2  ] = position;  // tail
    vertex[idx2+1] = position;  // head
}

/** This moves a single particle
    @param positions   Set of vectors for each particle
    @param positions_t The set of new vectors for each particle
    @param numXBins    The number of bins in the x axis
    @param numYBins    The number of bins in the y axis
    @param vectorField The field of vectors (mapped to an array), with normalized magnitude within the field
    @param seed     The random number seed
*/
__kernel void particleMove(  __global float2*   vertex    // position of each particle box
                           , float              dT          // Range from [0, 1]
                           , int numXBins, int  numYBins    // The size of the vector field
                           , __global float2*   vectorField // The data in the grid
                           , __global ulong*    seed        // A randomizer
                           )
{
    // The global id of the work item.  (the index i)
    int idx = get_global_id(0);

    // The positions are actually vertices (4)
    int idx2 = 2*idx;

    // The left two are the key ones
    // Get the particle position
    float2 position = vertex[idx2];
    float2 _v = vectorField[(int)position.x + ((int) position.y)*numXBins];
    float2 v =_v*dT;
    float2 origPosition = position;
    position = position+v;

    float2 position_t;
    // This to handle wrap around on either edge as elegantly as possible
    if (position.x < 0.0 && v.x < 0.0)
    {
        // We wrapped around to the "east end of the world"
        position.x = numXBins-0.1;

        // Sanity check the y coordinate (otherwise we can misindex)
        if (position.y >= numYBins) position.y = numYBins-0.5;
        else if (position.y < 0.0) position.y=0.0;

        position_t = position + v;
    }
    else if (position.x > numXBins && v.x > 0.0)
    {
        // We wrapped around to the "west end of the world"
        position.x =0.0;
        position.y = 0.5*(position.y+origPosition.y);
        // Sanity check the y coordinate (otherwise we can misindex)
        if (position.y >= numYBins) position.y = numYBins-0.5;
        else if (position.y < 0.0) position.y=0.0;
        position_t = position + v;
    }

    // Did the particle go out of bounds?
    else if (position.y< 0.0 || position.y >= numYBins-1.0)
    {
        // The particle is moving too fast or too slow; get rid of it
        position_t = particleRandomize(numXBins, numYBins, seed);
        position = position_t;
    }
    else
    {
        // vector at current position
        float m = _v.x*_v.x + _v.y*_v.y;

        position_t = position + v;

        if (m < 2.0)
        {
            // The particle is moving too fast or too slow; get rid of it
            position_t = particleRandomize(numXBins, numYBins, seed);
            position = position_t;
        }
    }

    // Path from (x,y) to (xt,yt) is visible, so add this particle to the appropriate draw bucket.
    vertex[idx2]   = position;
    vertex[idx2+1] = position_t;
}

/*
    Move the particles
    Copyright (c) 2013-2014, Randall Maas

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.

    Inpsired by Cameron Beccario, https://github.com/cambecc/earth
*/

#define M_PI   3.1415926535897932384626433832795f

// --- Randomize particle location -----------------------
/**Returns a random number
   @param seed The Seed to the random number generator
   @param max  The max number to return
   @returns A random number
 */
float random(__global ulong* seed, uint max)
{
    *seed = (*seed * 0x5DEECE66DL + 0xBL) & ((1L << 48) - 1);
    uint result = (*seed) >> 16;
    return result % max;
}

/** Find a random location for a particle
    @param numXBins The number of bins in the x axis
    @param numYBins The number of bins in the y axis
    @param seed     The random number seed
    @returns A {float2} vector
*/
float2 particleRandomize(
                       int numXBins, int numYBins,
                       __global ulong* seed
                       )
{
    float n = (random(seed, numYBins*10-1)/(numYBins*10.0f))*M_PI;
    float c = cos(n);
    float y = c*88.0f+91.0f;
    y *= (numYBins-2.0)/180.0f;
    float x =random(seed, (numXBins-1)*10)/10.0;
    return (float2){x,y};
}

/** This randomizes a single particle
    @param positions   Set of vectors for each particle
    @param positions_t The set of new vectors for each particle
    @param numXBins The number of bins in the x axis
    @param numYBins The number of bins in the y axis
    @param dataGrid The grid of vectors (mapped to an array)
    @param seed     The random number seed
 */
__kernel void particleInit( __global float2*   vertex   // position of each particle
                          , int numXBins, int  numYBins    // The size of the data grid
                          , __global ulong*    seed        // A randomizer
                          )
{
    // The global id of the work item.  (the index i)
    int idx = get_global_id(0);
    // Particle isn't visible, but it still moves through the field.
    // particle has escaped the grid, never to return...
    float2 position = particleRandomize(numXBins, numYBins, seed);

    // Path from (x,y) to (xt,yt) is visible, so add this particle to the appropriate draw bucket.
    // The positions are actually vertices (2)
    int idx2 = 2*idx;
    vertex[idx2  ] = position;  // tail
    vertex[idx2+1] = position;  // head
}

/** This moves a single particle
    @param positions   Set of vectors for each particle
    @param positions_t The set of new vectors for each particle
    @param numXBins    The number of bins in the x axis
    @param numYBins    The number of bins in the y axis
    @param vectorField The field of vectors (mapped to an array), with normalized magnitude within the field
    @param seed     The random number seed
*/
__kernel void particleMove(  __global float2*   vertex    // position of each particle box
                           , float              dT          // Range from [0, 1]
                           , int numXBins, int  numYBins    // The size of the vector field
                           , __global float2*   vectorField // The data in the grid
                           , __global ulong*    seed        // A randomizer
                           )
{
    // The global id of the work item.  (the index i)
    int idx = get_global_id(0);

    // The positions are actually vertices (4)
    int idx2 = 2*idx;

    // The left two are the key ones
    // Get the particle position
    float2 position = vertex[idx2]+dT*(vertex[idx2+1]-vertex[idx2]);
    float2 position_t;
    // This to handle wrap around on either edge as elegantly as possible
    if (position.x <= 0.0)
    {
        // We wrapped around
        float2 p = vertex[idx2+1];

        position.x = numXBins-0.01;
        // We could calculate the actual y position but this works better for now
        position.y = (position.y+p.y)*0.5;
    }
    else if (position.x >= numXBins-0.1)
    {
        float2 p = vertex[idx2+1];
        position.x = 0.0;
        // We could calculate the actual y position but this works better for now
        position.y = (position.y+p.y)*0.5;
    }

    // Did the particle go out of bounds?
    if (position.y< 0.0 || position.y >= numYBins-1.0)
    {
        // The particle is moving too fast or too slow; get rid of it
        position_t = particleRandomize(numXBins, numYBins, seed);
        position = position_t;
    }
    else
    {
        // vector at current position
        float2 v = vectorField[(int)position.x + ((int) position.y)*numXBins];
        float m = v.x*v.x + v.y*v.y;

        position_t = position + v;

        if (m <1.0)
        {
            // The particle is moving too fast or too slow; get rid of it
            position_t = particleRandomize(numXBins, numYBins, seed);
            position = position_t;
        }
    }

    // Path from (x,y) to (xt,yt) is visible, so add this particle to the appropriate draw bucket.
    vertex[idx2]   = position;
    vertex[idx2+1] = position_t;
}


Update bwa_hmm_opencl.cl

Update bwa_hmm_opencl.cl

/*
 * Copyright July 29, 2011 by Virginia Polytechnic Institute and State University
 * All rights reserved.
 *
 * Virginia Polytechnic Institute and State University (Virginia Tech) owns the
 * OpenCL and the 13 Dwarfs software and its associated documentation (Software).
 * You should carefully read the following terms and conditions before using this
 * software.  Your use of this Software indicates your acceptance of this license
 * agreement and all terms and conditions.
 *
 * You are hereby licensed to use the Software for Non-Commercial Purpose only.
 * Non-Commercial Purpose means the use of the Software solely for research.
 * Non-Commercial Purpose excludes, without limitation, any use of the Software, as
 * part of, or in any way in connection with a product or service which is sold,
 * offered for sale, licensed, leased, loaned, or rented.  Permission to use, copy,
 * modify, and distribute this compilation for Non-Commercial Purpose is hereby
 * granted without fee, subject to the following terms of this license.
 */

/************************************************************************/
/* OpenCL Kernels for BW Algorithm in Hidden Markov Model               */
/*                                                                      */
/************************************************************************/
#define SDOT_BLOCK_SIZE        (128)
#define SDOT_BLOCK_NUM         (80)

#define MVMUL_BLOCK_SIZE       (128)
#define MVMUL_BLOCK_NUM        (64)
#define TILE_SIZE              (32)

/* Initialize ones vector */
__kernel void init_ones_dev(  __global float *ones_s_d,  /* lth = nsymbols */
		int nsymbols)              /* number of symbols */
{
	//unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nsymbols)
	{
		ones_s_d[idx] = 1.0f;
	}
}

/* Initialize alpha variables */
__kernel void init_alpha_dev( __global float *b_d,      /* dim = nsymbols x nstates */
		__global  float *pi_d,    /* lth = nstates */
		int nstates,              /* number of states */
		__global float *alpha_d,  /* dim = length x nstates */
		__global float *ones_n_d, /* lth = nstates */
		int obs_t)                /* current observation */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		alpha_d[idx] = pi_d[idx] * b_d[(obs_t * nstates) + idx];
		ones_n_d[idx] = 1.0f;
	}
}

/* Calculate alpha variables */
__kernel void calc_alpha_dev( int nstates,             /* number of states */
		__global float *alpha_d, /* dim = length x nstates */
		int offset,              /* offset for alpha_d */
		__global float *b_d,     /* dim = nsymbols x nstates */
		int obs_t)               /* current observation */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		alpha_d[offset + idx] = alpha_d[offset + idx] * b_d[(obs_t * nstates) + idx];
	}
}

/* Scale alpha values */
__kernel void scale_alpha_dev( int nstates,             /* number of states */
		__global float *alpha_d, /* dim = length x nstates */
		int offset,              /* offset for alpha_d */
		float scale)             /* scaling value */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		alpha_d[offset + idx] = alpha_d[offset + idx] / scale;
	}
}

/* Initialize beta values */
__kernel void init_beta_dev(  int nstates,            /* number of states */
		__global float *beta_d, /* dim = length x nstates */
		int offset,             /* offset for beta_d */
		float scale)            /* scaling value */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		beta_d[offset + idx] = 1.0f / scale;
	}
}

/* Calculate beta variables */
__kernel void calc_beta_dev( __global float *beta_d, /* dim = length x nstates */
		__global float *b_d,    /* dim = nsymbols x nstates */
		float scale_t,          /* current scaling value */
		int nstates,            /* number of states */
		int obs_t,              /* current observation */
		int t)                  /* current time */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		beta_d[(t * nstates) + idx] = beta_d[((t + 1) * nstates) + idx] *
			b_d[(obs_t * nstates) + idx] / scale_t;
	}
}

/* Sum next iteration of gamma variables */
__kernel void calc_gamma_dev(__global float *gamma_sum_d, /* lth = nstates */
		__global float *alpha_d,     /* dim = length x nstates */
		__global float *beta_d,      /* dim = length x nstates */
		int nstates,                 /* number of states */
		int t)                       /* current time */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		gamma_sum_d[idx] += alpha_d[(t * nstates) + idx] *
			beta_d[(t * nstates) + idx];
	}
}

/* Sum next iteration of xi variables */
__kernel void calc_xi_dev(   __global float *xi_sum_d,  /* dim = nstates x nstates */
		__global float *a_d,       /* dim = nstates x nstates */
		__global float *b_d,       /* dim = nsymbols x nstates */
		__global float *alpha_d,   /* dim = length x nstates */
		__global float *beta_d,    /* dim = length x nstates */
		float sum_ab,              /* sum dot production of (alpha_d+t*nstates) and (beta_d+t*nstates) */
		int nstates,               /* number of states */
		int obs_t,                 /* next observation at t + 1 */
		int t)                     /* current time */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);
	// unsigned int idy = blockIdx.y * blockDim.y + threadIdx.y;
	unsigned int idy = get_group_id(1) * get_local_size(1) + get_local_id(1);

	if (idx < nstates && idy < nstates)
	{
		xi_sum_d[(idy * nstates) + idx] += alpha_d[(t * nstates) + idy] *
			a_d[(idy * nstates) + idx] *
			b_d[(obs_t * nstates) + idx] *
			beta_d[((t+1) * nstates) + idx] /
			sum_ab;
	}
}

/* Re-estimate A matrix */
__kernel void est_a_dev(  __global float *a_d,          /* dim = nstates x nstates */
		__global float *alpha_d,      /* dim = length x nstates */
		__global float *beta_d,       /* dim = length x nstates */
		__global float *xi_sum_d,     /* dim = nstates x nstates */
		__global float *gamma_sum_d,  /* lth = nstates */
		float sum_ab,                 /* sum of dot production of (alpha_d+(length-1)*nstates) and  */
		/* (beta_d+(length-1)*nstates)  */
		int nstates,                  /* number of states */
		int length)                   /* number of observations */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);
	// unsigned int idy = blockIdx.y * blockDim.y + threadIdx.y;
	unsigned int idy = get_group_id(1) * get_local_size(1) + get_local_id(1);

	if (idx < nstates && idy < nstates)
	{
		a_d[(idy * nstates) + idx] = xi_sum_d[(idy * nstates) + idx] /
			(gamma_sum_d[idy] -
			 alpha_d[(idy * nstates) + idx] *
			 beta_d[(idy * nstates) + idx] /
			 sum_ab);
	}
}

/* Normalize A matrix */
__kernel void scale_a_dev(  __global  float *a_d, /* dim = nstates x nstates */
		__global  float *c_d, /* lth = nstates, matrix-vector product of a_d and ones_n_d */
		int nstates)          /* number of states */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);
	// unsigned int idy = blockIdx.y * blockDim.y + threadIdx.y;
	unsigned int idy = get_group_id(1) * get_local_size(1) + get_local_id(1);

	if (idx < nstates && idy < nstates)
	{
		a_d[(idy * nstates) + idx] = a_d[(idy * nstates) + idx] / c_d[idy];
	}
}

/* Accumulate B values */
__kernel void acc_b_dev( __global float *b_d,      /* dim = nsymbols x nstates */
		__global float *alpha_d,  /* dim = length x nstates */
		__global float *beta_d,   /* dim = length x nstates */
		float sum_ab,             /* sum dot production of (alpha_d+t*nstates) and (beta_d+t*nstates) */
		int nstates,              /* number of states */
		int nsymbols,             /* number of symbols */
		int obs_t,                /* current observation */
		int t)                    /* current time */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);
	// unsigned int idy = blockIdx.y * blockDim.y + threadIdx.y;
	unsigned int idy = get_group_id(1) * get_local_size(1) + get_local_id(1);

	if (idy < nsymbols && idx < nstates && obs_t == idy)
	{
		b_d[(idy * nstates) + idx] += alpha_d[(t * nstates) + idx] *
			beta_d[(t * nstates) + idx] / sum_ab;
	}
}

/* Re-estimate B values */
__kernel void est_b_dev( __global float *b_d,         /* dim = nsymbols x nstates */
		__global float *gamma_sum_d, /* lth = nstates */
		int nstates,                 /* number of states */
		int nsymbols)                /* number of symbols */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);
	// unsigned int idy = blockIdx.y * blockDim.y + threadIdx.y;
	unsigned int idy = get_group_id(1) * get_local_size(1) + get_local_id(1);

	if (idy < nsymbols && idx < nstates)
	{
		b_d[(idy * nstates) + idx] = b_d[(idy * nstates) + idx] /
			gamma_sum_d[idx];
	}
}

/* Normalize B matrix */
__kernel void scale_b_dev(  __global float *b_d, /* dim = nsymbols x nstates */
		__global float *c_d, /* lth = nstates, matrix-vector product of b_d and ones_s_d */
		int nstates,         /* number of states */
		int nsymbols)        /* number of symbols */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);
	// unsigned int idy = blockIdx.y * blockDim.y + threadIdx.y;
	unsigned int idy = get_group_id(1) * get_local_size(1) + get_local_id(1);

	if (idx < nstates && idy < nsymbols)
	{
		if (b_d[(idx * nsymbols) + idy] == 0)
		{
			b_d[(idx * nsymbols) + idy] = 1e-10;
		}
		else
		{
			b_d[(idx * nsymbols) + idy] = b_d[(idx * nsymbols) + idy] / c_d[idx];
		}
	}
}

/* Re-estimate Pi values */
__kernel void est_pi_dev(__global float *pi_d,    /* lth = nstates */
		__global float *alpha_d, /* dim = length x nstates */
		__global float *beta_d,  /* dim = length x nstates */
		float sum_ab,            /* sum dot production of alpha_d and beta_d */
		int nstates)             /* number of states */
{
	// unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;
	unsigned int idx = get_group_id(0) * get_local_size(0) + get_local_id(0);

	if (idx < nstates)
	{
		pi_d[idx] = alpha_d[idx] * beta_d[idx] / sum_ab;
	}
}

/* dot production naive version */
__kernel void s_dot_kernel_naive(int n,                        /* number of elements */
		__global float *paramA,       /* first vector A */
		int offsetA,                  /* offset of vector A */
		__global float *paramB,       /* second vector B */
		int offsetB,                  /* offset of vector B */
		__global float *partialSum_d) /* memory space for partial sum */
{
	unsigned int i;
	unsigned int tid = get_local_id(0);
	unsigned int totalThreads = get_num_groups(0) * SDOT_BLOCK_SIZE;
	unsigned int offset = SDOT_BLOCK_SIZE * get_group_id(0);

	for(i = offset + tid; i < n; i += totalThreads)
	{
		partialSum_d[i] = paramA[offsetA + i] * paramB[offsetB + i];
	}
}

/* mat x vec naive verion */
__kernel void mvm_non_kernel_naive(int m,
		int n,
		__global float *A,
		int lda,
		__global float *x,
		int offsetX,
		__global float *y,
		int offsetY)
{
	unsigned int i, j;
	unsigned int tid = get_local_id(0);
	unsigned int totalThreads = get_num_groups(0) * MVMUL_BLOCK_SIZE;
	unsigned int offset = MVMUL_BLOCK_SIZE * get_group_id(0);
	int n_size, m_size;

	float sum;
	if(lda == m)
	{
		n_size = n;
		m_size = m;
	} else
	{
		n_size = m;
		m_size = n;
	}

	for(i = offset + tid; i < m_size; i += totalThreads)
	{
		sum = 0.f;
		for(j = 0; j < n_size; j++)
		{
			sum += A[i * n_size + j] * x[j+offsetX];
		}
		y[i+offsetY] = sum;
	}

}

/* mat x vec transposed naive verion */
__kernel void mvm_trans_kernel_naive(int m,
		int n,
		__global float *A,
		int lda,
		__global float *x,
		int offsetX,
		__global float *y,
		int offsetY)
{
	unsigned int i, j;
	unsigned int tid = get_local_id(0);
	unsigned int totalThreads = get_num_groups(0) * MVMUL_BLOCK_SIZE;
	unsigned int offset = MVMUL_BLOCK_SIZE * get_group_id(0);
	int n_size, m_size;

	float sum;
	if(lda == m)
	{
		n_size = n;
		m_size = m;
	} else
	{
		n_size = m;
		m_size = n;
	}

	for(i = offset + tid; i < m_size; i += totalThreads)
	{
		sum = 0.f;
		for(j = 0; j < n_size; j++)
		{
			sum += A[j * n_size + i] * x[j+offsetX];
		}
		y[i+offsetY] = sum;
	}

}


fixes fabs call in hessian kernel

fixes fabs call in hessian kernel

__kernel void hessian( __global float *xx, __global float* xy, __global float* yy, __global float* out, float sigmaD )
{
    int i = get_global_id(0);
    out[i] = fabs(((xx[i] * yy[i] - pow(xy[i],2)) / sigmaD ));
}

__kernel void find_keypoints(
        __global float *source_determinants,
        __global int* corner_counts,
        __global ushort* keypoints,
        __global int* hessian_determinant_indices,
        int width,
        int height )
{
    int i = get_global_id(0);

    keypoints[i] = 0;

    if( corner_counts[i] )
    {
        for( int j = 0; j < SCALE_COUNT; j++ )
        {
            float before = 0;
            float after = 0;
            float current = source_determinants[hessian_determinant_indices[j] * width * height + i];

            if( j > 0 )
            {
                before = source_determinants[hessian_determinant_indices[j-1] * width * height + i];
            }

            if( j < SCALE_COUNT - 1 )
            {
                after = source_determinants[hessian_determinant_indices[j+1] * width * height + i];
            }

            if( current > before && current > after && current > 10.0f )
            {
                keypoints[i] |= (1UL << j);
            }
        }
    }
}

__kernel void hessian( __global float *xx, __global float* xy, __global float* yy, __global float* out, float sigmaD )
{
    int i = get_global_id(0);
    out[i] = abs(((xx[i] * yy[i] - pow(xy[i],2)) / sigmaD ));
}

__kernel void find_keypoints(
        __global float *source_determinants,
        __global int* corner_counts,
        __global ushort* keypoints,
        __global int* hessian_determinant_indices,
        int width,
        int height )
{
    int i = get_global_id(0);

    keypoints[i] = 0;

    if( corner_counts[i] )
    {
        for( int j = 0; j < SCALE_COUNT; j++ )
        {
            float before = 0;
            float after = 0;
            float current = source_determinants[hessian_determinant_indices[j] * width * height + i];

            if( j > 0 )
            {
                before = source_determinants[hessian_determinant_indices[j-1] * width * height + i];
            }

            if( j < SCALE_COUNT - 1 )
            {
                after = source_determinants[hessian_determinant_indices[j+1] * width * height + i];
            }

            if( current > before && current > after && current > 10.0f )
            {
                keypoints[i] |= (1UL << j);
            }
        }
    }
}